Читайте также:
|
1. Scikit-learn: Machine Learning in Python, Pedregosa et al., JMLR 12, 2011, pp. 2825-2830.
2. C# 5.0 in a Nutshell 5th Edition, Albahari J., Albahari B., 2012, pp.319 – 423.
3. Нейронные сети. Полный курс, Хайкин С., Издательский дом "Вильямс", 2006, с. 417 – 458
4. Python manual [Электронный ресурс] - https://docs.python.org/3/
5. Причинно-следственный анализ [Электронный ресурс] - http://www.ideationtriz.com/ZZLab/DE/Cause_effect_analysis.htm
6. Инвестиции – Учебник, Уильям Ф. Шарп, Инфра-М, 2001
Приложение 1. Система предобработки информации.
Program.cs
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Xml.Linq;
namespace FactEventAnalyzer
{
public struct Parameters
{
public double ChangeThreshold;
public int SecIdFrom;
public int SecIdTo;
public int DateFrom;
public int DateTo;
public double SpikeThreshold;
public Logger fileLogger;
public int minerDateThreshold;
public string folderForMLFiles;
}
class Program
{
static void Main()
{
DateTime startDate = System.DateTime.Now;
Parameters curParam = readInitialParams(AppDomain.CurrentDomain.BaseDirectory);
curParam.fileLogger.writeToLog("Fact-Event analysis tool started");
FactMiner.runFactMiner(CurParam: curParam);
curParam.fileLogger.writeToLog("Fact-Event analysis tool completed, it took " + (System.DateTime.Now - startDate).TotalSeconds.ToString() + " seconds.");
}
static Parameters readInitialParams(string paramPass)
{
Parameters curParam = new Parameters();
XElement config = XElement.Load(paramPass + "\\Config.xml");
XElement LogConfig = config.Element("logging").Element("logFolder");
curParam.fileLogger = new Logger(LogConfig.Attribute("path").Value.ToString());
XElement factMinerSettings = config.Element("factMinerSettings");
curParam.ChangeThreshold = (double) factMinerSettings.Element("changeThreshold").Attribute("value");
curParam.SecIdFrom = (int)factMinerSettings.Element("securities").Attribute("from");
curParam.SecIdTo = (int)factMinerSettings.Element("securities").Attribute("to");
curParam.DateFrom = (int)factMinerSettings.Element("dates").Attribute("from");
curParam.DateTo = (int)factMinerSettings.Element("dates").Attribute("to");
curParam.SpikeThreshold = (double)factMinerSettings.Element("spikeThreshold").Attribute("value");
curParam.minerDateThreshold = (int)factMinerSettings.Element("dateThreshold").Attribute("value");
curParam.folderForMLFiles = (string)factMinerSettings.Element("filesForMl").Attribute("value");
string copyConfigFile = paramPass+"\\Log\\Config" + curParam.fileLogger.composeDateString(mode: 0) + ".xml";
System.IO.File.Copy(paramPass + "\\Config.xml", copyConfigFile);
curParam.fileLogger.writeToLog("Config loaded, copy of loaded config file:\n " + copyConfigFile);
return curParam;
}
}
}
FactMiner.cs
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Reflection;
namespace FactEventAnalyzer
{
static class FactMiner
{
enum ChangeType
{
Growing = 0,
Decreasing = 1,
Unchanged = 2,
PosSpike = 3,
NegSpike = 4
}
enum FieldType
{
Ask = 0,
Bid = 1,
Price = 2
}
public static void runFactMiner(Parameters CurParam)
{
DateTime startDate = DateTime.Now;
CurParam.fileLogger.writeToLog("Mining of Facts started");
FactEventAnalysisDBDataContext dbContext = new FactEventAnalysisDBDataContext();
dbContext.usp_clearFactsTables(dateFrom: CurParam.DateFrom, dateTo: CurParam.DateTo, secIdFrom: CurParam.SecIdFrom, secIdTo: CurParam.SecIdTo, isPrices: true, isCorax: false, isSpike: true);
CurParam.fileLogger.writeToLog("Price Facts and Spike Facts are cleared, it took " + (DateTime.Now - startDate).TotalSeconds);
var SecurityQuery = from security in dbContext.SecuritiesSources
orderby security.SecId
where security.SecId >= CurParam.SecIdFrom && security.SecId <= CurParam.SecIdTo && dbContext.PricesSources.Any(p => (p.SecId == security.SecId && p.DateTick >= CurParam.DateFrom && p.DateTick <= CurParam.DateTo))
select security;
int totalSecs = SecurityQuery.Count();
int curSecNum = 0;
foreach (FactEventAnalyzer.SecuritiesSource curSec in SecurityQuery)
{
curSecNum++;
CurParam.fileLogger.writeToLog(curSecNum.ToString() + "/" + totalSecs.ToString() + ", SecId is " + curSec.SecId);
analyzePriceField(DbContext: dbContext, CurSec: curSec, curParam: CurParam);
CurParam.fileLogger.writeToLog("Price completed");
analyzeAskField(DbContext: dbContext, CurSec: curSec, curParam: CurParam);
CurParam.fileLogger.writeToLog("Ask completed");
analyzeBidField(DbContext: dbContext, CurSec: curSec, curParam: CurParam);
CurParam.fileLogger.writeToLog("Bid completed");
if (CurParam.SpikeThreshold > 0)
{
produceSpikeFacts(DbContext: dbContext, CurSec: curSec, curParam: CurParam);
CurParam.fileLogger.writeToLog("Spikes analyzed");
}
}
CurParam.fileLogger.writeToLog("Price Facts are produced.\n Start time: " + startDate.ToString() +
"\n Finish time: " + System.DateTime.Now.ToString() +
"\n Securities processed: " + totalSecs.ToString() +
"\n Total runtime: " + (System.DateTime.Now - startDate).TotalSeconds.ToString());
CurParam.fileLogger.writeToLog("Starting Corax Facts production");
produceCoraxFacts(curParam: CurParam);
CurParam.fileLogger.writeToLog("Corax Facts are produced.Starting to save results to file.\n");
var hypothesisQuery = from hypothesis in dbContext.PriceFactsHypothesis
select hypothesis;
string resultsFile = CurParam.folderForMLFiles + "\\PriceFacts_" + CurParam.fileLogger.composeDateString(mode: 0) + ".csv";
string delimiter = ",";
Type type = typeof(PriceFactsHypothesi);
PropertyInfo[] properties = type.GetProperties();
string line = "";
foreach (PropertyInfo property in properties)
{
line += property.Name + delimiter;
}
System.IO.StreamWriter file = new System.IO.StreamWriter(resultsFile, true);
file.WriteLine(line);
file.Close();
foreach (var hypothesis in hypothesisQuery)
{
properties = type.GetProperties();
line = "";
foreach (PropertyInfo property in properties)
{
string value = property.GetValue(hypothesis)!= null
? property.GetValue(hypothesis).ToString()
: string.Empty; // or whatever you want the default text to be
line += value + delimiter;
}
file = new System.IO.StreamWriter(resultsFile, true);
file.WriteLine(line);
file.Close();
}
var hypothesisQuery2 = from hypothesis in dbContext.SpikeFactsHypothesis
select hypothesis;
resultsFile = CurParam.folderForMLFiles + "\\SpikeFacts_" + CurParam.fileLogger.composeDateString(mode: 0) + ".csv";
type = typeof(SpikeFactsHypothesi);
properties = type.GetProperties();
line = "";
foreach (PropertyInfo property in properties)
{
line += property.Name + delimiter;
}
file = new System.IO.StreamWriter(resultsFile, true);
file.WriteLine(line);
file.Close();
foreach (var hypothesis in hypothesisQuery2)
{
properties = type.GetProperties();
line = "";
foreach (PropertyInfo property in properties)
{
string value = property.GetValue(hypothesis)!= null
? property.GetValue(hypothesis).ToString()
: string.Empty; // or whatever you want the default text to be
line += value + delimiter;
}
file = new System.IO.StreamWriter(resultsFile, true);
file.WriteLine(line);
file.Close();
}
CurParam.fileLogger.writeToLog("Fact mining have completed.\n Start time: " + startDate.ToString() +
"\n Finish time: " + System.DateTime.Now.ToString() +
"\n Total runtime of Fact Miner: " + (System.DateTime.Now - startDate).TotalSeconds.ToString());
}
static void analyzePriceField(FactEventAnalysisDBDataContext DbContext, FactEventAnalyzer.SecuritiesSource CurSec, Parameters curParam)
{
var PriceQuery = from price in DbContext.PricesSources
where price.SecId.Equals(CurSec.SecId) && price.DateTick >= curParam.DateFrom && price.DateTick <= curParam.DateTo
orderby price.DateTick
select price;
bool isFirst = true;
var previousPrice = new FactEventAnalyzer.PricesSource();
var startPrice = new FactEventAnalyzer.PricesSource();
var lastPrice = new FactEventAnalyzer.PricesSource();
bool isSkip = false;
bool isSecond = false;
int skipNum = 0;
ChangeType curChange = ChangeType.Unchanged;
foreach (FactEventAnalyzer.PricesSource curPrice in PriceQuery)
{
lastPrice = curPrice;
if (isFirst == true)
{
startPrice = curPrice;
isFirst = false;
isSecond = true;
}
else
{
if (isSecond == true)
{
curChange = determineChangeType(prevPrice: startPrice.Price, curPrice: curPrice.Price, changeThreshold: curParam.ChangeThreshold);
previousPrice = curPrice;
isSecond = false;
}
else
{
if (isSkip == false)
{
if (determineChangeType(prevPrice: previousPrice.Price, curPrice: curPrice.Price, changeThreshold: curParam.ChangeThreshold)!= curChange)
{
isSkip = true;
}
else
{
previousPrice = curPrice;
}
}
else
{
if (skipNum < (curParam.minerDateThreshold - 1))
{
skipNum++;
}
else
{
skipNum = 0;
isSkip = false;
if (determineChangeType(prevPrice: previousPrice.Price, curPrice: curPrice.Price, changeThreshold: curParam.ChangeThreshold)!= curChange)
{
insertNewPriceFact(curChange, dateStart: startPrice.DateTick, dateEnd: previousPrice.DateTick, secId: CurSec.SecId, fieldType: FieldType.Price);
curChange = determineChangeType(prevPrice: previousPrice.Price, curPrice: curPrice.Price, changeThreshold: curParam.ChangeThreshold);
startPrice = previousPrice;
previousPrice = curPrice;
}
else
{
previousPrice = curPrice;
}
}
}
}
}
}
if (isSkip == true)
{
insertNewPriceFact(changeType: curChange, dateStart: startPrice.DateTick, dateEnd: previousPrice.DateTick, secId: CurSec.SecId, fieldType: FieldType.Price);
insertNewPriceFact(changeType: determineChangeType(previousPrice.Price, curPrice: lastPrice.Price, changeThreshold: curParam.ChangeThreshold), dateStart: previousPrice.DateTick, dateEnd: lastPrice.DateTick, secId: CurSec.SecId, fieldType: FieldType.Price);
}
else
{
if (isFirst!= true)
{
insertNewPriceFact(changeType: curChange, dateStart: startPrice.DateTick, dateEnd: lastPrice.DateTick, secId: CurSec.SecId, fieldType: FieldType.Price);
}
}
}
static void analyzeAskField(FactEventAnalysisDBDataContext DbContext, FactEventAnalyzer.SecuritiesSource CurSec, Parameters curParam)
{
var PriceQuery = from price in DbContext.PricesSources
where price.SecId.Equals(CurSec.SecId) && price.DateTick >= curParam.DateFrom && price.DateTick <= curParam.DateTo
orderby price.DateTick
select price;
bool isFirst = true;
var previousPrice = new FactEventAnalyzer.PricesSource();
var startPrice = new FactEventAnalyzer.PricesSource();
var lastPrice = new FactEventAnalyzer.PricesSource();
bool isSkip = false;
bool isSecond = false;
int skipNum = 0;
ChangeType curChange = ChangeType.Unchanged;
foreach (FactEventAnalyzer.PricesSource curPrice in PriceQuery)
{
lastPrice = curPrice;
if (isFirst == true)
{
startPrice = curPrice;
isFirst = false;
isSecond = true;
}
else
{
if (isSecond == true)
{
curChange = determineChangeType(prevPrice: startPrice.AskPrice, curPrice: curPrice.AskPrice, changeThreshold: curParam.ChangeThreshold);
previousPrice = curPrice;
isSecond = false;
}
else
{
if (isSkip == false)
{
if (determineChangeType(prevPrice: previousPrice.AskPrice, curPrice: curPrice.AskPrice, changeThreshold: curParam.ChangeThreshold)!= curChange)
{
isSkip = true;
}
else
{
previousPrice = curPrice;
}
}
else
{
if (skipNum < (curParam.minerDateThreshold - 1))
{
skipNum++;
}
else
{
skipNum = 0;
isSkip = false;
if (determineChangeType(prevPrice: previousPrice.AskPrice, curPrice: curPrice.AskPrice, changeThreshold: curParam.ChangeThreshold)!= curChange)
{
insertNewPriceFact(curChange, dateStart: startPrice.DateTick, dateEnd: previousPrice.DateTick, secId: CurSec.SecId, fieldType: FieldType.Ask);
curChange = determineChangeType(prevPrice: previousPrice.AskPrice, curPrice: curPrice.AskPrice, changeThreshold: curParam.ChangeThreshold);
startPrice = previousPrice;
previousPrice = curPrice;
}
else
{
previousPrice = curPrice;
}
}
}
}
}
}
if (isSkip == true)
{
insertNewPriceFact(changeType: curChange, dateStart: startPrice.DateTick, dateEnd: previousPrice.DateTick, secId: CurSec.SecId, fieldType: FieldType.Ask);
insertNewPriceFact(changeType: determineChangeType(previousPrice.AskPrice, curPrice: lastPrice.AskPrice, changeThreshold: curParam.ChangeThreshold), dateStart: previousPrice.DateTick, dateEnd: lastPrice.DateTick, secId: CurSec.SecId, fieldType: FieldType.Ask);
}
else
{
if (isFirst!= true)
{
insertNewPriceFact(changeType: curChange, dateStart: startPrice.DateTick, dateEnd: lastPrice.DateTick, secId: CurSec.SecId, fieldType: FieldType.Ask);
}
}
}
static void analyzeBidField(FactEventAnalysisDBDataContext DbContext, FactEventAnalyzer.SecuritiesSource CurSec, Parameters curParam)
{
var PriceQuery = from price in DbContext.PricesSources
where price.SecId.Equals(CurSec.SecId) && price.DateTick >= curParam.DateFrom && price.DateTick <= curParam.DateTo
orderby price.DateTick
select price;
bool isFirst = true;
var previousPrice = new FactEventAnalyzer.PricesSource();
var startPrice = new FactEventAnalyzer.PricesSource();
var lastPrice = new FactEventAnalyzer.PricesSource();
bool isSkip = false;
bool isSecond = false;
int skipNum = 1;
ChangeType curChange = ChangeType.Unchanged;
foreach (FactEventAnalyzer.PricesSource curPrice in PriceQuery)
{
lastPrice = curPrice;
if (isFirst == true)
{
startPrice = curPrice;
isFirst = false;
isSecond = true;
}
else
{
if (isSecond == true)
{
curChange = determineChangeType(prevPrice: startPrice.BidPrice, curPrice: curPrice.BidPrice, changeThreshold: curParam.ChangeThreshold);
previousPrice = curPrice;
isSecond = false;
}
else
{
if (isSkip == false)
{
if (determineChangeType(prevPrice: previousPrice.BidPrice, curPrice: curPrice.BidPrice, changeThreshold: curParam.ChangeThreshold)!= curChange)
{
isSkip = true;
}
else
{
previousPrice = curPrice;
}
}
else
{
if (skipNum < (curParam.minerDateThreshold - 1))
{
skipNum++;
}
else
{
skipNum = 1;
isSkip = false;
if (determineChangeType(prevPrice: previousPrice.BidPrice, curPrice: curPrice.BidPrice, changeThreshold: curParam.ChangeThreshold)!= curChange)
{
insertNewPriceFact(curChange, dateStart: startPrice.DateTick, dateEnd: previousPrice.DateTick, secId: CurSec.SecId, fieldType: FieldType.Bid);
startPrice = previousPrice;
previousPrice = curPrice;
curChange = determineChangeType(prevPrice: startPrice.BidPrice, curPrice: previousPrice.BidPrice, changeThreshold: curParam.ChangeThreshold);
}
else
{
previousPrice = curPrice;
}
}
}
}
}
}
if (isSkip == true)
{
insertNewPriceFact(changeType: curChange, dateStart: startPrice.DateTick, dateEnd: previousPrice.DateTick, secId: CurSec.SecId, fieldType: FieldType.Bid);
insertNewPriceFact(changeType: determineChangeType(previousPrice.BidPrice, curPrice: lastPrice.BidPrice, changeThreshold: curParam.ChangeThreshold), dateStart: previousPrice.DateTick, dateEnd: lastPrice.DateTick, secId: CurSec.SecId, fieldType: FieldType.Bid);
}
else
{
if (isFirst!= true)
{
insertNewPriceFact(changeType: curChange, dateStart: startPrice.DateTick, dateEnd: lastPrice.DateTick, secId: CurSec.SecId, fieldType: FieldType.Bid);
}
}
}
static void insertNewPriceFact(ChangeType changeType, int dateStart, int dateEnd, int secId, FieldType fieldType)
{
FactEventAnalysisDBDataContext dbContext = new FactEventAnalysisDBDataContext();
FactEventAnalyzer.PriceFactsTable newFact = new FactEventAnalyzer.PriceFactsTable();
newFact.ChangeType = changeType.ToString();
newFact.DateStart = dateStart;
newFact.DateEnd = dateEnd;
newFact.SecID = secId;
newFact.FieldType = fieldType.ToString();
dbContext.PriceFactsTables.InsertOnSubmit(newFact);
dbContext.SubmitChanges();
}
static void insertNewSpikeFact(ChangeType changeType, int date, int secId, FieldType fieldType)
{
FactEventAnalysisDBDataContext dbContext = new FactEventAnalysisDBDataContext();
FactEventAnalyzer.SpikeFactsTable newSpikeFact = new FactEventAnalyzer.SpikeFactsTable();
newSpikeFact.Date = date;
newSpikeFact.FieldType = fieldType.ToString();
newSpikeFact.SecId = secId;
newSpikeFact.SpikeType = changeType.ToString();
dbContext.SpikeFactsTables.InsertOnSubmit(newSpikeFact);
dbContext.SubmitChanges();
}
static void produceCoraxFacts(Parameters curParam)
{
int? insertedSecs = 0, insertedCoraxes = 0;
DateTime startDate = DateTime.Now;
FactEventAnalysisDBDataContext dbContext = new FactEventAnalysisDBDataContext();
dbContext.usp_clearFactsTables(dateFrom: curParam.DateFrom, dateTo: curParam.DateTo, secIdFrom: curParam.SecIdFrom, secIdTo: curParam.SecIdTo, isPrices: false, isCorax: true, isSpike: false);
curParam.fileLogger.writeToLog("Corax Facts are cleared, it took " + (DateTime.Now - startDate).TotalSeconds);
dbContext.usp_populateCoraxFactsTable(curParam.DateFrom, curParam.DateTo, curParam.SecIdFrom, curParam.SecIdTo, ref insertedSecs, ref insertedCoraxes);
curParam.fileLogger.writeToLog("Corax Facts are produced.\n Start time: " + startDate.ToString() +
"\n Finish time: " + System.DateTime.Now.ToString() +
"\n Securities processed: " + insertedSecs.ToString() +
"\n Facts inserted: " + insertedCoraxes.ToString() +
"\n Total runtime: " + (System.DateTime.Now - startDate).TotalSeconds.ToString());
}
static void produceSpikeFacts(FactEventAnalysisDBDataContext DbContext, FactEventAnalyzer.SecuritiesSource CurSec, Parameters curParam)
{
var PriceQuery = from price in DbContext.PricesSources
where price.SecId.Equals(CurSec.SecId) && price.DateTick >= curParam.DateFrom && price.DateTick <= curParam.DateTo
orderby price.DateTick
select price;
bool isFirst = true;
FactEventAnalyzer.PricesSource prevPrice = new FactEventAnalyzer.PricesSource();
foreach (FactEventAnalyzer.PricesSource curPrice in PriceQuery)
{
if (isFirst == true)
{
prevPrice = curPrice;
isFirst = false;
}
else
{
double changeAmount = (double)(100 * (curPrice.Price - prevPrice.Price) / prevPrice.Price);
if (Math.Abs(changeAmount) > curParam.SpikeThreshold)
{
if (changeAmount < 0)
{
insertNewSpikeFact(changeType: ChangeType.NegSpike, date: curPrice.DateTick, secId: CurSec.SecId, fieldType: FieldType.Price);
}
else
{
insertNewSpikeFact(changeType: ChangeType.PosSpike, date: curPrice.DateTick, secId: CurSec.SecId, fieldType: FieldType.Price);
}
}
changeAmount = (double)(100 * (curPrice.AskPrice - prevPrice.AskPrice) / prevPrice.AskPrice);
if (Math.Abs(changeAmount) > curParam.SpikeThreshold)
{
if (changeAmount < 0)
{
insertNewSpikeFact(changeType: ChangeType.NegSpike, date: curPrice.DateTick, secId: CurSec.SecId, fieldType: FieldType.Ask);
}
else
{
insertNewSpikeFact(changeType: ChangeType.PosSpike, date: curPrice.DateTick, secId: CurSec.SecId, fieldType: FieldType.Ask);
}
}
changeAmount = (double)(100 * (curPrice.BidPrice - prevPrice.BidPrice) / prevPrice.BidPrice);
if (Math.Abs(changeAmount) > curParam.SpikeThreshold)
{
if (changeAmount < 0)
{
insertNewSpikeFact(changeType: ChangeType.NegSpike, date: curPrice.DateTick, secId: CurSec.SecId, fieldType: FieldType.Bid);
}
else
{
insertNewSpikeFact(changeType: ChangeType.PosSpike, date: curPrice.DateTick, secId: CurSec.SecId, fieldType: FieldType.Bid);
}
}
prevPrice = curPrice;
}
}
}
static ChangeType determineChangeType(decimal prevPrice, decimal curPrice, double changeThreshold)
{
double changeAmount = (double)(100 * (curPrice - prevPrice) / prevPrice);
if (Math.Abs(changeAmount) < changeThreshold)
{
return ChangeType.Unchanged;
}
else
{
if (changeAmount < 0)
{
return ChangeType.Decreasing;
}
else
{
return ChangeType.Growing;
}
}
}
}
}
Logger.cs
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace FactEventAnalyzer
{
public class Logger
{
private Logger() { }
string logFile;
public Logger(string logFolder)
{
this.logFile = logFolder + "\\FactMiner_" + composeDateString(mode: 0) + ".log";
}
public string composeDateString(int mode=1)
{
DateTime tempDate = System.DateTime.Now;
if (mode == 0)
{
return (tempDate.Year.ToString() + tempDate.Month.ToString("D2") + tempDate.Day.ToString("D2")
+ "T" + tempDate.Hour.ToString("D2") + tempDate.Minute.ToString("D2") + tempDate.Second.ToString("D2")
+ "_" + tempDate.Millisecond.ToString("D3"));
}
else
{
return (tempDate.Year.ToString() + "." + tempDate.Month.ToString("D2") + "." + tempDate.Day.ToString("D2")
+ "T" + tempDate.Hour.ToString("D2") + ":" + tempDate.Minute.ToString("D2") + ":" + tempDate.Second.ToString("D2")
+ "." + tempDate.Millisecond.ToString("D3"));
}
}
public void writeToLog(string logMessage)
{
System.IO.StreamWriter file = new System.IO.StreamWriter(this.logFile, true);
file.WriteLine(composeDateString() + " " + logMessage);
file.Close();
}
}
}
Дата добавления: 2015-11-16; просмотров: 62 | Нарушение авторских прав
| <== предыдущая страница | | | следующая страница ==> |
| Машинный эксперимент | | | Приложение 2. Файл конфигурации системы предобработки информации |