Hole Fixing modules taken as example
This commit is contained in:
Luis Fernando Molina committed
1 parent
ee0c9fdb92
commit
4f4b80789b
6 files changed
+1994
-429
No files matched your search
Binary file not shown.
+86
-86
@@ -1,89 +1,89 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<SchemaVersion>2.0</SchemaVersion>
|
||||
<ProjectGuid>{045bbaef-3c35-4017-8458-e46aef438790}</ProjectGuid>
|
||||
<ProjectHome />
|
||||
<ProjectView>ShowAllFiles</ProjectView>
|
||||
<StartupFile>Azure File Storage.js</StartupFile>
|
||||
<WorkingDirectory>.</WorkingDirectory>
|
||||
<OutputPath>.</OutputPath>
|
||||
<ProjectTypeGuids>{3AF33F2E-1136-4D97-BBB7-1795711AC8B8};{349c5851-65df-11da-9384-00065b846f21};{9092AA53-FB77-4645-B42D-1CCCA6BD08BD}</ProjectTypeGuids>
|
||||
<VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0</VisualStudioVersion>
|
||||
<VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)' == 'Debug'" />
|
||||
<PropertyGroup Condition="'$(Configuration)' == 'Release'" />
|
||||
<ItemGroup>
|
||||
<Content Include="package-lock.json" />
|
||||
<Content Include="package.json" />
|
||||
<Content Include="README.md" />
|
||||
<Compile Include="Azure File Storage.js" />
|
||||
<Compile Include="Azure Web Job Shutdown.js" />
|
||||
<Compile Include="Debug Log.js" />
|
||||
<Compile Include="Markets.js" />
|
||||
<Compile Include="Poloniex API Client.js" />
|
||||
<Compile Include="Utilities.js" />
|
||||
<Content Include="logs\Dummy File.txt" />
|
||||
<Compile Include="One-Min-Daily-Candles\One Min Daily Candles.js" />
|
||||
<Compile Include="One-Min-Daily-Candles\run.js" />
|
||||
<Content Include="obj\Debug\AACarol.njsproj.FileListAbsolute.txt" />
|
||||
<Content Include="obj\Debug\AACharly.njsproj.FileListAbsolute.txt" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Folder Include="AABruce" />
|
||||
<Folder Include="bin" />
|
||||
<Folder Include="logs" />
|
||||
<Folder Include="logs\AA Masters" />
|
||||
<Folder Include="logs\AA Masters\Connectivity" />
|
||||
<Folder Include="logs\AA Masters\Connectivity\Carol" />
|
||||
<Folder Include="logs\AA Masters\Connectivity\Carol\1.0.0" />
|
||||
<Folder Include="logs\AA Masters\Connectivity\Carol\1.0.0\Order-Books" />
|
||||
<Folder Include="obj" />
|
||||
<Folder Include="obj\Debug" />
|
||||
<Folder Include="One-Min-Daily-Candles" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.Common.targets" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<!--Do not delete the following Import Project. While this appears to do nothing it is a marker for setting TypeScript properties before our import that depends on them.-->
|
||||
<Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\TypeScript\Microsoft.TypeScript.targets" Condition="False" />
|
||||
<Import Project="$(VSToolsPath)\Node.js Tools\Microsoft.NodejsTools.targets" />
|
||||
<ProjectExtensions>
|
||||
<VisualStudio>
|
||||
<FlavorProperties GUID="{349c5851-65df-11da-9384-00065b846f21}">
|
||||
<WebProjectProperties>
|
||||
<UseIIS>False</UseIIS>
|
||||
<AutoAssignPort>True</AutoAssignPort>
|
||||
<DevelopmentServerPort>0</DevelopmentServerPort>
|
||||
<DevelopmentServerVPath>/</DevelopmentServerVPath>
|
||||
<IISUrl>http://localhost:48022/</IISUrl>
|
||||
<NTLMAuthentication>False</NTLMAuthentication>
|
||||
<UseCustomServer>True</UseCustomServer>
|
||||
<CustomServerUrl>http://localhost:1337</CustomServerUrl>
|
||||
<SaveServerSettingsInUserFile>False</SaveServerSettingsInUserFile>
|
||||
</WebProjectProperties>
|
||||
</FlavorProperties>
|
||||
<FlavorProperties GUID="{349c5851-65df-11da-9384-00065b846f21}" User="">
|
||||
<WebProjectProperties>
|
||||
<StartPageUrl>
|
||||
</StartPageUrl>
|
||||
<StartAction>CurrentPage</StartAction>
|
||||
<AspNetDebugging>True</AspNetDebugging>
|
||||
<SilverlightDebugging>False</SilverlightDebugging>
|
||||
<NativeDebugging>False</NativeDebugging>
|
||||
<SQLDebugging>False</SQLDebugging>
|
||||
<ExternalProgram>
|
||||
</ExternalProgram>
|
||||
<StartExternalURL>
|
||||
</StartExternalURL>
|
||||
<StartCmdLineArguments>
|
||||
</StartCmdLineArguments>
|
||||
<StartWorkingDirectory>
|
||||
</StartWorkingDirectory>
|
||||
<EnableENC>False</EnableENC>
|
||||
<AlwaysStartWebServerOnDebug>False</AlwaysStartWebServerOnDebug>
|
||||
</WebProjectProperties>
|
||||
</FlavorProperties>
|
||||
</VisualStudio>
|
||||
</ProjectExtensions>
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<SchemaVersion>2.0</SchemaVersion>
|
||||
<ProjectGuid>{045bbaef-3c35-4017-8458-e46aef438790}</ProjectGuid>
|
||||
<ProjectHome />
|
||||
<ProjectView>ShowAllFiles</ProjectView>
|
||||
<StartupFile>One-Min-Daily-Candles\run.js</StartupFile>
|
||||
<WorkingDirectory>C:\Users\Luis\source\repos\AABruce\One-Min-Daily-Candles\</WorkingDirectory>
|
||||
<OutputPath>.</OutputPath>
|
||||
<ProjectTypeGuids>{3AF33F2E-1136-4D97-BBB7-1795711AC8B8};{349c5851-65df-11da-9384-00065b846f21};{9092AA53-FB77-4645-B42D-1CCCA6BD08BD}</ProjectTypeGuids>
|
||||
<VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0</VisualStudioVersion>
|
||||
<VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)' == 'Debug'" />
|
||||
<PropertyGroup Condition="'$(Configuration)' == 'Release'" />
|
||||
<ItemGroup>
|
||||
<Content Include="package-lock.json" />
|
||||
<Content Include="package.json" />
|
||||
<Content Include="README.md" />
|
||||
<Compile Include="Azure File Storage.js" />
|
||||
<Compile Include="Azure Web Job Shutdown.js" />
|
||||
<Compile Include="Debug Log.js" />
|
||||
<Compile Include="Markets.js" />
|
||||
<Compile Include="One-Min-Daily-Candles\Interval.js" />
|
||||
<Compile Include="Poloniex API Client.js" />
|
||||
<Compile Include="Utilities.js" />
|
||||
<Content Include="logs\Dummy File.txt" />
|
||||
<Compile Include="One-Min-Daily-Candles\One Min Daily Candles.js" />
|
||||
<Compile Include="One-Min-Daily-Candles\run.js" />
|
||||
<Content Include="obj\Debug\AACarol.njsproj.FileListAbsolute.txt" />
|
||||
<Content Include="obj\Debug\AACharly.njsproj.FileListAbsolute.txt" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Folder Include="bin" />
|
||||
<Folder Include="logs" />
|
||||
<Folder Include="logs\AA Masters" />
|
||||
<Folder Include="logs\AA Masters\Connectivity" />
|
||||
<Folder Include="logs\AA Masters\Connectivity\Carol" />
|
||||
<Folder Include="logs\AA Masters\Connectivity\Carol\1.0.0" />
|
||||
<Folder Include="logs\AA Masters\Connectivity\Carol\1.0.0\Order-Books" />
|
||||
<Folder Include="obj" />
|
||||
<Folder Include="obj\Debug" />
|
||||
<Folder Include="One-Min-Daily-Candles" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.Common.targets" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<!--Do not delete the following Import Project. While this appears to do nothing it is a marker for setting TypeScript properties before our import that depends on them.-->
|
||||
<Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\TypeScript\Microsoft.TypeScript.targets" Condition="False" />
|
||||
<Import Project="$(VSToolsPath)\Node.js Tools\Microsoft.NodejsTools.targets" />
|
||||
<ProjectExtensions>
|
||||
<VisualStudio>
|
||||
<FlavorProperties GUID="{349c5851-65df-11da-9384-00065b846f21}">
|
||||
<WebProjectProperties>
|
||||
<UseIIS>False</UseIIS>
|
||||
<AutoAssignPort>True</AutoAssignPort>
|
||||
<DevelopmentServerPort>0</DevelopmentServerPort>
|
||||
<DevelopmentServerVPath>/</DevelopmentServerVPath>
|
||||
<IISUrl>http://localhost:48022/</IISUrl>
|
||||
<NTLMAuthentication>False</NTLMAuthentication>
|
||||
<UseCustomServer>True</UseCustomServer>
|
||||
<CustomServerUrl>http://localhost:1337</CustomServerUrl>
|
||||
<SaveServerSettingsInUserFile>False</SaveServerSettingsInUserFile>
|
||||
</WebProjectProperties>
|
||||
</FlavorProperties>
|
||||
<FlavorProperties GUID="{349c5851-65df-11da-9384-00065b846f21}" User="">
|
||||
<WebProjectProperties>
|
||||
<StartPageUrl>
|
||||
</StartPageUrl>
|
||||
<StartAction>CurrentPage</StartAction>
|
||||
<AspNetDebugging>True</AspNetDebugging>
|
||||
<SilverlightDebugging>False</SilverlightDebugging>
|
||||
<NativeDebugging>False</NativeDebugging>
|
||||
<SQLDebugging>False</SQLDebugging>
|
||||
<ExternalProgram>
|
||||
</ExternalProgram>
|
||||
<StartExternalURL>
|
||||
</StartExternalURL>
|
||||
<StartCmdLineArguments>
|
||||
</StartCmdLineArguments>
|
||||
<StartWorkingDirectory>
|
||||
</StartWorkingDirectory>
|
||||
<EnableENC>False</EnableENC>
|
||||
<AlwaysStartWebServerOnDebug>False</AlwaysStartWebServerOnDebug>
|
||||
</WebProjectProperties>
|
||||
</FlavorProperties>
|
||||
</VisualStudio>
|
||||
</ProjectExtensions>
|
||||
</Project>
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio 15
|
||||
VisualStudioVersion = 15.0.26730.10
|
||||
MinimumVisualStudioVersion = 10.0.40219.1
|
||||
Project("{9092AA53-FB77-4645-B42D-1CCCA6BD08BD}") = "AABruce", "AABruce.njsproj", "{045BBAEF-3C35-4017-8458-E46AEF438790}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
Release|Any CPU = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{045BBAEF-3C35-4017-8458-E46AEF438790}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{045BBAEF-3C35-4017-8458-E46AEF438790}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{045BBAEF-3C35-4017-8458-E46AEF438790}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{045BBAEF-3C35-4017-8458-E46AEF438790}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
EndGlobalSection
|
||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||
SolutionGuid = {2A5094EC-199A-4833-894F-166CB38B4715}
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,66 +1,54 @@
|
||||
|
||||
|
||||
|
||||
exports.newPloniexOrderBooks = function newPloniexOrderBooks(BOT) {
|
||||
exports.newPloniexHoleFixing = function newPloniexHoleFixing(BOT) {
|
||||
|
||||
let bot = BOT;
|
||||
const ROOT_DIR = '../';
|
||||
|
||||
const INTERVAL_LAPSE = 60 * 1000;
|
||||
|
||||
const MODULE_NAME = "Poloniex Order Books";
|
||||
const LOG_INFO = false;
|
||||
const MODULE_NAME = "One Min Daily Candles";
|
||||
const LOG_INFO = true;
|
||||
|
||||
const EXCHANGE_NAME = "Poloniex";
|
||||
const EXCHANGE_ID = 1;
|
||||
|
||||
const RAW_FOLDER_NAME = "order-books";
|
||||
const AGG_FOLDER_NAME = "aggregatted-order-books";
|
||||
|
||||
const ORDER_BOOK_DEPTH = 100000; // This is the max amount of BIDs or ASKs we want the Exchange API to return us from each market.
|
||||
|
||||
const MARKETS_MODULE = require(ROOT_DIR + 'Markets');
|
||||
|
||||
const PloniexClient = require(ROOT_DIR + 'Poloniex API Client');
|
||||
const INTERVAL_MODULE = require('./Interval');
|
||||
|
||||
const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log');
|
||||
const logger = DEBUG_MODULE.newDebugLog();
|
||||
logger.fileName = MODULE_NAME;
|
||||
logger.bot = bot;
|
||||
|
||||
poloniexOrderBooks = {
|
||||
const INTERVAL_MAX_LOOPS = -1; // -1 for infinite loop.
|
||||
|
||||
poloniexHoleFixing = {
|
||||
initialize: initialize,
|
||||
start: start
|
||||
};
|
||||
|
||||
let markets;
|
||||
|
||||
const AZURE_FILE_STORAGE = require(ROOT_DIR + 'Azure File Storage');
|
||||
let azureFileStorage = AZURE_FILE_STORAGE.newAzureFileStorage(bot);
|
||||
const SHUTDOWN_EVENT = require(ROOT_DIR + 'Azure Web Job Shutdown');
|
||||
let shutdownEvent = SHUTDOWN_EVENT.newShutdownEvent(bot);
|
||||
|
||||
const UTILITIES = require(ROOT_DIR + 'Utilities');
|
||||
let utilities = UTILITIES.newUtilities(bot);
|
||||
|
||||
const SHUTDOWN_EVENT = require(ROOT_DIR + 'Azure Web Job Shutdown');
|
||||
let shutdownEvent = SHUTDOWN_EVENT.newShutdownEvent(bot);
|
||||
let year;
|
||||
let month;
|
||||
|
||||
return poloniexOrderBooks;
|
||||
return poloniexHoleFixing;
|
||||
|
||||
function initialize(callBackFunction) {
|
||||
function initialize(yearAssigend, monthAssigned, callBackFunction) {
|
||||
|
||||
try {
|
||||
|
||||
const logText = "[INFO] initialize - Entering function 'initialize' ";
|
||||
year = yearAssigend;
|
||||
month = monthAssigned;
|
||||
|
||||
month = utilities.pad(month, 2); // Adding a left zero when needed.
|
||||
|
||||
logger.fileName = MODULE_NAME + "-" + year + "-" + month;
|
||||
|
||||
const logText = "[INFO] initialize - Entering function 'initialize' " + " @ " + year + "-" + month;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
poloniexApiClient = new PloniexClient();
|
||||
|
||||
azureFileStorage.initialize();
|
||||
|
||||
markets = MARKETS_MODULE.newMarkets(bot);
|
||||
markets.initialize(callBackFunction);
|
||||
|
||||
callBackFunction();
|
||||
|
||||
} catch (err) {
|
||||
|
||||
@@ -72,82 +60,57 @@ exports.newPloniexOrderBooks = function newPloniexOrderBooks(BOT) {
|
||||
}
|
||||
|
||||
|
||||
|
||||
function start() {
|
||||
|
||||
let currentDate;
|
||||
|
||||
try {
|
||||
|
||||
let intervalId = setInterval(requestNewData, INTERVAL_LAPSE);
|
||||
if (LOG_INFO === true) {
|
||||
logger.write("[INFO] Entering function 'start', with year = " + year + " and month = " + month);
|
||||
}
|
||||
|
||||
requestNewData(); // The first execution we do it without waiting for the interval call.
|
||||
let intervalId;
|
||||
|
||||
function requestNewData() {
|
||||
if (INTERVAL_MAX_LOOPS === 0) {
|
||||
|
||||
intervalId = setInterval(startNewInterval, INTERVAL_LAPSE);
|
||||
|
||||
}
|
||||
|
||||
let intervalLoopCounter = 0;
|
||||
|
||||
startNewInterval(); // First, inmadiate run.
|
||||
|
||||
function startNewInterval() {
|
||||
|
||||
try {
|
||||
|
||||
const logText = "[INFO] start - Entering function 'requestNewData' , next execution in " + INTERVAL_LAPSE / 1000 / 60 + " minutes.";
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
if (LOG_INFO === true) {
|
||||
|
||||
if (INTERVAL_MAX_LOOPS !== 0) {
|
||||
|
||||
logger.write("[INFO] Entering function 'startNewInterval', next execution inmediately after the current Interval finishes.");
|
||||
|
||||
} else {
|
||||
|
||||
logger.write("[INFO] Entering function 'startNewInterval', next execution in " + INTERVAL_LAPSE / 1000 / 60 + " minutes.");
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
if (shutdownEvent.isShuttingDown() === false) {
|
||||
|
||||
currentDate = new Date();
|
||||
let fileProcessingInterval = INTERVAL_MODULE.newInterval(bot);
|
||||
|
||||
getOrderBookFromExchangeApi();
|
||||
fileProcessingInterval.initialize(year, month, onReadyToStart);
|
||||
|
||||
function getOrderBookFromExchangeApi() {
|
||||
function onReadyToStart() {
|
||||
|
||||
poloniexApiClient.returnOrderBooks(ORDER_BOOK_DEPTH, onReturnOrderBooks);
|
||||
fileProcessingInterval.start(loopControl);
|
||||
|
||||
}
|
||||
|
||||
function onReturnOrderBooks(err, apiResponse) {
|
||||
|
||||
try {
|
||||
|
||||
if (err || apiResponse.error !== undefined) {
|
||||
try {
|
||||
|
||||
if (err.message.indexOf("ETIMEDOUT") > 0) {
|
||||
|
||||
const logText = "[WARN] requestNewData - onReturnOrderBooks - Timeout reached while trying to access the Exchange API.";
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
/* We try to reconnect to the exchange and fetch the data again. */
|
||||
|
||||
getOrderBookFromExchangeApi();
|
||||
|
||||
} else {
|
||||
const logText = "[ERROR] requestNewData - onReturnOrderBooks - err.message.indexOf(ETIMEDOUT) <= 0 ' ERROR : " + err.message;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
return;
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
const logText = "[ERROR] requestNewData - onReturnOrderBooks ' RECEIVED ERROR : " + apiResponse.error;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
}
|
||||
return;
|
||||
|
||||
} else {
|
||||
processNextMessage(apiResponse);
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[ERROR] requestNewData - onReturnOrderBooks ' ERROR : " + err.message;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
else {
|
||||
|
||||
@@ -159,254 +122,47 @@ exports.newPloniexOrderBooks = function newPloniexOrderBooks(BOT) {
|
||||
|
||||
}
|
||||
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[ERROR] requestNewData ' ERROR : " + err.message;
|
||||
console.log(logText);
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'startNewInterval' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
}
|
||||
}
|
||||
|
||||
function loopControl(moreIntervalExecutionsNeeded) {
|
||||
|
||||
if (moreIntervalExecutionsNeeded === true) {
|
||||
|
||||
intervalLoopCounter++;
|
||||
|
||||
if (intervalLoopCounter === INTERVAL_MAX_LOOPS) {
|
||||
|
||||
const logText = "[WARN] 'INTERVAL_MAX_LOOPS' reached. Stopping execution. ";
|
||||
logger.write(logText);
|
||||
|
||||
} else {
|
||||
|
||||
const logText = "[WARN] 'loopControl'. Staring new Interval. ";
|
||||
logger.write(logText);
|
||||
|
||||
setTimeout(startNewInterval, INTERVAL_LAPSE);// Looped Execution.
|
||||
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
logger.write("[WARN] Received the signal from last Interval execution to not call it anymore. Stopping execution. ");
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
function processNextMessage(messageReceived) {
|
||||
|
||||
try {
|
||||
|
||||
let orderBooksMessage = messageReceived;
|
||||
|
||||
let dateForPath = currentDate.getUTCFullYear() + '/' + utilities.pad(currentDate.getUTCMonth() + 1, 2) + '/' + utilities.pad(currentDate.getUTCDate(), 2) + '/' + utilities.pad(currentDate.getUTCHours(), 2) + '/' + utilities.pad(currentDate.getUTCMinutes(), 2);
|
||||
|
||||
let rawFilePath = RAW_FOLDER_NAME + '/' + dateForPath;
|
||||
let aggregatedFilePath = AGG_FOLDER_NAME + '/' + dateForPath;
|
||||
|
||||
utilities.createFolderIfNeeded(rawFilePath, azureFileStorage, onFirstFolderCreated);
|
||||
|
||||
function onFirstFolderCreated() {
|
||||
|
||||
utilities.createFolderIfNeeded(aggregatedFilePath, azureFileStorage, processEachMarket);
|
||||
|
||||
}
|
||||
|
||||
function processEachMarket() {
|
||||
|
||||
for (let marketName in orderBooksMessage) {
|
||||
|
||||
let market = orderBooksMessage[marketName];
|
||||
|
||||
markets.getExistingOrNewMarketId(marketName, market, EXCHANGE_ID, onMarketIdReady);
|
||||
|
||||
function onMarketIdReady(marketName, market, pMarketId) {
|
||||
|
||||
try {
|
||||
|
||||
let bidsCounter = 0;
|
||||
let asksCounter = 0;
|
||||
|
||||
let newDate = new Date();
|
||||
|
||||
let asks = market.asks;
|
||||
let bids = market.bids;
|
||||
|
||||
let marketId = pMarketId;
|
||||
|
||||
let fileName = '' + EXCHANGE_ID + '_' + marketId + '.json';
|
||||
|
||||
let needSeparator;
|
||||
let separator;
|
||||
|
||||
needSeparator = false;
|
||||
|
||||
let maxRate = bids[0][0] * 1.1;
|
||||
let presicion = utilities.calculatePresicion(maxRate);
|
||||
|
||||
let truncatedRate;
|
||||
let asksMap = new Map();
|
||||
let bidsMap = new Map();
|
||||
|
||||
let fileContent = "";
|
||||
|
||||
generateFirstFile();
|
||||
|
||||
function generateFirstFile() {
|
||||
|
||||
/* First we create the raw file and aggregate the date for the aggregated file */
|
||||
|
||||
fileContent = fileContent + '[';
|
||||
fileContent = fileContent + '[';
|
||||
|
||||
for (let i = 0; i < asks.length; i++) {
|
||||
|
||||
if (needSeparator === false) {
|
||||
|
||||
needSeparator = true;
|
||||
separator = '';
|
||||
|
||||
} else {
|
||||
separator = ',';
|
||||
}
|
||||
|
||||
let rate = asks[i][0];
|
||||
let amount = asks[i][1];
|
||||
|
||||
fileContent = fileContent + separator + '[' + rate + ',' + amount + ']';
|
||||
|
||||
roundedRate = Math.round(rate / presicion) * presicion;
|
||||
|
||||
let currentAmount = asksMap.get(roundedRate) || 0;
|
||||
currentAmount = currentAmount + amount;
|
||||
|
||||
asksMap.set(roundedRate, currentAmount);
|
||||
|
||||
asksCounter = i;
|
||||
}
|
||||
|
||||
fileContent = fileContent + '],[';
|
||||
|
||||
needSeparator = false;
|
||||
|
||||
for (let i = 0; i < bids.length; i++) {
|
||||
|
||||
if (needSeparator === false) {
|
||||
|
||||
needSeparator = true;
|
||||
separator = '';
|
||||
|
||||
} else {
|
||||
separator = ',';
|
||||
}
|
||||
|
||||
let rate = bids[i][0];
|
||||
let amount = bids[i][1];
|
||||
|
||||
fileContent = fileContent + separator + '[' + rate + ',' + amount + ']';
|
||||
|
||||
truncatedRate = Math.trunc(rate / presicion) * presicion;
|
||||
|
||||
let currentAmount = bidsMap.get(truncatedRate) || 0;
|
||||
currentAmount = currentAmount + amount;
|
||||
|
||||
bidsMap.set(truncatedRate, currentAmount);
|
||||
|
||||
bidsCounter = i;
|
||||
}
|
||||
|
||||
fileContent = fileContent + ']';
|
||||
fileContent = fileContent + ']';
|
||||
|
||||
azureFileStorage.createTextFile(rawFilePath, fileName, fileContent + '\n', onFirstFileCreated);
|
||||
|
||||
function onFirstFileCreated() {
|
||||
|
||||
generateSecondFile();
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function generateSecondFile() {
|
||||
|
||||
|
||||
fileContent = "";
|
||||
|
||||
fileContent = fileContent + '[';
|
||||
fileContent = fileContent + '[';
|
||||
|
||||
needSeparator = false;
|
||||
|
||||
asksMap.forEach(addAsks);
|
||||
|
||||
function addAsks(value, key, map) {
|
||||
|
||||
if (needSeparator === false) {
|
||||
|
||||
needSeparator = true;
|
||||
separator = '';
|
||||
|
||||
} else {
|
||||
separator = ',';
|
||||
}
|
||||
|
||||
value = Math.trunc(value * 1000000) / 1000000;
|
||||
|
||||
fileContent = fileContent + separator + '[' + key + ',' + value + ']';
|
||||
}
|
||||
|
||||
fileContent = fileContent + '],[';
|
||||
|
||||
needSeparator = false;
|
||||
|
||||
bidsMap.forEach(addBids);
|
||||
|
||||
function addBids(value, key, map) {
|
||||
|
||||
if (needSeparator === false) {
|
||||
|
||||
needSeparator = true;
|
||||
separator = '';
|
||||
|
||||
} else {
|
||||
separator = ',';
|
||||
}
|
||||
|
||||
value = Math.trunc(value * 1000000) / 1000000;
|
||||
|
||||
fileContent = fileContent + separator + '[' + key + ',' + value + ']';
|
||||
}
|
||||
|
||||
fileContent = fileContent + ']';
|
||||
fileContent = fileContent + ']';
|
||||
|
||||
azureFileStorage.createTextFile(aggregatedFilePath, fileName, fileContent + '\n', onSecondFileCreated);
|
||||
|
||||
function onSecondFileCreated() {
|
||||
|
||||
const logText = "[WARN] Finished with " + marketName + ", " + asksCounter + " aks and " + bidsCounter + " bids written to file " + fileName;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[ERROR] processNextMessage - onMarketIdReady ' ERROR : " + err.message;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
messageReceived = null;
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[ERROR] processNextMessage ' ERROR : " + err.message;
|
||||
console.log(logText);
|
||||
logger.write(logText);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
|
||||
const logText = "[ERROR] Start - ' ERROR : " + err.message;
|
||||
console.log(logText);
|
||||
}
|
||||
catch (err) {
|
||||
const logText = "[ERROR] 'Start' - ERROR : " + err.message;
|
||||
logger.write(logText);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
};
|
||||
|
||||
|
||||
|
||||
@@ -11,9 +11,10 @@ const ROOT_DIR = '../';
|
||||
|
||||
const MODULE_NAME = "run";
|
||||
|
||||
const START_ONE_MIN_DAILY_CANDLES = true;
|
||||
const START_POLONIEX_HOLE_FIXING = true;
|
||||
const START_POLONIEX_HOLE_FIXING_AT_ONE_MONTH = false;
|
||||
|
||||
const POLONIEX_ORDE_BOOKS_MODULE = require('./One Min Daily Candles');
|
||||
const POLONIEX_HOLE_FIXING_MODULE = require('./Poloniex Hole Fixing');
|
||||
|
||||
const DEBUG_MODULE = require(ROOT_DIR + 'Debug Log');
|
||||
const logger = DEBUG_MODULE.newDebugLog();
|
||||
@@ -21,7 +22,7 @@ logger.fileName = MODULE_NAME;
|
||||
logger.bot = bot;
|
||||
|
||||
process.on('uncaughtException', function (err) {
|
||||
logger.write('uncaughtException' + err.message);
|
||||
logger.write('uncaughtException - ' + err.message);
|
||||
});
|
||||
|
||||
|
||||
@@ -35,27 +36,62 @@ process.on('exit', function (code) {
|
||||
});
|
||||
|
||||
|
||||
|
||||
try {
|
||||
if (START_ONE_MIN_DAILY_CANDLES === true) {
|
||||
if (START_POLONIEX_HOLE_FIXING === true) {
|
||||
|
||||
const poloniexOrderBooks = POLONIEX_ORDE_BOOKS_MODULE.newPloniexOrderBooks(bot);
|
||||
for (let year = 2018; year > 2014; year--) {
|
||||
|
||||
poloniexOrderBooks.initialize(onInitializeReady);
|
||||
for (let month = 12; month > 0; month--) {
|
||||
|
||||
function onInitializeReady() {
|
||||
let timeDelay = Math.random() * 100 * 1000; // We need a delay so as not to get banned from the exchange for requesting to many things at the same time.
|
||||
|
||||
poloniexOrderBooks.start();
|
||||
setTimeout(startProcess, timeDelay);
|
||||
|
||||
function startProcess() {
|
||||
|
||||
const newPloniexHoleFixing = POLONIEX_HOLE_FIXING_MODULE.newPloniexHoleFixing(bot);
|
||||
|
||||
newPloniexHoleFixing.initialize(year, month, onInitializeReady);
|
||||
|
||||
function onInitializeReady() {
|
||||
|
||||
newPloniexHoleFixing.start();
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
if (START_POLONIEX_HOLE_FIXING_AT_ONE_MONTH === true) {
|
||||
|
||||
startProcess();
|
||||
|
||||
function startProcess() {
|
||||
|
||||
const newPloniexHoleFixing = POLONIEX_HOLE_FIXING_MODULE.newPloniexHoleFixing(bot);
|
||||
|
||||
let month = 2;
|
||||
let year = 2015;
|
||||
|
||||
newPloniexHoleFixing.initialize(year, month, onInitializeReady);
|
||||
|
||||
function onInitializeReady() {
|
||||
|
||||
newPloniexHoleFixing.start();
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} catch (err) {
|
||||
console.log(err.message);
|
||||
logger.write(err.message);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user