364 lines
17 KiB
JavaScript
364 lines
17 KiB
JavaScript
exports.newTensorFlowBotModulesTensorFlowLibrary = function (processIndex) {
|
|
|
|
const MODULE_NAME = 'Time Series Forcasting Tensor Flow JS Learning Algorithm'
|
|
let thisObject = {
|
|
loadModel: loadModel,
|
|
saveModel: saveModel,
|
|
updateChart: updateChart,
|
|
mantain: mantain,
|
|
reset: reset,
|
|
run: run,
|
|
initialize: initialize,
|
|
finalize: finalize
|
|
}
|
|
|
|
let learningEngine
|
|
let learningSystem
|
|
let sessionParameters
|
|
|
|
/*
|
|
These 3 are the main data structures available to users
|
|
when writing conditions and formulas.
|
|
*/
|
|
let chart
|
|
let exchange
|
|
let market
|
|
|
|
const requiredErr = (str) => { throw new Error('Missing: ' + str); }
|
|
let fLen, lLen;
|
|
let tensorFlowBackend;
|
|
var tensorFlowModelsManager; // Initialized in setupModel()
|
|
var mlDataObj; // ML Data Manager Module.
|
|
var hasCondition;
|
|
|
|
return thisObject
|
|
|
|
function initialize(callbackFunction) {
|
|
learningSystem = TS.projects.foundations.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SIMULATION_STATE.learningSystem
|
|
learningEngine = TS.projects.foundations.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SIMULATION_STATE.learningEngine
|
|
sessionParameters = TS.projects.foundations.globals.processConstants.CONSTANTS_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_NODE.learningParameters
|
|
|
|
validations();
|
|
setupBackend()
|
|
setupEnvironmentFlags()
|
|
tensorFlowModelsManager = TS.projects.tensorflow.botModules.tensorFlowModelsManager.newTensorFlowModelsManager(processIndex, learningSystem);
|
|
callbackFunction(TS.projects.foundations.globals.standardResponses.DEFAULT_OK_RESPONSE)
|
|
fLen = learningSystem.machineLearningLibrary.typeOfLearning.typeOfModel.model.api.layersModel.inputLayer.inputFeatures.dataFeatures.length;
|
|
lLen = learningSystem.machineLearningLibrary.typeOfLearning.typeOfModel.model.api.layersModel.outputLayer.outputLabels.dataLabels.length;
|
|
mlDataObj = TS.projects.tensorflow.botModules.mlData.newMLDataObj(fLen, lLen);
|
|
hasCondition = (learningSystem.machineLearningLibrary.typeOfLearning.typeOfModel.model.api.layersModel.inputLayer.collectionCondition !== undefined) ? true : false;
|
|
|
|
function setupBackend() {
|
|
/* Setup the default backend: */
|
|
tensorFlowBackend = require("@tensorflow/tfjs-node")
|
|
|
|
if (
|
|
learningSystem.machineLearningLibrary.executionEnvironment !== undefined &&
|
|
learningSystem.machineLearningLibrary.executionEnvironment.backend !== undefined
|
|
) {
|
|
/* Replace the backend for the one defined at the UI: */
|
|
switch (learningSystem.machineLearningLibrary.executionEnvironment.backend.config.codeName) {
|
|
case 'node': {
|
|
// This is already the default.
|
|
break
|
|
}
|
|
case 'webgl': {
|
|
tensorFlowBackend = require("@tensorflow/tfjs-backend-webgl")
|
|
break
|
|
}
|
|
case 'wasm': {
|
|
tensorFlowBackend = require("@tensorflow/tfjs-backend-wasm")
|
|
break
|
|
}
|
|
case 'cpu': {
|
|
tensorFlowBackend = require("@tensorflow/tfjs-backend-cpu")
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function setupEnvironmentFlags() {
|
|
if (
|
|
learningSystem.machineLearningLibrary.executionEnvironment !== undefined &&
|
|
learningSystem.machineLearningLibrary.executionEnvironment.environmentFlags !== undefined &&
|
|
learningSystem.machineLearningLibrary.executionEnvironment.environmentFlags.mode
|
|
) {
|
|
switch (learningSystem.machineLearningLibrary.executionEnvironment.environmentFlags.mode.type) {
|
|
case 'Debug Mode': {
|
|
tensorFlowBackend.enableDebugMode()
|
|
break
|
|
}
|
|
case 'Debug Mode': {
|
|
tensorFlowBackend.enableProdMode()
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function finalize() {
|
|
learningEngine = undefined
|
|
learningSystem = undefined
|
|
sessionParameters = undefined
|
|
|
|
chart = undefined
|
|
exchange = undefined
|
|
market = undefined
|
|
|
|
tensorFlowBackend = undefined
|
|
tensorFlowModelsManager.model = undefined
|
|
tensorFlowData = undefined
|
|
}
|
|
|
|
async function loadModel() {
|
|
tensorFlowModelsManager.model = await tensorFlowModelsManager.api.loadLayersModel('file://' + getModelPath() + '/model.json');
|
|
tensorFlowModelsManager.compileModel();
|
|
}
|
|
|
|
async function saveModel() {
|
|
let fileLocation = getModelPath();
|
|
/* If necesary a folder or folders are created before writing the file to disk: */
|
|
SA.projects.foundations.utilities.filesAndDirectories.mkDirByPathSync(fileLocation);
|
|
|
|
const savedModel = await tensorFlowModelsManager.model.save('file://' + fileLocation);
|
|
}
|
|
|
|
function getModelPath() {
|
|
let fileName = learningSystem.machineLearningLibrary.typeOfLearning.typeOfModel.model.config.codeName
|
|
let filePath =
|
|
global.env.PATH_TO_DATA_STORAGE + '/' +
|
|
TS.projects.foundations.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).FILE_PATH_ROOT +
|
|
'/Output/' +
|
|
TS.projects.foundations.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).SESSION_FOLDER_NAME
|
|
|
|
return filePath + '/' + fileName
|
|
}
|
|
|
|
function updateChart(pChart, pExchange, pMarket) {
|
|
/*
|
|
We need these 3 data structures to be a local objects
|
|
accessible while evaluating conditions and formulas.
|
|
*/
|
|
chart = pChart
|
|
exchange = pExchange
|
|
market = pMarket
|
|
}
|
|
|
|
function mantain() {}
|
|
|
|
function reset() {}
|
|
|
|
async function run(callbackFunction) {
|
|
let layersModel = learningSystem.machineLearningLibrary.typeOfLearning.typeOfModel.model.api.layersModel;
|
|
let isAllowed = true;
|
|
if (hasCondition) {
|
|
learningSystem.evalConditions(layersModel.inputLayer.collectionCondition, layersModel.inputLayer.collectionCondition.type);
|
|
isAllowed = learningSystem.conditions.get(layersModel.inputLayer.collectionCondition.id);
|
|
}
|
|
|
|
if (isAllowed) {
|
|
let features = await getFeatures(layersModel);
|
|
|
|
/* We will store at the Learning Engine data structure the features we have calculated: */
|
|
for (let i = 0; i < layersModel.inputLayer.inputFeatures.dataFeatures.length; i++) {
|
|
if (learningEngine.features.features[i] === undefined) { continue }
|
|
learningEngine.features.features[i].featureValue.value = features[i];
|
|
learningEngine.features.features[i].begin.value = learningEngine.learningCurrent.learningEpisode.candle.begin.value
|
|
learningEngine.features.features[i].end.value = learningEngine.learningCurrent.learningEpisode.candle.end.value
|
|
}
|
|
|
|
/*
|
|
At this point we need to decide if we are going to learn or going to predict.
|
|
This is regulated by the backLearningPredictingRatio property at the config
|
|
of the Learning Algorithm of the Learning Session.
|
|
*/
|
|
let backLearningPredictingRatio = 0.8 // this is the default value.
|
|
|
|
if (sessionParameters.learningAlgorithm !== undefined) {
|
|
if (sessionParameters.learningAlgorithm.config.backLearningPredictingRatio !== undefined) {
|
|
backLearningPredictingRatio = sessionParameters.learningAlgorithm.config.backLearningPredictingRatio
|
|
}
|
|
}
|
|
|
|
let initialCandle = Math.trunc(sessionParameters.timeRange.config.initialDatetime / sessionParameters.timeFrame.config.value)
|
|
let finalCandle = Math.trunc(sessionParameters.timeRange.config.finalDatetime / sessionParameters.timeFrame.config.value)
|
|
let currentCandle = Math.trunc(learningEngine.learningCurrent.learningEpisode.candle.begin.value / sessionParameters.timeFrame.config.value)
|
|
let thresholdCandle = (finalCandle - initialCandle) * backLearningPredictingRatio + initialCandle;
|
|
|
|
|
|
if (currentCandle < thresholdCandle) {
|
|
await pushFeaturesLabels();
|
|
} else {
|
|
if (!tensorFlowModelsManager.isTrained) {
|
|
await mlDataObj.dataEngineering(learningSystem);
|
|
await tensorFlowModelsManager.trainModel(mlDataObj);
|
|
}
|
|
predict()
|
|
}
|
|
|
|
async function pushFeaturesLabels() {
|
|
let labels = await getLabels(layersModel);
|
|
|
|
/* We will store at the Learning Engine data structure the labels we have calculated. */
|
|
for (let i = 0; i < layersModel.outputLayer.outputLabels.dataLabels.length; i++) {
|
|
if (learningEngine.labels.labels[i] === undefined) {
|
|
console.log("Warning: learning engine has no space to record label #" + (i+1));
|
|
continue;
|
|
}
|
|
learningEngine.labels.labels[i].labelValue.value = labels[i];
|
|
learningEngine.labels.labels[i].begin.value = learningEngine.learningCurrent.learningEpisode.candle.begin.value
|
|
learningEngine.labels.labels[i].end.value = learningEngine.learningCurrent.learningEpisode.candle.end.value
|
|
}
|
|
|
|
// Push this features/labels array to the Data Manager:
|
|
mlDataObj.addFeatures(features);
|
|
mlDataObj.addLabels(labels);
|
|
}
|
|
|
|
async function predict() {
|
|
let featuresTensor = tensorFlowBackend.tensor([features])
|
|
let predictions = tensorFlowModelsManager.model.predict(featuresTensor, { verbose: 1 })
|
|
|
|
/*
|
|
We will download the predictions to an array, and we will store each one at the
|
|
learning engine data structure.
|
|
*/
|
|
let predictionsArray = predictions.dataSync()
|
|
predictions.dispose()
|
|
featuresTensor.dispose()
|
|
|
|
for (let i = 0; i < predictionsArray.length; i++) {
|
|
let prediction = predictionsArray[i]
|
|
if (isNaN(prediction)) {
|
|
prediction = 0
|
|
}
|
|
if (learningEngine.predictions.predictions[i] === undefined) {
|
|
console.log("Warning: learning engine has no space to record prediction #" + (i + 1));
|
|
continue;
|
|
}
|
|
learningEngine.predictions.predictions[i].predictionValue.value = prediction
|
|
learningEngine.predictions.predictions[i].begin.value = learningEngine.learningCurrent.learningEpisode.candle.begin.value
|
|
learningEngine.predictions.predictions[i].end.value = learningEngine.learningCurrent.learningEpisode.candle.end.value
|
|
}
|
|
}
|
|
|
|
/* Collecting the Features: */
|
|
async function getFeatures(layersModel) {
|
|
let features = []
|
|
|
|
learningSystem.evalFormulas(layersModel.inputLayer.inputFeatures, layersModel.inputLayer.inputFeatures.type);
|
|
|
|
for (let i = 0; i < fLen; i++) {
|
|
let dataFeature = layersModel.inputLayer.inputFeatures.dataFeatures[i];
|
|
if (dataFeature.featureFormula === undefined) { continue }
|
|
let featureValue = learningSystem.formulas.get(dataFeature.featureFormula.id);
|
|
features.push(featureValue);
|
|
}
|
|
return features;
|
|
}
|
|
|
|
/* Collecting the Labels: */
|
|
async function getLabels(layersModel) {
|
|
let labels = []
|
|
|
|
learningSystem.evalFormulas(layersModel.outputLayer.outputLabels, layersModel.outputLayer.outputLabels.type);
|
|
|
|
for (let i = 0; i < lLen; i++) {
|
|
let dataLabel = layersModel.outputLayer.outputLabels.dataLabels[i];
|
|
if (dataLabel.labelFormula === undefined) { continue }
|
|
let labelValue = learningSystem.formulas.get(dataLabel.labelFormula.id);
|
|
labels.push(labelValue);
|
|
}
|
|
return labels;
|
|
}
|
|
}
|
|
}
|
|
|
|
function badDefinitionUnhandledException(err, message, node) {
|
|
|
|
let docs = {
|
|
project: 'Foundations',
|
|
category: 'Topic',
|
|
type: 'TS LF Learning Bot Error - ' + message,
|
|
placeholder: {}
|
|
}
|
|
|
|
learningSystem.addError([node.id, message, docs])
|
|
|
|
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] -> " + message);
|
|
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] -> node.name = " + node.name);
|
|
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] -> node.type = " + node.type);
|
|
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] -> node.config = " + JSON.stringify(node.config));
|
|
if (err !== undefined) {
|
|
TS.projects.foundations.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME, "[ERROR] -> err.stack = " + err.stack);
|
|
}
|
|
throw 'Error Already Recorded'
|
|
}
|
|
|
|
|
|
/* validations() : This function validates that required nodes exist. */
|
|
function validations() {
|
|
let layersModel = learningSystem.machineLearningLibrary.typeOfLearning.typeOfModel.model.api.layersModel;
|
|
if (layersModel === undefined) {
|
|
requiredErr(">layers model<");
|
|
}
|
|
|
|
// Model Structure Validations:
|
|
if (layersModel.inputLayer === undefined) {
|
|
requiredErr(">input layer<");
|
|
}
|
|
if (layersModel.outputLayer === undefined) {
|
|
requiredErr(">output layer<");
|
|
}
|
|
if (layersModel.inputLayer.inputFeatures === undefined) {
|
|
requiredErr(">input features<");
|
|
}
|
|
if (layersModel.outputLayer.outputLabels === undefined) {
|
|
requiredErr(">output labels<");
|
|
}
|
|
if (layersModel.inputLayer.inputFeatures.dataFeatures.length === 0) {
|
|
requiredErr(">input data features<");
|
|
}
|
|
if (layersModel.outputLayer.outputLabels.dataLabels.length === 0) {
|
|
requiredErr(">output data labels<");
|
|
}
|
|
if (layersModel.layers.length === 0) {
|
|
requiredErr(">Could not find any layers<");
|
|
}
|
|
if (layersModel.inputLayer.referenceParent === undefined) {
|
|
requiredErr(">input layer missing reference parent<");
|
|
}
|
|
|
|
// Model Compiler Validations:
|
|
if (learningSystem.machineLearningLibrary.typeOfLearning.typeOfModel.model.compile === undefined) {
|
|
requiredErr(">compile node<");
|
|
}
|
|
if (learningSystem.machineLearningLibrary.typeOfLearning.typeOfModel.model.compile.lossFunction === undefined) {
|
|
requiredErr(">loss function<");
|
|
}
|
|
if (learningSystem.machineLearningLibrary.typeOfLearning.typeOfModel.model.compile.lossFunction.config.value === undefined) {
|
|
requiredErr(">loss function value<");
|
|
}
|
|
if (learningSystem.machineLearningLibrary.typeOfLearning.typeOfModel.model.compile.optimizer === undefined) {
|
|
requiredErr(">optimizer node<");
|
|
}
|
|
if (learningSystem.machineLearningLibrary.typeOfLearning.typeOfModel.model.compile.optimizer.optimizerType === undefined) {
|
|
requiredErr(">optimizer type node<");
|
|
}
|
|
|
|
//* Feature Validations: */
|
|
|
|
// Data Features must be Uniquely Named:
|
|
/*let duplicates = () => {
|
|
let names = [];
|
|
layersModel.inputLayer.inputFeatures.dataFeatures.forEach(e => names.push(e.name));
|
|
return names;
|
|
};
|
|
if (layersModel.inputLayer.inputFeatures.dataFeatures.length !==
|
|
new Set(duplicates()).size) {
|
|
requiredErr(">All Data Feature nodes must be uniquely named<");
|
|
}*/
|
|
}
|
|
} |