From f22a67904661bad466d92cb39a222f0d0c723edd Mon Sep 17 00:00:00 2001 From: Luis Fernando Molina Date: Mon, 8 Mar 2021 09:46:37 +0100 Subject: [PATCH] Machine Learning: Saving Model Fixed, crating necesary folders before the file. --- .../Low-Frequency-Learning/LearningRecords.js | 4 ++ .../Superalgos/TS/Task-Modules/FileStorage.js | 35 +--------------- .../TS/Utilities/MiscellaneousFunctions.js | 40 ++++++++++++++++++- .../TensorFlowLibrary.js | 6 ++- 4 files changed, 48 insertions(+), 37 deletions(-) diff --git a/Projects/Superalgos/TS/Bot-Modules/Learning-Bot/Low-Frequency-Learning/LearningRecords.js b/Projects/Superalgos/TS/Bot-Modules/Learning-Bot/Low-Frequency-Learning/LearningRecords.js index 2f3df136e..76b04bd86 100644 --- a/Projects/Superalgos/TS/Bot-Modules/Learning-Bot/Low-Frequency-Learning/LearningRecords.js +++ b/Projects/Superalgos/TS/Bot-Modules/Learning-Bot/Low-Frequency-Learning/LearningRecords.js @@ -45,6 +45,10 @@ exports.newSuperalgosBotModulesLearningRecords = function (processIndex) { let product = dataset.parentNode let outputDatasetArray = outputDatasetsMap.get(product.config.codeName) + if (product.config.codeName === 'Predictions') { + let a = 1 + } + if (TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).TRADING_PROCESSING_DAILY_FILES === true && dataset.config.type === 'Daily Files') { persistRecords() } diff --git a/Projects/Superalgos/TS/Task-Modules/FileStorage.js b/Projects/Superalgos/TS/Task-Modules/FileStorage.js index d47e8793f..ac58e6395 100644 --- a/Projects/Superalgos/TS/Task-Modules/FileStorage.js +++ b/Projects/Superalgos/TS/Task-Modules/FileStorage.js @@ -4,7 +4,6 @@ exports.newFileStorage = function newFileStorage(processIndex, host, port) { const MAX_RETRY = 10 const FAST_RETRY_TIME_IN_MILISECONDS = 500 const SLOW_RETRY_TIME_IN_MILISECONDS = 2000 - const path = require('path') let thisObject = { asyncGetTextFile: asyncGetTextFile, @@ -244,8 +243,7 @@ exports.newFileStorage = function newFileStorage(processIndex, host, port) { logger.write(MODULE_NAME, '[INFO] FileStorage -> createTextFile -> fileLocation: ' + fileLocation) /* If necesary a folder or folders are created before writing the file to disk. */ - let directoryPath = fileLocation.substring(0, fileLocation.lastIndexOf('/') + 1); - mkDirByPathSync(directoryPath) + TS.projects.superalgos.utilities.miscellaneousFunctions.mkDirByPathSync(fileLocation) /* Here we write the file with a temporary name so as to avoid dirty read from other processes. @@ -442,37 +440,6 @@ exports.newFileStorage = function newFileStorage(processIndex, host, port) { } } - /* Function to create folders of missing folders at any path. */ - function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) { - const sep = '/'; - const initDir = path.isAbsolute(targetDir) ? sep : ''; - const baseDir = isRelativeToScript ? __dirname : '.'; - - return targetDir.split(sep).reduce((parentDir, childDir) => { - const curDir = path.resolve(baseDir, parentDir, childDir); - try { - const fs = require('fs') - fs.mkdirSync(curDir); - } catch (err) { - if (err.code === 'EEXIST') { // curDir already exists! - return curDir; - } - - // To avoid `EISDIR` error on Mac and `EACCES`-->`ENOENT` and `EPERM` on Windows. - if (err.code === 'ENOENT') { // Throw the original parentDir error on curDir `ENOENT` failure. - throw new Error(`EACCES: permission denied, mkdir '${parentDir}'`); - } - - const caughtErr = ['EACCES', 'EPERM', 'EISDIR'].indexOf(err.code) > -1; - if (!caughtErr || caughtErr && curDir === path.resolve(targetDir)) { - throw err; // Throw if it's just the last created dir. - } - } - - return curDir; - }, initDir); - } - function getFileViaHTTP(filePath, callback, callBackFunction) { try { diff --git a/Projects/Superalgos/TS/Utilities/MiscellaneousFunctions.js b/Projects/Superalgos/TS/Utilities/MiscellaneousFunctions.js index 568b37dd3..8ea8a766c 100644 --- a/Projects/Superalgos/TS/Utilities/MiscellaneousFunctions.js +++ b/Projects/Superalgos/TS/Utilities/MiscellaneousFunctions.js @@ -4,12 +4,13 @@ exports.newSuperalgosUtilitiesMiscellaneousFunctions = function () { genereteUniqueId: genereteUniqueId, truncateToThisPrecision: truncateToThisPrecision, pad: pad, - asyncGetDatasetFile: asyncGetDatasetFile + asyncGetDatasetFile: asyncGetDatasetFile, + mkDirByPathSync: mkDirByPathSync } return thisObject - function genereteUniqueId () { + function genereteUniqueId() { return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) { var r = Math.random() * 16 | 0, v = c == 'x' ? r : (r & 0x3 | 0x8) return v.toString(16) @@ -45,4 +46,39 @@ exports.newSuperalgosUtilitiesMiscellaneousFunctions = function () { return promise } + + /* Function to create folders of missing folders at any path. */ + function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) { + const path = require('path') + + targetDir = targetDir.substring(0, targetDir.lastIndexOf('/') + 1); + + const sep = '/'; + const initDir = path.isAbsolute(targetDir) ? sep : ''; + const baseDir = isRelativeToScript ? __dirname : '.'; + + return targetDir.split(sep).reduce((parentDir, childDir) => { + const curDir = path.resolve(baseDir, parentDir, childDir); + try { + const fs = require('fs') + fs.mkdirSync(curDir); + } catch (err) { + if (err.code === 'EEXIST') { // curDir already exists! + return curDir; + } + + // To avoid `EISDIR` error on Mac and `EACCES`-->`ENOENT` and `EPERM` on Windows. + if (err.code === 'ENOENT') { // Throw the original parentDir error on curDir `ENOENT` failure. + throw new Error(`EACCES: permission denied, mkdir '${parentDir}'`); + } + + const caughtErr = ['EACCES', 'EPERM', 'EISDIR'].indexOf(err.code) > -1; + if (!caughtErr || caughtErr && curDir === path.resolve(targetDir)) { + throw err; // Throw if it's just the last created dir. + } + } + + return curDir; + }, initDir); + } } \ No newline at end of file diff --git a/Projects/TensorFlow/TS/Bot-Modules/Learning-Bot/Low-Frequency-Learning/TensorFlowLibrary.js b/Projects/TensorFlow/TS/Bot-Modules/Learning-Bot/Low-Frequency-Learning/TensorFlowLibrary.js index 34fa774a5..e9cafddb3 100644 --- a/Projects/TensorFlow/TS/Bot-Modules/Learning-Bot/Low-Frequency-Learning/TensorFlowLibrary.js +++ b/Projects/TensorFlow/TS/Bot-Modules/Learning-Bot/Low-Frequency-Learning/TensorFlowLibrary.js @@ -319,7 +319,11 @@ exports.newTensorFlowBotModulesTensorFlowLibrary = function (processIndex) { } async function saveModel() { - await tensorFlowModel.save('file://' + getModelPath()) + let fileLocation = getModelPath() + /* If necesary a folder or folders are created before writing the file to disk. */ + TS.projects.superalgos.utilities.miscellaneousFunctions.mkDirByPathSync(fileLocation) + + await tensorFlowModel.save('file://' + fileLocation) } function getModelPath() {