Merge remote-tracking branch 'upstream/develop' into develop
This commit is contained in:
Philip Love committed
commit
6b2d6847c7
30 files changed
+667
-454
No files matched your search
@@ -23,6 +23,26 @@
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Storing objects with named properties in the JSON format would be inappropriate as labels would repeat over and over, creating a ridiculous amount of unnecessary information. However, storing arrays in the JSON standard facilitates the handling of files in the context of JavaScript and Node.JS."
|
||||
},
|
||||
{
|
||||
"style": "Block",
|
||||
"text": "Record Properties"
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "What are the Record Properties used for?"
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "In general this is the mechanism to define what information is going to be available to the consumers of data of this product. "
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "When properties don't have a Formula, they define the format of the file that stores this data on disk. These record properties defines, the data type (string, number, etc), and in which order the data is saved. At the file there are no headers, just the data, because the header information is available here, for any process writing or reading the data from disk."
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "The formulas nodes there are used to expand the record definition adding fields that are not actually going to be saved on disk, to minimize space used, and they are going to be calculated at runtime. This allows proceeses that consume this data to have ell the fields available at runtime even though some were not actually saved on disk."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -5,10 +5,19 @@
|
||||
"updated": 1609788711861
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Include",
|
||||
"text": "Superalgos->Node->Record Definition->Record Properties",
|
||||
"updated": 1622292888747
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "How is a Record Formula used?",
|
||||
"updated": 1622292907320
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Formulas may use both the record and the variable objects made available by the calculations procedure and the data building procedure (see procedure loop).",
|
||||
"updated": 1609788725463
|
||||
"text": "Formulas may use both the record and the variable objects made available by the calculations procedure and the data building procedure (see procedure loop)."
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
|
||||
@@ -8,6 +8,11 @@
|
||||
"style": "Block",
|
||||
"text": "Content"
|
||||
},
|
||||
{
|
||||
"style": "Include",
|
||||
"text": "Superalgos->Node->Record Definition->Record Properties",
|
||||
"updated": 1622292794198
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Records may have as many properties as required."
|
||||
|
||||
+6
-2
@@ -12,10 +12,14 @@
|
||||
"text": "Dataset List",
|
||||
"updated": 1622052318412
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Dataset Types",
|
||||
"updated": 1622227827875
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "This is the complete list of Dataset types currently supported by the system:",
|
||||
"updated": 1622190817372
|
||||
"text": "This is the complete list of Dataset types currently supported by the system:"
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
|
||||
+6
-2
@@ -7,10 +7,14 @@
|
||||
"updated": 1622205220555
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Types of Dataset Conversions",
|
||||
"updated": 1622227863425
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "The type of conversions currently implemented within the system are:",
|
||||
"updated": 1622205237120
|
||||
"text": "The type of conversions currently implemented within the system are:"
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
|
||||
+178
@@ -0,0 +1,178 @@
|
||||
{
|
||||
"topic": "Data Mining",
|
||||
"pageNumber": 6,
|
||||
"type": "Data Mining - Data Mine Hierarchy",
|
||||
"definition": {
|
||||
"text": "Data mines enable the processing of data by consuming datasets others have produced, performing calculations, and outputting a new value-added data product that others may consume. In other words, Data Mines are user created packages of definitions. These definitions usually include Sensor Bot, Indicator Bot and Plotter definitions. The Data Mines shipped with Superalgos are packages contributed by users of the software.",
|
||||
"updated": 1622238150464
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Superalgos puts a powerful data processing and visualization system into your hands. Use it to the fullest and you will see your intelligence increase by orders of magnitude.",
|
||||
"updated": 1622235471834
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Anyone may access raw trades and order book data from exchanges. Anyone may access information in the form of standard indicators and technical studies."
|
||||
},
|
||||
{
|
||||
"style": "Callout",
|
||||
"text": "However, Superalgos allows you to perform any sort of processing over existing information across the whole span of the market, store the results into new datasets, and make them available to trading strategies. Moreover, it allows you to produce elaborate graphical representations of data, which is instantly browsable over the charts."
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Data mines’ functionality is highly flexible and works under the same paradigm used for building strategies. They provide a visual interface and a framework that leads you through the process of creating new bots, with minimal coding skills required."
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "The vast majority of the complexity of processing data is handled internally by the bots, and you don’t need to deal with it or even understand it. The system isolates complexity and only exposes you to the crucial elements that impact what you wish to accomplish."
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "This section of the documentation provides you with the technical understanding of how data is handled by the system. The core of the functionality this section unveils covers—in essence—the following points:"
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "The capacity to define any number of calculation processes that may feed on each other’s outputs, resulting in a wealth of information that may be consumed by trading strategies."
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "The properties and characteristics of datasets resulting from the execution of such processes—how and where they are stored, in what format, how they are consumed internally, and potentially, by other systems."
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "The capacity to visualize the data over the charts and browse it with all the features built in to the charting system.",
|
||||
"updated": 1622235587518
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "How Bots Use Data Mines"
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "It is important to undestand that a Data Mine is a package of definitions. When users run a Bot, they do it inside a Task. This task needs to know how to run the bot. In order to find this out, the bot within the task must have a reference to a Process Definition exisiting at a Data Mine. When the bot is launched inside the task, at that moment, it receives all the information regarding the task that is being used to launch the bot, while at the same time it also receives the information regarding the Bot's definition from the Data Mine."
|
||||
},
|
||||
{
|
||||
"style": "Important",
|
||||
"text": "Using both sides of this information (Task data and Bot definition) the Bot is able to do it's job.",
|
||||
"updated": 1622238315541
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Who Creates Data Mines?"
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Data Mines are created by power users and contributed to the project (the ones shipping with the software). Users might create Data Mines for their personal use and never share them. However, if they share them, they receive SA tokens for that, and also for adapting and mantaining them, if necessary, for each new release."
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Supported Bots"
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "There are three types of bots that can hold their definitions at a Data Mine."
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Sensor Bots: used to fetch data from some Internet source by coding a specialized algorithm."
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "API Data Fetcher Bots: used to fetch data from an Internet API without needing to code."
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Indicator Bots: used to process data."
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Data Mine",
|
||||
"updated": 1611407778880
|
||||
},
|
||||
{
|
||||
"style": "Include",
|
||||
"text": "Superalgos->Node->Data Mine->Definition",
|
||||
"updated": 1611407782909
|
||||
},
|
||||
{
|
||||
"style": "Include",
|
||||
"text": "Superalgos->Node->Data Mine->Content",
|
||||
"updated": 1611407786577
|
||||
},
|
||||
{
|
||||
"style": "Include",
|
||||
"text": "Superalgos->Concept->Reusable Snippets->Note for Hierarchy Tables No Discussion"
|
||||
},
|
||||
{
|
||||
"style": "Hierarchy",
|
||||
"text": "Superalgos->Data Mine->2",
|
||||
"updated": 1611407832651
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": ""
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Sensor Bot",
|
||||
"updated": 1611407910652
|
||||
},
|
||||
{
|
||||
"style": "Include",
|
||||
"text": "Superalgos->Node->Sensor Bot->Definition",
|
||||
"updated": 1611407914247
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Indicator Bot",
|
||||
"updated": 1611407935038
|
||||
},
|
||||
{
|
||||
"style": "Include",
|
||||
"text": "Superalgos->Node->Indicator Bot->Definition",
|
||||
"updated": 1611407948292
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Plotter",
|
||||
"updated": 1611407956192
|
||||
},
|
||||
{
|
||||
"style": "Include",
|
||||
"text": "Superalgos->Node->Plotter->Definition",
|
||||
"updated": 1611407972840
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Interdependency of Data Mines"
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Data Mines might depend on other Data Mines. What this means is the bots within a Data Mine may depend on bots belonging to other Data Mines. Further on in this Topic, you will learn more about how a bot can depend on other bots in detail. In a nutshell, they can depend in 3 ways. They can have a:"
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Data Dependency: This means that a Bot uses as an input the data produced by another bot."
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Status Dependency: This means that a Bot uses the status of another Bot as parameters for it's own processing."
|
||||
},
|
||||
{
|
||||
"style": "Success",
|
||||
"text": "The most common use case is when a Bot needs the date on which a market starts or ends. This information is automatically fetched from the Status Report of the bot that produces the raw data because it knows which data is the oldest and the newest at all times.",
|
||||
"updated": 1622237821439
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Execution Dependency: This means that a Bot depends on other Bot execution to finish in order for it to start it's own execution."
|
||||
},
|
||||
{
|
||||
"style": "Important",
|
||||
"text": "When a Data Mine depends on another Data Mine, both need to be present at the Workspace where the Task executing the Bot is located.",
|
||||
"updated": 1622237827110
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
{
|
||||
"topic": "Data Mining",
|
||||
"pageNumber": 6,
|
||||
"type": "Data Mining - Data Mines",
|
||||
"definition": {
|
||||
"text": "Data Mines are user created packages of definitions. These definitions usually include Sensor Bot, Indicator Bot and Plotter definitions. The Data Mines shipped with Superalgos are packages contributed by users of the software.",
|
||||
"updated": 1621708561783
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "It is important to undestand that a Data Mine is a package of definitions. When users run a Bot, they do it inside a Task and they need to reference a Process Definition exisiting at one Data Mine. When the bot is launched inside the Task, it receives at that moment all the information regarding the Task that is being used to launch the bot, but at the same time it receives all the information regarding the the Bot's definitions at the Data Mine.",
|
||||
"updated": 1621709219036
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "With all that information the Bot is able to do it's job."
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Who creates them?"
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Data Mines are created by power users and contributed to the project (the ones shipping with the software). Users might create Data Mines for their personal use and never share them. If they share them, they receive SA tokens for that, and also for mantaining them to adapt them to every new release, if necesary.",
|
||||
"updated": 1622188509380
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Supported Bots"
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "There are three types of bots that can hold their definitions at a Data Mine."
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Sensor Bots: used to fetch data from some source at the Internet by coding a specialized algorithm for that."
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "API Data Fetcher Bots: used to fetch data from an Internet API without needing to code."
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Indicator Bots: used to process data."
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Interdependency of Data Mines",
|
||||
"updated": 1621709799491
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Data Mines might depend of other Data Mines if any of their bots depend on bots belonging to other Data Mines. You will learn further down this Topic how a bot can depend on other bots in detail. In a nutshell, they can depend in 3 ways. They can have a:",
|
||||
"updated": 1621903084681
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Data Dependency: This means that a Bot uses as an input the data produced by another bot."
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Status Dependency: This means that a Bot uses the status of another Bot as parameters for it's own process. The most common use case is when a Bot needs for instance the date where a market starts or ends. This information is automatically fetched from the Status Report of the bot that produces the raw data and knows which data is the oldest and the newest all the time."
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Execution Dependency: This means that a Bot depends on other Bot execution to finish in order for it to start it's own execution.",
|
||||
"updated": 1621709849717
|
||||
},
|
||||
{
|
||||
"style": "Note",
|
||||
"text": "When a Data Mine depends on another Data Mine, both need to be present at the Workspace where the Task defining the execution of the Bot is located.",
|
||||
"updated": 1621903174117
|
||||
}
|
||||
]
|
||||
}
|
||||
+52
-23
@@ -3,58 +3,87 @@
|
||||
"pageNumber": 7,
|
||||
"type": "Data Mining - Managing Execution",
|
||||
"definition": {
|
||||
"text": "Bots can execute at their own pace or they can wait for other bots to finish executing in order for them to start.",
|
||||
"updated": 1621903220369
|
||||
"text": "Bots can execute at their own pace or they can wait for other bots to finish executing before starting.",
|
||||
"updated": 1622214711890
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Usually when a bot produces raw data, it executes itself based on the configuration of it's Process Definition node. These bots are usually the root of an execution tree formed by all the bots that will process and reprocess the raw data into more and more elaborate datasets.",
|
||||
"updated": 1621710144112
|
||||
"style": "Title",
|
||||
"text": "Bot Execution Tree",
|
||||
"updated": 1622215214005
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "There are two types of execution managment."
|
||||
"text": "A helpful way to think about managing bot execution is to think of the image of a tree. The roots support the trunk which supports the branches. In the same way, Superalgos uses many bots to process data each relying on the output of the last to form a tree. As data is passed through the tree the raw data is processed and reprocessed to build more and more elaborate datasets.",
|
||||
"updated": 1622215593975
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Types of Execution Managment"
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "There are two types of execution managment:"
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Standalone Bots: These bots are the root of the Bot Execution Tree. They don't depend on any other bot to run. They usually run on a pre-defined schedulle, in most cases every one minute.",
|
||||
"updated": 1621710324631
|
||||
"text": "Standalone Bots: These bots are the root of the Bot Execution Tree. They don't depend on any other bot to run but use the configuration of it's Process Definition node to run on a pre-defined schedule (in most cases every one minute).",
|
||||
"updated": 1622215038717
|
||||
},
|
||||
{
|
||||
"style": "Note",
|
||||
"text": "Usually when a bot produces raw data, it operates as a standalone bot. ",
|
||||
"updated": 1622215509477
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Execution Dependent Bots: These bots are the branches of the Bot Execution Tree and they need to wait for another Bot to finish it's execution in order for it to start.",
|
||||
"updated": 1621710352983
|
||||
"text": "Execution Dependent Bots: These bots are the branches of the Bot Execution Tree. They need to wait for other Bots to finish their execution in order to start.",
|
||||
"updated": 1622215195801
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Configuring a Standalone Bot",
|
||||
"updated": 1622215700788
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "This is an example of how to configure a bot to be a Standalone Bot from the execution point of view. As you can see in the example below, the normalWaitTime property has a non zero value, in this case 60 seconds.",
|
||||
"updated": 1621710692139
|
||||
"text": "This is an example of how to configure a bot to execute as a Standalone Bot. Within the Process Definition of a Bot, the normalWaitTime property has a non zero value, in this case 60 seconds.",
|
||||
"updated": 1622215895631
|
||||
},
|
||||
{
|
||||
"style": "Json",
|
||||
"text": "{\n \"codeName\": \"Historic-OHLCVs\",\n \"description\": \"Retrieves and saves historical maket data\",\n \"deadWaitTime\": 0,\n \"normalWaitTime\": 60000,\n \"retryWaitTime\": 10000,\n \"botModule\": \"Exchange Raw Data Historic OHLCVs\"\n}",
|
||||
"updated": 1621710641664
|
||||
"text": "{\n \"codeName\": \"Historic-OHLCVs\",\n \"description\": \"Retrieves and saves historical market data\",\n \"deadWaitTime\": 0,\n \"normalWaitTime\": 60000,\n \"retryWaitTime\": 10000,\n \"botModule\": \"Exchange Raw Data Historic OHLCVs\"\n}",
|
||||
"updated": 1622215784605
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Configuring an Execution Dependent Bot",
|
||||
"updated": 1622215826932
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "This is how to configure the Process Definition of a Bot that it's execution depends on other Bot. Here the normalWaitTime has a value of zero, which will force the Bot to inmediatelly run the next loop, but since there is an Execution Started Event defined, that will make it wait until the referenced Bot finishes.",
|
||||
"updated": 1621903624630
|
||||
"text": "In order to configure a bot to execute as an Execution Dependent Bot two things need to be in place. ",
|
||||
"updated": 1622216289412
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "First, the bot's definition will need to have an Execution Started Event defined. After this, the Execution Started Event will need to have a reference to the Execution Finished Event of the Bot on which it is waiting.",
|
||||
"updated": 1622216544766
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Second, the Bot's Process Definition will need a zero in the normalWaitTime property. This will force the Bot to immediately run the next loop, but since there is an Execution Started Event defined, that will make it wait until the referenced Bot finishes.",
|
||||
"updated": 1622216345546
|
||||
},
|
||||
{
|
||||
"style": "Json",
|
||||
"text": "{\n \"codeName\": \"Multi-Time-Frame-Market\",\n \"normalWaitTime\": 0,\n \"retryWaitTime\": 10000,\n \"framework\": {\n \"name\": \"Multi-Time-Frame-Market\"\n }\n}",
|
||||
"updated": 1621710596317
|
||||
},
|
||||
{
|
||||
"style": "Success",
|
||||
"text": "In this case users must also define which bot to wait for. To do this they stablish a reference from the Execution Started Event node of the Bot that needs to wait, to the Execution Finished Event of the Bot they are waiting for to finish.",
|
||||
"updated": 1621710539966
|
||||
},
|
||||
{
|
||||
"style": "Note",
|
||||
"text": "From the deployment point of view, the dependant bots can be physically located at the same machine, or at multiple different machines. Superalgos handles the syncronization across multiples machines.",
|
||||
"updated": 1621903914878
|
||||
"text": "From a deployment point of view, the dependant bots can be physically located on the same machine, or on multiple different machines. Superalgos handles syncronization across multiples machines.",
|
||||
"updated": 1622216626010
|
||||
}
|
||||
]
|
||||
}
|
||||
+27
-14
@@ -3,35 +3,48 @@
|
||||
"pageNumber": 8,
|
||||
"type": "Data Mining - Data Dependencies",
|
||||
"definition": {
|
||||
"text": "Indicator Bots have data dependencies. That means that they have as their input a Dataset created by other Bot. ",
|
||||
"updated": 1621715782447
|
||||
"text": "Indicator Bots have data dependencies. This means that they receive their input as Datasets created by other Bots.",
|
||||
"updated": 1622216765574
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "You can also think about data dependencies as a type of relationship between bots. Bots producing raw data (Sensor Bot and API Data Fetcher Bot) don't have them, since their job is to extract data from an external source. At the same time they are the root of a tree or graph of data dependencies relationships.",
|
||||
"updated": 1621715986533
|
||||
"style": "Title",
|
||||
"text": "Data Dependency Tree",
|
||||
"updated": 1622219215680
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "The data dependencias are in fact at the Process Definition level. Each process might have their own set of data dependencies. "
|
||||
"text": "You can think about data dependencies as a type of relationship between bots. Bots producing raw data (Sensor Bot and API Data Fetcher Bot) don't have them, since their job is to extract data from an external source. At the same time, they are also the root of the data dependancy tree of relationships. This tree is very similar to the execution tree but instead of describing how bots run, it describes how bots send and receive Data Products.",
|
||||
"updated": 1622217397887
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Defining Data Dependencies",
|
||||
"updated": 1622217419549
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "The definition of data dependencias is held at the Process Definition level. Each process might have its own set of data dependencies. This means that a single bot may have many varying dependencies depending on the number of Process Definitions it contains."
|
||||
},
|
||||
{
|
||||
"style": "Note",
|
||||
"text": "It is possible for a single Bot's process to have more than one Data Dependency, from the same Bot or from different Bot's. That is particulary usefull to aggregate data from many datasets into another one. The dependent datasets could belong to Bots at the same Data Mine or at bots at other Data Mines. From the deployment point of view, the dependant datasets could be physically located at the same machine, or at multiple different machines."
|
||||
"text": "In addtion to this, it is possible for a single Bot's process to have more than one Data Dependency (from the same Bot or from different Bots). That is particulary useful to aggregate data from many datasets into another one. The dependent datasets could belong to Bots at the same Data Mine or at bots at other Data Mines.",
|
||||
"updated": 1622217595798
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "The first declared dependency is called the Main Dependency and has a special meaning. The process of the Bot is going to iterate the Dataset of the Main Dependency to produce the output Dataset. All other data dependencies will be used to extract data from."
|
||||
"text": "The first declared dependency is called the Main Dependency and has a special meaning. It is used by the process of the Bot to iterate. What this means is that the process will loop over the Dataset of the Main Dependency to produce the output Dataset. All the other data dependencies will be used each iteration to extract additional data.",
|
||||
"updated": 1622218002546
|
||||
},
|
||||
{
|
||||
"style": "Success",
|
||||
"text": "As with bot execution, the dependant datasets can be physically located on the same machine, or at multiple different machines. However, when a Bot has a Data Dependency of a dataset produced by a Bot defined at a different Data Mine, both Data Mines must be present at the Workspace where the Bot is going to be launched.",
|
||||
"updated": 1622218098435
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "When a Bot has a Data Dependency of a dataset produced by a Bot defined at a different Data Mine, then both Data Mines must be present at the Workspace where the Bot is going to be lauched at.",
|
||||
"updated": 1621903950116
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "The Masters Data Mine is the most referenced Data Mine because is the one that defines the Candles-Volumes Indicator Bot, that produces the most used information for trading systems: Candles."
|
||||
"text": "The Candles Data Mine is the most referenced Data Mine because it defines the Candles-Volumes Indicator Bot, that produces the most used information for trading systems: Candles.",
|
||||
"updated": 1622218178970
|
||||
}
|
||||
]
|
||||
}
|
||||
+20
-10
@@ -3,23 +3,33 @@
|
||||
"pageNumber": 9,
|
||||
"type": "Data Mining - Status Reports",
|
||||
"definition": {
|
||||
"text": "Status Reports are files produced by each Bot's processes with some key information that the Bot needs to save in order to remember it the next time is run.",
|
||||
"updated": 1621904046327
|
||||
"text": "Status Reports are files produced by each Bot's processes with key information the Bot needs to save in order to remember the next time it is run.",
|
||||
"updated": 1622218351726
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Most Bots in Superalgos are run by the Task Server and they implement a Main Loop. At each loop execution, bots do their job and at the end they go to sleep. How much time they sleep and other details are configured at the Bot's Process Definition.",
|
||||
"updated": 1621904131214
|
||||
"style": "Title",
|
||||
"text": "The Task Server Main Loop",
|
||||
"updated": 1622218578406
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Before going to sleep, all the memory is flushed from all information related to the Bot, so that hardware resources are freed for other bots in the same machine to use. There is some info Bots need to remember between every execution of the Main Loop, and between executions of the Task itself. That info is stored in a JSON / text file called the Status Report.",
|
||||
"updated": 1621904194069
|
||||
"text": "Most Bots in Superalgos are run by the Task Server and implement a Main Loop. At each loop execution, bots do their job and at the end they go to sleep. ",
|
||||
"updated": 1622218637972
|
||||
},
|
||||
{
|
||||
"style": "Note",
|
||||
"text": "How much time they sleep and other details are configured at the Bot's Process Definition."
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Status Report Mechanics",
|
||||
"updated": 1622218611081
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "The name Status was used since the information stored is often the status of the Bot's process. Here is an example of a simple Status Report file."
|
||||
"text": "Before a bot goes to sleep, computer memory is flushed of all information related to the Bot. This is important to free up hardware resources for other bots running on the same machine. However, there is some information Bots need to remember between each execution of the Main Loop, and between executions of the Task itself. That info is stored in a JSON / text file called the Status Report. The name Status was used since the information stored is often the status of the Bot's process. Here is an example of a simple Status Report file:",
|
||||
"updated": 1622218847283
|
||||
},
|
||||
{
|
||||
"style": "Json",
|
||||
@@ -28,8 +38,8 @@
|
||||
},
|
||||
{
|
||||
"style": "Note",
|
||||
"text": "Status Report files also might include a data structure called Inter Execution Memory. Status information is kind of standard for all types of bots. Inter Execution Memory on the other hand, depends on each Bot type and what is actually processing.",
|
||||
"updated": 1621904260686
|
||||
"text": "Status Report files might also include a data structure called Inter Execution Memory. Status information is standard for all types of bots. Inter Execution Memory, on the other hand, depends on each Bot's type and what is actually being processed.",
|
||||
"updated": 1622218782345
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
|
||||
+26
-17
@@ -3,24 +3,32 @@
|
||||
"pageNumber": 10,
|
||||
"type": "Data Mining - Status Dependencies",
|
||||
"definition": {
|
||||
"text": "Status Dependencies is a type of relationship between Bots. Sometimes bots don't depend on the Datasets produced by other bots, but they depend directly on the status of other bots.",
|
||||
"updated": 1621717479906
|
||||
"text": "Status Dependencies are another type of relationship between Bots. Sometimes bots don't depend on the Datasets produced by other bots, but they depend directly on the status of other bots.",
|
||||
"updated": 1622219077845
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "A good way to undestand why Status Dependencies are needed is with an example:",
|
||||
"updated": 1621717585017
|
||||
"style": "Title",
|
||||
"text": "Status Dependency Tree",
|
||||
"updated": 1622219244127
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "The Indicator Bot Bollinger-Bands depends on Candles-Volumes Dataset in order to calculate the Bollinger Bands, but it depend's on Exchange Raw Data Sensor Bot in order to know where the market begins, which is the date of the first piece of data. That information is part of the Status information that Exchange Raw Data Sensor Bot keeps at it's Satus Report.",
|
||||
"updated": 1621717669486
|
||||
"text": "Status dependencies operate using the same idea of a tree as execution and data dependencies. Any Bot can have from zero to many status dependencies. For each Status Dependency, the Bot's process at runtime will load the Status Report file of the dependent Bot and will extract from there the information it needs.",
|
||||
"updated": 1622222998840
|
||||
},
|
||||
{
|
||||
"style": "Note",
|
||||
"text": "Bots that produce Raw Data usually do not depend on any other Bot's status and are usually the root of a status dependency tree. "
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Bots that produce Raw Data usually do not depend on any other Bot's status and they are usually the root of a dependency tree or graph. Any Bot can have from zero to many status dependencies. For each Status Dependency, the Bot's process at runtime will load the Status Report file of the dependent Bot and will extract from there the information it needs.",
|
||||
"updated": 1621904417277
|
||||
"text": "A good way to undestand why Status Dependencies are needed is with an example:"
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "The Indicator Bot Bollinger-Bands depends on Candles-Volumes Dataset in order to calculate the Bollinger Bands, but it depend's on Exchange Raw Data Sensor Bot in order to know where the market begins, which is the date of the first piece of data. That information is part of the Status information that Exchange Raw Data Sensor Bot keeps at it's Status Report.",
|
||||
"updated": 1622223640276
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
@@ -29,21 +37,22 @@
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "A Status Dependency can be classified by it's utility. This make it easier for processes to handle them in code. There are currently three different utilities:",
|
||||
"updated": 1621904453267
|
||||
"text": "A Status Dependency can be classified by it's utility. This makes it easier for processes to handle them in code. There are currently three different utilities:",
|
||||
"updated": 1622223177318
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Self Reference: this is used to load it's own Status Report to extract data from it or modify it."
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Market Ending Point: this means that the referenced Status Report is to be used to extract the information of where the market ends, meaning which date the market ends."
|
||||
"text": "Self Reference: this is used to load a bot's Status Report to extract data from it or modify it.",
|
||||
"updated": 1622223714025
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Market Starting Point: this means that the referenced Status Report is to be used to extract the information of where the market begins.",
|
||||
"updated": 1621718175842
|
||||
"updated": 1622223349370
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Market Ending Point: this means that the referenced Status Report is to be used to extract the information of where the market ends, meaning which date the market ends."
|
||||
},
|
||||
{
|
||||
"style": "Important",
|
||||
|
||||
+16
-14
@@ -3,19 +3,19 @@
|
||||
"pageNumber": 11,
|
||||
"type": "Data Mining - Deployments",
|
||||
"definition": {
|
||||
"text": "Data Minign Bots can be deployed into one or many machines at the same time and they will automatically find the way too coordinate their actions.",
|
||||
"updated": 1621718512909
|
||||
"text": "Data Mining Bots can be deployed into one or many machines at the same time and they will automatically find the way too coordinate their actions.",
|
||||
"updated": 1622225530181
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Once you are done defining your data mining Bots, you can move the Tasks that execute them to other Network Nodes and some other node structures at the Network hierarchy and the bot's will still run even from different machines on the network.",
|
||||
"updated": 1621719049976
|
||||
"text": "Once you are done defining your data mining Bots, you can move the Tasks that execute them to other Network Nodes and some other node structures within the Network hierarchy. The bot's will still run even from different machines on the network.",
|
||||
"updated": 1622226742772
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "How does Superalgos know how to coordinate Bots?",
|
||||
"updated": 1621904714762
|
||||
"text": "How Does Superalgos Know How to Coordinate Bots?",
|
||||
"updated": 1622226151968
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
@@ -24,26 +24,28 @@
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "In Data Mines: The definitions at the Data Mine contain the 3 types of relationships between Bots. Execution, Data Dependency and Status Dependency. "
|
||||
"text": "In Data Mines: The definitions at the Data Mine contain the 3 types of relationships between Bots: Execution Dependency, Data Dependency and Status Dependency.",
|
||||
"updated": 1622226774180
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "At the Network: The Network hierarchy contains all the information needed to know where the dependent bots are running (looking at the Tasks we can know at which machine in the network) and where the Datasets are in the network (looking at the Data Storage section)."
|
||||
"text": "At the Network: The Network hierarchy contains all the information needed to know where the dependent bots are running. Looking at the Tasks on the network node, we can know which bots are running on a particular machine in the network. Looking at the Data Storage hierarchy, we can know where Datasets are being storied in the network.",
|
||||
"updated": 1622226083877
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "How does a Bot know where are it's dependencies running?",
|
||||
"updated": 1621904824161
|
||||
"text": "How Does a Bot Know Where Its Dependencies are Running?",
|
||||
"updated": 1622226168096
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Everytime a Task is executed, the whole Network hierarchy, together with the bot's definitions at Data Mines are sent to the Task Server. This includes not only the Bot to be ran definitions, but also all the definitions of the dependent bots. ",
|
||||
"updated": 1621904939603
|
||||
"text": "Everytime a Task is executed, the whole Network hierarchy, together with the bot definitions at the Data Mines are sent to the Task Server. This includes not only the running Bot's definition, but also all the definitions of the dependent bots.",
|
||||
"updated": 1622226410465
|
||||
},
|
||||
{
|
||||
"style": "Success",
|
||||
"text": "At runtime, Superalgos will provide the Bot with the Datasets defined at Data Depencies, the Status Reports defined as Status Dependencies and if necesary, it iwll help the Bot wait for the Bot defined as an Execution Dependency, no matter where all those resources are within the physical network.",
|
||||
"updated": 1621905102590
|
||||
"text": "At runtime, Superalgos will provide the Bot with the Datasets defined at Data Depencies, the Status Reports defined as Status Dependencies, and if necessary, it will help the Bot wait for the Bot defined as an Execution Dependency, no matter where all those resources are within the physical network.",
|
||||
"updated": 1622226870002
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -3,14 +3,14 @@
|
||||
"pageNumber": 12,
|
||||
"type": "Data Mining - Products",
|
||||
"definition": {
|
||||
"text": "All Datasets of any type of bot within the Superalgos system are considered products of the Bot that produced them. Why products? Because users will be able to sell hese datasets in the form of subscriptions once the Superalgos Network is implemented.",
|
||||
"updated": 1621905194343
|
||||
"text": "All Datasets of any type created by bots within the Superalgos system are considered data products of the Bot that produced them. Why products? Because users will be able to sell these datasets in the form of subscriptions once the Superalgos Network is implemented.",
|
||||
"updated": 1622226983424
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "In the future, users will be able to offer products to the rest of the users of the Superalgos Network / Superalgos Marketplace. A single bot might produce as many products as the bot designer wishes. Each product might contain one or more datasets.",
|
||||
"updated": 1621905284676
|
||||
"text": "In the future, users will be able to offer products to the rest of the users within the Superalgos Network / Superalgos Marketplace. A single bot might produce as many products as the bot designer wishes. Each product might contain one or more datasets.",
|
||||
"updated": 1622227043268
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -3,14 +3,14 @@
|
||||
"pageNumber": 13,
|
||||
"type": "Data Mining - Plotting",
|
||||
"definition": {
|
||||
"text": "Each product might have one Plotter Module associated with it. This Plotter Module has all the definitions needed to interpret the Product's datasets and render the information at the charts.",
|
||||
"updated": 1621905321330
|
||||
"text": "Each product may have one Plotter Module associated with it. This Plotter Module has all the definitions needed to interpret the Product's datasets and render the information on a chart.",
|
||||
"updated": 1622227086801
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Superalgos has the capability to allow it's users to create plotters without writing any code at all. The ploter is defined at the Designer Space, and by the means of defining vertices and polygons, users can create custom charts for any Dataset.",
|
||||
"updated": 1621905478694
|
||||
"text": "Superalgos has the capability to allow it's users to create plotters without writing any code. The plotter is defined at the Designer Space. This is done by means of defining vertices and polygons. Users can create custom charts for any Dataset.",
|
||||
"updated": 1622227235452
|
||||
}
|
||||
]
|
||||
}
|
||||
+64
-53
@@ -3,8 +3,8 @@
|
||||
"pageNumber": 14,
|
||||
"type": "Data Mining - Fetching Data From APIs",
|
||||
"definition": {
|
||||
"text": "Superalgos allow it's users to fetch data from any API on the Internet without needing to write code to do it. Instead, users design an API Map, which is a set of definitions of how a Bot can access a certain API and how to interpret the API Server responses. With this API Maps, a API Data Fetcher Bot can be configured to extract data from any API Server and stored inside the system as Raw Data that later can processed and consumed by other Bots.",
|
||||
"updated": 1621906349521
|
||||
"text": "Superalgos allows its users to fetch data from any API on the Internet without needing to write code. Instead, users design an API Map, which is a set of definitions for how a Bot can access a certain API and how to interpret the API Server responses. With this API Map, an API Data Fetcher Bot can be configured to extract data from any API Server and store Raw Data inside the system. This raw data can later be processed and consumed by other Bots.",
|
||||
"updated": 1622228621778
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
@@ -14,24 +14,28 @@
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "How to fetch data from an API?",
|
||||
"updated": 1621951311144
|
||||
"text": "How to Fetch Data from an API?",
|
||||
"updated": 1622228699547
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "The main workflow to pull this out is quite straight forward and it does not require coding at all. Follow this guide and you will be using data from any source at the Internet from your trading strategies in no time."
|
||||
"text": "The main workflow to pull this off is quite straight forward. It does not require coding at all. Follow this guide and you will be using data from any source on the Internet for your trading strategies in no time.",
|
||||
"updated": 1622228741015
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "1. Choose an API: Current version is limited to public APIs only. Secured APIs comming soon. "
|
||||
"text": "1. Choose an API: The current version is limited to public APIs only. Secured APIs coming soon.",
|
||||
"updated": 1622229777229
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "2. Design an API Map: Read the APIs online documentation and identify the Endpoints that returns the data you would like to fetch. Carefully map those Endpoints defining the parameters the API Server expects at each call, and also how the API Server response is formatted."
|
||||
"text": "2. Design an API Map: Read the APIs online documentation and identify the Endpoints that return the data you would like to fetch. Carefully map the Endpoints defining the parameters the API Server expects at each call, and also how the API Server response is formatted.",
|
||||
"updated": 1622229840237
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "3. Setup an API Data Fetcher Bot: Create a Data Mine for this API and create there an API Data Fetcher Bot. From that Bot's Record Properties you will need to reference the fields returned by the API Server defined at the API Map, defining in this way which fields from each response you are going to save to disk. You might also need to define query or path parameters here. "
|
||||
"text": "3. Setup an API Data Fetcher Bot: Create a Data Mine for this API and add an API Data Fetcher Bot. From that Bot's Record Properties you will need to reference the fields returned by the API Server defined at the API Map. This is how you define which fields from each response you are going to save to disk. You might also need to define query or path parameters here.",
|
||||
"updated": 1622229993447
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
@@ -40,11 +44,13 @@
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "5. Setup Indicator Bots: Once the data is converted, you can extract from it specific Datasets defining a set of indicators you can build from that data. These Indicators can be just a reorganization of the data, with some adjustments on the names of fields and splitting the data in different indicators, or it can also include some data processing / transformations if you wish (this would require some coding)."
|
||||
"text": "5. Setup Indicator Bots: Once the data is converted, you can extract specific Datasets from it. These datasets can be used to define a set of indicators you can build from that data. These Indicators can be as simple as reorganizations of the data (with some adjustments to the names of fields and splitting the data into different indicators), or it can include some data processing / transformations if you wish (this would require some coding).",
|
||||
"updated": 1622230478744
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "6. Setup Plotters: These last indicators can have their plotters if you wish."
|
||||
"text": "6. Setup Plotters: After you have set up your indicators. You can design plotters for them if you wish.",
|
||||
"updated": 1622230566915
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
@@ -52,20 +58,18 @@
|
||||
},
|
||||
{
|
||||
"style": "Success",
|
||||
"text": "There is a Workspace called API Data Fetcher Bot Demo with a simple working example of the previous steps you can take as reference."
|
||||
},
|
||||
{
|
||||
"style": "Block",
|
||||
"text": "Data Aggregation"
|
||||
"text": "There is a Workspace called API Data Fetcher Bot Demo with a simple working example of the previous steps you can use as a reference.",
|
||||
"updated": 1622230619069
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "How does data aggregation works?"
|
||||
"text": "How Does Data Aggregation Work?",
|
||||
"updated": 1622230634850
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "When Raw Data is fetched from the Internet, it needs to be converted into a Dataset Type either of Multi-Time-Frame-Daily and Multi-Time-Frame-Market types. Usually Raw Data is a timestamped dataset with a timesStamp field or it has a begin and end properties if they are fragmented in time slots of one minute.",
|
||||
"updated": 1621973202254
|
||||
"text": "When Raw Data is fetched from the Internet, it needs to be converted into a Dataset Type either of Multi-Time-Frame-Daily or Multi-Time-Frame-Market types. Usually Raw Data is a timestamped dataset with a timesStamp field, or it has begin and end properties if they are fragmented in time slots of one minute.",
|
||||
"updated": 1622230917402
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
@@ -73,8 +77,8 @@
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "At the Bot where this conversion happens, the Record Property nodes needs to have a config property agregattionMode that defines how this is going to be handled. Here is a list of the supported aggregated methods that can be used during this conversion procedure.",
|
||||
"updated": 1621973294298
|
||||
"text": "At the Bot where this conversion happens, the Record Property nodes needs to have an aggregationMethod config property that defines how this is going to be handled. Here is a list of the supported aggregation methods that can be used during this conversion procedure:",
|
||||
"updated": 1622235156415
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
@@ -86,11 +90,13 @@
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Min: This means that the mathematical minimun value from all values found in the time range will be used as the value for the Record Proeprty."
|
||||
"text": "Min: This means that the mathematical minimum value from all values found in the time range will be used as the value for the Record Proeprty.",
|
||||
"updated": 1622231381194
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Max: This means that the mathematical maximun value from all values found in the time range will be used as the value for the Record Proeprty."
|
||||
"text": "Max: This means that the mathematical maximum value from all values found in the time range will be used as the value for the Record Proeprty.",
|
||||
"updated": 1622231375974
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
@@ -100,13 +106,10 @@
|
||||
"style": "List",
|
||||
"text": "Avg: This means that the average of all values found in the time range will be used as the value for the Record Proeprty."
|
||||
},
|
||||
{
|
||||
"style": "Block",
|
||||
"text": "Step By Step"
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Step by step guite to fetch data from an API"
|
||||
"text": "Step by Step Guide to Fetch Data from an API",
|
||||
"updated": 1622231458352
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
@@ -120,13 +123,13 @@
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Step #2: Open a second UI (this will prevent the previous one to save the changes, but you don't need to do that). You will use the first UI to check from time to time how nodes are configured there, and the second UI to build your own data mining operation from scratch. The rest of the staps you will do it at the second UI.",
|
||||
"updated": 1621976658929
|
||||
"text": "Step #2: Open a second UI (this will prevent the previous one from saving the changes, but you don't need to do that). You will use the first UI to check from time to time how nodes are configured there, and the second UI to build your own data mining operation from scratch. The rest of the steps you will do within the second UI.",
|
||||
"updated": 1622231623225
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Step #3: Select an API from the Internet with data that might be usefull at your Trading Strategy.",
|
||||
"updated": 1621976668506
|
||||
"text": "Step #3: Select an API from the Internet with data that might be useful at your Trading Strategy.",
|
||||
"updated": 1622231649073
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
@@ -150,13 +153,13 @@
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Step #8: Define one API Endpoint first, once you have the first one working, go for the rest of the enpoints available.",
|
||||
"updated": 1621976699911
|
||||
"text": "Step #8: Define one API Endpoint first, once you have the first one working, go for the rest of the endpoints available.",
|
||||
"updated": 1622231734738
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Step #9: Define the query or path parameters and the the responses. Find at each node's help the details on how to do it right.",
|
||||
"updated": 1621976710636
|
||||
"text": "Step #9: Define the query, or path parameters, and the responses. Read each node's help for details on how to do it right.",
|
||||
"updated": 1622231862303
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
@@ -165,18 +168,18 @@
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Step #11: Create a new Data Mine woth the name of the company that host the API.",
|
||||
"updated": 1621976722023
|
||||
"text": "Step #11: Create a new Data Mine with the name of the company that hosts the API.",
|
||||
"updated": 1622231912331
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Step #12: Create at that Data Mine and a API Data Fetcher Bot. Configure the Bot's nodes following the example at the UI #1.",
|
||||
"updated": 1621976915804
|
||||
"text": "Step #12: Within the Data Mine add an API Data Fetcher Bot. Configure the Bot's nodes following the example at the UI #1.",
|
||||
"updated": 1622231954570
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Step #13: Install a new market. This procedure is going to setup many things for you. It will detect the API Map, and the Data Mine you created and will create appropiate Tasks to run your API Fertcher Bot.",
|
||||
"updated": 1621976960858
|
||||
"text": "Step #13: Install a new market. This procedure is going to setup many things for you. It will detect the API Map, and the Data Mine you created and will create appropiate Tasks to run your API Fetcher Bot.",
|
||||
"updated": 1622232103844
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
@@ -184,7 +187,8 @@
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Step #15: Define at your API Data Fetcher Bot Product Definition Record Properties the fields that you would like to save from the API Server response. You don't need to save everything, only what makes sense to you."
|
||||
"text": "Step #15: At your API Data Fetcher Bot's Product Definition Record Properties, define the fields that you would like to save from the API Server response. You don't need to save everything, only what makes sense to you.",
|
||||
"updated": 1622232590430
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
@@ -192,12 +196,13 @@
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Step #17: Pay attention to the special case of the needed timeStamp Record Property, specially because it might be pointing to the header of the API response. Read the docs of the API Response Field Reference node to set up the nodePath property if necesary."
|
||||
"text": "Step #17: Pay attention to the special case of the required timeStamp Record Property. Specifically because it might be pointing to the header of the API response. Read the docs of the API Response Field Reference node to set up the nodePath property if necessary.",
|
||||
"updated": 1622232714229
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Step #18: Setup an your Data Mine an Indicator Bot and configure their Process Definition nodes to be of the framework that converts from One-Min-Daily to Multi-Time-Frame-Daily and Multi-Perio-Market types of Datasets. See how it was done at the UI #1.",
|
||||
"updated": 1621977642805
|
||||
"text": "Step #18: At your Data Mine, set up an Indicator Bot and configure its Process Definition nodes to the framework that will convert from One-Min-Daily to Multi-Time-Frame-Daily and Multi-Perio-Market types of Datasets. See how it was done at the UI #1.",
|
||||
"updated": 1622232795794
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
@@ -205,11 +210,13 @@
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Step #20: Configure at each Record Proeprty the aggregationMode property. Check the docs of that node for details."
|
||||
"text": "Step #20: Configure the aggregationMethod property for each Record Property. Check the docs of that node for details.",
|
||||
"updated": 1622235178187
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Step #21: If you wish you can create multiple other Indicator Bots to cut the dataset in different groups of Record Properties."
|
||||
"text": "Step #21: If you wish, you can create multiple Indicator Bots to cut the dataset in different groups of Record Properties.",
|
||||
"updated": 1622233072603
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
@@ -225,24 +232,28 @@
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Step #25: Iterate through step #24 until everything is working. Re read node's documentations until you get the details of what it explains there. Reread this guide or this whole Topic from the start, until the concepts explained sinks in. "
|
||||
"text": "Step #25: Iterate through step #24 until everything is working. Re read node's documentation until you get the details of what it explains there. Reread this guide or this whole Topic from the start, until the concepts explained sinks in.",
|
||||
"updated": 1622233708371
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Step #26: If after digesting the docs, and following all this advice you stil can not put it to work, go to the Telegram Support group and ask for help."
|
||||
"text": "Step #26: If after digesting the docs, and following all this advice you still cannot get it to work, go to the Telegram Support group and ask for help.",
|
||||
"updated": 1622233735826
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Step #27: Once all is running well, set up your Trading Stragegy in a way that allows you to test if you have acces to the prcessed data, the one provided by the latest Indicator Bots created."
|
||||
"text": "Step #27: Once all is running well, set up your Trading Strategy in a way that allows you to test if you have access to the processed data (the one provided by the latest Indicator Bots created).",
|
||||
"updated": 1622233777598
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Step #29: Once the data is correctly consumed by your Trading Strategy iterate throug this guide by adding another API Endpoint to the API Map. You will need for that a new API Fetcher Bot.",
|
||||
"updated": 1621978484377
|
||||
"text": "Step #29: Once the data is correctly consumed by your Trading Strategy, iterate through this guide by adding another API Endpoint to the API Map. You will need to add an addtional API Fetcher Bot.",
|
||||
"updated": 1622233824073
|
||||
},
|
||||
{
|
||||
"style": "List",
|
||||
"text": "Step #30: Once you have sucessfully mapped the whole API and you have all the needed API Data Fetcher, Indicator Bots and Plotters tested, you are ready to share your API Map and Data Mine with the community as plugins, and get in exchange SA tokens for your contribution. "
|
||||
"text": "Step #30: Once you have successfully mapped the whole API and you have all the needed API Data Fetchers, Indicator Bots and Plotters tested, you are ready to share your API Map and Data Mine with the community as a plugin, and receive SA tokens in exchange for your contribution.",
|
||||
"updated": 1622233875203
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
|
||||
+4
-3
@@ -3,13 +3,14 @@
|
||||
"pageNumber": 15,
|
||||
"type": "Data Mining - Consuming Data From Strategies",
|
||||
"definition": {
|
||||
"text": "Traing Bots can have as Data Dependencies datasets from any Indicator Bots. This allows Trading Strategies to act upon the data processed by Indicator Bots.",
|
||||
"updated": 1621721963137
|
||||
"text": "Trading Bots can have Data Dependencies from any Indicator Bot that puts out data products. This allows Trading Strategies to act upon the data processed by Indicator Bots.",
|
||||
"updated": 1622234087314
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Right click and Edit to enter edit mode and change this text. ENTER to write new paragraphs. ESC to exit edit mode. Then you can use docs.save to save your changes and app.contribute to send it to the main repository."
|
||||
"text": "This is the heart and soul of the data mining process allowing users to create trading bots that act upon sophisticated data and data analysis logic.",
|
||||
"updated": 1622234159617
|
||||
}
|
||||
]
|
||||
}
|
||||
-106
@@ -1,106 +0,0 @@
|
||||
{
|
||||
"topic": "Data Mining",
|
||||
"pageNumber": 16,
|
||||
"type": "Data Mining - Data Mine Hierarchy",
|
||||
"definition": {
|
||||
"text": "Data mines enable the processing of data by consuming datasets others have produced, performing calculations, and outputting a new value-added data product that others may consume.",
|
||||
"updated": 1611407680590
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Superalgos puts a powerful data processing and visualization system in your hands. Use it to the fullest and you will see your intelligence increase by orders of magnitude.",
|
||||
"updated": 1611407700153
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Anyone may access raw trades and order book data from exchanges. Anyone may access information in the form of standard indicators and technical studies."
|
||||
},
|
||||
{
|
||||
"style": "Callout",
|
||||
"text": "However, Superalgos allows you to perform any sort of processing over existing information across the whole span of the market, store the results into new datasets, and make them available to trading strategies. Moreover, it allows you to produce elaborate graphical representations of data, which is instantly browsable over the charts."
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Data mines’ functionality is highly flexible and works under the same paradigm used for building strategies. They provide a visual interface and a framework that leads you through the process of creating new bots, with minimal coding skills required."
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "The vast majority of the complexity of processing data is handled internally by the bots, and you don’t need to deal with it or even understand it. The system isolates complexity and only exposes you to the crucial elements that impact what you wish to accomplish."
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "This section of the documentation provides you with the technical understanding of how data is handled by the system. The core of the functionality this section unveils covers—in essence—the following points:"
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "The capacity to define any number of calculation processes that may feed on each other’s outputs, resulting in a wealth of information that may be consumed by trading strategies."
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "The properties and characteristics of datasets resulting from the execution of such processes—how and where they are stored, in what format, how they are consumed internally, and potentially, by other systems."
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "The capacity to visualize the data over the charts and browse it with all the features built in the charting system."
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Data Mine",
|
||||
"updated": 1611407778880
|
||||
},
|
||||
{
|
||||
"style": "Include",
|
||||
"text": "Superalgos->Node->Data Mine->Definition",
|
||||
"updated": 1611407782909
|
||||
},
|
||||
{
|
||||
"style": "Include",
|
||||
"text": "Superalgos->Node->Data Mine->Content",
|
||||
"updated": 1611407786577
|
||||
},
|
||||
{
|
||||
"style": "Include",
|
||||
"text": "Superalgos->Concept->Reusable Snippets->Note for Hierarchy Tables No Discussion"
|
||||
},
|
||||
{
|
||||
"style": "Hierarchy",
|
||||
"text": "Superalgos->Data Mine->2",
|
||||
"updated": 1611407832651
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": ""
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Sensor Bot",
|
||||
"updated": 1611407910652
|
||||
},
|
||||
{
|
||||
"style": "Include",
|
||||
"text": "Superalgos->Node->Sensor Bot->Definition",
|
||||
"updated": 1611407914247
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Indicator Bot",
|
||||
"updated": 1611407935038
|
||||
},
|
||||
{
|
||||
"style": "Include",
|
||||
"text": "Superalgos->Node->Indicator Bot->Definition",
|
||||
"updated": 1611407948292
|
||||
},
|
||||
{
|
||||
"style": "Title",
|
||||
"text": "Plotter",
|
||||
"updated": 1611407956192
|
||||
},
|
||||
{
|
||||
"style": "Include",
|
||||
"text": "Superalgos->Node->Plotter->Definition",
|
||||
"updated": 1611407972840
|
||||
}
|
||||
]
|
||||
}
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"topic": "Data Mining",
|
||||
"pageNumber": 17,
|
||||
"pageNumber": 16,
|
||||
"type": "Data Mining - Indicator Bots",
|
||||
"definition": {
|
||||
"text": "Indicator bots enable the processing of data by consuming datasets others have produced, performing calculations, and outputting a new value-added data product that others may consume.",
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"topic": "Data Mining",
|
||||
"pageNumber": 18,
|
||||
"pageNumber": 17,
|
||||
"type": "Data Mining - Indicator Bots - Process Definition",
|
||||
"definition": {
|
||||
"text": "The process definition is the compilation of configurations, dependencies, and references that make up a process.",
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"topic": "Data Mining",
|
||||
"pageNumber": 19,
|
||||
"pageNumber": 18,
|
||||
"type": "Data Mining - Indicator Bots - Product Definition",
|
||||
"definition": {
|
||||
"text": "The product definition is the compilation of configurations, references, and JavaScript code that make up the data product.",
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"topic": "Data Mining",
|
||||
"pageNumber": 20,
|
||||
"pageNumber": 19,
|
||||
"type": "Data Mining - Indicator Bots - Data Mining Workflow",
|
||||
"definition": {
|
||||
"text": "When running on a data mining operation, indicators follow a cycle designed to fulfill their purpose: to process data others have produced and make the result available for others to consume.",
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"topic": "Data Mining",
|
||||
"pageNumber": 21,
|
||||
"pageNumber": 20,
|
||||
"type": "Data Mining - Indicator Bots - Coding the Indicator's Logic",
|
||||
"definition": {
|
||||
"text": "Learn how to use data dependencies, the variable object, and how to deal with indicators with irregular periods.",
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"topic": "Data Mining",
|
||||
"pageNumber": 22,
|
||||
"pageNumber": 21,
|
||||
"type": "Data Mining - Plotters",
|
||||
"definition": {
|
||||
"text": "Plotters read data products created by bots and map certain coordinate points to use them as vertices for polygons, which are assigned styles to create graphic representations of the underlying data.",
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"topic": "Data Mining",
|
||||
"pageNumber": 23,
|
||||
"pageNumber": 22,
|
||||
"type": "Data Mining - Plotters - Shapes",
|
||||
"definition": {
|
||||
"text": "Shapes define coordinate points to use as vertex for polygons, which are assigned styles.",
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"topic": "Data Mining",
|
||||
"pageNumber": 24,
|
||||
"pageNumber": 23,
|
||||
"type": "Data Mining - How to Build an Indicator - Video Series",
|
||||
"definition": {
|
||||
"text": "This video series goes through the entire process of building an indicator, step by step.",
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"topic": "Frequently Asked Questions",
|
||||
"pageNumber": "2",
|
||||
"type": "How does the system handle changes to trading system during live trading?",
|
||||
"definition": {
|
||||
"text": "You need to restart the Trading Session and sometimes the Task in order for the Bot to get the new definitions.",
|
||||
"updated": 1622293227672
|
||||
},
|
||||
"paragraphs": [
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "Users cannot change the definitions and feed them to a running bot in real time. If the definitions are at the Trading System level, or the Session Parameters level, you can change them and they will be sent to the running Task the next time you stop and run the Trading Session again.",
|
||||
"updated": 1622293262444
|
||||
},
|
||||
{
|
||||
"style": "Text",
|
||||
"text": "It the definitions are at the Task level or upwards, like a change of market, you will need to stop the Task and run it again in order to be recognized."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -386,10 +386,11 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
}
|
||||
|
||||
let separator = ""
|
||||
pathString = '/'
|
||||
|
||||
parametersMap.forEach(addParameterToPath)
|
||||
|
||||
function addParameterToPath(value, key, map) {
|
||||
if (pathString === "") { pathString = '/' }
|
||||
let apiPathParameter = value
|
||||
let parameterValue = key // This is the default value
|
||||
if (apiPathParameter.config.replaceBy !== undefined) {
|
||||
@@ -517,7 +518,7 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
Now that we have the endpointNode, the parameters and all the information
|
||||
needed to place the call to the API.
|
||||
*/
|
||||
let httpClient = require('https')
|
||||
const fetch = require('node-fetch')
|
||||
let url = protocol + '://' +
|
||||
hostName +
|
||||
portNumber +
|
||||
@@ -532,140 +533,192 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
This is how we call the API.
|
||||
*/
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[INFO] start -> startProcess -> fetchAPIData -> httpClient.get -> url = " + url)
|
||||
"[INFO] start -> startProcess -> fetchAPIData -> url = " + url)
|
||||
|
||||
httpClient.get(url, (response) => {
|
||||
fetch(url)
|
||||
.then((response) => {
|
||||
|
||||
const chunks = []
|
||||
let apiResponseSchemaNode
|
||||
let errorCodeReceived
|
||||
|
||||
let apiResponseSchemaNode
|
||||
let errorCodeReceived
|
||||
|
||||
/*
|
||||
At this point we have received the first http reponse from the API server,
|
||||
and from it we can extract the status code, which is the html code of the
|
||||
response of our request.
|
||||
|
||||
With that code we can get the API Response Schema, which will tell us how
|
||||
to extract data from the response.
|
||||
*/
|
||||
getApiResponseSchema(response.statusCode)
|
||||
|
||||
if (apiResponseSchemaNode === undefined) {
|
||||
errorCodeReceived = true
|
||||
TS.projects.superalgos.utilities.errorHandlingFunctions.throwHandledException(
|
||||
processIndex,
|
||||
MODULE_NAME,
|
||||
'API Data Fetcher Bot',
|
||||
{ resposeCodeReceived: response.statusCode, detais: 'The response code ' + response.statusCode + ' received is not associated with any API Query Response node.' },
|
||||
'Unexpected API Response Code',
|
||||
endpointNode
|
||||
)
|
||||
}
|
||||
|
||||
function getApiResponseSchema(responseCode) {
|
||||
/*
|
||||
The API Response Schema we will depend on the response code received from the
|
||||
API server. Here we will locate the right Api Response Schema.
|
||||
At this point we have received the first http reponse from the API server,
|
||||
and from it we can extract the status code, which is the html code of the
|
||||
response of our request.
|
||||
|
||||
With that code we can get the API Response Schema, which will tell us how
|
||||
to extract data from the response.
|
||||
*/
|
||||
for (let j = 0; j < apiMap.apiVersions.length; j++) {
|
||||
let version = apiMap.apiVersions[j]
|
||||
for (let k = 0; k < version.apiEndpoints.length; k++) {
|
||||
let endpoint = version.apiEndpoints[k]
|
||||
if (endpointNode.id === endpoint.id) {
|
||||
for (let i = 0; i < endpoint.apiQueryResponses.apiQueryResponses.length; i++) {
|
||||
let apiQueryResponse = endpoint.apiQueryResponses.apiQueryResponses[i]
|
||||
getApiResponseSchema(response.status)
|
||||
|
||||
if (Number(apiQueryResponse.config.codeName) === responseCode) {
|
||||
apiResponseSchemaNode = apiQueryResponse.apiResponseSchema
|
||||
if (apiResponseSchemaNode === undefined) {
|
||||
errorCodeReceived = true
|
||||
TS.projects.superalgos.utilities.errorHandlingFunctions.throwHandledException(
|
||||
processIndex,
|
||||
MODULE_NAME,
|
||||
'API Data Fetcher Bot',
|
||||
{ resposeCodeReceived: response.status, errorDetails: 'Calling the URL ' + url + '. The response code ' + response.status + ' received is not associated with any API Query Response node.' },
|
||||
'Unexpected API Response Code',
|
||||
endpointNode
|
||||
)
|
||||
} else {
|
||||
getResponseBody()
|
||||
}
|
||||
|
||||
/*
|
||||
We will check that what we received is not an error code.
|
||||
*/
|
||||
if (apiQueryResponse.config.isError === true) {
|
||||
errorCodeReceived = true
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> getApiResponseSchema -> API Response is an Error -> responseCode = " + responseCode)
|
||||
function getApiResponseSchema(responseCode) {
|
||||
/*
|
||||
The API Response Schema we will depend on the response code received from the
|
||||
API server. Here we will locate the right Api Response Schema.
|
||||
*/
|
||||
for (let j = 0; j < apiMap.apiVersions.length; j++) {
|
||||
let version = apiMap.apiVersions[j]
|
||||
for (let k = 0; k < version.apiEndpoints.length; k++) {
|
||||
let endpoint = version.apiEndpoints[k]
|
||||
if (endpointNode.id === endpoint.id) {
|
||||
for (let i = 0; i < endpoint.apiQueryResponses.apiQueryResponses.length; i++) {
|
||||
let apiQueryResponse = endpoint.apiQueryResponses.apiQueryResponses[i]
|
||||
|
||||
if (Number(apiQueryResponse.config.codeName) === responseCode) {
|
||||
apiResponseSchemaNode = apiQueryResponse.apiResponseSchema
|
||||
|
||||
/*
|
||||
We will check that what we received is not an error code.
|
||||
*/
|
||||
if (apiQueryResponse.config.isError === true) {
|
||||
errorCodeReceived = true
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> getApiResponseSchema -> API Response is an Error -> responseCode = " + responseCode)
|
||||
}
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
response.on('data', (chunk) => {
|
||||
chunks.push(chunk)
|
||||
});
|
||||
|
||||
response.on('end', (d) => {
|
||||
apiResponseReceivedText = Buffer.concat(chunks).toString('utf8')
|
||||
/*
|
||||
If we received an errror code, we abort the processing at this point.
|
||||
*/
|
||||
if (errorCodeReceived === true) {
|
||||
resolve('ERROR_CODE_RECEIVED')
|
||||
return
|
||||
}
|
||||
/*
|
||||
Here we will validate that the overall format is what we are expecting.
|
||||
*/
|
||||
switch (apiResponseSchemaNode.apiResponseFields.config.fieldType) {
|
||||
case 'object': {
|
||||
function getResponseBody() {
|
||||
response.text().then(body => {
|
||||
apiResponseReceivedText = body
|
||||
/*
|
||||
If we did not received an object, that probably means something is not
|
||||
good, and we got an HTML with the reason inside.
|
||||
If we received an errror code, we abort the processing at this point.
|
||||
*/
|
||||
if (apiResponseReceivedText.substring(0, 1) !== "{") {
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an JSON Object. -> apiResponseReceivedText = " + apiResponseReceivedText)
|
||||
resolve('UNEXPECTED_API_RESPONSE')
|
||||
if (errorCodeReceived === true) {
|
||||
resolve('ERROR_CODE_RECEIVED')
|
||||
return
|
||||
}
|
||||
break
|
||||
}
|
||||
case 'array': {
|
||||
/*
|
||||
If we did not received an array, that probably means something is not
|
||||
good, and we got an HTML with the reason inside.
|
||||
Here we will validate that the overall format is what we are expecting.
|
||||
*/
|
||||
if (apiResponseReceivedText.substring(0, 1) !== "[") {
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an Array with Data. -> apiResponseReceivedText = " + apiResponseReceivedText)
|
||||
resolve('UNEXPECTED_API_RESPONSE')
|
||||
switch (apiResponseSchemaNode.apiResponseFields.config.fieldType) {
|
||||
case 'object': {
|
||||
/*
|
||||
If we did not received an object, that probably means something is not
|
||||
good, and we got an HTML with the reason inside.
|
||||
*/
|
||||
if (apiResponseReceivedText.substring(0, 1) !== "{") {
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an JSON Object. -> apiResponseReceivedText = " + apiResponseReceivedText)
|
||||
resolve('UNEXPECTED_API_RESPONSE')
|
||||
return
|
||||
}
|
||||
break
|
||||
}
|
||||
case 'array': {
|
||||
/*
|
||||
If we did not received an array, that probably means something is not
|
||||
good, and we got an HTML with the reason inside.
|
||||
*/
|
||||
if (apiResponseReceivedText.substring(0, 1) !== "[") {
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an Array with Data. -> apiResponseReceivedText = " + apiResponseReceivedText)
|
||||
resolve('UNEXPECTED_API_RESPONSE')
|
||||
return
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
/*
|
||||
The actual data we need could be anywhere within the data structure received.
|
||||
The exact place is configured at the apiResponseSchemaNode property nodePath.
|
||||
We will eval the nodePath property (this assumes that the apiResponseReceivedObject is defined)
|
||||
*/
|
||||
apiResponseReceivedObject = JSON.parse(apiResponseReceivedText)
|
||||
apiData = eval(apiResponseSchemaNode.config.nodePath)
|
||||
/*
|
||||
We will expect the apiData to be an Array. Depending if it has data or not we will return
|
||||
NO_MORE_PAGES or PAGE_FETCHED so that pagination procedure knows when to stop.
|
||||
*/
|
||||
if (apiData.length === 0) {
|
||||
resolve('NO_MORE_PAGES')
|
||||
} else {
|
||||
resolve('PAGE_FETCHED')
|
||||
}
|
||||
/*
|
||||
If we received an errror code, we abort the processing at this point.
|
||||
*/
|
||||
if (errorCodeReceived === true) {
|
||||
resolve('ERROR_CODE_RECEIVED')
|
||||
return
|
||||
}
|
||||
break
|
||||
}
|
||||
/*
|
||||
Here we will validate that the overall format is what we are expecting.
|
||||
*/
|
||||
switch (apiResponseSchemaNode.apiResponseFields.config.fieldType) {
|
||||
case 'object': {
|
||||
/*
|
||||
If we did not received an object, that probably means something is not
|
||||
good, and we got an HTML with the reason inside.
|
||||
*/
|
||||
if (apiResponseReceivedText.substring(0, 1) !== "{") {
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an JSON Object. -> apiResponseReceivedText = " + apiResponseReceivedText)
|
||||
resolve('UNEXPECTED_API_RESPONSE')
|
||||
return
|
||||
}
|
||||
break
|
||||
}
|
||||
case 'array': {
|
||||
/*
|
||||
If we did not received an array, that probably means something is not
|
||||
good, and we got an HTML with the reason inside.
|
||||
*/
|
||||
if (apiResponseReceivedText.substring(0, 1) !== "[") {
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[WARN] start -> startProcess -> fetchAllPages -> fetchAPIData -> onResponse -> onEnd -> Unexpected Response. Not an Array with Data. -> apiResponseReceivedText = " + apiResponseReceivedText)
|
||||
resolve('UNEXPECTED_API_RESPONSE')
|
||||
return
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
/*
|
||||
The actual data we need could be anywhere within the data structure received.
|
||||
The exact place is configured at the apiResponseSchemaNode property nodePath.
|
||||
We will eval the nodePath property (this assumes that the apiResponseReceivedObject is defined)
|
||||
*/
|
||||
apiResponseReceivedObject = JSON.parse(apiResponseReceivedText)
|
||||
apiData = eval(apiResponseSchemaNode.config.nodePath)
|
||||
/*
|
||||
We will expect the apiData to be an Array. Depending if it has data or not we will return
|
||||
NO_MORE_PAGES or PAGE_FETCHED so that pagination procedure knows when to stop.
|
||||
*/
|
||||
if (apiData.length === 0) {
|
||||
resolve('NO_MORE_PAGES')
|
||||
} else {
|
||||
resolve('PAGE_FETCHED')
|
||||
}
|
||||
})
|
||||
}
|
||||
/*
|
||||
The actual data we need could be anywhere within the data structure received.
|
||||
The exact place is configured at the apiResponseSchemaNode property nodePath.
|
||||
We will eval the nodePath property (this assumes that the apiResponseReceivedObject is defined)
|
||||
*/
|
||||
apiResponseReceivedObject = JSON.parse(apiResponseReceivedText)
|
||||
apiData = eval(apiResponseSchemaNode.config.nodePath)
|
||||
/*
|
||||
We will expect the apiData to be an Array. Depending if it has data or not we will return
|
||||
NO_MORE_PAGES or PAGE_FETCHED so that pagination procedure knows when to stop.
|
||||
*/
|
||||
if (apiData.length === 0) {
|
||||
resolve('NO_MORE_PAGES')
|
||||
} else {
|
||||
resolve('PAGE_FETCHED')
|
||||
}
|
||||
});
|
||||
|
||||
}).on('error', (err) => {
|
||||
console.error(err);
|
||||
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[ERROR] start -> httpClient.get -> err = " + err.stack)
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[ERROR] start -> httpClient.get -> Connection to the API Server Failed. Maybe there is no Internet connection. Retrying later. ")
|
||||
resolve('NO_CONNECTION')
|
||||
});
|
||||
})
|
||||
.catch(err => {
|
||||
TS.projects.superalgos.globals.processVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).UNEXPECTED_ERROR = err
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[ERROR] start -> fetch -> err = " + err.stack)
|
||||
TS.projects.superalgos.globals.loggerVariables.VARIABLES_BY_PROCESS_INDEX_MAP.get(processIndex).BOT_MAIN_LOOP_LOGGER_MODULE_OBJECT.write(MODULE_NAME,
|
||||
"[ERROR] start -> fetch -> Connection to the API Server Failed. Maybe there is no Internet connection. Retrying later. ")
|
||||
resolve('NO_CONNECTION')
|
||||
})
|
||||
|
||||
}
|
||||
)
|
||||
@@ -947,7 +1000,7 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
/*
|
||||
We will need to get the nodePath for this property, representing the path
|
||||
to the value on the received data. There are 2 ways to get this nodePath.
|
||||
|
||||
|
||||
1) It is explicitly declared at the node. If this exists then we will use it.
|
||||
2) It was calculated automatically.
|
||||
*/
|
||||
@@ -1009,10 +1062,16 @@ exports.newSuperalgosBotModulesFetchingProcess = function (processIndex) {
|
||||
thisReport.file = {
|
||||
lastRun: (new Date()).toISOString(),
|
||||
lastPage: lastPage,
|
||||
lastFile: contextVariables.lastFile.toISOString(),
|
||||
beginingOfMarket: contextVariables.beginingOfMarket
|
||||
}
|
||||
|
||||
/*
|
||||
For Single-File Datasets there is no lastFile defined.
|
||||
*/
|
||||
if (contextVariables.lastFile !== undefined) {
|
||||
thisReport.file.lastFile = contextVariables.lastFile.toISOString()
|
||||
}
|
||||
|
||||
if (thisReport.file.beginingOfMarket === undefined) {
|
||||
thisReport.file.beginingOfMarket = (new Date()).toISOString()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"requires": true,
|
||||
"lockfileVersion": 1,
|
||||
"dependencies": {
|
||||
"node-fetch": {
|
||||
"version": "2.6.1",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
|
||||
"integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw=="
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"name": "api-data-fetcher-bot",
|
||||
"version": "1.0.0",
|
||||
"description": "Api Fetcher Bot",
|
||||
"main": "FetchingProcess.js",
|
||||
"author": "Luis Molina, Harrellbm",
|
||||
"dependencies": {
|
||||
"node-fetch": "2.6.1"
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,7 @@ if (process.argv.includes("noShortcuts")) {
|
||||
let nodeModulesDirs = [
|
||||
path.join( process.cwd(), "Client"),
|
||||
path.join( process.cwd(), "Projects", "Superalgos", "TS", "Bot-Modules", "Sensor-Bot", "Exchange-Raw-Data"),
|
||||
path.join( process.cwd(), "Projects", "Superalgos", "TS", "Bot-Modules", "API-Data-Fetcher-Bot"),
|
||||
path.join( process.cwd(), "Projects", "Superalgos", "TS", "Bot-Modules", "Trading-Bot", "Announcements"),
|
||||
path.join( process.cwd(), "Projects", "Superalgos", "TS", "Bot-Modules", "Trading-Bot", "Low-Frequency-Trading", "APIs"),
|
||||
path.join( process.cwd(), "Projects", "Superalgos", "TS", "Task-Modules"),
|
||||
|
||||
Reference in new issue
Block a user