diff --git a/Projects/Superalgos/Schemas/Docs-Topics/data-mining-006-data-mining-data-mines.json b/Projects/Superalgos/Schemas/Docs-Topics/data-mining-006-data-mining-data-mines.json index f3e5d16cf..9e1a686cc 100644 --- a/Projects/Superalgos/Schemas/Docs-Topics/data-mining-006-data-mining-data-mines.json +++ b/Projects/Superalgos/Schemas/Docs-Topics/data-mining-006-data-mining-data-mines.json @@ -9,21 +9,23 @@ "paragraphs": [ { "style": "Text", - "text": "It is important to undestand that a Data Mine is a package of definitions. When users run a Bot, they do it inside a Task and they need to reference a Process Definition exisiting at one Data Mine. When the bot is launched inside the Task, it receives at that moment all the information regarding the Task that is being used to launch the bot, but at the same time it receives all the information regarding the the Bot's definitions at the Data Mine.", - "updated": 1621709219036 + "text": "It is important to undestand that a Data Mine is a package of definitions. When users run a Bot, they do it inside a Task. This task needs to know how to run the bot. In order to find this out, the bot within the task must have a reference to a Process Definition exisiting at a Data Mine. When the bot is launched inside the task, at that moment, it receives all the information regarding the task that is being used to launch the bot, while at the same time it also receives the information regarding the Bot's definition from the Data Mine.", + "updated": 1622213888931 }, { - "style": "Text", - "text": "With all that information the Bot is able to do it's job." + "style": "Important", + "text": "Using both sides of this information (Task data and Bot definition) the Bot is able to do it's job.", + "updated": 1622213831832 }, { "style": "Title", - "text": "Who creates them?" + "text": "Who Creates Data Mines?", + "updated": 1622213931643 }, { "style": "Text", - "text": "Data Mines are created by power users and contributed to the project (the ones shipping with the software). Users might create Data Mines for their personal use and never share them. If they share them, they receive SA tokens for that, and also for mantaining them to adapt them to every new release, if necesary.", - "updated": 1622188509380 + "text": "Data Mines are created by power users and contributed to the project (the ones shipping with the software). Users might create Data Mines for their personal use and never share them. However, if they share them, they receive SA tokens for that, and also for adapting and mantaining them, if necessary, for each new release.", + "updated": 1622214117754 }, { "style": "Title", @@ -35,7 +37,8 @@ }, { "style": "List", - "text": "Sensor Bots: used to fetch data from some source at the Internet by coding a specialized algorithm for that." + "text": "Sensor Bots: used to fetch data from some Internet source by coding a specialized algorithm.", + "updated": 1622214170520 }, { "style": "List", @@ -52,8 +55,8 @@ }, { "style": "Text", - "text": "Data Mines might depend of other Data Mines if any of their bots depend on bots belonging to other Data Mines. You will learn further down this Topic how a bot can depend on other bots in detail. In a nutshell, they can depend in 3 ways. They can have a:", - "updated": 1621903084681 + "text": "Data Mines might depend on other Data Mines. What this means is the bots within a Data Mine may depend on bots belonging to other Data Mines. Further on in this Topic, you will learn more about how a bot can depend on other bots in detail. In a nutshell, they can depend in 3 ways. They can have a:", + "updated": 1622214335992 }, { "style": "List", @@ -61,7 +64,13 @@ }, { "style": "List", - "text": "Status Dependency: This means that a Bot uses the status of another Bot as parameters for it's own process. The most common use case is when a Bot needs for instance the date where a market starts or ends. This information is automatically fetched from the Status Report of the bot that produces the raw data and knows which data is the oldest and the newest all the time." + "text": "Status Dependency: This means that a Bot uses the status of another Bot as parameters for it's own processing.", + "updated": 1622214428120 + }, + { + "style": "Success", + "text": "The most common use case is when a Bot needs the date on which a market starts or ends. This information is automatically fetched from the Status Report of the bot that produces the raw data because it knows which data is the oldest and the newest at all times.", + "updated": 1622214512428 }, { "style": "List", @@ -69,9 +78,9 @@ "updated": 1621709849717 }, { - "style": "Note", - "text": "When a Data Mine depends on another Data Mine, both need to be present at the Workspace where the Task defining the execution of the Bot is located.", - "updated": 1621903174117 + "style": "Important", + "text": "When a Data Mine depends on another Data Mine, both need to be present at the Workspace where the Task executing the Bot is located.", + "updated": 1622214646178 } ] } \ No newline at end of file diff --git a/Projects/Superalgos/Schemas/Docs-Topics/data-mining-007-data-mining-managing-execution.json b/Projects/Superalgos/Schemas/Docs-Topics/data-mining-007-data-mining-managing-execution.json index fa6e945b0..6ba10d917 100644 --- a/Projects/Superalgos/Schemas/Docs-Topics/data-mining-007-data-mining-managing-execution.json +++ b/Projects/Superalgos/Schemas/Docs-Topics/data-mining-007-data-mining-managing-execution.json @@ -3,58 +3,87 @@ "pageNumber": 7, "type": "Data Mining - Managing Execution", "definition": { - "text": "Bots can execute at their own pace or they can wait for other bots to finish executing in order for them to start.", - "updated": 1621903220369 + "text": "Bots can execute at their own pace or they can wait for other bots to finish executing before starting.", + "updated": 1622214711890 }, "paragraphs": [ { - "style": "Text", - "text": "Usually when a bot produces raw data, it executes itself based on the configuration of it's Process Definition node. These bots are usually the root of an execution tree formed by all the bots that will process and reprocess the raw data into more and more elaborate datasets.", - "updated": 1621710144112 + "style": "Title", + "text": "Bot Execution Tree", + "updated": 1622215214005 }, { "style": "Text", - "text": "There are two types of execution managment." + "text": "A helpful way to think about managing bot execution is to think of the image of a tree. The roots support the trunk which supports the branches. In the same way, Superalgos uses many bots to process data each relying on the output of the last to form a tree. As data is passed through the tree the raw data is processed and reprocessed to build more and more elaborate datasets.", + "updated": 1622215593975 + }, + { + "style": "Title", + "text": "Types of Execution Managment" + }, + { + "style": "Text", + "text": "There are two types of execution managment:" }, { "style": "List", - "text": "Standalone Bots: These bots are the root of the Bot Execution Tree. They don't depend on any other bot to run. They usually run on a pre-defined schedulle, in most cases every one minute.", - "updated": 1621710324631 + "text": "Standalone Bots: These bots are the root of the Bot Execution Tree. They don't depend on any other bot to run but use the configuration of it's Process Definition node to run on a pre-defined schedule (in most cases every one minute).", + "updated": 1622215038717 + }, + { + "style": "Note", + "text": "Usually when a bot produces raw data, it operates as a standalone bot. ", + "updated": 1622215509477 }, { "style": "List", - "text": "Execution Dependent Bots: These bots are the branches of the Bot Execution Tree and they need to wait for another Bot to finish it's execution in order for it to start.", - "updated": 1621710352983 + "text": "Execution Dependent Bots: These bots are the branches of the Bot Execution Tree. They need to wait for other Bots to finish their execution in order to start.", + "updated": 1622215195801 + }, + { + "style": "Title", + "text": "Configuring a Standalone Bot", + "updated": 1622215700788 }, { "style": "Text", - "text": "This is an example of how to configure a bot to be a Standalone Bot from the execution point of view. As you can see in the example below, the normalWaitTime property has a non zero value, in this case 60 seconds.", - "updated": 1621710692139 + "text": "This is an example of how to configure a bot to execute as a Standalone Bot. Within the Process Definition of a Bot, the normalWaitTime property has a non zero value, in this case 60 seconds.", + "updated": 1622215895631 }, { "style": "Json", - "text": "{\n \"codeName\": \"Historic-OHLCVs\",\n \"description\": \"Retrieves and saves historical maket data\",\n \"deadWaitTime\": 0,\n \"normalWaitTime\": 60000,\n \"retryWaitTime\": 10000,\n \"botModule\": \"Exchange Raw Data Historic OHLCVs\"\n}", - "updated": 1621710641664 + "text": "{\n \"codeName\": \"Historic-OHLCVs\",\n \"description\": \"Retrieves and saves historical market data\",\n \"deadWaitTime\": 0,\n \"normalWaitTime\": 60000,\n \"retryWaitTime\": 10000,\n \"botModule\": \"Exchange Raw Data Historic OHLCVs\"\n}", + "updated": 1622215784605 + }, + { + "style": "Title", + "text": "Configuring an Execution Dependent Bot", + "updated": 1622215826932 }, { "style": "Text", - "text": "This is how to configure the Process Definition of a Bot that it's execution depends on other Bot. Here the normalWaitTime has a value of zero, which will force the Bot to inmediatelly run the next loop, but since there is an Execution Started Event defined, that will make it wait until the referenced Bot finishes.", - "updated": 1621903624630 + "text": "In order to configure a bot to execute as an Execution Dependent Bot two things need to be in place. ", + "updated": 1622216289412 + }, + { + "style": "List", + "text": "First, the bot's definition will need to have an Execution Started Event defined. After this, the Execution Started Event will need to have a reference to the Execution Finished Event of the Bot on which it is waiting.", + "updated": 1622216544766 + }, + { + "style": "List", + "text": "Second, the Bot's Process Definition will need a zero in the normalWaitTime property. This will force the Bot to immediately run the next loop, but since there is an Execution Started Event defined, that will make it wait until the referenced Bot finishes.", + "updated": 1622216345546 }, { "style": "Json", "text": "{\n \"codeName\": \"Multi-Time-Frame-Market\",\n \"normalWaitTime\": 0,\n \"retryWaitTime\": 10000,\n \"framework\": {\n \"name\": \"Multi-Time-Frame-Market\"\n }\n}", "updated": 1621710596317 }, - { - "style": "Success", - "text": "In this case users must also define which bot to wait for. To do this they stablish a reference from the Execution Started Event node of the Bot that needs to wait, to the Execution Finished Event of the Bot they are waiting for to finish.", - "updated": 1621710539966 - }, { "style": "Note", - "text": "From the deployment point of view, the dependant bots can be physically located at the same machine, or at multiple different machines. Superalgos handles the syncronization across multiples machines.", - "updated": 1621903914878 + "text": "From a deployment point of view, the dependant bots can be physically located on the same machine, or on multiple different machines. Superalgos handles syncronization across multiples machines.", + "updated": 1622216626010 } ] } \ No newline at end of file diff --git a/Projects/Superalgos/Schemas/Docs-Topics/data-mining-008-data-mining-data-dependencies.json b/Projects/Superalgos/Schemas/Docs-Topics/data-mining-008-data-mining-data-dependencies.json index 3db6b8759..b850edfad 100644 --- a/Projects/Superalgos/Schemas/Docs-Topics/data-mining-008-data-mining-data-dependencies.json +++ b/Projects/Superalgos/Schemas/Docs-Topics/data-mining-008-data-mining-data-dependencies.json @@ -3,35 +3,48 @@ "pageNumber": 8, "type": "Data Mining - Data Dependencies", "definition": { - "text": "Indicator Bots have data dependencies. That means that they have as their input a Dataset created by other Bot. ", - "updated": 1621715782447 + "text": "Indicator Bots have data dependencies. This means that they receive their input as Datasets created by other Bots.", + "updated": 1622216765574 }, "paragraphs": [ { - "style": "Text", - "text": "You can also think about data dependencies as a type of relationship between bots. Bots producing raw data (Sensor Bot and API Data Fetcher Bot) don't have them, since their job is to extract data from an external source. At the same time they are the root of a tree or graph of data dependencies relationships.", - "updated": 1621715986533 + "style": "Title", + "text": "Data Dependency Tree", + "updated": 1622219215680 }, { "style": "Text", - "text": "The data dependencias are in fact at the Process Definition level. Each process might have their own set of data dependencies. " + "text": "You can think about data dependencies as a type of relationship between bots. Bots producing raw data (Sensor Bot and API Data Fetcher Bot) don't have them, since their job is to extract data from an external source. At the same time, they are also the root of the data dependancy tree of relationships. This tree is very similar to the execution tree but instead of describing how bots run, it describes how bots send and receive Data Products.", + "updated": 1622217397887 + }, + { + "style": "Title", + "text": "Defining Data Dependencies", + "updated": 1622217419549 + }, + { + "style": "Text", + "text": "The definition of data dependencias is held at the Process Definition level. Each process might have its own set of data dependencies. This means that a single bot may have many varying dependencies depending on the number of Process Definitions it contains." }, { "style": "Note", - "text": "It is possible for a single Bot's process to have more than one Data Dependency, from the same Bot or from different Bot's. That is particulary usefull to aggregate data from many datasets into another one. The dependent datasets could belong to Bots at the same Data Mine or at bots at other Data Mines. From the deployment point of view, the dependant datasets could be physically located at the same machine, or at multiple different machines." + "text": "In addtion to this, it is possible for a single Bot's process to have more than one Data Dependency (from the same Bot or from different Bots). That is particulary useful to aggregate data from many datasets into another one. The dependent datasets could belong to Bots at the same Data Mine or at bots at other Data Mines.", + "updated": 1622217595798 }, { "style": "Text", - "text": "The first declared dependency is called the Main Dependency and has a special meaning. The process of the Bot is going to iterate the Dataset of the Main Dependency to produce the output Dataset. All other data dependencies will be used to extract data from." + "text": "The first declared dependency is called the Main Dependency and has a special meaning. It is used by the process of the Bot to iterate. What this means is that the process will loop over the Dataset of the Main Dependency to produce the output Dataset. All the other data dependencies will be used each iteration to extract additional data.", + "updated": 1622218002546 + }, + { + "style": "Success", + "text": "As with bot execution, the dependant datasets can be physically located on the same machine, or at multiple different machines. However, when a Bot has a Data Dependency of a dataset produced by a Bot defined at a different Data Mine, both Data Mines must be present at the Workspace where the Bot is going to be launched.", + "updated": 1622218098435 }, { "style": "Text", - "text": "When a Bot has a Data Dependency of a dataset produced by a Bot defined at a different Data Mine, then both Data Mines must be present at the Workspace where the Bot is going to be lauched at.", - "updated": 1621903950116 - }, - { - "style": "Text", - "text": "The Masters Data Mine is the most referenced Data Mine because is the one that defines the Candles-Volumes Indicator Bot, that produces the most used information for trading systems: Candles." + "text": "The Candles Data Mine is the most referenced Data Mine because it defines the Candles-Volumes Indicator Bot, that produces the most used information for trading systems: Candles.", + "updated": 1622218178970 } ] } \ No newline at end of file diff --git a/Projects/Superalgos/Schemas/Docs-Topics/data-mining-009-data-mining-status-reports.json b/Projects/Superalgos/Schemas/Docs-Topics/data-mining-009-data-mining-status-reports.json index 5991c9f65..92bd93dca 100644 --- a/Projects/Superalgos/Schemas/Docs-Topics/data-mining-009-data-mining-status-reports.json +++ b/Projects/Superalgos/Schemas/Docs-Topics/data-mining-009-data-mining-status-reports.json @@ -3,23 +3,33 @@ "pageNumber": 9, "type": "Data Mining - Status Reports", "definition": { - "text": "Status Reports are files produced by each Bot's processes with some key information that the Bot needs to save in order to remember it the next time is run.", - "updated": 1621904046327 + "text": "Status Reports are files produced by each Bot's processes with key information the Bot needs to save in order to remember the next time it is run.", + "updated": 1622218351726 }, "paragraphs": [ { - "style": "Text", - "text": "Most Bots in Superalgos are run by the Task Server and they implement a Main Loop. At each loop execution, bots do their job and at the end they go to sleep. How much time they sleep and other details are configured at the Bot's Process Definition.", - "updated": 1621904131214 + "style": "Title", + "text": "The Task Server Main Loop", + "updated": 1622218578406 }, { "style": "Text", - "text": "Before going to sleep, all the memory is flushed from all information related to the Bot, so that hardware resources are freed for other bots in the same machine to use. There is some info Bots need to remember between every execution of the Main Loop, and between executions of the Task itself. That info is stored in a JSON / text file called the Status Report.", - "updated": 1621904194069 + "text": "Most Bots in Superalgos are run by the Task Server and implement a Main Loop. At each loop execution, bots do their job and at the end they go to sleep. ", + "updated": 1622218637972 + }, + { + "style": "Note", + "text": "How much time they sleep and other details are configured at the Bot's Process Definition." + }, + { + "style": "Title", + "text": "Status Report Mechanics", + "updated": 1622218611081 }, { "style": "Text", - "text": "The name Status was used since the information stored is often the status of the Bot's process. Here is an example of a simple Status Report file." + "text": "Before a bot goes to sleep, computer memory is flushed of all information related to the Bot. This is important to free up hardware resources for other bots running on the same machine. However, there is some information Bots need to remember between each execution of the Main Loop, and between executions of the Task itself. That info is stored in a JSON / text file called the Status Report. The name Status was used since the information stored is often the status of the Bot's process. Here is an example of a simple Status Report file:", + "updated": 1622218847283 }, { "style": "Json", @@ -28,8 +38,8 @@ }, { "style": "Note", - "text": "Status Report files also might include a data structure called Inter Execution Memory. Status information is kind of standard for all types of bots. Inter Execution Memory on the other hand, depends on each Bot type and what is actually processing.", - "updated": 1621904260686 + "text": "Status Report files might also include a data structure called Inter Execution Memory. Status information is standard for all types of bots. Inter Execution Memory, on the other hand, depends on each Bot's type and what is actually being processed.", + "updated": 1622218782345 }, { "style": "Text", diff --git a/Projects/Superalgos/Schemas/Docs-Topics/data-mining-010-data-mining-status-dependencies.json b/Projects/Superalgos/Schemas/Docs-Topics/data-mining-010-data-mining-status-dependencies.json index 5d872b662..d53a45dbb 100644 --- a/Projects/Superalgos/Schemas/Docs-Topics/data-mining-010-data-mining-status-dependencies.json +++ b/Projects/Superalgos/Schemas/Docs-Topics/data-mining-010-data-mining-status-dependencies.json @@ -3,24 +3,32 @@ "pageNumber": 10, "type": "Data Mining - Status Dependencies", "definition": { - "text": "Status Dependencies is a type of relationship between Bots. Sometimes bots don't depend on the Datasets produced by other bots, but they depend directly on the status of other bots.", - "updated": 1621717479906 + "text": "Status Dependencies are another type of relationship between Bots. Sometimes bots don't depend on the Datasets produced by other bots, but they depend directly on the status of other bots.", + "updated": 1622219077845 }, "paragraphs": [ { - "style": "Text", - "text": "A good way to undestand why Status Dependencies are needed is with an example:", - "updated": 1621717585017 + "style": "Title", + "text": "Status Dependency Tree", + "updated": 1622219244127 }, { "style": "Text", - "text": "The Indicator Bot Bollinger-Bands depends on Candles-Volumes Dataset in order to calculate the Bollinger Bands, but it depend's on Exchange Raw Data Sensor Bot in order to know where the market begins, which is the date of the first piece of data. That information is part of the Status information that Exchange Raw Data Sensor Bot keeps at it's Satus Report.", - "updated": 1621717669486 + "text": "Status dependencies operate using the same idea of a tree as execution and data dependencies. Any Bot can have from zero to many status dependencies. For each Status Dependency, the Bot's process at runtime will load the Status Report file of the dependent Bot and will extract from there the information it needs.", + "updated": 1622222998840 + }, + { + "style": "Note", + "text": "Bots that produce Raw Data usually do not depend on any other Bot's status and are usually the root of a status dependency tree. " }, { "style": "Text", - "text": "Bots that produce Raw Data usually do not depend on any other Bot's status and they are usually the root of a dependency tree or graph. Any Bot can have from zero to many status dependencies. For each Status Dependency, the Bot's process at runtime will load the Status Report file of the dependent Bot and will extract from there the information it needs.", - "updated": 1621904417277 + "text": "A good way to undestand why Status Dependencies are needed is with an example:" + }, + { + "style": "Text", + "text": "The Indicator Bot Bollinger-Bands depends on Candles-Volumes Dataset in order to calculate the Bollinger Bands, but it depend's on Exchange Raw Data Sensor Bot in order to know where the market begins, which is the date of the first piece of data. That information is part of the Status information that Exchange Raw Data Sensor Bot keeps at it's Status Report.", + "updated": 1622223640276 }, { "style": "Title", @@ -29,21 +37,22 @@ }, { "style": "Text", - "text": "A Status Dependency can be classified by it's utility. This make it easier for processes to handle them in code. There are currently three different utilities:", - "updated": 1621904453267 + "text": "A Status Dependency can be classified by it's utility. This makes it easier for processes to handle them in code. There are currently three different utilities:", + "updated": 1622223177318 }, { "style": "List", - "text": "Self Reference: this is used to load it's own Status Report to extract data from it or modify it." - }, - { - "style": "List", - "text": "Market Ending Point: this means that the referenced Status Report is to be used to extract the information of where the market ends, meaning which date the market ends." + "text": "Self Reference: this is used to load a bot's Status Report to extract data from it or modify it.", + "updated": 1622223714025 }, { "style": "List", "text": "Market Starting Point: this means that the referenced Status Report is to be used to extract the information of where the market begins.", - "updated": 1621718175842 + "updated": 1622223349370 + }, + { + "style": "List", + "text": "Market Ending Point: this means that the referenced Status Report is to be used to extract the information of where the market ends, meaning which date the market ends." }, { "style": "Important",