Pull code live from Github during build

.gitignore

@@ -20,4 +20,5 @@ yarn-debug.log*
 yarn-error.log*
 
 ./yarn.lock
-.idea/runConfigurations.xml
+.idea
+/gitrepos

codeblocks/codeblocks.json

@@ -1 +1 @@
-{"https://github.com/conductor-sdk/orkes-java-springboot2-example/blob/main/src/main/java/io/orkes/example/banking/service/WorkflowService.java---1":"{`        StartWorkflowRequest request = new StartWorkflowRequest();\n        request.setName(\"deposit_payment\");\n        Map<String, Object> inputData = new HashMap<>();\n        inputData.put(\"amount\", depositDetail.getAmount());\n        inputData.put(\"accountId\", depositDetail.getAccountId());\n        request.setInput(inputData);\n\n        String workflowId = workflowClient.startWorkflow(request);\n        log.info(\"Workflow id: {}\", workflowId);`}","https://github.com/conductor-sdk/orkes-java-springboot2-example/blob/main/src/main/java/io/orkes/example/banking/service/WorkflowService.java---1-lines":"#L22-L30"}
+{"https://github.com/conductor-sdk/orkes-java-springboot2-example/blob/main/src/main/java/io/orkes/example/banking/service/WorkflowService.java---1":"{`\n        StartWorkflowRequest request = new StartWorkflowRequest();\n        request.setName(\"deposit_payment\");\n        Map<String, Object> inputData = new HashMap<>();\n        inputData.put(\"amount\", depositDetail.getAmount());\n        inputData.put(\"accountId\", depositDetail.getAccountId());\n        request.setInput(inputData);\n\n        String workflowId = workflowClient.startWorkflow(request);\n        log.info(\"Workflow id: {}\", workflowId);\n`}","https://github.com/conductor-sdk/orkes-java-springboot2-example/blob/main/src/main/java/io/orkes/example/banking/service/WorkflowService.java---1-lines":"#L22-L32","https://github.com/conductor-sdk/orkes-java-springboot2-example/blob/main/src/main/java/io/orkes/example/banking/workers/ConductorWorkers.java---1":"{`\n    /** Note: Using this setting, up to 5 tasks will run in parallel, with tasks being polled every 200ms */\n    @WorkerTask(value = \"fraud-check\", threadCount = 5, pollingInterval = 200)\n    public FraudCheckResult checkForFraudTask(DepositDetail depositDetail) {\n        return fraudCheckService.checkForFraud(depositDetail);\n    }\n`}","https://github.com/conductor-sdk/orkes-java-springboot2-example/blob/main/src/main/java/io/orkes/example/banking/workers/ConductorWorkers.java---1-lines":"#L17-L23"}

docs/getting-started/step2.md

@@ -24,12 +24,6 @@ Let's look at some __code examples__ of how to trigger a workflow by it's name.
 <TabItem value="Java" label="Java">
 
 ```java dynamic https://github.com/conductor-sdk/orkes-java-springboot2-example/blob/main/src/main/java/io/orkes/example/banking/service/WorkflowService.java section=1 .../service/WorkflowService.java
-StartWorkflowRequest request = new StartWorkflowRequest();
-request.setName("deposit-cash");
-request.setInput(Map.of("amount", 100, "account", "account-id"));
-
-String workflowId = workflowClient.startWorkflow(request);
-log.info("Workflow id: {}", workflowId);
 ```
 
 </TabItem>

docs/getting-started/step3.md

@@ -4,7 +4,7 @@ sidebar_position: 1
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 import Install from '@site/src/components/install.mdx';
-
+import CodeBlock from '@theme/CodeBlock';
 
 # Step 3: Adding Custom Code Worker
 
@@ -38,24 +38,14 @@ Continuing the use case from the previous step, we now have a requirement to add
 We can see that when we run this workflow for amounts > $50,000, it runs a fraud check. If we named the task `fraud-check`, we'd notice that it actually executed (in playground env), but how? 
 That's because there is a pre-defined task that is polling and running all the tasks named `fraud-check`. 
 
-So how can we implement this task? First let's rename the task to a new unique name for ourselves - for ex: `fraud-check-<replace-with-your-name>`. And now let’s see how this custom fraud check can be implemented:
+So how can we implement this task for ourselves? First let's rename the task to a new unique name for ourselves - for ex: `fraud-check-<replace-with-your-name>`. And now let’s see how this custom fraud check can be implemented:
 
 View our documentation on [Conductor Clients & SDKs](/content/category/sdks)  list and how to import the required dependencies in our applications.
 
 <Tabs>
 <TabItem value="Java" label="Java">
 
-```java
-
-    @WorkerTask("fraud-check-<replace-with-your-name>")
-    public String checkForFraud(@InputParam("amount") BigDecimal amount, @InputParam("accountId") String accountId) {
-        boolean isFraud = fraudService.isFraudulentTxn(accountId, amount); 
-        if(isFraud) {
-            return "This transaction cannot be processed as its flagged for review.";
-        }
-        return "Deposit of " + amount + " has processed successfully";
-    }
-
+```java dynamic https://github.com/conductor-sdk/orkes-java-springboot2-example/blob/main/src/main/java/io/orkes/example/banking/workers/ConductorWorkers.java section=1 ../workers/ConductorWorkers.java
 ```
 
 </TabItem>

package.json

@@ -23,6 +23,7 @@
     "@fortawesome/react-fontawesome": "^0.2.0",
     "@mdx-js/react": "^1.6.22",
     "axios": "^1.3.5",
+    "child_process": "^1.0.2",
     "classnames": "^2.3.2",
     "clsx": "^1.2.1",
     "docusaurus-theme-search-typesense": "^0.9.0",

plugin/plugin.js

@@ -12,8 +12,12 @@ function plugin(options = {}) {
     return function transformer(tree) {
         let includesImportCodeBlock = false;
         let hasCodeBlock = false;
-
+        let totalCount = 1;
         visit(tree, ["code", "import"], (node, index, parent) => {
+            totalCount++;
+            if (totalCount > 10000) {
+                return;
+            }
             if (is(node, "import") && node.value.includes("@theme/CodeBlock")) {
                 includesImportCodeBlock = true;
                 return;
@@ -27,8 +31,8 @@ function plugin(options = {}) {
                 let key = `${url}---${section}`
                 let linesKey = `${url}---${section}-lines`
                 let urlLabel = metaSplit[3];
-                let language = 'java';
-                tree.children.splice(7, 1, {
+                let language = 'java'; // TODO Fix
+                tree.children.splice(index, 1, {
                         type: "jsx",
                         value: `<CodeBlock language="${language}" title="${url}${codeblocks[linesKey]}" titleType="url" titleLabel="${urlLabel}">${codeblocks[key]}`,
                     },

preprocesscodeblocks.js

@@ -1,47 +1,88 @@
 const fs = require('fs');
 const path = require('path');
 const axios = require('axios');
+const {exec} = require("child_process");
 
 const startMarker = 'docs-marker-start-';
 const endMarker = 'docs-marker-end-';
 
-function extractContent(rawUrl, section) {
+function extractContentFromFile(file, repo, section) {
     return new Promise((resolve, reject) => {
+        console.log('file ', file, repo)
         const result = [];
         const startMarkerSec = `${startMarker}${section}`;
         const endMarkerSec = `${endMarker}${section}`;
-
-        axios.get(rawUrl).then((response) => {
-            const lines = response.data.split('\n');
-            let foundStartMarker = false;
-            let lineStart = 0;
-            let lineEnd = 0;
-            let index = 0;
-            lines.forEach((line) => {
-                index++;
-                if (line.includes(endMarkerSec)) {
-                    foundStartMarker = false;
-                    lineEnd = index - 1;
+        const repoFolder = `./gitrepos/${repo}/${file}`
+        try {
+            fs.readFile(repoFolder, "utf8", (error, data) => {
+                if (error) {
+                    console.error(error.message);
+                    return;
                 }
+                const lines = data.split('\n');
+                let foundStartMarker = false;
+                let lineStart = 0;
+                let lineEnd = 0;
+                let index = 0;
+                lines.forEach((line) => {
+                    index++;
+                    if (line.includes(endMarkerSec)) {
+                        foundStartMarker = false;
+                        lineEnd = index - 1;
+                    }
 
-                if (foundStartMarker) {
-                    // result.push(line.replaceAll('>', '&gt;').replaceAll('<', '&lt;'));
-                    result.push(line);
-                }
+                    if (foundStartMarker) {
+                        result.push(line);
+                    }
 
-                if (line.includes(startMarkerSec)) {
-                    foundStartMarker = true;
-                    lineStart = index + 1;
-                }
+                    if (line.includes(startMarkerSec)) {
+                        foundStartMarker = true;
+                        lineStart = index + 1;
+                    }
+                });
+                resolve([result, lineStart, lineEnd]);
             });
-            resolve([result, lineStart, lineEnd]);
-        }).catch((error) => {
+        } catch (error) {
             console.error(error);
             reject(error);
-        });
+        }
     });
 }
 
+const getGitUrl = (url) => {
+    const rawUrl = url.substring(0, url.indexOf(`/blob/`));
+    return `${rawUrl}.git`;
+}
+
+const cloneRepoIfNeeded = (gitUrl) => {
+    return new Promise((resolve, reject) => {
+        const gitFolder = gitUrl.substring(gitUrl.lastIndexOf('/'), gitUrl.indexOf('.git'));
+        const destFolder = `./gitrepos/${gitFolder}`;
+        let command;
+        if (fs.existsSync(destFolder)) {
+            command = `git -C ${destFolder} pull`;
+        } else {
+            command = `git clone ${gitUrl} ${destFolder}`;
+        }
+        try {
+            exec(command, (error, stdout, stderr) => {
+                if (error) {
+                    console.error(`Error executing command: ${command}`);
+                    console.error(error.message);
+                    return;
+                }
+                console.log(stdout);
+                console.error(stderr);
+                console.log(command, "successful")
+                resolve(true);
+            });
+        } catch (e) {
+            console.error(e);
+            reject(e);
+        }
+    });
+};
+
 const searchDirectory = async (dirPath, codeBlocks) => {
     const files = fs.readdirSync(dirPath);
     for (const file of files) {
@@ -59,12 +100,18 @@ const searchDirectory = async (dirPath, codeBlocks) => {
                     if (line.startsWith('```') && line.indexOf(' dynamic ') > 0) {
                         const lineContent = line.split(' ');
                         const originalUrl = lineContent[2];
-                        const rawUrl = originalUrl.replace("github.com", "raw.githubusercontent.com").replace("/blob/", "/");
-                        const section = lineContent[3].split('=')[1];
-                        const [extractedContent, lineStart, lineEnd] = await extractContent(rawUrl, section);
-                        console.log(`Found code block with url ${rawUrl} and section ${section} and content`, extractedContent);
-                        codeBlocks[`${originalUrl}---${section}`] = `{\`${extractedContent.join('\n')}\`}`;
-                        codeBlocks[`${originalUrl}---${section}-lines`] = `#L${lineStart}-L${lineEnd}`;
+                        const gitUrl = getGitUrl(originalUrl);
+                        await cloneRepoIfNeeded(gitUrl).then(async () => {
+                            console.log("Repo clone/update successful")
+                            const section = lineContent[3].split('=')[1];
+                            const rawUrl = originalUrl.substring(originalUrl.indexOf('/blob/main/') + 10);
+                            let l = originalUrl.substring(0, originalUrl.indexOf('/blob/main/'));
+                            const repo = l.substring(l.lastIndexOf('/') + 1)
+                            const [extractedContent, lineStart, lineEnd] = await extractContentFromFile(rawUrl, repo, section);
+                            console.log(`Found code block with url ${rawUrl} and section ${section} and content`, extractedContent);
+                            codeBlocks[`${originalUrl}---${section}`] = `{\`${extractedContent.join('\n')}\`}`;
+                            codeBlocks[`${originalUrl}---${section}-lines`] = `#L${lineStart}-L${lineEnd}`;
+                        });
                     }
                 }
             }

yarn.lock

@@ -3161,6 +3161,11 @@ cheerio@^1.0.0-rc.12:
     parse5 "^7.0.0"
     parse5-htmlparser2-tree-adapter "^7.0.0"
 
+child_process@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/child_process/-/child_process-1.0.2.tgz#b1f7e7fc73d25e7fd1d455adc94e143830182b5a"
+  integrity sha512-Wmza/JzL0SiWz7kl6MhIKT5ceIlnFPJX+lwUGj7Clhy5MMldsSoJR0+uvRzOS5Kv45Mq7t1PoE8TsOA9bzvb6g==
+
 chokidar@^3.4.2, chokidar@^3.5.3:
   version "3.5.3"
   resolved "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz"