feat: ✨ add support for JSONL file parsing (#1031)

microsoft · Jan 20, 2025 · c91f3f2 · c91f3f2
1 parent d510741
commit c91f3f2
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 12 deletions.
diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts
@@ -50,6 +50,7 @@ export const MDX_REGEX = /\.mdx$/i
 export const MJS_REGEX = /\.mjs$/i
 export const JS_REGEX = /\.js$/i
 export const JSON5_REGEX = /\.json5?$/i
+export const JSONL_REGEX = /\.jsonl$/i
 export const PROMPTY_REGEX = /\.prompty$/i
 export const TOOL_NAME = "GenAIScript"
 export const SERVER_PORT = 8003

diff --git a/packages/core/src/data.ts b/packages/core/src/data.ts
@@ -1,4 +1,3 @@
-import { readFile } from "fs/promises"
 import {
     XLSX_REGEX,
     CSV_REGEX,
@@ -9,19 +8,21 @@ import {
     XML_REGEX,
     MD_REGEX,
     MDX_REGEX,
+    JSONL_REGEX,
 } from "./constants"
-import { CSVParse, CSVTryParse } from "./csv"
+import { CSVTryParse } from "./csv"
 import { splitMarkdown } from "./frontmatter"
-import { INIParse, INITryParse } from "./ini"
-import { JSON5parse } from "./json5"
-import { TOMLParse } from "./toml"
+import { INITryParse } from "./ini"
+import { JSON5TryParse } from "./json5"
+import { TOMLTryParse } from "./toml"
 import { XLSXParse } from "./xlsx"
-import { XMLParse } from "./xml"
-import { YAMLParse } from "./yaml"
+import { XMLTryParse } from "./xml"
+import { YAMLTryParse } from "./yaml"
 import { resolveFileContent } from "./file"
 import { TraceOptions } from "./trace"
 import { host } from "./host"
 import { fromBase64 } from "./base64"
+import { JSONLTryParse } from "./jsonl"
 
 export async function dataTryParse(
     file: WorkspaceFile,
@@ -40,13 +41,14 @@ export async function dataTryParse(
     else {
         if (CSV_REGEX.test(filename)) data = CSVTryParse(content, options)
         else if (INI_REGEX.test(filename)) data = INITryParse(content, options)
-        else if (TOML_REGEX.test(filename)) data = TOMLParse(content)
+        else if (TOML_REGEX.test(filename)) data = TOMLTryParse(content)
         else if (JSON5_REGEX.test(filename))
-            data = JSON5parse(content, { repair: true })
-        else if (YAML_REGEX.test(filename)) data = YAMLParse(content)
-        else if (XML_REGEX.test(filename)) data = XMLParse(content, options)
+            data = JSON5TryParse(content, { repair: true })
+        else if (YAML_REGEX.test(filename)) data = YAMLTryParse(content)
+        else if (XML_REGEX.test(filename)) data = XMLTryParse(content, options)
+        else if (JSONL_REGEX.test(filename)) data = JSONLTryParse(content)
         else if (MD_REGEX.test(filename) || MDX_REGEX.test(filename))
-            data = YAMLParse(splitMarkdown(content).frontmatter)
+            data = YAMLTryParse(splitMarkdown(content).frontmatter)
         else {
             return undefined // unknown
         }