add url support

csv_to_json/README.md

@@ -6,8 +6,10 @@ The CSV to JSON mod provides a convenient and efficient way to convert CSV data,
 
 - Run the pipeline with the appropriate arguments.
 
-- For a local csv file run, `flowpipe pipeline run csv_to_json --arg local_csv_file_path="/Users/bob/Desktop/students.csv"`
+- For a local csv file run, `flowpipe pipeline run local_csv_file_to_json --arg local_csv_file_path="/Users/bob/Desktop/students.csv"`
 
 - For csv locals, update the locals.fp file and run `flowpipe pipeline run local_csv_values_to_json`
 
 - For a remote Google Spreadsheet, run `flowpipe pipeline run remote_public_spreadsheet_to_json --arg spreadsheet_id="1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms"`
+
+- For a remote URL with csv data, run `flowpipe pipeline run remote_csv_url_to_json --arg url="https://data.nasa.gov/resource/nj3a-8wq3.csv"`

csv_to_json/local_csv_file_to_json.fp

@@ -10,16 +10,11 @@ pipeline "local_csv_file_to_json" {
 
   # Using a local file
   step "transform" "local_csv_file_to_json" {
-    value = file("${param.local_csv_file_path}")
+    value = jsonencode(csvdecode(file("${param.local_csv_file_path}")))
   }
 
   output "local_csv_file_to_json" {
-    value = csvdecode(step.transform.local_csv_file_to_json.value)
+    value = step.transform.local_csv_file_to_json.value
   }
 
-
 }
-
-
-
-

csv_to_json/local_csv_values_to_json.fp

@@ -4,7 +4,7 @@ pipeline "local_csv_values_to_json" {
 
   # Using local CSV values
   step "transform" "local_csv_values_to_json" {
-    value = csvdecode(local.csv_data)
+    value = jsonencode(csvdecode(local.csv_data))
   }
 
   output "local_csv_values_to_json" {

csv_to_json/remote_csv_url_to_json.fp

@@ -0,0 +1,17 @@
+pipeline "remote_csv_url_to_json" {
+  title       = "Remote CSV URL file To JSON"
+  description = "Read CSV data from a remote URL and convert the contents to JSON."
+
+  param "url" {
+    type        = string
+    description = "The URL which contains the CSV data."
+  }
+
+  step "http" "remote_csv_url_to_json" {
+    url = param.url
+  }
+
+  output "remote_csv_url_to_json" {
+    value = jsonencode(csvdecode(step.http.remote_csv_url_to_json.response_body))
+  }
+}

csv_to_json/remote_public_spreadsheet_to_json.fp

@@ -11,10 +11,11 @@ pipeline "remote_public_spreadsheet_to_json" {
   # Using a remote CSV file without authentication
   # Default example: https://docs.google.com/spreadsheets/d/1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms/
   step "http" "remote_public_spreadsheet_to_json" {
-    url = "https://docs.google.com/spreadsheets/d/${param.spreadsheet_id}/export?format=csv"
+    # url = "https://docs.google.com/spreadsheets/d/${param.spreadsheet_id}/export?format=csv"
+    url = "https://github.com/datablist/sample-csv-files/raw/main/files/customers/customers-10000.csv"
   }
 
   output "remote_public_spreadsheet_to_json" {
-    value = csvdecode(step.http.remote_public_spreadsheet_to_json.response_body)
+    value = jsonencode(csvdecode(step.http.remote_public_spreadsheet_to_json.response_body))
   }
 }

csv_to_json/users.csv

@@ -0,0 +1,6 @@
+Username,Identifier,First name,Last name
+booker12,9012,Rachel,Booker
+grey07,2070,Laura,Grey
+johnson81,4081,Craig,Johnson
+jenkins46,9346,Mary,Jenkins
+smith79,5079,Jamie,Smith