bruteforce save roundtrips transit-json

deps.edn

@@ -4,7 +4,7 @@
         de.otto/nom {:mvn/version "0.3.0"}
         tick/tick {:mvn/version "0.6.2"}
         org.pinkgorilla/timbre {:mvn/version "0.0.7"}
-        scicloj/tablecloth {:mvn/version "7.021"} ; brings techml-dataset
+        scicloj/tablecloth {:mvn/version "7.029.2"} ; ensure clj-transit is there for sure.
         babashka/fs {:mvn/version "0.5.22"} ; create log dir
         io.github.clojure-quant/quanta {:mvn/version "0.4.866"}
         io.github.clojure-quant/quanta-dag-algo {:mvn/version "0.1.8"}}

dev/src/dev/algo_bollinger_bruteforce.clj

@@ -49,6 +49,7 @@
 (defn get-pf [r]
   (-> r :metrics :roundtrip :pf))
 
+
 (defn show-fn [r]
   (-> r :metrics :roundtrip (select-keys [:trades])))
 

dev/src/dev/lib/dataset.clj

@@ -0,0 +1,16 @@
+(ns dev.lib.dataset
+  (:require 
+    [tablecloth.api :as tc]
+   [quanta.trade.backtest.store :refer [save-ds-transit-safe]]
+   )
+
+  )
+
+
+(def ds (tc/dataset {:a [1 2 3]
+                     :b [:a :b :C]
+                     }))
+
+ds
+
+(save-ds-transit-safe ds ".data/test.transit-json")

src/quanta/trade/backtest/store.clj

@@ -0,0 +1,31 @@
+(ns quanta.trade.backtest.store
+  (:require
+   [nano-id.core :refer [nano-id]]
+   [tech.v3.libs.clj-transit :as tech-transit]
+   [tech.v3.io :as io]))
+
+(defn ds->transit-json-file
+  [fname ds]
+  (with-open [outs (io/output-stream! fname)]
+    (tech-transit/dataset->transit ds outs :json tech-transit/java-time-write-handlers)))
+
+(defn transit-json-file->ds
+  [fname]
+  (with-open [ins (io/input-stream fname)]
+    (tech-transit/transit->dataset ins :json tech-transit/java-time-write-handlers)))
+
+(defn ds->nippy [filename ds]
+  (let [s (io/gzip-output-stream! filename)]
+    (io/put-nippy! s ds)))
+
+(defn nippy->ds [filename]
+  (let [s (io/gzip-input-stream filename)
+        ds (io/get-nippy s)]
+    ds))
+
+(defn save-ds-transit-safe [fname ds]
+  (let [fname-tmp "/tmp/ds.nippy.gz"
+        _ (ds->nippy fname-tmp ds)
+        ds-safe (nippy->ds fname-tmp)
+        ]
+    (ds->transit-json-file fname ds-safe)))

src/quanta/trade/bruteforce.clj

@@ -6,7 +6,7 @@
    [missionary.core :as m]
    [nano-id.core :refer [nano-id]]
    [babashka.fs :as fs]
-   [ta.db.bars.nippy :refer [save-ds]]
+   [quanta.trade.backtest.store :refer [ds->nippy nippy->ds ds->transit-json-file]]
    [quanta.dag.core :as dag]
    [quanta.algo.core :as algo]
    [quanta.algo.options :refer [create-algo-variations]]))
@@ -53,6 +53,14 @@
     (catch Exception ex
       {})))
 
+(defn save-ds [report-dir id ds]
+    (let [fname-nippy (str report-dir id "-roundtrips.nippy.gz")
+          fname-transit (str report-dir id "-roundtrips.transit-json")
+          _ (ds->nippy fname-nippy ds)
+          ds-safe (nippy->ds fname-nippy)]
+      (ds->transit-json-file fname-transit ds-safe)))
+
+
 (defn create-algo-task [dag-env algo cell-id dt variations target-fn show-fn report-dir]
   ; needs to throw so it can fail.
   (m/via m/cpu
@@ -65,7 +73,7 @@
            (when report-dir 
               (spit (str report-dir id "-result.edn") (pr-str report))  
               (spit (str report-dir id "-raw.txt") (with-out-str (println result)))
-              (save-ds (str report-dir id "-roundtrips.nippy.gz") (:roundtrip-ds result)))
+              (save-ds report-dir id (:roundtrip-ds result)))
            report
            )))