trade backtest metrics improved (a lot)

lib/algo/src/ta/algo/spec/type/bar_strategy.clj

@@ -47,9 +47,8 @@
   (assert algo)
   (let [algo-fn (chain/make-chain algo)
         load-fn (create-trailing-bar-loader spec)]
-    (assert algo-fn)
+    (assert algo-fn (str "could not compile algo-fn: " algo))
     (fn [env _spec time]
-       ;(println "calculating barstrategy for time: " time)
       (when time
         (let [ds-bars (load-fn env spec time)]
           (if (nom/anomaly? ds-bars)

lib/calendar/src/ta/calendar/align.clj

@@ -1,6 +1,5 @@
 (ns ta.calendar.align
   (:require
-   [taoensso.timbre :refer [trace debug info warn error]]
    [tablecloth.api :as tc]
    [tech.v3.datatype.argops :as argops]
    [tech.v3.tensor :as dtt]))

lib/db/resources/quanta/notebook/bardb_duck_delete.clj

@@ -0,0 +1,17 @@
+(ns quanta.notebook.bardb-duck-delete
+  (:require
+   [tick.core :as t]
+   [tablecloth.api :as tc]
+   [ta.db.bars.protocol :as b]
+   [ta.db.bars.duckdb :as duck]
+   [ta.db.bars.dynamic.overview-db :refer [remove-asset]]
+   [modular.system]))
+
+(def ddb (modular.system/system :bardb-dynamic))
+
+(def db (modular.system/system :duckdb))
+
+
+(duck/delete-bars db [:crypto :d] "ETHUSDT")
+
+(remove-asset (:overview-db ddb) {:asset "ETHUSDT" :calendar [:crypto :d]})

lib/db/resources/quanta/notebook/bardb_duck.clj → lib/db/resources/quanta/notebook/docs/bardb_duck.clj

@@ -1,4 +1,4 @@
-(ns quanta.notebook.bardb-duck
+(ns quanta.notebook.docs.bardb-duck
   (:require
    [tick.core :as t]
    [tablecloth.api :as tc]

lib/db/src/ta/db/bars/multi_asset.clj

@@ -1,5 +1,6 @@
 (ns ta.db.bars.multi-asset
   (:require
+   [taoensso.timbre :as timbre :refer [debug info warn error]]
    [de.otto.nom.core :as nom]
    [tick.core :as t]
    [tablecloth.api :as tc]
@@ -16,24 +17,37 @@
 (defn- align-to-calendar2 [ds-bars ds-cal]
   (align/align-to-calendar ds-cal ds-bars))
 
+(defn- debug-ds [ds]
+  (info "aligned ds: " ds)
+  ds)
+
 (defn load-aligned [bardb {:keys [asset] :as opts} window ds-cal]
   (nom/let-nom> [ds-bars (b/get-bars bardb opts window)]
                 (-> ds-bars
                     (tc/set-dataset-name asset)
+                    ;debug-ds
                     (align-to-calendar2 ds-cal)
+                    ;debug-ds
                     (align/fill-missing-close)
                     (tc/add-column :asset asset))))
 
 (defn load-aligned-assets [bardb opts assets cal-seq]
-  (let [window (calendar-seq->window cal-seq)
-        date-ds (calendar-seq->date-ds cal-seq)
-        load-asset (fn [asset]
-                     [asset (load-aligned bardb (assoc opts :asset asset) window date-ds)])]
-    (->> (map load-asset assets)
-         (into {}))))
+  (try
+    (let [window (calendar-seq->window cal-seq)
+          date-ds (calendar-seq->date-ds cal-seq)
+          load-asset (fn [asset]
+                       [asset (load-aligned bardb (assoc opts :asset asset) window date-ds)])]
+      (->> (map load-asset assets)
+           (into {})))
+    (catch Exception ex
+      (error "exception: " ex)
+      (nom/fail ::load-aligned-bars {}))))
 
 (defn is-valid? [result asset]
-  (->  result (get asset) nom/anomaly? not))
+  (if (nom/anomaly? result)
+    false
+    (let [ds (get result asset)]
+      (->  ds  nom/anomaly? not))))
 
 (comment
   (require '[modular.system])

lib/helper/src/ta/helper/date.clj

@@ -198,6 +198,10 @@
       (subtract-days n)
       ;(t/date)
       ))
+(defn fmt-yyyymmdd [dt]
+  (if dt
+    (t/format (t/formatter "YYYY-MM-dd")  (t/zoned-date-time dt))
+    ""))
 
 ; *****************************************************************************
 (comment
@@ -301,5 +305,8 @@
 
 ;java.time.LocalDateTime
 
+  (-> (t/instant)
+      (fmt-yyyymmdd))
+
 ;
   )

lib/import/resources/quanta/notebook/docs/import_bardb_dynamic.clj

@@ -0,0 +1,25 @@
+(ns quanta.notebook.docs.import-bardb-dynamic
+  (:require
+   [tick.core :as t]
+   [modular.system]
+   [ta.calendar.core :as cal]
+   [ta.db.bars.protocol :as b]))
+
+(def db (modular.system/system :bardb-dynamic))
+
+(def window-daily
+  (cal/trailing-range [:us :d] 200
+                      (t/zoned-date-time "2024-05-02T17:30-05:00[America/New_York]")))
+
+window-daily
+
+
+(b/get-bars db {:asset "AEE.AU"
+                :calendar [:us :d]
+                :import :eodhd}
+            window-daily)
+
+(b/get-bars db {:asset "BTCUSDT"
+                :calendar [:us :d]
+                :import :bybit}
+            window-daily)

lib/import/resources/quanta/notebook/import_manager.clj → lib/import/resources/quanta/notebook/docs/import_manager.clj

@@ -1,4 +1,4 @@
-(ns quanta.notebook.import-manager
+(ns quanta.notebook.docs.import-manager
   (:require
    [tick.core :as t]
    [ta.db.bars.protocol :as b]
@@ -8,11 +8,11 @@
 
 im
 
-(def dt (t/instant "2024-02-01T00:00:00Z"))
+(def dt (t/instant "2024-05-01T00:00:00Z"))
 dt
 
 ;; BYBIT
-(b/get-bars im {:asset "BTCUSDT" ; crypto
+(b/get-bars im {:asset "ETHUSDT" ; crypto
                 :calendar [:crypto :d]
                 :import :bybit}
             {:start  (t/instant "2020-01-01T00:00:00Z")
@@ -29,12 +29,12 @@ dt
   (-> ds :date meta :datatype))
 
 ;; KIBOT
-(-> (b/get-bars im {:asset "NG0" ; future
-                    :calendar [:us :d]
-                    :import :kibot}
-                {:start  (t/instant "2020-01-01T00:00:00Z")
-                 :end (t/instant "2024-01-01T00:00:00Z")})
-    date-type)
+
+(b/get-bars im {:asset "NG0" ; future
+                :calendar [:us :d]
+                :import :kibot}
+            {:start  (t/instant "2020-01-01T00:00:00Z")
+             :end (t/instant "2024-01-01T00:00:00Z")})
 
 (b/get-bars im
             {:asset "EUR/USD" ; forex
@@ -62,7 +62,13 @@ dt
             {:asset "AEE.AU"
              :calendar [:us :d]
              :import :eodhd}
-            ; fails because more than a year ago
             {:start (-> "2023-12-01T00:00:00Z" t/instant)
              :end (-> "2024-04-01T00:00:00Z" t/instant)})
 
+(b/get-bars im
+            {:asset "AEE.AU"
+             :calendar [:us :d]
+             :import :eodhd}
+            ; fails because more than a year ago
+            {:start (-> "2020-12-01T00:00:00Z" t/instant)
+             :end (-> "2024-04-01T00:00:00Z" t/instant)})

lib/import/src/ta/db/bars/dynamic/overview_db.clj

@@ -94,6 +94,14 @@
     (info "overview tx: " tx)
     (d/transact conn [tx])))
 
+(defn remove-asset [conn {:keys [asset calendar] :as opts}]
+  (let [[market interval] calendar
+        id (find-id conn opts)]
+    (info "removing overview-db asset: " asset " " calendar " db-id: " id)
+    (d/transact
+     conn
+     {:tx-data [[:db/retractEntity id]]})))
+
 (comment
   (def conn (start-overview-db "/tmp/datahike-overview"))
   conn

lib/import/src/ta/import/provider/bybit/ds.clj

@@ -141,11 +141,28 @@
          (reduce or-fn false)
          not)))
 
+(defn set-close-time [dt]
+  (-> dt
+      (t/date)
+      (t/at (t/time "23:59:59"))
+      (t/in "UTC")
+      (t/instant)))
+
+(defn set-close-time-vec [dt-vec]
+  (map set-close-time dt-vec))
+
+(defn set-daily-time [{:keys [asset calendar] :as opts} ds]
+  (let [frequency (second calendar)]
+    (if (= frequency :d)
+      (tc/update-columns ds {:date set-close-time-vec})
+      ds)))
+
 (defn consolidate-datasets [opts range datasets]
   (if (all-ds-valid datasets)
     (->> datasets
          (apply tc/concat)
-         (sort-ds))
+         (sort-ds)
+         (set-daily-time opts))
     (nom/fail ::consolidate-datasets {:message "paged request failed!"
                                       :opts opts
                                       :range range})))

lib/import/src/ta/import/provider/eodhd/ds.clj

@@ -44,7 +44,7 @@
       ))
 
 (defn error? [body]
-  (-> body first :warning))
+  (-> body last :warning))
 
 (defn get-bars-eodhd [api-token {:keys [asset calendar] :as opts} {:keys [start end] :as window}]
   (warn "get-bars: " opts window)

lib/import/src/ta/import/provider/eodhd/raw.clj

@@ -1,9 +1,7 @@
 (ns ta.import.provider.eodhd.raw
   (:require
-   [clojure.string :as str]
    [clojure.set]
    [taoensso.timbre :refer [info warn error]]
-   [clojure.edn :as edn]
    [cheshire.core :as cheshire] ; JSON Encoding
    [de.otto.nom.core :as nom]
    [ta.import.helper :refer [str->float http-get]]
@@ -51,12 +49,24 @@
 (defn get-exchange-tickers [api-token exchange-code]
   (make-request api-token (str "exchange-symbol-list/" exchange-code) {}))
 
+(defn warning [result]
+  (-> result last :warning))
+
 (comment
   (def d (get-bars "65f0ad82c56400.56029279"
                    "MCD.US"
                    "2024-01-01"
                    "2024-03-15"))
 
   d
+
+  (require '[clojure.pprint :refer [print-table]])
+
+  (->> (get-bars "65f0ad82c56400.56029279"
+                 "MCD.US"
+                 "2020-01-01"
+                 "2024-03-15")
+       warning)
+
 ;  
   )

lib/trade/resources/quanta/notebook/docs/mark2market.clj

@@ -0,0 +1,30 @@
+(ns quanta.notebook.docs.mark2market
+  (:require
+   [tick.core :as t]
+   [modular.system]
+   [ta.trade.roundtrip.nav.mark2market :refer [portfolio]]
+   [ta.viz.trade.m2m.core :refer [m2m-chart]]
+   [ta.db.bars.protocol :as b]))
+
+(def bardb (modular.system/system :bardb-dynamic))
+
+(def rts [{:asset "ETHUSDT"
+           :side :long
+           :qty 100.0
+           :entry-date (t/instant "2023-03-09T23:59:59Z")
+           :entry-price 1400.78
+           :exit-date (t/instant "2023-03-16T23:59:59Z")
+           :exit-price 1600.92}])
+
+
+(-> (portfolio bardb rts {:calendar [:crypto :d]
+                          :import :bybit})
+    (m2m-chart)
+ ;pr-str
+  )
+
+; {:open# 0, :long$ 0.0, :short$ 0.0, :net$ 0.0, :pl-u 0.0, 
+; :pl-r 0.0, :date #inst \"2023-03-06T23:59:59.000000000-00:00\",
+; :pl-r-cum 0.0, :pl-cum 0.0}
+
+

lib/trade/resources/quanta/notebook/trade_entry_exit.clj → lib/trade/resources/quanta/notebook/docs/trade_entry_exit.clj

@@ -1,10 +1,10 @@
-(ns quanta.notebook.trade-entry-exit
+(ns quanta.notebook.docs.trade-entry-exit
   (:require
    [tick.core :as t]
    [tablecloth.api :as tc]
    [ta.trade.backtest.from-entry :refer [entry-signal->roundtrips]]
-   [ta.trade.roundtrip.core :refer [metrics]]
-   [ta.viz.ds.metrics :refer [metrics-render-spec-impl]]))
+   [ta.trade.roundtrip.core :refer [roundtrip-stats]]
+   [ta.viz.trade.core :refer [roundtrip-stats-ui]]))
 
 (def ds (tc/dataset {:date (repeatedly 6 #(t/instant))
                      :close [100.0 104.0 106.0 103.0 102.0 108.0]
@@ -24,6 +24,6 @@ ds
 
 rts
 
-(-> (metrics rts)
-    (metrics-render-spec-impl)
+(-> (roundtrip-stats rts)
+    (roundtrip-stats-ui)
  )