Merge branch 'master' into feature/iss-304-other-names

WormBase · Aug 21, 2020 · a016404 · a016404
2 parents bab867a + c8def80
commit a016404
Show file tree

Hide file tree

Showing 14 changed files with 163 additions and 97 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,17 +1,18 @@
 # Change-log
 
-# [current-beta]
+## [current-beta]
  - Improved & complemented docs
  - Deployment scripts & procedure fixes & updates (#310)
+ - Follow-up fixes (#308, #313) on merge reference side swap (done for #306 fix)
 
-# [1.4.2]
+## [1.4.2]
  - Updated version of wormbase/ids to 0.6.1
 
-# [1.4.1]
+## [1.4.1]
  - Revert attempt to display historic gene merges that produced no
    data change for the gene being merged into (gh issue #306).
 
-# [1.4.0]
+## [1.4.0]
  - Fix: Merge history bugfix (gh issue #306).
 
 ## [1.3.0]

diff --git a/README.md b/README.md
@@ -60,20 +60,20 @@ which uses a GPG2 key to encrypt tokens.
 
 
 ### REST API
-Run with:
+To be able to run the REST API locally, one must define the (local) datomic DB URI as the env variable `WB_DB_URI`.
 
-```bash
-export WB_DB_URI="[datomic-uri]"
-```
-
-An example of `[datomic-uri]` may be `datomic:mem://localhost:4334/names`. No transactor setup is needed for this in-memory database URI.
-For a persistent database (like `ddb-local`), a transactor needs to be configured, in which case the `[datomic-uri]` is based the your transactor configuration and database name. Make sure to define the ```DATOMIC_EXT_CLASSPATH``` env variable to point to the wormbase/ids jar when setting up the transactor (see [these instruction](./ids/README.md#Build) to build the ids jar).
+An example of a valid datomic URI may be `datomic:mem://localhost:4334/names`. No transactor setup is needed for this in-memory database URI.
+For a persistent database (like `ddb-local`), a transactor needs to be configured, in which case the `WB_DB_URI` is based on your transactor configuration and database name. Make sure to define the `DATOMIC_EXT_CLASSPATH` env variable to point to the wormbase/ids jar when setting up the transactor (see [these instruction](./ids/README.md#Build) to build the ids jar).
 
 ```bash
 export DATOMIC_EXT_CLASSPATH="$HOME/git/wormbase-names/ids/target/wbids.jar"
 ```
 
-Run with `make run-dev-webserver PORT=[port] WB_DB_URI=[datomic-uri]`.
+When using a `ddb-local` transactor, ensure to have set AWS environment variables with mock credentials,
+then run the following command to launch the local REST API service:
+```bash
+make run-dev-webserver PORT=[port] WB_DB_URI=[datomic-uri]
+```
 
 To allow the UI webpackDevServer to proxy to the ring server, the ring server has to be run at the host and port configured in the `"proxy"` section in [client/package.json](client/package.json) (standardly 4010 is used).
 
@@ -152,7 +152,8 @@ PORT=[PORT] npm run start
 ## Testing
 Use built-in testing utilities as provided by your environment, else use the `make` command
 below to run all tests.
-Ensure to run all tests and check they pass before submitting new pull requests.
+Ensure to run all tests and check they pass before committing large code changes,
+before submitting new pull requests and before deploying to any live AWS environment (test or production).
 
 ```bash
 make run-tests

diff --git a/ids/README.md b/ids/README.md
@@ -61,9 +61,8 @@ make release
 ```
 
 ### Update Datomic transactors
-For instructions on how to update the datomic transactors
-so they would use a new release of the IDs library,
-see the [wormbase-architecture/transactor README](https://github.com/WormBase/wormbase-architecture/tree/develop/transactor#rolling-updates-console).
+To update the datomic transactors so they would use a new release of the IDs library,
+follow the instructions in the [wormbase-architecture/transactor README](https://github.com/WormBase/wormbase-architecture/tree/develop/transactor#templateparameter-updates-cli-update) and update the appropriate CF parameter to match the new IDs package release number.
 
 ### Update the main name-service application dependency
 Update the main name-service to use the new version of the `wormbase/ids` dependency by:

diff --git a/src/wormbase/db.clj b/src/wormbase/db.clj
@@ -84,9 +84,15 @@
          (remove (fn [[e _ _ tx _]]
                    (= e tx)))
          (map (fn [[e a v tx added?]]
-                (if-let [fact (fact-mapper e a v tx added?)]
-                  fact
-                  [(if added? :db/retract :db/add) e a v]))))
+                (let [fact (fact-mapper e a v tx added?)]
+                  (cond
+                    (vector? fact)     fact
+                    (true? fact)       [(if added? :db/retract :db/add) e a v]
+                    (not (nil? fact))  (throw (ex-info "fact-mapper returned invalid type"
+                                                       {:tx tx
+                                                        :type ::invert-tx-problem
+                                                        :return-type (type fact)}))))))
+         (remove nil?))
         conj
         [provenance]
         datoms)
@@ -95,7 +101,7 @@
                         :type ::invert-tx-problem
                         :range (d/tx-range log t (inc t))})))))
   ([log tx provenance]
-   (invert-tx log tx provenance (constantly nil))))
+   (invert-tx log tx provenance (constantly true))))
 
 (defn extract-id [tx-result identity-kw]
   (some->> (:tx-data tx-result)
@@ -126,9 +132,10 @@
 
 (def txfn-schema (partial edn-definition "tx-fns"))
 
+(defn fmt-pull-result [db result]
+  (->> result
+       (wu/elide-importer-info)
+       (wu/elide-db-internals db)))
+
 (defn pull [db expr & args]
-  (let [format-result (fn [result]
-                        (->> result
-                             (wu/elide-importer-info)
-                             (wu/elide-db-internals db)))]
-    (format-result (apply d/pull db expr args))))
+  (fmt-pull-result db (apply d/pull db expr args)))
diff --git a/src/wormbase/names/batch/gene.clj b/src/wormbase/names/batch/gene.clj
@@ -4,6 +4,7 @@
    [compojure.api.sweet :as sweet]
    [ring.util.http-response :refer [bad-request! created ok]]
    [spec-tools.core :as stc]
+   [wormbase.db :as wdb]
    [wormbase.ids.batch :as wbids-batch]
    [wormbase.names.batch.generic :as wnbg]
    [wormbase.names.entity :as wne]   
@@ -75,13 +76,14 @@
        :parameters {:body-params {:data ::wsg/update-batch
                                   :prov ::prov}}
        :handler (fn update-handler [request]
-                  (wnbg/update-entities :gene/id
-                                        [{:gene/species [[:species/latin-name]]}]
-                                        :event/update-gene
-                                        ::wsg/update-batch
-                                        wnu/conform-data
-                                        (partial wnv/validate-names request)
-                                        request))}
+                  (let [get-info-fn #(wdb/pull %1 [{:gene/species [[:species/latin-name]]}] %2)]
+                    (wnbg/update-entities :gene/id
+                                          get-info-fn
+                                          :event/update-gene
+                                          ::wsg/update-batch
+                                          wnu/conform-data
+                                          (partial wnv/validate-names request)
+                                          request)))}
       :post
       {:summary "Assign identifiers and associate names, creating new genes."
        :x-name ::batch-new-genes

diff --git a/src/wormbase/names/batch/generic.clj b/src/wormbase/names/batch/generic.clj
@@ -170,19 +170,19 @@
   "Perform a batch update.
 
   - uiident : Uniquely Identifing Ident - A datomic entity attribute.
-  - item-pull-expr: a datomic pull expression describing the data for each entity
-                    that's fetched as defaults when the data isn't supplied in the request.
+  - get-info-fn: a fn to pull the datomic data for each entity
+                 that's fetched as defaults when the data isn't supplied in the request.
   - event-type: A datomic ident that identifies the kind of event.
   - spec: The spec desribing the shape of the data.
   - conformer: function taking spec and data, responsible for coercion of input data.
   - validator: function to validate the coerced data.
   - request: the HTTP request."
-  [uiident item-pull-expr event-type spec _ validator request]
+  [uiident get-info-fn event-type spec _ validator request]
   (let [ent-ns (namespace uiident)
         data-transform (fn valdiating-conformer [_ data]
                          (let [{db :db} request
                                qdata (map #(wnu/qualify-keys % ent-ns) data)
-                               db-data (map #(wdb/pull db item-pull-expr (find % uiident)) qdata)
+                               db-data (map #(get-info-fn db (find % uiident)) qdata)
                                transformed (->> qdata
                                                 (map merge db-data)
                                                 (map wne/transform-ident-ref-values))]
@@ -293,9 +293,9 @@
        :handler (fn handle-update [request]
                   (let [ent-ident (keyword entity-type "id")
                         event-ident (keyword "event" (str "update-" entity-type))
-                        pull-expr (wne/make-summary-pull-expr entity-type)]
+                        summary-fn (partial wne/pull-ent-summary entity-type)]
                     (update-entities ent-ident
-                                     pull-expr
+                                     summary-fn
                                      event-ident
                                      ::wse/update-batch
                                      wnu/conform-data

diff --git a/src/wormbase/names/coercion.clj b/src/wormbase/names/coercion.clj
@@ -5,7 +5,7 @@
    [spec-tools.core :as st]))
 
 ;;; Modified copies of the compojure-api's coercion to prevent stripping of keys.
-;;; TODO!!: This neesd to be kept in sync with compojure-api releases if the implementation changes.
+;;; TODO!!: This needs to be kept in sync with compojure-api releases if the implementation changes.
 ;;;         Add developer documentation.
 
 (def string-transformer

diff --git a/src/wormbase/names/entity.clj b/src/wormbase/names/entity.clj
@@ -57,10 +57,14 @@
              data
              data))
 
-(defn make-summary-pull-expr [entity-type]
-  (let [pe '[*]
-        attr-spec {(keyword entity-type "status") [:db/ident]}]
-    (conj pe attr-spec)))
+(defn pull-ent-summary
+  "Pull an entity's attributes, dereferencing the entity-type's status."
+  [entity-type db eid]
+  (let
+    [pe '[*]
+     attr-spec {(keyword entity-type "status") [:db/ident]}
+     pull-exp  (conj pe attr-spec)]
+    (wdb/pull db pull-exp eid)))
 
 (defn identify
   "Return an lookup ref and entity for a given identifier.
@@ -108,7 +112,7 @@
 
 (defn creator
   "Return an endpoint handler for new entity creation."
-  [uiident conform-spec-fn event summary-pull-expr & [validate]]
+  [uiident conform-spec-fn event get-info-fn & [validate]]
   (fn handle-new [request]
     (let [{payload :body-params db :db conn :conn} request
           ent-ns (namespace uiident)
@@ -135,7 +139,7 @@
             dba (:db-after tx-res)]
         (when dba
           (let [new-id (wdb/extract-id tx-res uiident)
-                emap (wdb/pull dba summary-pull-expr [uiident new-id])
+                emap  (get-info-fn dba [uiident new-id])
                 emap* (reduce-kv (fn [m k v]
                                    (if (qualified-keyword? v)
                                      (assoc m k (name v))
@@ -158,13 +162,13 @@
            data)))
 
 (defn updater
-  [identify-fn uiident conform-spec-fn event summary-pull-expr & [validate ref-resolver-fn]]
+  [identify-fn uiident conform-spec-fn event get-info-fn & [validate ref-resolver-fn]]
   (fn handle-update [request identifier]
     (let [{db :db conn :conn payload :body-params} request
           ent-ns (namespace uiident)
           [lur entity] (identify-fn request identifier)]
       (when entity
-        (let [ent-data (wdb/pull db summary-pull-expr lur)
+        (let [ent-data (get-info-fn db lur)
               names-validator (if validate
                                 (partial validate request)
                                 identity)
@@ -190,7 +194,7 @@
                 txes [['wormbase.ids.core/cas-batch lur cdata] prov]
                 tx-result @(d/transact-async conn txes)]
             (when-let [db-after (:db-after tx-result)]
-              (if-let [updated (wdb/pull db-after summary-pull-expr lur)]
+              (if-let [updated (get-info-fn db-after lur)]
                 (ok {:updated (wnu/unqualify-keys updated ent-ns)})
                 (not-found
                  (format "%s '%s' does not exist" ent-ns (last lur)))))))))))
@@ -287,7 +291,7 @@
                                   :precondition-failure-msg precond-fail-msg)]
     (resurrect request identifier)))
 
-(defn summarizer [identify-fn pull-expr ref-attrs]
+(defn summarizer [identify-fn get-info-fn ref-attrs]
   (fn handle-summary [request identifier]
     (let [{db :db conn :conn} request
           log (d/log conn)
@@ -301,7 +305,7 @@
                                 (assoc m k (name v))
                                 (assoc m k v)))
                             {}
-                            (-> (wdb/pull db pull-expr lur)
+                            (-> (get-info-fn db lur)
                                 (wnu/unqualify-keys ent-ns)))
             prov (map
                   (fn [prov-data]
@@ -567,11 +571,11 @@
                   (let [id-ident (keyword entity-type "id")
                         event-ident (keyword "event" (str "new-" entity-type))
                         conformer (partial wnu/conform-data ::wse/new)
-                        spe (make-summary-pull-expr entity-type)
+                        summary-fn (partial pull-ent-summary entity-type)
                         new-entity (creator id-ident
                                             conformer
                                             event-ident
-                                            spe
+                                            summary-fn
                                             validate-names)]
                     (new-entity request)))}})
     (sweet/context "/:identifier" []
@@ -598,22 +602,22 @@
                     (let [id-ident (keyword entity-type "id")
                           update-spec ::wse/update
                           event-ident (keyword "event" (str "update-" entity-type))
-                          summary-pull-expr (make-summary-pull-expr entity-type)
+                          summary-fn (partial pull-ent-summary entity-type)
                           update-handler (updater (partial identify ::wse/identifier entity-type)
                                                   id-ident
                                                   (partial wnu/conform-data update-spec)
                                                   event-ident
-                                                  summary-pull-expr
+                                                  summary-fn
                                                   validate-names)]
                       (update-handler request identifier)))}
         :get
         {:summary "Summarise an entity."
          :x-name ::about-entity
          :responses (wnu/http-responses-for-read {:schema ::wse/summary})
          :handler (fn handle-entity-summary [request]
-                    (let [summary-pull-expr (make-summary-pull-expr entity-type)
+                    (let [summary-fn (partial pull-ent-summary entity-type)
                           summarize (summarizer (partial identify ::wse/identifier entity-type)
-                                                summary-pull-expr
+                                                summary-fn
                                                 #{})]
                       (summarize request identifier)))}})
       (sweet/context "/resurrect" []