Merge pull request #366 from WormBase/PR-validation

Automated PR validation

.github/workflows/PR-validation.yml

@@ -0,0 +1,53 @@
+name: PR validation
+on:
+  pull_request:
+    types: [synchronize, opened, reopened, edited]
+    branches:
+      - master
+jobs:
+  test-api:
+    permissions:
+      id-token: write # Required for authentication through OIDC to AWS
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Report workflow details
+        run: |
+          echo "Repository ${{ github.repository }}."
+          echo "Trigger ref ${{ github.ref }}, base-ref ${{ github.base_ref }}, head_ref ${{ github.head_ref }}."
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: Report files updated in PR
+        run: |
+          git fetch -q origin ${{ github.base_ref }} ${{ github.head_ref }}
+          git diff --name-only origin/${{ github.base_ref }} origin/${{ github.head_ref }}
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'temurin'
+      - name: Install clojure and clojure cli (clj)
+        uses: DeLaGuardo/setup-clojure@12.3
+        with:
+          cli: 1.10.1.536
+      - name: Report runtime details
+        run: |
+          echo "Github runner OS: ${{ runner.os }}"
+      - name: AWS credentials configuration
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
+          role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-test-api
+          aws-region: us-east-1
+      - name: Download and install Datomic Pro
+        run: |
+          aws s3 cp s3://wormbase/datomic-pro/distro/datomic-pro-1.0.6165.zip ./
+          unzip datomic-pro-1.0.6165.zip
+          cd datomic-pro-1.0.6165/
+          bin/maven-install
+      - name: Generate pom file
+        run: |
+          clojure -Spom
+      - name: Run Integration tests
+        run: |
+          make run-tests GOOGLE_APP_PROFILE=dev
+  #TODO: add UI and API build and container packaging test

Makefile

@@ -235,7 +235,7 @@ run-tests: google-oauth2-secrets \
 	@ export API_GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID} && \
 	  export API_GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET} && \
 	  export GOOGLE_REDIRECT_URI=${LOCAL_GOOGLE_REDIRECT_URI} && \
-	  clj -A:datomic-pro:logging:webassets:dev:test:run-tests
+	  clojure -A:datomic-pro:logging:webassets:dev:test:run-tests
 
 .PHONY: run-dev-server
 run-dev-webserver: PORT := 4010

deps.edn

@@ -21,9 +21,11 @@
   metosin/compojure-api {:mvn/version "2.0.0-alpha30"
                          :exclusions [frankiesardo/linked]}
   metosin/muuntaja {:mvn/version "0.6.8"}
+  metosin/ring-http-response {:mvn/version "0.9.3"}
   ikitommi/linked {:mvn/version "1.3.1-alpha1"}
   mount/mount {:mvn/version "0.1.17"}
   phrase/phrase {:mvn/version "0.3-alpha4"}
+  ring/ring-core {:mvn/version "1.9.4"}
   ring/ring-codec {:mvn/version "1.2.0"}
   ring/ring-defaults {:mvn/version "0.4.0"}
   ring/ring-jetty-adapter {:mvn/version "1.10.0"}

export/src/wormbase/names/export.clj

@@ -2,7 +2,6 @@
   (:require
    [clojure.java.io :as io]
    [clojure.data.csv :as cd-csv]
-   [clojure.set :as set]
    [clojure.string :as str]
    [clojure.tools.cli :as cli]
    [datomic.api :as d]

ids/src/wormbase/ids/batch.clj

@@ -19,7 +19,7 @@
    [clojure.core :as cc]
    [clojure.set :as set]
    [datomic.api :as d]
-   [wormbase.ids.core :refer [attr-schema-unique? identifier-format]]))
+   [wormbase.ids.core :refer [identifier-format]]))
 
 (defn- assoc-prov
   "Attach an identifier to `prov` making this a mapping suitable for tracking provenance for a batch.
@@ -32,12 +32,6 @@
   (when tx-data
     (cons prov tx-data)))
 
-(defn db-error? [exc]
-  (some->> (ex-data exc)
-           (keys)
-           (filter (fn has-db-ns? [k]
-                     (#{"db" "db.error"} (namespace k))))))
-
 (defrecord BatchResult [tx-result errors])
 
 (defrecord BatchError [error-type exc])

ids/src/wormbase/ids/core.clj

@@ -9,7 +9,7 @@
   (try
     (let [s-attr (:db/unique (d/entity db attr))]
       (#{:db.unique/identity :db.unique/value} s-attr))
-    (catch IllegalArgumentException iae)))
+    (catch IllegalArgumentException _)))
 
 (defn entids->data [db m k v]
     (assoc m

src/wormbase/names/importers/processing.clj

@@ -130,7 +130,7 @@
       (jt/zoned-date-time tz)
       (jt/with-zone-same-instant tz)
       (jt/instant)
-      (jt/to-java-date)))
+      (jt/java-date)))
 
 (defn check-environ! []
   (when-not (environ/env :token)

src/wormbase/names/provenance.clj

@@ -47,7 +47,7 @@
         whence (-> (get prov :provenance/when (jt/instant))
                    (jt/zoned-date-time tz)
                    (jt/with-zone-same-instant tz)
-                   (jt/to-java-date))
+                   (jt/java-date))
         how (-> request :headers wna/identify)
         why (:provenance/why prov)
         prov {:db/id "datomic.tx"

src/wormbase/names/recent.clj

@@ -10,8 +10,6 @@
    [wormbase.util :as wu])
   (:import (java.util Date)))
 
-(def conf (:recent (wu/read-app-config)))
-
 (defn- find-max-imported-date [db]
   (let [max-tx-inst (d/q '[:find (max ?inst) .
                            :where
@@ -23,7 +21,7 @@
                    (jt/instant))]
     (-> max-date
         (jt/plus (jt/seconds 1))
-        (jt/to-java-date))))
+        (jt/java-date))))
 
 (def imported-date (memoize find-max-imported-date))
 
@@ -45,7 +43,7 @@
          from-t (if (and from (>= (compare from import-date) 0))
                   from
                   import-date)
-         until-t (or until (jt/to-java-date (jt/instant)))]
+         until-t (or until (jt/java-date (jt/instant)))]
      ;; Timings for the `tx-ids` query below with default configured time window (60 days)
      ;; (excluding pull expressions)
      ;; jvm (cold): 107.266427 msecs
@@ -108,16 +106,14 @@
 (def batch-rules '[[(filter-events ?tx ?needle)
                     [?tx :batch/id _ _ ]]])
 
-(def response-schema (wnu/response-map ok {:schema {:activities ::wsr/activities}}))
-
 (defn handle
   ([request rules puller needle from until]
    (handle request rules puller needle from until #{:agent/console :agent/web}))
   ([request rules puller needle from until how]
    (let [{conn :conn db :db} request
          log (d/log conn)
          from* (or from (wu/days-ago wsr/*default-days-ago*))
-         until* (or until (jt/to-java-date (jt/instant)))
+         until* (or until (jt/java-date (jt/instant)))
          items (activities db log rules puller (or needle "") how from* until*)
          etag (some-> items first :t wnu/encode-etag)]
      (some-> {:from from* :until until*}
@@ -179,7 +175,7 @@
   (binding [db (d/db conn) log (d/log conn)
             pull-prov-only (prov-only-puller db log)
             pull-changes-and-prov (changes-and-prov-puller db log)
-            from (jt/to-java-date (jt/instant))
+            from (jt/java-date (jt/instant))
             until (wu/days-ago 2)]
     (activities db log entity-rules pull-changes-and-prov "gene")
     (activities db log entity-rules-rules pull-changes-and-prov "gene" from until)

src/wormbase/names/service.clj

@@ -23,7 +23,8 @@
    [ring.middleware.gzip :as ring-gzip]
    [ring.middleware.not-modified :as rmnm]
    [ring.middleware.resource :as ring-resource]
-   [ring.util.http-response :as http-response]))
+   [ring.util.http-response :as http-response]
+   [ring.util.response :as ring-response]))
 
 (defn- wrap-not-found
   "Fallback 404 handler."
@@ -36,9 +37,9 @@
         (http-response/not-found {:message "Resource not found (fallback)"})
 
         :else
-        (-> (http-response/resource-response "client_build/index.html")
-            (http-response/content-type "text/html")
-            (http-response/status 200))))))
+        (-> (ring-response/resource-response "client_build/index.html")
+            (ring-response/content-type "text/html")
+            (ring-response/status 200))))))
 
 (def ^:private swagger-validator-url
   "The URL used to validate the swagger JSON produced by the application."

src/wormbase/names/stats.clj

@@ -3,7 +3,6 @@
    [clojure.walk :as w]
    [compojure.api.sweet :as sweet]
    [datomic.api :as d]
-   [ring.middleware.not-modified :as rmnm]
    [ring.util.http-response :refer [ok]]
    [wormbase.names.util :as wnu]
    [wormbase.specs.stats :as wsst]))
@@ -29,14 +28,11 @@
          (w/stringify-keys))))
 
 (defn handle-summary [request]
-  (let [etag (some-> request :db d/basis-t wnu/encode-etag)]
-    (-> (summary request)
-        (ok)
-        (wnu/add-etag-header-maybe etag))))
+  (-> (summary request)
+      (ok)))
 
 (def routes (sweet/routes
              (sweet/context "/stats" []
-               :middleware [rmnm/wrap-not-modified]
                :tags ["stats"]
                (sweet/resource
                 {:get

src/wormbase/names/util.clj

@@ -6,20 +6,15 @@
    [clojure.string :as str]
    [clojure.walk :as w]
    [buddy.core.codecs :as codecs]
-   [buddy.core.codecs.base64 :as b64]
    [datomic.api :as d]
    [expound.alpha :refer [expound-str]]
    [phrase.alpha :as ph]
-   [ring.util.http-response :refer [bad-request conflict header not-found not-modified ok]]
+   [ring.util.http-response :refer [bad-request conflict not-found not-modified ok]]
+   [ring.util.response :refer [header]]
    [wormbase.db :as wdb]
    [wormbase.specs.common :as wsc]
    [wormbase.specs.validation :as wsv]))
 
-(defn- nsify [domain kw]
-  (if (namespace kw)
-    kw
-    (keyword domain (name kw))))
-
 ;; trunc and datom-table taken from day-of-datomic repo (congnitect).
 
 (defn trunc
@@ -143,7 +138,7 @@
             bid)))
 
 (defn encode-etag [latest-t]
-  (some-> latest-t str b64/encode codecs/bytes->str))
+  (some-> latest-t codecs/long->bytes codecs/bytes->b64 codecs/bytes->str))
 
 (defn add-etag-header-maybe [response etag]
   (if (seq etag)

src/wormbase/util.clj

@@ -49,7 +49,7 @@
 (defn days-ago [n]
   (-> (jt/instant)
       (jt/minus (jt/days n))
-      (jt/to-java-date)))
+      (jt/java-date)))
 
 (defn format-java-date [dt & {:keys [tz fmt]
                               :or {tz (jt/zone-id)
@@ -73,7 +73,7 @@
   ([tz]
    (-> (jt/instant)
        (jt/zoned-date-time (jt/zone-id tz))
-       (jt/to-java-date)))
+       (jt/java-date)))
   ([]
    (now "UTC")))
 

test/integration/service_test.clj

@@ -2,7 +2,7 @@
   (:require
    [clojure.test :as t]
    [ring.util.http-predicates :as ru-hp]
-   [ring.util.http-response :as http-response]
+   [ring.util.response :as ring-response]
    [wormbase.db-testing :as db-testing]
    [wormbase.fake-auth]
    [wormbase.names.service :as service]))
@@ -19,5 +19,5 @@
                      :request-method :get})]
       (t/is (not (nil? response)))
       (t/is (ru-hp/ok? response))
-      (t/is (http-response/get-header response "content-type") "text/html"))))
+      (t/is (ring-response/get-header response "content-type") "text/html"))))
 

test/integration/stats_test.clj

@@ -16,11 +16,6 @@
 
 (t/deftest test-summary
   (t/testing "Stats summary renders with ok response on first access."
-    (let [response (stats-summary)
-          etag (get-in response [:headers "etag"])]
+    (let [response (stats-summary)]
       (t/is (ru-hp/ok? response))
-      (t/is (set/subset? #{:gene :variation :sequence-feature} (:body response)))
-      (t/testing "Status summary renders a not-modified response when nothing has changed."
-        (let [extra-headers {"if-none-match" etag}
-              response2 (stats-summary extra-headers)]
-          (t/is (ru-hp/not-modified? response2)))))))
+      (t/is (set/subset? #{:gene :variation :sequence-feature} (:body response))))))

test/wormbase/gen_specs/person.clj

@@ -2,7 +2,6 @@
   (:require
    [clojure.spec.alpha :as s]
    [miner.strgen :as sg]
-   [wormbase.gen-specs.util :as util]
    [wormbase.specs.person :as wsp]))
 
 (def email (sg/string-generator wsp/email-regexp))

test/wormbase/test_utils.clj

@@ -238,7 +238,7 @@
 (defn provenance
   [_ & {:keys [how what whence why person _ batch-id]
         :or {how :agent/console
-             whence (jt/to-java-date (jt/instant))
+             whence (jt/java-date (jt/instant))
              what :event/test-fixture-assertion
              person [:person/email "tester@wormbase.org"]}}]
   (merge {:db/id "datomic.tx"