Go Api Migration (#1057)

* feat: initialize

feat: aggregator

feat: add input validation, error api, data api

feat: add cors middleware, add apis, refactors

fix: remove unnecessary file

docs: add todos

feat: add apis and update readme

feat: handle timestamptz format & add testcodes

test: add tests

feat: add testcodes, fix redisClient error

wip: add encryption&decryption, check formats

fix: modify formats

feat: custom types for compatibility

feat: update element types based on customTypes

fix: rollback timestamp type handling

feat: update consitency test

wip

feat: update models

feat: set pointer types for struct elements

test: update test based on updated struct elements

test: update tests

fix: .env file is now optional

feat: add dockerfile, fix from docker-compose err

fix: fix err

feat: docker compose file comment to run go

docs: add explanation to run docker-compose for go

feat: add github actions

feat: add linter on image build, fix on lint

fix: rollback nest js query logging

test: github action

fix: indent fix

test: tmp file to trigger test

fix: update go-migrate installation

test: remove tmp file to trigger github action

test: fix typo

test: add tmp file to trigger GA

test: go dependency cache setting for GA

test: fix tests

fix: rename struct element names

feat: rename struct element, GA script update,

fix: remove struct embedding, lint fix

fix: fix typo for version reference

fix: update based on feedback(logging)

fix: change path before getting version from GA

feat: embed `.version` into go project

feat: use recover middleware

fix: update based on feedback

fix: add missing newline at end of files

fix: declare payload as pointer types

fix: improve readability, remove inline structs

fix: updates based on feedbacks

fix: updates based on feedbacks

fix: update github action

test: update to trigger github action

fix: fix typo

test: tmp file to trigger GA

fix: remove typo

fix: fix typo, remove return after panic

fix: fix typo and add tmp file to trigger GA

feat: Add check on build

tmp: remove tmp to trigger GA

fix: run vet from go-api directory

fix: remove comments

fix: simplify date parse from `UnmarshalJSON`

docs: update docs

fix: strict datetime format checking

fix: fix err

fix: update based on feedbacks

fix: updates based on feedback

fix: add missing env var in `.env.local`

fix: modify docker image to to read env variable properly

fix: update docs

fix: update dockerfile command

fix: update docker image command

fix: set pointer for location to take null value

fix: update datetime formatting

fix: add more error logging

fix: update to use pgxPool and Panic

* fix: remove unnecessary log

* Improve api logging (#1108)

* fix: remove logger middleware

* fix: update logging and error handling

* fix: better logging

* fix: fix typo

.github/workflows/goapi.image+upload.yaml

@@ -0,0 +1,97 @@
+name: GO API Deploy to Amazon ECR
+
+on:
+  push:
+    braches:
+      - master
+    paths:
+      - "go-api/**"
+  workflow_dispatch:
+
+env:
+  ecr_url: public.ecr.aws/bisonai/orakl-goapi
+
+jobs:
+  prepare:
+    name: Prepare Build
+    runs-on: ubuntu-latest
+
+    outputs:
+      tag_date: ${{ steps.tag.outputs.date }}
+      tag_git_hash: ${{ steps.tag.outputs.git_hash }}
+      version: ${{ steps.package.outputs.version }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Get time TAG
+        id: tag
+        run: |
+          echo "date=$(date +'%Y%m%d.%H%M')" >> $GITHUB_OUTPUT
+          echo "git_hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Get package version
+        id: package
+        run: |
+          version=$(cat "./go-api/.version")
+          echo "version=${version}" >> $GITHUB_OUTPUT
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    needs: prepare
+
+    permissions:
+      id-token: write
+      contents: read
+
+    outputs:
+      tag_date: ${{ steps.tag.outputs.date }}
+      tag_git_hash: ${{ steps.tag.outputs.git_hash }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.21.5"
+          check-latest: true
+          cache-dependency-path: |
+            ./go-api/go.sum
+
+      - name: Run lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.54
+          working-directory: go-api
+          skip-pkg-cache: true
+          skip-build-cache: true
+          args: --timeout=10m
+
+      - name: Run Vet
+        run: |
+          cd ./go-api
+          go install golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow@latest
+          go vet
+          go vet -vettool=$(which shadow)
+
+      - name: Docker build orakl-goapi
+        run: SERVICE_NAME=orakl-goapi docker-compose -f docker-compose.build.yaml build
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          aws-region: us-east-1
+          role-to-assume: ${{ secrets.ROLE_ARN }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr-public
+        uses: aws-actions/amazon-ecr-login@v1
+        with:
+          registry-type: public
+
+      - name: Publish Image to ECR(api)
+        run: |
+          docker tag orakl-goapi ${{ env.ecr_url }}:v${{ needs.prepare.outputs.version }}.${{ needs.prepare.outputs.tag_date }}.${{ needs.prepare.outputs.tag_git_hash }}
+          docker push ${{ env.ecr_url }}:v${{ needs.prepare.outputs.version }}.${{ needs.prepare.outputs.tag_date }}.${{ needs.prepare.outputs.tag_git_hash }}

.github/workflows/goapi.test.yaml

@@ -0,0 +1,74 @@
+name: "goapi: test"
+
+on:
+  push:
+    branches-ignore:
+      - "master"
+    paths:
+      - "go-api/**"
+  workflow_dispatch:
+
+jobs:
+  core-build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 3
+
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: orakl-test
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      redis:
+        image: redis
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.21.5"
+          check-latest: true
+          cache-dependency-path: |
+            ./go-api/go.sum
+      - name: Install golang-migrate
+        run: |
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.17.0/migrate.linux-amd64.tar.gz | tar xvz
+          sudo mv ./migrate /usr/bin
+      - name: Migrate up
+        run: |
+          cd ./go-api
+          migrate -database "postgresql://postgres:postgres@localhost:5432/orakl-test?search_path=public&sslmode=disable" -verbose -path ./migrations up
+      - name: Install dependencies
+        run: |
+          cd ./go-api
+          go mod tidy
+      - name: Build
+        run: |
+          cd ./go-api
+          go build
+      - name: Run test
+        run: |
+          cd ./go-api
+          go test ./tests -v
+        env:
+          DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/orakl-test?search_path=public"
+          ENCRYPT_PASSWORD: "abc123"
+          REDIS_HOST: "localhost"
+          REDIS_PORT: "6379"
+          TEST_MODE: true

.gitignore

@@ -1,4 +1,5 @@
 node_modules/
 contracts/scripts/**/tmp
 yarn-error.log
-dockerfiles/local-data-feed/tmp/**
+.DS_Store
+dockerfiles/local-data-feed/tmp/**

docker-compose.local-data-feed.yaml

@@ -61,7 +61,7 @@ services:
   api:
     build:
       context: .
-      dockerfile: dockerfiles/orakl-api.Dockerfile
+      dockerfile: dockerfiles/orakl-goapi.Dockerfile
     image: orakl-api
     expose:
       - "3000"

dockerfiles/local-data-feed/envs/.api.env

@@ -1,4 +1,4 @@
-DATABASE_URL=postgresql://testuser@postgres:5432/test?schema=public
+DATABASE_URL=postgresql://testuser@postgres:5432/test?search_path=public
 APP_PORT=3000
 ENCRYPT_PASSWORD=anything
 

dockerfiles/orakl-goapi.Dockerfile

@@ -0,0 +1,35 @@
+
+FROM golang:1.21.5-bullseye as builder
+
+RUN apt-get update && apt-get install -y curl
+
+WORKDIR /app
+
+COPY go-api go-api
+
+WORKDIR /app/go-api
+
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o apibin -ldflags="-w -s" .
+
+# debian:bullseye-slim
+FROM debian@sha256:4b48997afc712259da850373fdbc60315316ee72213a4e77fc5a66032d790b2a
+
+RUN apt-get update && apt-get install -y curl
+
+RUN curl -L https://github.com/golang-migrate/migrate/releases/download/v4.17.0/migrate.linux-amd64.tar.gz | tar xvz && \
+    mv ./migrate /usr/bin
+
+WORKDIR /app
+
+RUN mkdir /app/migrations
+
+COPY --from=builder /app/go-api/migrations /app/migrations
+
+COPY --from=builder /app/go-api/apibin /usr/bin
+
+CMD sh -c '_DATABASE_URL=$DATABASE_URL; if echo $_DATABASE_URL | grep -q "\?"; then \
+              _DATABASE_URL="${_DATABASE_URL}&sslmode=disable"; \
+           else \
+              _DATABASE_URL="${_DATABASE_URL}?sslmode=disable"; \
+           fi && \
+           migrate -database "$_DATABASE_URL" -verbose -path ./migrations up && apibin'

go-api/.env.example

@@ -0,0 +1,6 @@
+APP_PORT=
+DATABASE_URL=
+REDIS_HOST=
+REDIS_PORT=
+TEST_MODE=
+ENCRYPT_PASSWORD=

go-api/.env.local

@@ -0,0 +1,6 @@
+APP_PORT=3111
+DATABASE_URL=postgresql://${USER}@localhost:5432/orakl?search_path=public
+REDIS_HOST=localhost
+REDIS_PORT=6379
+TEST_MODE=true
+ENCRYPT_PASSWORD=anything

go-api/.gitignore

@@ -0,0 +1,22 @@
+# If you prefer the allow list template instead of the deny list, see community template:
+# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+.env

go-api/.version

@@ -0,0 +1 @@
+0.0.1