feat: dockerize blockfrost-ryo and enable blockfrost providers in e2e…

… CI tests

.github/workflows/continuous-integration-e2e-blockfrost.yaml

@@ -0,0 +1,111 @@
+name: Continuous Integration - E2E (Blockfrost Providers)
+
+env:
+  TL_DEPTH: ${{ github.event.pull_request.head.repo.fork && '0' || fromJson(vars.TL_DEPTH) }}
+  TL_LEVEL: ${{ github.event.pull_request.head.repo.fork && 'info' || vars.TL_LEVEL }}
+  # -----------------------------------------------------------------------------------------
+  KEY_MANAGEMENT_PROVIDER: 'inMemory'
+  KEY_MANAGEMENT_PARAMS: '{"bip32Ed25519": "Sodium", "accountIndex": 0, "chainId":{"networkId": 0, "networkMagic": 888}, "passphrase":"some_passphrase","mnemonic":"vacant violin soft weird deliver render brief always monitor general maid smart jelly core drastic erode echo there clump dizzy card filter option defense"}'
+  TEST_CLIENT_ASSET_PROVIDER: 'http'
+  TEST_CLIENT_ASSET_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4014/"}'
+  TEST_CLIENT_CHAIN_HISTORY_PROVIDER: 'http'
+  TEST_CLIENT_CHAIN_HISTORY_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4000/"}'
+  DB_SYNC_CONNECTION_STRING: 'postgresql://postgres:doNoUseThisSecret!@localhost:5435/cexplorer'
+  TEST_CLIENT_HANDLE_PROVIDER: 'http'
+  TEST_CLIENT_HANDLE_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4011/"}'
+  STAKE_POOL_CONNECTION_STRING: 'postgresql://postgres:doNoUseThisSecret!@localhost:5435/stake_pool'
+  STAKE_POOL_TEST_CONNECTION_STRING: 'postgresql://postgres:doNoUseThisSecret!@localhost:5435/stake_pool_test'
+  TEST_CLIENT_NETWORK_INFO_PROVIDER: 'ws'
+  TEST_CLIENT_NETWORK_INFO_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4000/"}'
+  OGMIOS_URL: 'ws://localhost:1340/'
+  TEST_CLIENT_REWARDS_PROVIDER: 'http'
+  TEST_CLIENT_REWARDS_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4000/"}'
+  TEST_CLIENT_TX_SUBMIT_PROVIDER: 'http'
+  TEST_CLIENT_TX_SUBMIT_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4000/"}'
+  TEST_CLIENT_UTXO_PROVIDER: 'http'
+  TEST_CLIENT_UTXO_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4000/"}'
+  TEST_CLIENT_STAKE_POOL_PROVIDER: 'http'
+  TEST_CLIENT_STAKE_POOL_PROVIDER_PARAMS: '{"baseUrl":"http://localhost:4000/"}'
+  WS_PROVIDER_URL: 'http://localhost:4100/ws'
+  # enable Blockfrost backed providers, the rest will use defaults
+  ASSET_PROVIDER: 'blockfrost'
+  UTXO_PROVIDER: 'blockfrost'
+  CHAIN_HISTORY_PROVIDER: 'blockfrost'
+  REWARDS_PROVIDER: 'blockfrost'
+  NETWORK_INFO_PROVIDER: 'blockfrost'
+  TX_SUBMIT_PROVIDER: 'blockfrost'
+
+on:
+  pull_request:
+  push:
+    branches: ['master']
+    tags: ['*.*.*']
+
+jobs:
+  build_and_test:
+    strategy:
+      matrix:
+        os: [ubuntu-20.04]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: 📥 Checkout repository
+        uses: actions/checkout@v3
+
+      - name: 🧰 Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18.12.0
+
+      - name: 🔨 Build
+        run: |
+          yarn install --immutable --inline-builds --mode=skip-build
+          yarn workspace @cardano-sdk/cardano-services-client build:cjs
+          yarn workspace @cardano-sdk/cardano-services build:cjs
+          yarn workspace @cardano-sdk/e2e build:cjs
+          yarn workspace @cardano-sdk/util-dev build:cjs
+          docker build --no-cache .
+        env:
+          NODE_OPTIONS: '--max_old_space_size=8192'
+
+      - name: 🌐 Setup local test network
+        working-directory: packages/e2e
+        run: |
+          yarn local-network:blockfrost:up -d
+        env:
+          CARDANO_NODE_CHAINDB_LOG_LEVEL: 'Warning'
+          CARDANO_NODE_LOG_LEVEL: 'Warning'
+          OGMIOS_PORT: '1340'
+          OGMIOS_URL: 'ws://ogmios:1340'
+          POSTGRES_PORT: '5435'
+
+      - name: Wait for network init
+        run: |
+          yarn workspace @cardano-sdk/e2e wait-for-network-init
+
+      - name: 🔬 Test - e2e - wallet at epoch 0
+        run: |
+          yarn workspace @cardano-sdk/e2e test:wallet:epoch0
+          yarn workspace @cardano-sdk/e2e test:projection
+          yarn workspace @cardano-sdk/e2e test:ws
+
+      - name: Wait for epoch 3
+        run: |
+          yarn workspace @cardano-sdk/e2e wait-for-network-epoch-3
+
+      - name: 🔬 Test - e2e - wallet at epoch 3
+        run: |
+          yarn workspace @cardano-sdk/e2e test:wallet:epoch3
+          yarn workspace @cardano-sdk/e2e test:pg-boss
+          yarn workspace @cardano-sdk/e2e test:providers
+        env:
+          STAKE_POOL_PROVIDER_URL: 'http://localhost:4000/'
+
+      - name: 🔬 Test - e2e - wallet - typeorm stake pool provider
+        run: |
+          yarn workspace @cardano-sdk/e2e test:providers -t StakePoolProvider
+        env:
+          STAKE_POOL_PROVIDER_URL: 'http://localhost:4010/'
+
+      - name: Dump docker logs
+        if: ${{ cancelled() || failure() }}
+        uses: jwalton/gh-docker-logs@v2

compose/common.yml

@@ -69,6 +69,13 @@ x-provider-server-environment: &provider-server-environment
   SUBMIT_API_URL: ${SUBMIT_API_URL:-http://cardano-submit-api:8090/}
   USE_BLOCKFROST: ${USE_BLOCKFROST:-false}
   USE_SUBMIT_API: ${USE_SUBMIT_API:-false}
+  ASSET_PROVIDER: ${ASSET_PROVIDER:-dbsync}
+  UTXO_PROVIDER: ${UTXO_PROVIDER:-dbsync}
+  CHAIN_HISTORY_PROVIDER: ${CHAIN_HISTORY_PROVIDER:-dbsync}
+  REWARDS_PROVIDER: ${REWARDS_PROVIDER:-dbsync}
+  NETWORK_INFO_PROVIDER: ${NETWORK_INFO_PROVIDER:-dbsync}
+  TX_SUBMIT_PROVIDER: ${TX_SUBMIT_PROVIDER:-submit-node}
+  STAKE_POOL_PROVIDER: ${STAKE_POOL_PROVIDER:-dbsync}
 
 x-sdk-environment: &sdk-environment
   LOGGER_MIN_SEVERITY: ${LOGGER_MIN_SEVERITY:-info}
@@ -110,6 +117,18 @@ x-sdk-environment: &sdk-environment
   WEB_SOCKET_API_URL: ws://ws-server:3000/ws
 
 services:
+  blockfrost-ryo:
+    build:
+      context: "https://github.com/ginnun/blockfrost-backend-ryo.git#feat/custom-network-support"
+      dockerfile: Dockerfile
+    environment:
+      BLOCKFROST_CONFIG_SERVER_LISTEN_ADDRESS: 0.0.0.0
+    depends_on:
+      cardano-db-sync:
+        condition: service_started
+    ports:
+      - "3015:3000"
+
   cardano-db-sync:
     <<:
       - *logging

packages/cardano-services/blockfrost/ryo/local-blockfrost-ryo.yaml

@@ -0,0 +1,11 @@
+server:
+  listenAddress: "0.0.0.0"
+  port: 3000
+  debug: false
+dbSync:
+  host: "postgres"
+  port: 5432
+  database: "cexplorer"
+  user: "postgres"
+  password: "doNoUseThisSecret!"
+  maxConnections: 5

packages/cardano-services/docker-compose.yml

@@ -9,6 +9,18 @@ x-sdk-environment: &sdk-environment
   HANDLE_POLICY_IDS: ${HANDLE_POLICY_IDS:-f0ff48bbb7bbe9d59a40f1ce90e9e9d0ff5002ec48f232b49ca0fb9a}
 
 services:
+
+  blockfrost-ryo:
+    environment:
+      <<: *sdk-environment
+      NODE_ENV: local-blockfrost-ryo
+      BLOCKFROST_CONFIG_NETWORK: ${NETWORK:-mainnet}
+      BLOCKFROST_CONFIG_TOKEN_REGISTRY_URL: ${TOKEN_METADATA_SERVER_URL:-https://tokens.cardano.org}
+
+    volumes:
+      - ./blockfrost/ryo:/app/config
+    profiles: [ blockfrost-ryo ]
+
   cardano-node:
     volumes:
       - ./config/network/${NETWORK:-mainnet}/cardano-node:/config
@@ -74,7 +86,7 @@ services:
 
 secrets:
   blockfrost_key:
-    file: ./blockfrost-keys/${NETWORK:-mainnet}.key
+    file: ./blockfrost/keys/${NETWORK:-mainnet}.key
 
 volumes:
   postgres-data:

packages/cardano-services/package.json

@@ -29,7 +29,7 @@
     "cleanup": "rm -rf dist node_modules",
     "cli": "ts-node --transpile-only src/cli.ts",
     "compose:single:up": "yarn compose:up cardano-node ogmios postgres",
-    "compose:up": "__FIX_UMASK__=$(chmod -R a+r ../../compose/placeholder-secrets) docker compose --env-file environments/.env.$NETWORK -p cardano-services-$NETWORK -f docker-compose.yml -f ../../compose/common.yml -f ../../compose/pg-agent.yml -f ../../compose/$(uname -m).yml ${FILES:-} up",
+    "compose:up": "__FIX_UMASK__=$(chmod -R a+r ../../compose/placeholder-secrets) docker compose --env-file environments/.env.$NETWORK -p cardano-services-$NETWORK -f docker-compose.yml -f ../../compose/common.yml -f ../../compose/pg-agent.yml -f ../../compose/$(uname -m).yml ${FILES:-}  --profile ${DOCKER_COMPOSE_PROFILE:-none} up",
     "compose:down": "docker compose -p cardano-services-$NETWORK -f docker-compose.yml -f ../../compose/common.yml -f ../../compose/pg-agent.yml -f ../../compose/$(uname -m).yml down -t 120",
     "coverage": "yarn test --coverage || true",
     "circular-deps:check": "madge --circular dist/cjs",
@@ -38,17 +38,21 @@
     "generate-migration": "typeorm-ts-node-commonjs migration:generate src/Projection/migrations/migrations -d src/migrationDataSource.ts",
     "mainnet:single:up": "NETWORK=mainnet yarn compose:single:up",
     "mainnet:up": "NETWORK=mainnet SUBMIT_API_ARGS=--mainnet yarn compose:up",
+    "mainnet:blockfrost:up": "DOCKER_COMPOSE_PROFILE='blockfrost-ryo' yarn mainnet:up",
     "mainnet:down": "NETWORK=mainnet yarn compose:down",
     "prepack": "yarn build",
     "preprod:single:up": "NETWORK=preprod yarn compose:single:up",
     "preprod:up": "NETWORK=preprod SUBMIT_API_ARGS='--testnet-magic 1' yarn compose:up",
+    "preprod:blockfrost:up": "DOCKER_COMPOSE_PROFILE='blockfrost-ryo' yarn preprod:up",
     "preprod:down": "NETWORK=preprod yarn compose:down",
     "pretest": "yarn build",
     "preview:single:up": "NETWORK=preview yarn compose:single:up",
     "preview:up": "NETWORK=preview SUBMIT_API_ARGS='--testnet-magic 2' yarn compose:up",
+    "preview:blockfrost:up": "DOCKER_COMPOSE_PROFILE='blockfrost-ryo' yarn preview:up",
     "preview:down": "NETWORK=preview yarn compose:down",
     "sanchonet:single:up": "NETWORK=sanchonet yarn compose:single:up",
     "sanchonet:up": "NETWORK=sanchonet SUBMIT_API_ARGS='--testnet-magic 4' yarn compose:up",
+    "sanchonet:blockfrost:up": "DOCKER_COMPOSE_PROFILE='blockfrost-ryo' yarn sanchonet:up",
     "sanchonet:down": "NETWORK=sanchonet yarn compose:down",
     "test": "jest --runInBand -c ./jest.config.js --selectProjects unit",
     "test:build:verify": "tsc --build ./test",

packages/cardano-services/src/Program/options/providers.ts

@@ -22,58 +22,52 @@ export const ProviderImplementationDescription = 'Select one of the available pr
 const argParser = (impl: string) => ProviderImplementation[impl.toUpperCase() as keyof typeof ProviderImplementation];
 export const providerSelectionOptions = [
   newOption(
-    '--asset-provider <implementation>',
+    '--asset-provider <assetProvider>',
     ProviderImplementationDescription,
     'ASSET_PROVIDER',
     argParser,
     ProviderImplementation.DBSYNC
-  )
-    .conflicts('useTypeormAssetProvider')
-    .choices([ProviderImplementation.BLOCKFROST, ProviderImplementation.DBSYNC, ProviderImplementation.TYPEORM]),
+  ).choices([ProviderImplementation.BLOCKFROST, ProviderImplementation.DBSYNC, ProviderImplementation.TYPEORM]),
   newOption(
-    '--stake-pool-provider <implementation>',
+    '--stake-pool-provider <stakePoolProvider>',
     ProviderImplementationDescription,
     'STAKE_POOL_PROVIDER',
     argParser,
     ProviderImplementation.DBSYNC
-  )
-    .conflicts('useTypeormStakePoolProvider')
-    .choices([ProviderImplementation.DBSYNC, ProviderImplementation.TYPEORM]),
+  ).choices([ProviderImplementation.DBSYNC, ProviderImplementation.TYPEORM]),
   newOption(
-    '--utxo-provider <implementation>',
+    '--utxo-provider <utxoProvider>',
     ProviderImplementationDescription,
     'UTXO_PROVIDER',
     argParser,
     ProviderImplementation.DBSYNC
   ).choices([ProviderImplementation.BLOCKFROST, ProviderImplementation.DBSYNC]),
   newOption(
-    '--chain-history-provider <implementation>',
+    '--chain-history-provider <chainHistoryProvider>',
     ProviderImplementationDescription,
     'CHAIN_HISTORY_PROVIDER',
     argParser,
     ProviderImplementation.DBSYNC
   ).choices([ProviderImplementation.BLOCKFROST, ProviderImplementation.DBSYNC]),
   newOption(
-    '--rewards-provider <implementation>',
+    '--rewards-provider <rewardsProvider>',
     ProviderImplementationDescription,
     'REWARDS_PROVIDER',
     argParser,
     ProviderImplementation.DBSYNC
   ).choices([ProviderImplementation.BLOCKFROST, ProviderImplementation.DBSYNC]),
   newOption(
-    '--network-info-provider <implementation>',
+    '--network-info-provider <networkInfoProvider>',
     ProviderImplementationDescription,
     'NETWORK_INFO_PROVIDER',
     argParser,
     ProviderImplementation.DBSYNC
   ).choices([ProviderImplementation.BLOCKFROST, ProviderImplementation.DBSYNC]),
   newOption(
-    '--tx-submit-provider <implementation>',
+    '--tx-submit-provider <txSubmitProvider>',
     ProviderImplementationDescription,
     'TX_SUBMIT_PROVIDER',
     argParser,
     ProviderImplementation.SUBMIT_NODE
-  )
-    .conflicts('useSubmitApi')
-    .choices([ProviderImplementation.BLOCKFROST, ProviderImplementation.SUBMIT_API, ProviderImplementation.SUBMIT_NODE])
+  ).choices([ProviderImplementation.BLOCKFROST, ProviderImplementation.SUBMIT_API, ProviderImplementation.SUBMIT_NODE])
 ];

packages/e2e/docker-compose.yml

@@ -6,6 +6,17 @@ x-logging: &logging
       max-file: '10'
 
 services:
+
+  blockfrost-ryo:
+    depends_on:
+      local-testnet:
+        condition: service_healthy
+    environment:
+      NODE_ENV: local-network
+    volumes:
+      - ./local-network/config/network/blockfrost-ryo:/app/config
+    profiles: [blockfrost-ryo]
+
   local-testnet:
     <<: *logging
     build:

packages/e2e/local-network/scripts/make-babbage.sh

@@ -303,7 +303,7 @@ for NODE in ${SP_NODES}; do
   echo "${ROOT}/${NODE}.sh"
 done
 
-echo "Update start time in genesis files"
+echo "Update start time in genesis files with ${timeUnix}"
 sed_i -E "s/\"startTime\": [0-9]+/\"startTime\": ${timeUnix}/" ${ROOT}/genesis/byron/genesis.json
 sed_i -E "s/\"systemStart\": \".*\"/\"systemStart\": \"${timeISO}\"/" ${ROOT}/genesis/shelley/genesis.json
 
@@ -359,6 +359,8 @@ cp "${ROOT}"/genesis/shelley/genesis.conway.json ./config/network/genesis/conway
 
 mkdir -p "${ROOT}/run"
 
+echo "${timeUnix}" > "${ROOT}/run/system_start"
+
 echo "#!/usr/bin/env bash" >"${ROOT}/run/all.sh"
 echo "" >>"${ROOT}/run/all.sh"
 echo "" >>"${ROOT}/run/all.sh"

packages/e2e/local-network/scripts/prepare_blockfrost_ryo.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+timeUnix=$(cat network-files/run/system_start)
+
+echo "Update start time in blockfrost genesis file with ${timeUnix}"
+
+mkdir -p ./config/network/blockfrost-ryo/
+
+cp ./templates/blockfrost-ryo/genesis.json ./config/network/blockfrost-ryo/genesis.json
+cp ./templates/blockfrost-ryo/byron_genesis.json ./config/network/blockfrost-ryo/byron_genesis.json
+cp ./templates/blockfrost-ryo/local-network.yaml ./config/network/blockfrost-ryo/local-network.yaml
+
+sed -i -E "s/\"system_start\": [0-9]+/\"system_start\": ${timeUnix}/" ./config/network/blockfrost-ryo/genesis.json

packages/e2e/local-network/scripts/start.sh

@@ -26,6 +26,7 @@ trap 'kill 0' INT
 
 echo "Run"
 ./scripts/make-babbage.sh
+./scripts/prepare_blockfrost_ryo.sh
 ./network-files/run/all.sh &
 
 if [ -d /sdk-ipc ] ; then cp -a config/network /sdk-ipc/config ; fi

packages/e2e/local-network/templates/blockfrost-ryo/byron_genesis.json

@@ -0,0 +1,6 @@
+{
+  "epoch_length": 100,
+  "slot_length": 1,
+  "safe_zone": 864,
+  "end_epoch": 0
+}

packages/e2e/local-network/templates/blockfrost-ryo/genesis.json

@@ -0,0 +1,12 @@
+{
+  "active_slots_coefficient": 0.05,
+  "update_quorum": 5,
+  "max_lovelace_supply": "45000000000000000",
+  "network_magic": 888,
+  "epoch_length": 1000,
+  "system_start": 1000000,
+  "slots_per_kes_period": 129600,
+  "slot_length": 0.2,
+  "max_kes_evolutions": 62,
+  "security_param": 10
+}

packages/e2e/local-network/templates/blockfrost-ryo/local-network.yaml

@@ -0,0 +1,14 @@
+server:
+  listenAddress: "0.0.0.0"
+  port: 3000
+  debug: true
+dbSync:
+  host: "postgres"
+  port: 5432
+  database: "cexplorer"
+  user: "postgres"
+  password: "doNoUseThisSecret!"
+  maxConnections: 5
+network: "custom"
+genesisDataFolder: '/app/config'
+tokenRegistryUrl: "https://metadata.cardano-testnet.iohkdev.io"

packages/e2e/package.json

@@ -46,8 +46,9 @@
     "test:web-extension:watch": "run-s test:web-extension:build test:web-extension:watch:bg",
     "test:web-extension:watch:bg": "run-p test:web-extension:watch:build test:web-extension:watch:run",
     "test:ws": "jest -c jest.config.js --forceExit --selectProjects ws-server --runInBand --verbose",
-    "local-network:common": "DISABLE_DB_CACHE=${DISABLE_DB_CACHE:-true} SUBMIT_API_ARGS='--testnet-magic 888' USE_BLOCKFROST=false __FIX_UMASK__=$(chmod -R a+r ../../compose/placeholder-secrets) docker compose --env-file ../cardano-services/environments/.env.local -p local-network-e2e -f docker-compose.yml -f ../../compose/common.yml -f ../../compose/$(uname -m).yml $FILES up",
+    "local-network:common": "DISABLE_DB_CACHE=${DISABLE_DB_CACHE:-true} SUBMIT_API_ARGS='--testnet-magic 888' USE_BLOCKFROST=false __FIX_UMASK__=$(chmod -R a+r ../../compose/placeholder-secrets) docker compose --env-file ../cardano-services/environments/.env.local -p local-network-e2e -f docker-compose.yml -f ../../compose/common.yml -f ../../compose/$(uname -m).yml $FILES --profile ${DOCKER_COMPOSE_PROFILE:-none} up",
     "local-network:up": "FILES='' yarn local-network:common",
+    "local-network:blockfrost:up": "FILES='' DOCKER_COMPOSE_PROFILE='blockfrost-ryo' yarn local-network:common",
     "local-network:single:up": "FILES='' yarn local-network:common cardano-node file-server local-testnet ogmios postgres",
     "local-network:profile:up": "FILES='-f ../../compose/pg-agent.yml' yarn local-network:common",
     "local-network:down": "docker compose -p local-network-e2e -f docker-compose.yml -f ../../compose/common.yml -f ../../compose/pg-agent.yml down -v --remove-orphans",