feat(event-db): WIP building with latest postgresql builder

catalyst-gateway/event-db/Earthfile

@@ -13,6 +13,8 @@ builder:
         --sqlfluff_cfg=./../../+repo-config/repo/.sqlfluff
 
     COPY ./../../+repo-config-2/repo/.sqlfluff .
+    COPY setup-db.sql /sql/
+    COPY --dir json_schemas .
 
 # check if the sqlfiles are properly formatted and pass lint quality checks.
 #   CI target : true
@@ -29,20 +31,13 @@ format:
 
     DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:fix/postgres-builders+FORMAT --src=$(echo ${PWD}/../../)   
 
-# build an event db docker image.
+# build - an event db docker image.
 #   CI target : true
-# Arguments:
-#  * tag: docker image `tag`.
-#  * with_historic_data: add historic data from the `historic_data` dir (default `true`).
-#  * with_test_data: add test data from the `test_data` dir (default `true`).
 build:
-    ARG with_historic_data=true
-    ARG with_test_data=true
-    ARG tag="latest"
-
-    FROM +builder --with_historic_data=$with_historic_data --with_test_data=$with_test_data
+    FROM +builder
 
-    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:fix/postgres-builders+BUILD --tag=$tag --image_name=event-db
+    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:fix/postgres-builders+BUILD --image_name=event-db
+    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:fix/postgres-builders+DOCS --image_name=event-db
 
 # test the event db database schema
 #   CI target : true

catalyst-gateway/event-db/diagrams.json

@@ -0,0 +1,16 @@
+{
+    "name": "Catalyst Gateway - Event DB",
+    "all_schema": {
+        "comments": true,
+        "included_tables": [],
+        "excluded_tables": [
+            "refinery_schema_history"
+        ],
+        "column_description_wrap": 50,
+        "table_description_wrap": 120
+    },
+    "full_schema": {
+        "title": "Full Schema",
+        "comments": false
+    }
+}

catalyst-gateway/event-db/migrations/V1__config_tables.sql

@@ -1,6 +1,8 @@
 -- Catalyst Voices Database - Configuration Data
 -- sqlfluff:dialect:postgres
 
+-- Title : Configuration Data
+
 -- Configuration Tables
 
 -- -------------------------------------------------------------------------------------------------
@@ -120,15 +122,15 @@ at the app level to allow for querying groups of data.';
 -- * This will be removed in the future.
 
 -- Add the Initial Schemas for configuration.
-INSERT INTO json_schema_type (id, type, name, schema)
-VALUES
-(
-  'd899cd44-3513-487b-ab46-fdca662a724d', -- Fix the Schema ID so that it is consistent.
-  'config',
-  'dbsync',
-  (SELECT jsonb FROM PG_READ_FILE('../json_schemas/config/dbsync.json')),
-  '62d614c0-97a7-41ec-a976-91294b8f4384', -- Fix the Schema ID so that it is consistent.
-  'config',
-  'registration',
-  (SELECT jsonb FROM PG_READ_FILE('../json_schemas/config/registration.json'))
-);
+--INSERT INTO json_schema_type (id, type, name, schema)
+--VALUES
+--(
+--  'd899cd44-3513-487b-ab46-fdca662a724d', -- Fix the Schema ID so that it is consistent.
+--  'config',
+--  'dbsync',
+--  (SELECT PG_READ_FILE('../json_schemas/config/dbsync.json'))::JSONB), (
+--  '62d614c0-97a7-41ec-a976-91294b8f4384', -- Fix the Schema ID so that it is consistent.
+--  'config',
+--  'registration',
+--  (SELECT PG_READ_FILE('../json_schemas/config/registration.json'))::JSONB
+--);

catalyst-gateway/event-db/migrations/V2__event_tables.sql

@@ -1,6 +1,8 @@
 -- Catalyst Voices Database - Event Data
 -- sqlfluff:dialect:postgres
 
+-- Title : Event Data
+
 -- `id` mapping.
 -- * `id` is a UUID, in the past it was an auto incrementing value.
 -- * it is changed to a UUID so that the data can be generated independently and it is more friendly 
@@ -50,42 +52,42 @@ COMMENT ON COLUMN event_type.data_schema IS
 -- Run as required after migrations.
 
 -- Add Event Schemas to the known schema types.
-INSERT INTO json_schema_type_names (id)
-VALUES
-('event_description'), -- Event Description schemas
-('event_data');        -- Event Data Schemas
+-- INSERT INTO json_schema_type_names (id)
+-- VALUES
+-- ('event_description'), -- Event Description schemas
+-- ('event_data');        -- Event Data Schemas
 
 -- Add the Initial Schemas for events.
-INSERT INTO json_schema_type (id, type, name, schema)
-VALUES
-(
-  'd899cd44-3513-487b-ab46-fdca662a724d', -- From the schema file.
-  'event_description',
-  'multiline_text',
-  (
-    SELECT jsonb
-    FROM PG_READ_FILE('../json_schemas/event/description/multiline_text.json')
-  )
-),
-(
-  '9c5df318-fa9a-4310-80fa-490f46d1cc43', -- From the schema file.
-  'event_data',
-  'catalyst_v1',
-  (
-    SELECT jsonb
-    FROM PG_READ_FILE('../json_schemas/event/description/catalyst_v1.json')
-  )
-);
+-- INSERT INTO json_schema_type (id, type, name, schema)
+-- VALUES
+--(
+--  'd899cd44-3513-487b-ab46-fdca662a724d', -- From the schema file.
+--  'event_description',
+--  'multiline_text',
+--  (
+--    SELECT jsonb
+--    FROM PG_READ_FILE('../json_schemas/event/description/multiline_text.json')
+--  )
+--),
+--(
+-- '9c5df318-fa9a-4310-80fa-490f46d1cc43', -- From the schema file.
+--  'event_data',
+--  'catalyst_v1',
+--  (
+--    SELECT jsonb
+--    FROM PG_READ_FILE('../json_schemas/event/description/catalyst_v1.json')
+--  )
+--);
 
 -- Define a Catalyst V1 Event.
 
-INSERT INTO event_type (name, description_schema, data_schema)
-VALUES
-(
-  'Catalyst V1',
-  'd899cd44-3513-487b-ab46-fdca662a724d',
-  '9c5df318-fa9a-4310-80fa-490f46d1cc43'
-);
+--INSERT INTO event_type (name, description_schema, data_schema)
+--VALUES
+--(
+--  'Catalyst V1',
+--  'd899cd44-3513-487b-ab46-fdca662a724d',
+--  '9c5df318-fa9a-4310-80fa-490f46d1cc43'
+--);
 
 -- -------------------------------------------------------------------------------------------------
 

catalyst-gateway/event-db/migrations/V3__objective_tables.sql

@@ -1,5 +1,7 @@
 -- Catalyst Event Database
 
+-- Title: Objective Tables
+
 -- objective types table - Defines all currently known objectives types.
 CREATE TABLE objective_category
 (
@@ -68,12 +70,12 @@ VALUES
 CREATE TABLE goal
 (
   id SERIAL PRIMARY KEY,
-  event_id INTEGER NOT NULL,
+  event_id UUID NOT NULL,
 
   idx INTEGER NOT NULL,
   name VARCHAR NOT NULL,
 
-  FOREIGN KEY (event_id) REFERENCES event (row_id) ON DELETE CASCADE
+  FOREIGN KEY (event_id) REFERENCES event (id) ON DELETE CASCADE
 );
 
 CREATE UNIQUE INDEX goal_index ON goal (event_id, idx);
@@ -94,7 +96,7 @@ CREATE TABLE objective
   row_id SERIAL PRIMARY KEY,
 
   id INTEGER NOT NULL,
-  event INTEGER NOT NULL,
+  event UUID NOT NULL,
 
   category TEXT NOT NULL,
   title TEXT NOT NULL,
@@ -108,7 +110,7 @@ CREATE TABLE objective
 
   extra JSONB,
 
-  FOREIGN KEY (event) REFERENCES event (row_id) ON DELETE CASCADE,
+  FOREIGN KEY (event) REFERENCES event (id) ON DELETE CASCADE,
   FOREIGN KEY (category) REFERENCES objective_category (name) ON DELETE CASCADE,
   FOREIGN KEY (rewards_currency) REFERENCES currency (name) ON DELETE CASCADE,
   FOREIGN KEY (vote_options) REFERENCES vote_options (id) ON DELETE CASCADE

catalyst-gateway/event-db/migrations/V4__proposal_tables.sql

@@ -1,5 +1,7 @@
 -- Catalyst Event Database
 
+-- Title : Proposals
+
 -- Proposals Table
 
 CREATE TABLE proposal
@@ -23,10 +25,10 @@ CREATE TABLE proposal
   proposer_relevant_experience TEXT NOT NULL,
   bb_proposal_id BYTEA,
 
-  FOREIGN KEY (objective) REFERENCES objective (row_id) ON DELETE CASCADE,
-  FOREIGN KEY (bb_vote_options) REFERENCES vote_options (
-    objective
-  ) ON DELETE CASCADE
+  FOREIGN KEY (objective) REFERENCES objective (row_id) ON DELETE CASCADE
+  --FOREIGN KEY (bb_vote_options) REFERENCES vote_options (
+  --  objective
+  --) ON DELETE CASCADE
 );
 
 CREATE UNIQUE INDEX proposal_index ON proposal (id, objective);
@@ -75,9 +77,9 @@ CREATE TABLE reviewer_level (
     total_reward_pct <= 100 AND total_reward_pct >= 0
   ),
 
-  event_id INTEGER NOT NULL,
+  event_id UUID NOT NULL,
 
-  FOREIGN KEY (event_id) REFERENCES event (row_id) ON DELETE CASCADE
+  FOREIGN KEY (event_id) REFERENCES event (id) ON DELETE CASCADE
 );
 
 COMMENT ON TABLE reviewer_level IS

catalyst-gateway/event-db/migrations/V5__vote_plan.sql

@@ -1,5 +1,7 @@
 -- Catalyst Event Database
 
+-- Title : Vote Plan
+
 -- Vote Plan Categories
 
 CREATE TABLE voteplan_category

catalyst-gateway/event-db/migrations/V6__registration.sql

@@ -1,6 +1,8 @@
 -- Catalyst Voices Database - Role Registration Data
 -- sqlfluff:dialect:postgres
 
+-- Title : Role Registration Data
+
 -- cspell: words utxo stxo
 -- Configuration Tables
 
@@ -27,7 +29,7 @@ CREATE TABLE cardano_update_state (
 
 CREATE INDEX cardano_update_state_idx ON cardano_update_state (id, network);
 
-COMMENT ON INDEX cardano_update_state_time_idx IS
+COMMENT ON INDEX cardano_update_state_idx IS
 'Index to allow us to efficiently get find an update state record by its id.
 This index can be used to find the latest state record for a particular network.';
 
@@ -137,16 +139,27 @@ CREATE TABLE cardano_utxo (
 
   stake_credential BYTEA NOT NULL,
 
+  spent_slot_no BIGINT NULL,
+  spent_tx_id BIGINT NULL,
+
   PRIMARY KEY (tx_id, index, network),
-  FOREIGN KEY (slot_no, network) REFERENCES slot_index (slot_no, network)
+  FOREIGN KEY (slot_no, network) REFERENCES slot_index (slot_no, network),
+  FOREIGN KEY (spent_slot_no, network) REFERENCES slot_index (slot_no, network)
+
 );
 
-CREATE INDEX cardano_utxo_stake_credential_idx ON utxo (
+CREATE INDEX cardano_utxo_stake_credential_idx ON cardano_utxo (
   stake_credential, slot_no
 );
 COMMENT ON INDEX cardano_utxo_stake_credential_idx IS
 'Index to allow us to efficiently lookup a set of UTXOs by stake credential relative to a slot_no.';
 
+CREATE INDEX cardano_stxo_slot_idx ON cardano_utxo (spent_slot_no, network);
+
+COMMENT ON INDEX cardano_stxo_slot_idx IS
+'Index to allow us to efficiently lookup a set of Spent TX Outputs by slot_no.';
+
+
 COMMENT ON TABLE cardano_utxo IS
 'This table holds all UTXOs for any transaction which is tied to a stake address.
 This data allows us to calculate staked ADA at any particular instant in time.';
@@ -164,51 +177,14 @@ COMMENT ON COLUMN cardano_utxo.slot_no IS
 'The slot number the UTXO appeared in.';
 COMMENT ON COLUMN cardano_utxo.stake_credential IS
 'The stake credential of the address which owns the UTXO.';
-
-
--- -------------------------------------------------------------------------------------------------
-
--- STXO Table -- Spent TX Outputs.
--- Populated with this query from db-sync
---     SELECT tx_out_id as tx_id, tx_out_index as index, tx_in.id as spent_tx_id, slot_no, time  FROM tx_in
---        INNER JOIN tx ON tx_in.tx_in_id = tx.id
---        INNER JOIN block ON tx.block_id = block.id;
--- network needs to be supplied, as dbsync only has a single database per network.
-CREATE TABLE cardano_stxo (
-  tx_id BIGINT NOT NULL,
-  index INTEGER NOT NULL,
-  network TEXT NOT NULL,
-
-  spent_tx_id BIGINT NOT NULL,
-  spent_slot_no BIGINT NOT NULL,
-
-  PRIMARY KEY (tx_id, index, network),
-  FOREIGN KEY (tx_id, index, network) REFERENCES utxo (tx_id, index, network),
-  FOREIGN KEY (spent_slot_no, network) REFERENCES slot_index (slot_no, network)
-);
-
-CREATE INDEX cardano_stxo_slot_idx ON stxo (spent_slot_no, network);
-
-COMMENT ON INDEX cardano_stxo_slot_idx IS
-'Index to allow us to efficiently lookup a set of Spent TX Outputs by slot_no.';
-
-COMMENT ON TABLE cardano_stxo IS
-'This table holds all Spent TX Outputs.
-This data allows us to calculate staked ADA at any particular instant in time.';
-
-COMMENT ON COLUMN cardano_stxo.tx_id IS
-'The ID of the transaction containing the original UTXO.';
-COMMENT ON COLUMN cardano_stxo.index IS
-'The index of the UTXO within the original transaction.';
-COMMENT ON COLUMN cardano_stxo.network IS
-'The Cardano network for the original UTXO.';
-COMMENT ON COLUMN cardano_stxo.spent_tx_id IS
+COMMENT ON COLUMN cardano_utxo.spent_tx_id IS
 'The ID of the transaction which Spent the TX Output.';
-COMMENT ON COLUMN cardano_stxo.spent_slot_no IS
+COMMENT ON COLUMN cardano_utxo.spent_slot_no IS
 'The slot number the TX Output was spent in.
 This lets us determine when the TX Output was spent, so we can calculate staked ada at a particular slot.';
 
 
+
 -- -------------------------------------------------------------------------------------------------
 
 -- Rewards Table -- Earned Rewards
@@ -232,7 +208,7 @@ CREATE TABLE cardano_reward (
   FOREIGN KEY (slot_no, network) REFERENCES slot_index (slot_no, network)
 );
 
-CREATE INDEX cardano_rewards_stake_credential_idx ON cardano_rewards (
+CREATE INDEX cardano_rewards_stake_credential_idx ON cardano_reward (
   stake_credential, slot_no
 );
 
@@ -338,14 +314,7 @@ CREATE INDEX cardano_voter_registration_voting_key_idx ON cardano_voter_registra
 COMMENT ON INDEX cardano_voter_registration_voting_key_idx IS
 'Optimize lookups for "public_voting_key" or "public_voting_key"+"nonce" or "public_voting_key"+"nonce"+"valid".';
 
-
-CREATE INDEX cardano_voter_registration_voting_key_idx ON cardano_voter_registration (
-  public_voting_key, nonce, valid
-);
-COMMENT ON INDEX cardano_voter_registration_voting_key_idx IS
-'Optimize lookups for "public_voting_key" or "public_voting_key"+"nonce" or "public_voting_key"+"nonce"+"valid".';
-
-COMMENT ON TABLE cardano_voter_registrations IS
+COMMENT ON TABLE cardano_voter_registration IS
 'All CIP15/36 Voter Registrations that are on-chain.
 This tables stores all found registrations, even if they are invalid, or have been rolled back.';