diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic index 35ad3b30be5..f36169492e1 100644 --- a/.config/dictionaries/project.dic +++ b/.config/dictionaries/project.dic @@ -53,6 +53,7 @@ rustc saibatizoku seckey slotno +sqlfluff stevenj tacho thiserror diff --git a/.sqlfluff b/.sqlfluff index c114ac005fe..29f79e503ee 100644 --- a/.sqlfluff +++ b/.sqlfluff @@ -1,5 +1,7 @@ [sqlfluff] +# cspell: words templater cpus ctes capitalisation organisation + # Supported dialects https://docs.sqlfluff.com/en/stable/dialects.html # Or run 'sqlfluff dialects' # We do not set a global SQL Dialect, so that we can support multiple diff --git a/catalyst-gateway/event-db/json_schemas/.cspell.json b/catalyst-gateway/event-db/json_schemas/.cspell.json new file mode 100644 index 00000000000..067d8536007 --- /dev/null +++ b/catalyst-gateway/event-db/json_schemas/.cspell.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json", + "version": "0.2", + "enabled": true, + "language": "en,en-US", + "overrides": [ + { + "filename": "**/*.json", + "allowCompoundWords": true, + "words": [ + "sshuser", + "cexplorer", + "Cardano", + "jormungandr" + ] + } + ] +} \ No newline at end of file diff --git a/catalyst-gateway/event-db/json_schemas/config/dbsync_connection.jsonc b/catalyst-gateway/event-db/json_schemas/config/dbsync.json similarity index 91% rename from catalyst-gateway/event-db/json_schemas/config/dbsync_connection.jsonc rename to catalyst-gateway/event-db/json_schemas/config/dbsync.json index f112690e886..a30b7b420da 100644 --- a/catalyst-gateway/event-db/json_schemas/config/dbsync_connection.jsonc +++ b/catalyst-gateway/event-db/json_schemas/config/dbsync.json @@ -1,10 +1,12 @@ { "$schema": "http://json-schema.org/draft-07/schema#", + "$comment": "Custom URI schema: catalyst_schema:////", + "$id": "catalyst_schema://d899cd44-3513-487b-ab46-fdca662a724d/config/dbsync", "title": "DBSync Connection", "type": "object", "properties": { - // Optional SSH tunnel needed to communicate to the DBSync Postgres database. "ssh": { + "$comment": "Optional SSH tunnel needed to communicate to the DBSync Postgres database.", "type": "object", "properties": { "sshHost": { @@ -41,8 +43,8 @@ "sshUser" ] }, - // DBSync connection "conn": { + "$comment": "DBSync connection", "type": "object", "properties": { "host": { @@ -83,7 +85,7 @@ ], "description": "The Cardano network to connect to.", "default": "mainnet" - }, + } }, "required": [ "password" diff --git a/catalyst-gateway/event-db/json_schemas/event/data/catalyst_v1.json b/catalyst-gateway/event-db/json_schemas/event/data/catalyst_v1.json new file mode 100644 index 00000000000..324896109f1 --- /dev/null +++ b/catalyst-gateway/event-db/json_schemas/event/data/catalyst_v1.json @@ -0,0 +1,165 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$comment": "Custom URI schema: catalyst_schema:////", + "$id": "catalyst_schema://9c5df318-fa9a-4310-80fa-490f46d1cc43/event_data/catalyst_v1", + "title": "Catalyst V1 Event", + "description": "Traditional Project Catalyst Event Configuration", + "type": "object", + "properties": { + "timeline": { + "$comment": "allows additionalProperties so the schema can be extended in a backwards compatible way in the future", + "type": "object", + "description": "The timeline of the various stages of the event.", + "additionalProperties": true, + "properties": { + "registration_deadline": { + "description": "The deadline for registrations.\nThe Time (UTC) Registrations are taken from the Cardano blockchain.\nRegistrations after this date are not valid for voting on the event.\nNULL = Not yet defined or Not Applicable.'When the Voting power snapshot is set for.", + "type": "string", + "format": "date-time" + }, + "snapshot_stable": { + "description": "The Time (UTC) Registrations taken from Cardano blockchain are considered stable.\nThis is not the Time of the Registration Snapshot.\nThis is the time after which the registration snapshot will be stable.\nNULL = Not yet defined or Not Applicable.", + "type": "string", + "format": "date-time" + }, + "insight_sharing_start": { + "description": "TODO.\nNULL = Not yet defined.", + "type": "string", + "format": "date-time" + }, + "proposal_submission_start": { + "description": "The Time (UTC) proposals can start to be submitted for the event.\nNULL = Not yet defined, or Not applicable.", + "type": "string", + "format": "date-time" + }, + "refine_proposals_start": { + "description": "TODO.\nNULL = Not yet defined.", + "type": "string", + "format": "date-time" + }, + "finalize_proposals_start": { + "description": "The Time (UTC) when all proposals must be finalized by.\nNULL = Not yet defined, or Not applicable.", + "type": "string", + "format": "date-time" + }, + "proposal_assessment_start": { + "description": "The Time (UTC) when PA Assessors can start assessing proposals.\nNULL = Not yet defined, or Not applicable.", + "type": "string", + "format": "date-time" + }, + "assessment_qa_start": { + "description": "The Time (UTC) when vPA Assessors can start assessing assessments.\nNULL = Not yet defined, or Not applicable.", + "type": "string", + "format": "date-time" + }, + "voting_start": { + "description": "The earliest time that registered wallets with sufficient voting power can place votes in the event.\nNULL = Not yet defined.\nTypically this is aligned with Backing Start of the event.", + "type": "string", + "format": "date-time" + }, + "voting_end": { + "description": "The latest time that registered wallets with sufficient voting power can place votes in the event.\nNULL = Not yet defined.\nTypically aligned with backing End of the Event.", + "type": "string", + "format": "date-time" + }, + "tallying_end": { + "description": "The latest time that tallying the event can complete by.\nNULL = Not yet defined.", + "type": "string", + "format": "date-time" + } + } + }, + "registration": { + "$comment": "Allows additionalProperties so the schema can be extended in a backwards compatible way in the future.", + "type": "object", + "description": "The options which control the individual selection power of each participant in the event.", + "additionalProperties": true, + "properties": { + "stake_participation_threshold": { + "description": "The Minimum number of Lovelace needed to be staked at the time of snapshot, to be eligible to participate.\nNULL = Not yet defined.", + "type": "number", + "minimum": 0 + }, + "max_stake_percent": { + "description": "The Maximum Percentage of all registered staked ADA which can be used to participate in the event.", + "type": "number", + "minimum": 0, + "maximum": 100, + "default": 100 + }, + "network": { + "description": "The Cardano network the event allows registrations from. Can be one or more networks.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "mainnet", + "preprod", + "testnet" + ] + }, + "default": [ + "mainnet" + ] + } + } + }, + "rewards": { + "$comment": "Allows additionalProperties so the schema can be extended in a backwards compatible way in the future.", + "type": "object", + "description": "The options which control the reward distribution of the event.", + "additionalProperties": true, + "properties": { + "review": { + "description": "The total reward pool (in lovelace) to pay for community reviewers for their valid reviews of the proposals assigned to this event.", + "type": "integer", + "format": "int64", + "minimum": 0 + } + } + }, + "tally": { + "$comment": "Allows additionalProperties so the schema can be extended in a backwards compatible way in the future.", + "type": "object", + "description": "The options which control the results tally of the event.", + "additionalProperties": true, + "properties": { + "committee_size": { + "description": "The Size of the Tally committee.\n0 = No Committee, and all votes are therefore public.", + "type": "integer", + "format": "", + "minimum": 0, + "default": 0 + }, + "committee_threshold": { + "description": "The Minimum Size of the Tally committee to perform the Tally.\nMust be <= `committee_size`.\n0 is equivalent to ALL Committee members are required.", + "type": "integer", + "minimum": 0, + "default": 0 + } + } + }, + "jormungandr": { + "$comment": "Allows additionalProperties so the schema can be extended in a backwards compatible way in the future.", + "type": "object", + "description": "The Events Jormungandr configuration.", + "additionalProperties": true, + "properties": { + "block0": { + "description": "The BASE64 encoded copy of Block 0 used to start the Blockchain.\nIf not defined then the Blockchain is not yet configured.", + "type": "string", + "contentEncoding": "base64", + "contentMediaType": "application/octet-stream" + }, + "block0_hash": { + "description": "The BASE64 encoded hash of Block 0 used to start the Blockchain.\nIf not defined then the Blockchain is not yet configured.", + "type": "string", + "contentEncoding": "base64", + "contentMediaType": "application/octet-stream" + } + } + } + } +} \ No newline at end of file diff --git a/catalyst-gateway/event-db/json_schemas/event/description/multiline_text.json b/catalyst-gateway/event-db/json_schemas/event/description/multiline_text.json new file mode 100644 index 00000000000..1bd65e43be1 --- /dev/null +++ b/catalyst-gateway/event-db/json_schemas/event/description/multiline_text.json @@ -0,0 +1,11 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$comment": "Custom URI schema: catalyst_schema:////", + "$id": "catalyst_schema://d8f79cbc-9777-4fee-bbbe-397ce412a75c/event_description/multiline_text", + "title": "Simple Catalyst Event Description", + "description": "A simple multi-line description used for a Catalyst type event.", + "type": "string", + "contentEncoding": "utf-8", + "contentMediaType": "text/plain", + "format": "multiline" +} \ No newline at end of file diff --git a/catalyst-gateway/event-db/migrations/V1__config_tables.sql b/catalyst-gateway/event-db/migrations/V1__config_tables.sql index e29e35c66bf..55893688660 100644 --- a/catalyst-gateway/event-db/migrations/V1__config_tables.sql +++ b/catalyst-gateway/event-db/migrations/V1__config_tables.sql @@ -24,9 +24,9 @@ CREATE TABLE json_schema_type_names ( id TEXT PRIMARY KEY ); -COMMENT ON TABLE json_schema_type_names IS 'All known Json Schema Types.'; +COMMENT ON TABLE json_schema_type_names IS 'All known Json Schema Type Names.'; --- Known Schema Types are inserted when the Table which uses that type is created. +-- Known Schema Type Names are inserted when the Table which uses that type is created. -- ------------------------------------------------------------------------------------------------- @@ -35,7 +35,7 @@ COMMENT ON TABLE json_schema_type_names IS 'All known Json Schema Types.'; -- Json Schemas used to validate the contents of JSONB fields in this database. -- Catalyst Event Database CREATE TABLE json_schema_type ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + id UUID PRIMARY KEY, type TEXT NOT NULL, name TEXT NOT NULL, schema JSONB NOT NULL, @@ -50,13 +50,20 @@ COMMENT ON TABLE json_schema_type IS 'Library of defined json schemas used to validate JSONB field contents.'; COMMENT ON COLUMN json_schema_type.id IS -'Synthetic Unique ID for each json_schema_type (UUIDv4).'; +'Synthetic Unique ID for each json_schema_type (UUIDv4). +Must match $id URI inside the schema.'; COMMENT ON COLUMN json_schema_type.type IS 'The type of the json schema type. -eg. "Event Description"'; +eg. "event" +Must match $id URI inside the schema.'; COMMENT ON COLUMN json_schema_type.name IS 'The name of the json schema type. -eg. "Catalyst V1"'; +eg. "catalyst_v1" +Must match $id URI inside the schema.'; + +-- Known Schema Types are inserted when the Table which uses that type is created. +-- Or can be added by migrations as the database evolves. +-- They could also be added outside of the schema setup by inserting directly into the database. -- ------------------------------------------------------------------------------------------------- @@ -113,8 +120,8 @@ VALUES INSERT INTO json_schema_type (id, type, name, schema) VALUES ( - 'd899cd44-3513-487b-ab46-fdca662a724d', + 'd899cd44-3513-487b-ab46-fdca662a724d', -- Fix the Schema ID so that it is consistent. 'config', 'dbsync', - (SELECT jsonb FROM pg_read_file('../json_schemas/config/dbsync_connection.json')) + (SELECT jsonb FROM pg_read_file('../json_schemas/config/dbsync.json')) ); diff --git a/catalyst-gateway/event-db/migrations/V2__event_tables.sql b/catalyst-gateway/event-db/migrations/V2__event_tables.sql index 9918c775dc8..77b78b98c39 100644 --- a/catalyst-gateway/event-db/migrations/V2__event_tables.sql +++ b/catalyst-gateway/event-db/migrations/V2__event_tables.sql @@ -1,170 +1,102 @@ -- Catalyst Event Database -CREATE TABLE -event_type ( + +CREATE TABLE event_type ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - "name" TEXT NOT NULL, - description_schema JSONB NOT NULL, - extra_data_schema JSONB NOT NULL + name TEXT NOT NULL, + description_schema UUID NOT NULL, + data_schema UUID NOT NULL, + + FOREIGN KEY (description_schema) REFERENCES json_schema_type (id) ON DELETE CASCADE, + FOREIGN KEY (data_schema) REFERENCES json_schema_type (id) ON DELETE CASCADE ); CREATE UNIQUE INDEX event_type_name_idx ON event_type (name); -COMMENT ON TABLE event_type IS 'The types of event which have been defined.'; - -COMMENT ON COLUMN event_type.id IS 'Synthetic Unique ID for each event_type (UUIDv4).'; +COMMENT ON TABLE event_type IS +'The types of event which have been defined.'; -COMMENT ON COLUMN event_type.name IS 'The name of the event type. +COMMENT ON COLUMN event_type.id IS +'Synthetic Unique ID for each event_type (UUIDv4).'; +COMMENT ON COLUMN event_type.name IS +'The name of the event type. eg. "Catalyst V1"'; +COMMENT ON COLUMN event_type.description_schema IS +'The JSON Schema which defines the structure of the data in the `description` field in the event record.'; +COMMENT ON COLUMN event_type.data_schema IS +'The JSON Schema which defines the structure of the data in the `extra_data` field in the event record.'; + +-- TODO: Would be better to read the schemas, extract the ID, and add or update new schemas. +-- Run as required after migrations. + +-- Add Event Schemas to the known schema types. +INSERT INTO json_schema_type_names (id) +VALUES +('event_description'), -- Event Description schemas +('event_data'); -- Event Data Schemas + +-- Add the Initial Schemas for events. +INSERT INTO json_schema_type (id, type, name, schema) +VALUES +( + 'd899cd44-3513-487b-ab46-fdca662a724d', -- From the schema file. + 'event_description', + 'multiline_text', + (SELECT jsonb FROM pg_read_file('../json_schemas/event/description/multiline_text.json')) +), +( + '9c5df318-fa9a-4310-80fa-490f46d1cc43', -- From the schema file. + 'event_data', + 'catalyst_v1', + (SELECT jsonb FROM pg_read_file('../json_schemas/event/description/catalyst_v1.json')) +); -COMMENT ON COLUMN event_type.description_schema IS 'The JSON Schema which defines the structure of the data in the `description` field in the event record.'; - -COMMENT ON COLUMN event_type.extra_data_schema IS 'The JSON Schema which defines the structure of the data in the `extra_data` field in the event record.'; -- Event Table - Defines each voting or decision event -CREATE TABLE -"event" ( +CREATE TABLE "event" ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - "type" UUID REFERENCES event_type (id), - "name" TEXT NOT NULL, + type UUID REFERENCES event_type (id), + name TEXT NOT NULL, description JSONB NOT NULL, start_time TIMESTAMP, backing_start TIMESTAMP, backing_end TIMESTAMP, end_time TIMESTAMP, - extra_data JSONB NOT NULL + data JSONB NOT NULL ); CREATE UNIQUE INDEX event_name_idx ON event (name); -COMMENT ON TABLE event IS 'The basic parameters of a related set of funding campaigns.'; +COMMENT ON TABLE event IS +'The basic parameters of a related set of funding campaigns.'; -COMMENT ON COLUMN event.id IS 'Synthetic Unique ID for each event (UUIDv4).'; +COMMENT ON COLUMN event.id IS +'Synthetic Unique ID for each event (UUIDv4).'; -COMMENT ON COLUMN event.name IS 'The name of the event. +COMMENT ON COLUMN event.name IS +'The name of the event. eg. "Fund9" or "SVE1"'; -COMMENT ON COLUMN event.type IS 'The type of the event.'; +COMMENT ON COLUMN event.type IS +'The type of the event.'; -COMMENT ON COLUMN event.description IS 'A detailed description of the purpose of the event. +COMMENT ON COLUMN event.description IS +'A detailed description of the purpose of the event. Must conform to the JSON Schema defined by `event_type.description_schema.`'; - -COMMENT ON COLUMN event.start_time IS 'The time (UTC) the event starts. +COMMENT ON COLUMN event.start_time IS +'The time (UTC) the event starts. NULL = Not yet defined.'; - -COMMENT ON COLUMN event.backing_start IS 'The time (UTC) when backers may start backing the campaigns in the event. This must >= event.start_time. +COMMENT ON COLUMN event.backing_start IS +'The time (UTC) when backers may start backing the campaigns in the event. +This must >= event.start_time. NULL = Not yet defined.'; - -COMMENT ON COLUMN event.backing_end IS 'The time (UTC) when backers may no longer back the campaigns in the event. This must > event.backing_start and <= event.end_time. +COMMENT ON COLUMN event.backing_end IS +'The time (UTC) when backers may no longer back the campaigns in the event. +This must > event.backing_start and <= event.end_time. NULL = Not yet defined.'; - -COMMENT ON COLUMN event.end_time IS 'The time (UTC) the event ends. Must be >= event.backing_end. +COMMENT ON COLUMN event.end_time IS +'The time (UTC) the event ends. +Must be >= event.backing_end. NULL = Not yet defined.'; - -COMMENT ON COLUMN event.extra_data IS 'Extra event type specific data defined about the event. +COMMENT ON COLUMN event.data IS +'Event Type specific data defined about the event. Must conform to the JSON Schema defined by `event_type.extra_data_schema.`'; - ---COMMENT ON COLUMN event.registration_snapshot_time IS ---'The Time (UTC) Registrations are taken from Cardano main net. ---Registrations after this date are not valid for voting on the event. ---NULL = Not yet defined or Not Applicable.'; ---COMMENT ON COLUMN event.snapshot_start IS ---'The Time (UTC) Registrations taken from Cardano main net are considered stable. ---This is not the Time of the Registration Snapshot, ---This is the time after which the registration snapshot will be stable. ---NULL = Not yet defined or Not Applicable.'; ---COMMENT ON COLUMN event.voting_power_threshold IS ---'The Minimum number of Lovelace staked at the time of snapshot, to be eligible to vote. ---NULL = Not yet defined.'; ---COMMENT ON COLUMN event.review_rewards IS 'The total reward pool to pay for community reviewers for their valid reviews of the proposals assigned to this event.'; ---COMMENT ON COLUMN event.start_time IS ---'The time (UTC) the event starts. ---NULL = Not yet defined.'; ---COMMENT ON COLUMN event.end_time IS ---'The time (UTC) the event ends. ---NULL = Not yet defined.'; ---COMMENT ON COLUMN event.insight_sharing_start IS ---'TODO. ---NULL = Not yet defined.'; ---COMMENT ON COLUMN event.proposal_submission_start IS ---'The Time (UTC) proposals can start to be submitted for the event. ---NULL = Not yet defined, or Not applicable.'; ---COMMENT ON COLUMN event.refine_proposals_start IS ---'TODO. ---NULL = Not yet defined.'; ---COMMENT ON COLUMN event.finalize_proposals_start IS ---'The Time (UTC) when all proposals must be finalized by. ---NULL = Not yet defined, or Not applicable.'; ---COMMENT ON COLUMN event.proposal_assessment_start IS ---'The Time (UTC) when PA Assessors can start assessing proposals. ---NULL = Not yet defined, or Not applicable.'; ---COMMENT ON COLUMN event.assessment_qa_start IS ---'The Time (UTC) when vPA Assessors can start assessing assessments. ---NULL = Not yet defined, or Not applicable.'; ---COMMENT ON COLUMN event.voting_start IS ---'The earliest time that registered wallets with sufficient voting power can place votes in the event. ---NULL = Not yet defined.'; ---COMMENT ON COLUMN event.voting_end IS ---'The latest time that registered wallets with sufficient voting power can place votes in the event. ---NULL = Not yet defined.'; ---COMMENT ON COLUMN event.tallying_end IS ---'The latest time that tallying the event can complete by. ---NULL = Not yet defined.'; ---COMMENT ON COLUMN event.block0 IS ---'The copy of Block 0 used to start the Blockchain. ---NULL = Blockchain not started yet.'; ---COMMENT ON COLUMN event.block0_hash IS ---'The hash of block 0. ---NULL = Blockchain not started yet.'; ---COMMENT ON COLUMN event.committee_size IS ---'The size of the tally committee. ---0 = No Committee, and all votes are therefore public.'; ---COMMENT ON COLUMN event.committee_threshold IS ---'The minimum size of the tally committee to perform the tally. ---Must be <= `committee_size`'; ---COMMENT ON COLUMN event.cast_to IS ---'Json Map defining parameters which control where the vote is to be cast. ---Multiple destinations can be defined simultaneously. ---In this case the vote gets cast to all defined destinations. ---`NULL` = Default Jormungandr Blockchain. ---```jsonc ---"jorm" : { // Voting on Jormungandr Blockchain --- chain_id: , // Jormungandr chain id. Defaults to 0. --- // Other parameters TBD. ---}, ---"cardano" : { // Voting on Cardano Directly --- chain_id: , // 0 = pre-prod, 1 = mainnet. --- // Other parameters TBD. ---}, ---"postgres" : { // Store votes in Web 2 postgres backed DB only. --- url: "" --- // Other parameters TBD. --- // Note: Votes that arrive in the Cat1 system are always stored in the DB. --- // This Option only allows the vote storage DB to be tuned. ---}, ---"cat2" : { // Store votes to the Catalyst 2.0 P2P Network. --- gateway: "= 0), --- insight_sharing_start TIMESTAMP, --- proposal_submission_start TIMESTAMP, --- refine_proposals_start TIMESTAMP, --- finalize_proposals_start TIMESTAMP, --- proposal_assessment_start TIMESTAMP, --- assessment_qa_start TIMESTAMP, --- voting_start TIMESTAMP, --- voting_end TIMESTAMP, --- tallying_end TIMESTAMP, --- block0 BYTEA NULL, --- block0_hash TEXT NULL, --- committee_size INTEGER NOT NULL, --- committee_threshold INTEGER NOT NULL, --- extra JSONB, --- cast_to JSONB