From 49005c00868cd93d07628c43aad8b913f812ca40 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Thu, 14 Nov 2024 15:11:08 +0100 Subject: [PATCH 01/33] wip --- .../tests/schemathesis_tests/Earthfile | 51 +- .../schemathesis_tests/node1-scylla.yaml | 627 ++++++++++++++++++ .../schemathesis_tests/node2-scylla.yaml | 624 +++++++++++++++++ .../schemathesis_tests/node3-scylla.yaml | 624 +++++++++++++++++ .../schemathesis_tests/node4-scylla.yaml | 624 +++++++++++++++++ .../schemathesis-docker-compose.yml | 61 +- utilities/local-scylla/Earthfile | 22 + 7 files changed, 2618 insertions(+), 15 deletions(-) create mode 100644 catalyst-gateway/tests/schemathesis_tests/node1-scylla.yaml create mode 100644 catalyst-gateway/tests/schemathesis_tests/node2-scylla.yaml create mode 100644 catalyst-gateway/tests/schemathesis_tests/node3-scylla.yaml create mode 100644 catalyst-gateway/tests/schemathesis_tests/node4-scylla.yaml create mode 100644 utilities/local-scylla/Earthfile diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index a10bd3e1103..469692f4af7 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -29,25 +29,48 @@ package-schemathesis: # test-fuzzer-api - Fuzzy test cat-gateway using openapi specs. # Disabled in CI, to enable it rename to `test-fuzzer-api`. fuzzer-api: - FROM earthly/dind:alpine-alpine-3.20-docker-26.1.5-r0 + FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 RUN apk update && apk add iptables-legacy # workaround for https://github.com/earthly/earthly/issues/3784 - RUN apk add yq zstd + RUN apk add yq zstd curl COPY schemathesis-docker-compose.yml . + COPY node1-scylla.yaml . + COPY node2-scylla.yaml . + COPY node3-scylla.yaml . + COPY node4-scylla.yaml . + ENV HOST_IP + RUN export HOST_IP="$(ifconfig $(route -n | grep ^0.0.0.0 | awk '{print $NF}') | grep inet | grep -v inet6 | awk '{print $2}')" LET OPENAPI_SPEC="http://127.0.0.1:3030/docs/cat-gateway.json" + ENV LOG_LEVEL=error + ENV DATA_REFRESH_TICK=5 + ENV CHECK_CONFIG_TICK=5 + ENV MACHINE_ID="UID" + ENV CHAIN_FOLLOWER_SYNC_TASKS="16" + ENV RUST_LOG="error,cat_gateway=debug,cardano_chain_follower=debug,mithril-client=debug" + ENV CHAIN_NETWORK="Preprod" WITH DOCKER \ --compose schemathesis-docker-compose.yml \ - --load schemathesis:latest=(+package-schemathesis --openapi_spec=$OPENAPI_SPEC) \ - --load event-db:latest=(../../event-db+build) \ - --load cat-gateway:latest=(../../+package-cat-gateway) \ - --service event-db \ - --service cat-gateway \ + #--load schemathesis:latest=(+package-schemathesis --openapi_spec=$OPENAPI_SPEC) \ + #--load event-db:latest=(../../event-db+build) \ + --load cat-gateway:latest=(../../+package) \ + #--service event-db \ + --service scylla-node1 \ + --service scylla-node2 \ + --service scylla-node3 \ + --service scylla-node4 \ + #--service cat-gateway \ --allow-privileged - RUN docker run --net=host --name=st schemathesis:latest && \ - docker cp st:/results/junit-report.xml junit-report.xml && \ - docker cp st:/results/cassette.yaml cassette.yaml - END - WAIT - SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis.junit-report.xml - SAVE ARTIFACT cassette.yaml AS LOCAL cassette.yaml + RUN docker run --net=default_cluster \ + -e LOG_LEVEL \ + -e DATA_REFRESH_TICK \ + -e CHECK_CONFIG_TICK \ + -e MACHINE_ID \ + -e CHAIN_FOLLOWER_SYNC_TASKS \ + -e RUST_LOG="error,cat_gateway=debug,cardano_chain_follower=debug,mithril-client=debug" \ + -e CHAIN_NETWORK="Preprod" \ + cat-gateway:latest END + # WAIT + # SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis.junit-report.xml + # SAVE ARTIFACT cassette.yaml AS LOCAL cassette.yaml + # END diff --git a/catalyst-gateway/tests/schemathesis_tests/node1-scylla.yaml b/catalyst-gateway/tests/schemathesis_tests/node1-scylla.yaml new file mode 100644 index 00000000000..63267c2bcad --- /dev/null +++ b/catalyst-gateway/tests/schemathesis_tests/node1-scylla.yaml @@ -0,0 +1,627 @@ +# Scylla storage config YAML + +# cspell: words fsyncs rackdc partitioner mbean certficate degraade defragment + +####################################### +# This file is split to two sections: +# 1. Supported parameters +# 2. Unsupported parameters: reserved for future use or backwards +# compatibility. +# Scylla will only read and use the first segment +####################################### + +### Supported Parameters + +# The name of the cluster. This is mainly used to prevent machines in +# one logical cluster from joining another. +# It is recommended to change the default value when creating a new cluster. +# You can NOT modify this value for an existing cluster +#cluster_name: 'Test Cluster' + +# This defines the number of tokens randomly assigned to this node on the ring +# The more tokens, relative to other nodes, the larger the proportion of data +# that this node will store. You probably want all nodes to have the same number +# of tokens assuming they have equal hardware capability. +num_tokens: 256 + +# Directory where Scylla should store all its files, which are commitlog, +# data, hints, view_hints and saved_caches subdirectories. All of these +# subs can be overridden by the respective options below. +# If unset, the value defaults to /var/lib/scylla +# workdir: /var/lib/scylla + +# Directory where Scylla should store data on disk. +# data_file_directories: +# - /var/lib/scylla/data + +# commit log. when running on magnetic HDD, this should be a +# separate spindle than the data directories. +# commitlog_directory: /var/lib/scylla/commitlog + +# schema commit log. A special commitlog instance +# used for schema and system tables. +# When running on magnetic HDD, this should be a +# separate spindle than the data directories. +# schema_commitlog_directory: /var/lib/scylla/commitlog/schema + +# commitlog_sync may be either "periodic" or "batch." +# +# When in batch mode, Scylla won't ack writes until the commit log +# has been fsynced to disk. It will wait +# commitlog_sync_batch_window_in_ms milliseconds between fsyncs. +# This window should be kept short because the writer threads will +# be unable to do extra work while waiting. (You may need to increase +# concurrent_writes for the same reason.) +# +# commitlog_sync: batch +# commitlog_sync_batch_window_in_ms: 2 +# +# the other option is "periodic" where writes may be acked immediately +# and the CommitLog is simply synced every commitlog_sync_period_in_ms +# milliseconds. +commitlog_sync: periodic +commitlog_sync_period_in_ms: 10000 + +# The size of the individual commitlog file segments. A commitlog +# segment may be archived, deleted, or recycled once all the data +# in it (potentially from each columnfamily in the system) has been +# flushed to sstables. +# +# The default size is 32, which is almost always fine, but if you are +# archiving commitlog segments (see commitlog_archiving.properties), +# then you probably want a finer granularity of archiving; 8 or 16 MB +# is reasonable. +commitlog_segment_size_in_mb: 32 + +# The size of the individual schema commitlog file segments. +# +# The default size is 128, which is 4 times larger than the default +# size of the data commitlog. It's because the segment size puts +# a limit on the mutation size that can be written at once, and some +# schema mutation writes are much larger than average. +schema_commitlog_segment_size_in_mb: 128 + +# seed_provider class_name is saved for future use. +# A seed address is mandatory. +seed_provider: + # The addresses of hosts that will serve as contact points for the joining node. + # It allows the node to discover the cluster ring topology on startup (when + # joining the cluster). + # Once the node has joined the cluster, the seed list has no function. + - class_name: org.apache.cassandra.locator.SimpleSeedProvider + parameters: + # In a new cluster, provide the address of the first node. + # In an existing cluster, specify the address of at least one existing node. + # If you specify addresses of more than one node, use a comma to separate them. + # For example: ",," + - seeds: "127.0.0.1" + +# Address to bind to and tell other Scylla nodes to connect to. +# You _must_ change this if you want multiple nodes to be able to communicate! +# +# If you leave broadcast_address (below) empty, then setting listen_address +# to 0.0.0.0 is wrong as other nodes will not know how to reach this node. +# If you set broadcast_address, then you can set listen_address to 0.0.0.0. +listen_address: localhost + +# Address to broadcast to other Scylla nodes +# Leaving this blank will set it to the same value as listen_address +# broadcast_address: 1.2.3.4 + + +# When using multiple physical network interfaces, set this to true to listen on broadcast_address +# in addition to the listen_address, allowing nodes to communicate in both interfaces. +# Ignore this property if the network configuration automatically routes between the public and private networks such as EC2. +# +# listen_on_broadcast_address: false + +# port for the CQL native transport to listen for clients on +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# To disable the CQL native transport, remove this option and configure native_transport_port_ssl. +native_transport_port: 9042 + +# Like native_transport_port, but clients are forwarded to specific shards, based on the +# client-side port numbers. +native_shard_aware_transport_port: 19042 + +# Enabling native transport encryption in client_encryption_options allows you to either use +# encryption for the standard port or to use a dedicated, additional port along with the unencrypted +# standard native_transport_port. +# Enabling client encryption and keeping native_transport_port_ssl disabled will use encryption +# for native_transport_port. Setting native_transport_port_ssl to a different value +# from native_transport_port will use encryption for native_transport_port_ssl while +# keeping native_transport_port unencrypted. +#native_transport_port_ssl: 9142 + +# Like native_transport_port_ssl, but clients are forwarded to specific shards, based on the +# client-side port numbers. +#native_shard_aware_transport_port_ssl: 19142 + +# How long the coordinator should wait for read operations to complete +read_request_timeout_in_ms: 5000 + +# How long the coordinator should wait for writes to complete +write_request_timeout_in_ms: 2000 +# how long a coordinator should continue to retry a CAS operation +# that contends with other proposals for the same row +cas_contention_timeout_in_ms: 1000 + +# phi value that must be reached for a host to be marked down. +# most users should never need to adjust this. +# phi_convict_threshold: 8 + +# IEndpointSnitch. The snitch has two functions: +# - it teaches Scylla enough about your network topology to route +# requests efficiently +# - it allows Scylla to spread replicas around your cluster to avoid +# correlated failures. It does this by grouping machines into +# "datacenters" and "racks." Scylla will do its best not to have +# more than one replica on the same "rack" (which may not actually +# be a physical location) +# +# IF YOU CHANGE THE SNITCH AFTER DATA IS INSERTED INTO THE CLUSTER, +# YOU MUST RUN A FULL REPAIR, SINCE THE SNITCH AFFECTS WHERE REPLICAS +# ARE PLACED. +# +# Out of the box, Scylla provides +# - SimpleSnitch: +# Treats Strategy order as proximity. This can improve cache +# locality when disabling read repair. Only appropriate for +# single-datacenter deployments. +# - GossipingPropertyFileSnitch +# This should be your go-to snitch for production use. The rack +# and datacenter for the local node are defined in +# cassandra-rackdc.properties and propagated to other nodes via +# gossip. If cassandra-topology.properties exists, it is used as a +# fallback, allowing migration from the PropertyFileSnitch. +# - PropertyFileSnitch: +# Proximity is determined by rack and data center, which are +# explicitly configured in cassandra-topology.properties. +# - Ec2Snitch: +# Appropriate for EC2 deployments in a single Region. Loads Region +# and Availability Zone information from the EC2 API. The Region is +# treated as the datacenter, and the Availability Zone as the rack. +# Only private IPs are used, so this will not work across multiple +# Regions. +# - Ec2MultiRegionSnitch: +# Uses public IPs as broadcast_address to allow cross-region +# connectivity. (Thus, you should set seed addresses to the public +# IP as well.) You will need to open the storage_port or +# ssl_storage_port on the public IP firewall. (For intra-Region +# traffic, Scylla will switch to the private IP after +# establishing a connection.) +# - RackInferringSnitch: +# Proximity is determined by rack and data center, which are +# assumed to correspond to the 3rd and 2nd octet of each node's IP +# address, respectively. Unless this happens to match your +# deployment conventions, this is best used as an example of +# writing a custom Snitch class and is provided in that spirit. +# +# You can use a custom Snitch by setting this to the full class name +# of the snitch, which will be assumed to be on your classpath. +endpoint_snitch: SimpleSnitch + +# The address or interface to bind the Thrift RPC service and native transport +# server to. +# +# Set rpc_address OR rpc_interface, not both. Interfaces must correspond +# to a single address, IP aliasing is not supported. +# +# Leaving rpc_address blank has the same effect as on listen_address +# (i.e. it will be based on the configured hostname of the node). +# +# Note that unlike listen_address, you can specify 0.0.0.0, but you must also +# set broadcast_rpc_address to a value other than 0.0.0.0. +# +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# +# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address +# you can specify which should be chosen using rpc_interface_prefer_ipv6. If false the first ipv4 +# address will be used. If true the first ipv6 address will be used. Defaults to false preferring +# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. +rpc_address: localhost +# rpc_interface: eth1 +# rpc_interface_prefer_ipv6: false + +# port for Thrift to listen for clients on +rpc_port: 9160 + +# port for REST API server +api_port: 10000 + +# IP for the REST API server +api_address: 127.0.0.1 + +# Log WARN on any batch size exceeding this value. 128 kiB per batch by default. +# Caution should be taken on increasing the size of this threshold as it can lead to node instability. +batch_size_warn_threshold_in_kb: 128 + +# Fail any multiple-partition batch exceeding this value. 1 MiB (8x warn threshold) by default. +batch_size_fail_threshold_in_kb: 1024 + +# Authentication backend, identifying users +# Out of the box, Scylla provides org.apache.cassandra.auth.{AllowAllAuthenticator, +# PasswordAuthenticator}. +# +# - AllowAllAuthenticator performs no checks - set it to disable authentication. +# - PasswordAuthenticator relies on username/password pairs to authenticate +# users. It keeps usernames and hashed passwords in system_auth.credentials table. +# Please increase system_auth keyspace replication factor if you use this authenticator. +# - com.scylladb.auth.TransitionalAuthenticator requires username/password pair +# to authenticate in the same manner as PasswordAuthenticator, but improper credentials +# result in being logged in as an anonymous user. Use for upgrading clusters' auth. +# authenticator: AllowAllAuthenticator + +# Authorization backend, implementing IAuthorizer; used to limit access/provide permissions +# Out of the box, Scylla provides org.apache.cassandra.auth.{AllowAllAuthorizer, +# CassandraAuthorizer}. +# +# - AllowAllAuthorizer allows any action to any user - set it to disable authorization. +# - CassandraAuthorizer stores permissions in system_auth.permissions table. Please +# increase system_auth keyspace replication factor if you use this authorizer. +# - com.scylladb.auth.TransitionalAuthorizer wraps around the CassandraAuthorizer, using it for +# authorizing permission management. Otherwise, it allows all. Use for upgrading +# clusters' auth. +# authorizer: AllowAllAuthorizer + +# initial_token allows you to specify tokens manually. While you can use # it with +# vnodes (num_tokens > 1, above) -- in which case you should provide a +# comma-separated list -- it's primarily used when adding nodes # to legacy clusters +# that do not have vnodes enabled. +# initial_token: + +# RPC address to broadcast to drivers and other Scylla nodes. This cannot +# be set to 0.0.0.0. If left blank, this will be set to the value of +# rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must +# be set. +# broadcast_rpc_address: 1.2.3.4 + +# Uncomment to enable experimental features +# experimental_features: +# - udf +# - alternator-streams +# - broadcast-tables +# - keyspace-storage-options +# - tablets + +# The directory where hints files are stored if hinted handoff is enabled. +# hints_directory: /var/lib/scylla/hints + +# The directory where hints files are stored for materialized-view updates +# view_hints_directory: /var/lib/scylla/view_hints + +# See https://docs.scylladb.com/architecture/anti-entropy/hinted-handoff +# May either be "true" or "false" to enable globally, or contain a list +# of data centers to enable per-datacenter. +# hinted_handoff_enabled: DC1,DC2 +# hinted_handoff_enabled: true + +# this defines the maximum amount of time a dead host will have hints +# generated. After it has been dead this long, new hints for it will not be +# created until it has been seen alive and gone down again. +# max_hint_window_in_ms: 10800000 # 3 hours + + +# Validity period for permissions cache (fetching permissions can be an +# expensive operation depending on the authorizer, CassandraAuthorizer is +# one example). Defaults to 10000, set to 0 to disable. +# Will be disabled automatically for AllowAllAuthorizer. +# permissions_validity_in_ms: 10000 + +# Refresh interval for permissions cache (if enabled). +# After this interval, cache entries become eligible for refresh. Upon next +# access, an async reload is scheduled and the old value returned until it +# completes. If permissions_validity_in_ms is non-zero, then this also must have +# a non-zero value. Defaults to 2000. It's recommended to set this value to +# be at least 3 times smaller than the permissions_validity_in_ms. +# permissions_update_interval_in_ms: 2000 + +# The partitioner is responsible for distributing groups of rows (by +# partition key) across nodes in the cluster. You should leave this +# alone for new clusters. The partitioner can NOT be changed without +# reloading all data, so when upgrading you should set this to the +# same partitioner you were already using. +# +# Murmur3Partitioner is currently the only supported partitioner, +# +partitioner: org.apache.cassandra.dht.Murmur3Partitioner + +# Total space to use for commitlogs. +# +# If space gets above this value (it will round up to the next nearest +# segment multiple), Scylla will flush every dirty CF in the oldest +# segment and remove it. So a small total commitlog space will tend +# to cause more flush activity on less-active columnfamilies. +# +# A value of -1 (default) will automatically equate it to the total amount of memory +# available for Scylla. +commitlog_total_space_in_mb: -1 + +# TCP port, for commands and data +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# storage_port: 7000 + +# SSL port, for encrypted communication. Unused unless enabled in +# encryption_options +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# ssl_storage_port: 7001 + +# listen_interface: eth0 +# listen_interface_prefer_ipv6: false + +# Whether to start the native transport server. +# Please note that the address on which the native transport is bound is the +# same as the rpc_address. The port however is different and specified below. +# start_native_transport: true + +# The maximum size of allowed frame. Frame (requests) larger than this will +# be rejected as invalid. The default is 256MB. +# native_transport_max_frame_size_in_mb: 256 + +# Whether to start the thrift rpc server. +# start_rpc: true + +# enable or disable keepalive on rpc/native connections +# rpc_keepalive: true + +# Set to true to have Scylla create a hard link to each sstable +# flushed or streamed locally in a backups/ subdirectory of the +# keyspace data. Removing these links is the operator's +# responsibility. +# incremental_backups: false + +# Whether or not to take a snapshot before each compaction. Be +# careful using this option, since Scylla won't clean up the +# snapshots for you. Mostly useful if you're paranoid when there +# is a data format change. +# snapshot_before_compaction: false + +# Whether or not a snapshot is taken of the data before keyspace truncation +# or dropping of column families. The STRONGLY advised default of true +# should be used to provide data safety. If you set this flag to false, you will +# lose data on truncation or drop. +# auto_snapshot: true + +# When executing a scan, within or across a partition, we need to keep the +# tombstones seen in memory so we can return them to the coordinator, which +# will use them to make sure other replicas also know about the deleted rows. +# With workloads that generate a lot of tombstones, this can cause performance +# problems and even exhaust the server heap. +# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets) +# Adjust the thresholds here if you understand the dangers and want to +# scan more tombstones anyway. These thresholds may also be adjusted at runtime +# using the StorageService mbean. +# tombstone_warn_threshold: 1000 +# tombstone_failure_threshold: 100000 + +# Granularity of the collation index of rows within a partition. +# Increase if your rows are large, or if you have a very large +# number of rows per partition. The competing goals are these: +# 1) a smaller granularity means more index entries are generated +# and looking up rows within the partition by collation column +# is faster +# 2) but, Scylla will keep the collation index in memory for hot +# rows (as part of the key cache), so a larger granularity means +# you can cache more hot rows +# column_index_size_in_kb: 64 + +# Auto-scaling of the promoted index prevents running out of memory +# when the promoted index grows too large (due to partitions with many rows +# vs. too small column_index_size_in_kb). When the serialized representation +# of the promoted index grows by this threshold, the desired block size +# for this partition (initialized to column_index_size_in_kb) +# is doubled, to decrease the sampling resolution by half. +# +# To disable promoted index auto-scaling, set the threshold to 0. +# column_index_auto_scale_threshold_in_kb: 10240 + +# Log a warning when writing partitions larger than this value +# compaction_large_partition_warning_threshold_mb: 1000 + +# Log a warning when writing rows larger than this value +# compaction_large_row_warning_threshold_mb: 10 + +# Log a warning when writing cells larger than this value +# compaction_large_cell_warning_threshold_mb: 1 + +# Log a warning when row number is larger than this value +# compaction_rows_count_warning_threshold: 100000 + +# Log a warning when writing a collection containing more elements than this value +# compaction_collection_elements_count_warning_threshold: 10000 + +# How long the coordinator should wait for seq or index scans to complete +# range_request_timeout_in_ms: 10000 +# How long the coordinator should wait for writes to complete +# counter_write_request_timeout_in_ms: 5000 +# How long a coordinator should continue to retry a CAS operation +# that contends with other proposals for the same row +# cas_contention_timeout_in_ms: 1000 +# How long the coordinator should wait for truncates to complete +# (This can be much longer, because unless auto_snapshot is disabled +# we need to flush first so we can snapshot before removing the data.) +# truncate_request_timeout_in_ms: 60000 +# The default timeout for other, miscellaneous operations +# request_timeout_in_ms: 10000 + +# Enable or disable inter-node encryption. +# You must also generate keys and provide the appropriate key and trust store locations and passwords. +# +# The available internode options are : all, none, dc, rack +# If set to dc scylla will encrypt the traffic between the DCs +# If set to rack scylla will encrypt the traffic between the racks +# +# SSL/TLS algorithm and ciphers used can be controlled by +# the priority_string parameter. Info on priority string +# syntax and values is available at: +# https://gnutls.org/manual/html_node/Priority-Strings.html +# +# The require_client_auth parameter allows you to +# restrict access to service based on certificate +# validation. Client must provide a certificate +# accepted by the used trust store to connect. +# +# server_encryption_options: +# internode_encryption: none +# certificate: conf/scylla.crt +# keyfile: conf/scylla.key +# truststore: +# certficate_revocation_list: +# require_client_auth: False +# priority_string: + +# enable or disable client/server encryption. +# client_encryption_options: +# enabled: false +# certificate: conf/scylla.crt +# keyfile: conf/scylla.key +# truststore: +# certficate_revocation_list: +# require_client_auth: False +# priority_string: + +# internode_compression controls whether traffic between nodes is +# compressed. +# can be: all - all traffic is compressed +# dc - traffic between different datacenters is compressed +# none - nothing is compressed. +# internode_compression: none + +# Enable or disable tcp_nodelay for inter-dc communication. +# Disabling it will result in larger (but fewer) network packets being sent, +# reducing overhead from the TCP protocol itself, at the cost of increasing +# latency if you block for cross-datacenter responses. +# inter_dc_tcp_nodelay: false + +# Relaxation of environment checks. +# +# Scylla places certain requirements on its environment. If these requirements are +# not met, performance and reliability can be degraded. +# +# These requirements include: +# - A filesystem with good support for asynchronous I/O (AIO). Currently, +# this means XFS. +# +# false: strict environment checks are in place; do not start if they are not met. +# true: relaxed environment checks; performance and reliability may degraade. +# +# developer_mode: false + + +# Idle-time background processing +# +# Scylla can perform certain jobs in the background while the system is otherwise idle, +# freeing processor resources when there is other work to be done. +# +# defragment_memory_on_idle: true +# +# prometheus port +# By default, Scylla opens prometheus API port on port 9180 +# setting the port to 0 will disable the prometheus API. +# prometheus_port: 9180 +# +# prometheus address +# Leaving this blank will set it to the same value as listen_address. +# This means that by default, Scylla listens to the prometheus API on the same +# listening address (and therefore network interface) used to listen for +# internal communication. If the monitoring node is not in this internal +# network, you can override prometheus_address explicitly - e.g., setting +# it to 0.0.0.0 to listen on all interfaces. +# prometheus_address: 1.2.3.4 + +# Distribution of data among cores (shards) within a node +# +# Scylla distributes data within a node among shards, using a round-robin +# strategy: +# [shard0] [shard1] ... [shardN-1] [shard0] [shard1] ... [shardN-1] ... +# +# Scylla versions 1.6 and below used just one repetition of the pattern; +# this interfered with data placement among nodes (vnodes). +# +# Scylla versions 1.7 and above use 4096 repetitions of the pattern; this +# provides for better data distribution. +# +# the value below is log (base 2) of the number of repetitions. +# +# Set to 0 to avoid rewriting all data when upgrading from Scylla 1.6 and +# below. +# +# Keep at 12 for new clusters. +murmur3_partitioner_ignore_msb_bits: 12 + +# Bypass in-memory data cache (the row cache) when performing reversed queries. +# reversed_reads_auto_bypass_cache: false + +# Use a new optimized algorithm for performing reversed reads. +# Set to `false` to fall-back to the old algorithm. +# enable_optimized_reversed_reads: true + +# Use on a new, parallel algorithm for performing aggregate queries. +# Set to `false` to fall-back to the old algorithm. +# enable_parallelized_aggregation: true + +# Time for which task manager task is kept in memory after it completes. +# task_ttl_in_seconds: 0 + +# In materialized views, restrictions are allowed only on the view's primary key columns. +# In old versions Scylla mistakenly allowed IS NOT NULL restrictions on columns which were not part +# of the view's primary key. These invalid restrictions were ignored. +# This option controls the behavior when someone tries to create a view with such invalid IS NOT NULL restrictions. +# +# Can be true, false, or warn. +# * `true`: IS NOT NULL is allowed only on the view's primary key columns, +# trying to use it on other columns will cause an error, as it should. +# * `false`: Scylla accepts IS NOT NULL restrictions on regular columns, but they're silently ignored. +# It's useful for backwards compatibility. +# * `warn`: The same as false, but there's a warning about invalid view restrictions. +# +# To preserve backwards compatibility on old clusters, Scylla's default setting is `warn`. +# New clusters have this option set to `true` by scylla.yaml (which overrides the default `warn`) +# to make sure that trying to create an invalid view causes an error. +strict_is_not_null_in_views: true + +# The Unix Domain Socket the node uses for maintenance socket. +# The possible options are: +# * ignore: the node will not open the maintenance socket, +# * workdir: the node will open the maintenance socket on the path /cql.m, +# where is a path defined by the workdir configuration option, +# * : the node will open the maintenance socket on the path . +maintenance_socket: ignore + +# If set to true, configuration parameters defined with LiveUpdate option can be updated in runtime with CQL +# by updating system.config virtual table. If we don't want any configuration parameter to be changed in runtime +# via CQL, this option should be set to false. This parameter doesn't impose any limits on other mechanisms updating +# configuration parameters in runtime, e.g. sending SIGHUP or using API. This option should be set to false +# e.g. for cloud users, for whom scylla's configuration should be changed only by support engineers. +# live_updatable_config_params_changeable_via_cql: true + +# **************** +# * GUARDRAILS * +# **************** + +# Guardrails to warn or fail when Replication Factor is smaller/greater than the threshold. +# Please note that the value of 0 is always allowed, +# which means that having no replication at all, i.e. RF = 0, is always valid. +# A guardrail value smaller than 0, e.g. -1, means that the guardrail is disabled. +# Commenting out a guardrail also means it is disabled. +# minimum_replication_factor_fail_threshold: -1 +# minimum_replication_factor_warn_threshold: 3 +# maximum_replication_factor_warn_threshold: -1 +# maximum_replication_factor_fail_threshold: -1 + +# Guardrails to warn about or disallow creating a keyspace with specific replication strategy. +# Each of these 2 settings is a list storing replication strategies considered harmful. +# The replication strategies to choose from are: +# 1) SimpleStrategy, +# 2) NetworkTopologyStrategy, +# 3) LocalStrategy, +# 4) EverywhereStrategy +# +# replication_strategy_warn_list: +# - SimpleStrategy +# replication_strategy_fail_list: + +# This enables tablets on newly created keyspaces +enable_tablets: true +api_ui_dir: /opt/scylladb/swagger-ui/dist/ +api_doc_dir: /opt/scylladb/api/api-doc/ \ No newline at end of file diff --git a/catalyst-gateway/tests/schemathesis_tests/node2-scylla.yaml b/catalyst-gateway/tests/schemathesis_tests/node2-scylla.yaml new file mode 100644 index 00000000000..d0f15402524 --- /dev/null +++ b/catalyst-gateway/tests/schemathesis_tests/node2-scylla.yaml @@ -0,0 +1,624 @@ +# Scylla storage config YAML + +# cspell: words fsyncs rackdc partitioner mbean certficate degraade defragment + +####################################### +# This file is split to two sections: +# 1. Supported parameters +# 2. Unsupported parameters: reserved for future use or backwards +# compatibility. +# Scylla will only read and use the first segment +####################################### + +### Supported Parameters + +# The name of the cluster. This is mainly used to prevent machines in +# one logical cluster from joining another. +# It is recommended to change the default value when creating a new cluster. +# You can NOT modify this value for an existing cluster +#cluster_name: 'Test Cluster' + +# This defines the number of tokens randomly assigned to this node on the ring +# The more tokens, relative to other nodes, the larger the proportion of data +# that this node will store. You probably want all nodes to have the same number +# of tokens assuming they have equal hardware capability. +num_tokens: 256 + +# Directory where Scylla should store all its files, which are commitlog, +# data, hints, view_hints and saved_caches subdirectories. All of these +# subs can be overridden by the respective options below. +# If unset, the value defaults to /var/lib/scylla +# workdir: /var/lib/scylla + +# Directory where Scylla should store data on disk. +# data_file_directories: +# - /var/lib/scylla/data + +# commit log. when running on magnetic HDD, this should be a +# separate spindle than the data directories. +# commitlog_directory: /var/lib/scylla/commitlog + +# schema commit log. A special commitlog instance +# used for schema and system tables. +# When running on magnetic HDD, this should be a +# separate spindle than the data directories. +# schema_commitlog_directory: /var/lib/scylla/commitlog/schema + +# commitlog_sync may be either "periodic" or "batch." +# +# When in batch mode, Scylla won't ack writes until the commit log +# has been fsynced to disk. It will wait +# commitlog_sync_batch_window_in_ms milliseconds between fsyncs. +# This window should be kept short because the writer threads will +# be unable to do extra work while waiting. (You may need to increase +# concurrent_writes for the same reason.) +# +# commitlog_sync: batch +# commitlog_sync_batch_window_in_ms: 2 +# +# the other option is "periodic" where writes may be acked immediately +# and the CommitLog is simply synced every commitlog_sync_period_in_ms +# milliseconds. +commitlog_sync: periodic +commitlog_sync_period_in_ms: 10000 + +# The size of the individual commitlog file segments. A commitlog +# segment may be archived, deleted, or recycled once all the data +# in it (potentially from each columnfamily in the system) has been +# flushed to sstables. +# +# The default size is 32, which is almost always fine, but if you are +# archiving commitlog segments (see commitlog_archiving.properties), +# then you probably want a finer granularity of archiving; 8 or 16 MB +# is reasonable. +commitlog_segment_size_in_mb: 32 + +# The size of the individual schema commitlog file segments. +# +# The default size is 128, which is 4 times larger than the default +# size of the data commitlog. It's because the segment size puts +# a limit on the mutation size that can be written at once, and some +# schema mutation writes are much larger than average. +schema_commitlog_segment_size_in_mb: 128 + +# seed_provider class_name is saved for future use. +# A seed address is mandatory. +seed_provider: + # The addresses of hosts that will serve as contact points for the joining node. + # It allows the node to discover the cluster ring topology on startup (when + # joining the cluster). + # Once the node has joined the cluster, the seed list has no function. + - class_name: org.apache.cassandra.locator.SimpleSeedProvider + parameters: + # In a new cluster, provide the address of the first node. + # In an existing cluster, specify the address of at least one existing node. + # If you specify addresses of more than one node, use a comma to separate them. + # For example: ",," + - seeds: "127.0.0.1" + +# Address to bind to and tell other Scylla nodes to connect to. +# You _must_ change this if you want multiple nodes to be able to communicate! +# +# If you leave broadcast_address (below) empty, then setting listen_address +# to 0.0.0.0 is wrong as other nodes will not know how to reach this node. +# If you set broadcast_address, then you can set listen_address to 0.0.0.0. +listen_address: localhost + +# Address to broadcast to other Scylla nodes +# Leaving this blank will set it to the same value as listen_address +# broadcast_address: 1.2.3.4 + +# When using multiple physical network interfaces, set this to true to listen on broadcast_address +# in addition to the listen_address, allowing nodes to communicate in both interfaces. +# Ignore this property if the network configuration automatically routes between the public and private networks such as EC2. +# +# listen_on_broadcast_address: false + +# port for the CQL native transport to listen for clients on +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# To disable the CQL native transport, remove this option and configure native_transport_port_ssl. +native_transport_port: 9043 + +# Like native_transport_port, but clients are forwarded to specific shards, based on the +# client-side port numbers. +native_shard_aware_transport_port: 19043 + +# Enabling native transport encryption in client_encryption_options allows you to either use +# encryption for the standard port or to use a dedicated, additional port along with the unencrypted +# standard native_transport_port. +# Enabling client encryption and keeping native_transport_port_ssl disabled will use encryption +# for native_transport_port. Setting native_transport_port_ssl to a different value +# from native_transport_port will use encryption for native_transport_port_ssl while +# keeping native_transport_port unencrypted. +#native_transport_port_ssl: 9142 + +# Like native_transport_port_ssl, but clients are forwarded to specific shards, based on the +# client-side port numbers. +#native_shard_aware_transport_port_ssl: 19142 + +# How long the coordinator should wait for read operations to complete +read_request_timeout_in_ms: 5000 + +# How long the coordinator should wait for writes to complete +write_request_timeout_in_ms: 2000 +# how long a coordinator should continue to retry a CAS operation +# that contends with other proposals for the same row +cas_contention_timeout_in_ms: 1000 + +# phi value that must be reached for a host to be marked down. +# most users should never need to adjust this. +# phi_convict_threshold: 8 + +# IEndpointSnitch. The snitch has two functions: +# - it teaches Scylla enough about your network topology to route +# requests efficiently +# - it allows Scylla to spread replicas around your cluster to avoid +# correlated failures. It does this by grouping machines into +# "datacenters" and "racks." Scylla will do its best not to have +# more than one replica on the same "rack" (which may not actually +# be a physical location) +# +# IF YOU CHANGE THE SNITCH AFTER DATA IS INSERTED INTO THE CLUSTER, +# YOU MUST RUN A FULL REPAIR, SINCE THE SNITCH AFFECTS WHERE REPLICAS +# ARE PLACED. +# +# Out of the box, Scylla provides +# - SimpleSnitch: +# Treats Strategy order as proximity. This can improve cache +# locality when disabling read repair. Only appropriate for +# single-datacenter deployments. +# - GossipingPropertyFileSnitch +# This should be your go-to snitch for production use. The rack +# and datacenter for the local node are defined in +# cassandra-rackdc.properties and propagated to other nodes via +# gossip. If cassandra-topology.properties exists, it is used as a +# fallback, allowing migration from the PropertyFileSnitch. +# - PropertyFileSnitch: +# Proximity is determined by rack and data center, which are +# explicitly configured in cassandra-topology.properties. +# - Ec2Snitch: +# Appropriate for EC2 deployments in a single Region. Loads Region +# and Availability Zone information from the EC2 API. The Region is +# treated as the datacenter, and the Availability Zone as the rack. +# Only private IPs are used, so this will not work across multiple +# Regions. +# - Ec2MultiRegionSnitch: +# Uses public IPs as broadcast_address to allow cross-region +# connectivity. (Thus, you should set seed addresses to the public +# IP as well.) You will need to open the storage_port or +# ssl_storage_port on the public IP firewall. (For intra-Region +# traffic, Scylla will switch to the private IP after +# establishing a connection.) +# - RackInferringSnitch: +# Proximity is determined by rack and data center, which are +# assumed to correspond to the 3rd and 2nd octet of each node's IP +# address, respectively. Unless this happens to match your +# deployment conventions, this is best used as an example of +# writing a custom Snitch class and is provided in that spirit. +# +# You can use a custom Snitch by setting this to the full class name +# of the snitch, which will be assumed to be on your classpath. +endpoint_snitch: SimpleSnitch + +# The address or interface to bind the Thrift RPC service and native transport +# server to. +# +# Set rpc_address OR rpc_interface, not both. Interfaces must correspond +# to a single address, IP aliasing is not supported. +# +# Leaving rpc_address blank has the same effect as on listen_address +# (i.e. it will be based on the configured hostname of the node). +# +# Note that unlike listen_address, you can specify 0.0.0.0, but you must also +# set broadcast_rpc_address to a value other than 0.0.0.0. +# +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# +# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address +# you can specify which should be chosen using rpc_interface_prefer_ipv6. If false the first ipv4 +# address will be used. If true the first ipv6 address will be used. Defaults to false preferring +# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. +rpc_address: localhost +# rpc_interface: eth1 +# rpc_interface_prefer_ipv6: false + +# port for Thrift to listen for clients on +rpc_port: 9160 + +# port for REST API server +api_port: 10000 + +# IP for the REST API server +api_address: 127.0.0.1 + +# Log WARN on any batch size exceeding this value. 128 kiB per batch by default. +# Caution should be taken on increasing the size of this threshold as it can lead to node instability. +batch_size_warn_threshold_in_kb: 128 + +# Fail any multiple-partition batch exceeding this value. 1 MiB (8x warn threshold) by default. +batch_size_fail_threshold_in_kb: 1024 + +# Authentication backend, identifying users +# Out of the box, Scylla provides org.apache.cassandra.auth.{AllowAllAuthenticator, +# PasswordAuthenticator}. +# +# - AllowAllAuthenticator performs no checks - set it to disable authentication. +# - PasswordAuthenticator relies on username/password pairs to authenticate +# users. It keeps usernames and hashed passwords in system_auth.credentials table. +# Please increase system_auth keyspace replication factor if you use this authenticator. +# - com.scylladb.auth.TransitionalAuthenticator requires username/password pair +# to authenticate in the same manner as PasswordAuthenticator, but improper credentials +# result in being logged in as an anonymous user. Use for upgrading clusters' auth. +# authenticator: AllowAllAuthenticator + +# Authorization backend, implementing IAuthorizer; used to limit access/provide permissions +# Out of the box, Scylla provides org.apache.cassandra.auth.{AllowAllAuthorizer, +# CassandraAuthorizer}. +# +# - AllowAllAuthorizer allows any action to any user - set it to disable authorization. +# - CassandraAuthorizer stores permissions in system_auth.permissions table. Please +# increase system_auth keyspace replication factor if you use this authorizer. +# - com.scylladb.auth.TransitionalAuthorizer wraps around the CassandraAuthorizer, using it for +# authorizing permission management. Otherwise, it allows all. Use for upgrading +# clusters' auth. +# authorizer: AllowAllAuthorizer + +# initial_token allows you to specify tokens manually. While you can use # it with +# vnodes (num_tokens > 1, above) -- in which case you should provide a +# comma-separated list -- it's primarily used when adding nodes # to legacy clusters +# that do not have vnodes enabled. +# initial_token: + +# RPC address to broadcast to drivers and other Scylla nodes. This cannot +# be set to 0.0.0.0. If left blank, this will be set to the value of +# rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must +# be set. +# broadcast_rpc_address: 1.2.3.4 + +# Uncomment to enable experimental features +# experimental_features: +# - udf +# - alternator-streams +# - broadcast-tables +# - keyspace-storage-options +# - tablets + +# The directory where hints files are stored if hinted handoff is enabled. +# hints_directory: /var/lib/scylla/hints + +# The directory where hints files are stored for materialized-view updates +# view_hints_directory: /var/lib/scylla/view_hints + +# See https://docs.scylladb.com/architecture/anti-entropy/hinted-handoff +# May either be "true" or "false" to enable globally, or contain a list +# of data centers to enable per-datacenter. +# hinted_handoff_enabled: DC1,DC2 +# hinted_handoff_enabled: true + +# this defines the maximum amount of time a dead host will have hints +# generated. After it has been dead this long, new hints for it will not be +# created until it has been seen alive and gone down again. +# max_hint_window_in_ms: 10800000 # 3 hours + +# Validity period for permissions cache (fetching permissions can be an +# expensive operation depending on the authorizer, CassandraAuthorizer is +# one example). Defaults to 10000, set to 0 to disable. +# Will be disabled automatically for AllowAllAuthorizer. +# permissions_validity_in_ms: 10000 + +# Refresh interval for permissions cache (if enabled). +# After this interval, cache entries become eligible for refresh. Upon next +# access, an async reload is scheduled and the old value returned until it +# completes. If permissions_validity_in_ms is non-zero, then this also must have +# a non-zero value. Defaults to 2000. It's recommended to set this value to +# be at least 3 times smaller than the permissions_validity_in_ms. +# permissions_update_interval_in_ms: 2000 + +# The partitioner is responsible for distributing groups of rows (by +# partition key) across nodes in the cluster. You should leave this +# alone for new clusters. The partitioner can NOT be changed without +# reloading all data, so when upgrading you should set this to the +# same partitioner you were already using. +# +# Murmur3Partitioner is currently the only supported partitioner, +# +partitioner: org.apache.cassandra.dht.Murmur3Partitioner + +# Total space to use for commitlogs. +# +# If space gets above this value (it will round up to the next nearest +# segment multiple), Scylla will flush every dirty CF in the oldest +# segment and remove it. So a small total commitlog space will tend +# to cause more flush activity on less-active columnfamilies. +# +# A value of -1 (default) will automatically equate it to the total amount of memory +# available for Scylla. +commitlog_total_space_in_mb: -1 + +# TCP port, for commands and data +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# storage_port: 7000 + +# SSL port, for encrypted communication. Unused unless enabled in +# encryption_options +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# ssl_storage_port: 7001 + +# listen_interface: eth0 +# listen_interface_prefer_ipv6: false + +# Whether to start the native transport server. +# Please note that the address on which the native transport is bound is the +# same as the rpc_address. The port however is different and specified below. +# start_native_transport: true + +# The maximum size of allowed frame. Frame (requests) larger than this will +# be rejected as invalid. The default is 256MB. +# native_transport_max_frame_size_in_mb: 256 + +# Whether to start the thrift rpc server. +# start_rpc: true + +# enable or disable keepalive on rpc/native connections +# rpc_keepalive: true + +# Set to true to have Scylla create a hard link to each sstable +# flushed or streamed locally in a backups/ subdirectory of the +# keyspace data. Removing these links is the operator's +# responsibility. +# incremental_backups: false + +# Whether or not to take a snapshot before each compaction. Be +# careful using this option, since Scylla won't clean up the +# snapshots for you. Mostly useful if you're paranoid when there +# is a data format change. +# snapshot_before_compaction: false + +# Whether or not a snapshot is taken of the data before keyspace truncation +# or dropping of column families. The STRONGLY advised default of true +# should be used to provide data safety. If you set this flag to false, you will +# lose data on truncation or drop. +# auto_snapshot: true + +# When executing a scan, within or across a partition, we need to keep the +# tombstones seen in memory so we can return them to the coordinator, which +# will use them to make sure other replicas also know about the deleted rows. +# With workloads that generate a lot of tombstones, this can cause performance +# problems and even exhaust the server heap. +# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets) +# Adjust the thresholds here if you understand the dangers and want to +# scan more tombstones anyway. These thresholds may also be adjusted at runtime +# using the StorageService mbean. +# tombstone_warn_threshold: 1000 +# tombstone_failure_threshold: 100000 + +# Granularity of the collation index of rows within a partition. +# Increase if your rows are large, or if you have a very large +# number of rows per partition. The competing goals are these: +# 1) a smaller granularity means more index entries are generated +# and looking up rows within the partition by collation column +# is faster +# 2) but, Scylla will keep the collation index in memory for hot +# rows (as part of the key cache), so a larger granularity means +# you can cache more hot rows +# column_index_size_in_kb: 64 + +# Auto-scaling of the promoted index prevents running out of memory +# when the promoted index grows too large (due to partitions with many rows +# vs. too small column_index_size_in_kb). When the serialized representation +# of the promoted index grows by this threshold, the desired block size +# for this partition (initialized to column_index_size_in_kb) +# is doubled, to decrease the sampling resolution by half. +# +# To disable promoted index auto-scaling, set the threshold to 0. +# column_index_auto_scale_threshold_in_kb: 10240 + +# Log a warning when writing partitions larger than this value +# compaction_large_partition_warning_threshold_mb: 1000 + +# Log a warning when writing rows larger than this value +# compaction_large_row_warning_threshold_mb: 10 + +# Log a warning when writing cells larger than this value +# compaction_large_cell_warning_threshold_mb: 1 + +# Log a warning when row number is larger than this value +# compaction_rows_count_warning_threshold: 100000 + +# Log a warning when writing a collection containing more elements than this value +# compaction_collection_elements_count_warning_threshold: 10000 + +# How long the coordinator should wait for seq or index scans to complete +# range_request_timeout_in_ms: 10000 +# How long the coordinator should wait for writes to complete +# counter_write_request_timeout_in_ms: 5000 +# How long a coordinator should continue to retry a CAS operation +# that contends with other proposals for the same row +# cas_contention_timeout_in_ms: 1000 +# How long the coordinator should wait for truncates to complete +# (This can be much longer, because unless auto_snapshot is disabled +# we need to flush first so we can snapshot before removing the data.) +# truncate_request_timeout_in_ms: 60000 +# The default timeout for other, miscellaneous operations +# request_timeout_in_ms: 10000 + +# Enable or disable inter-node encryption. +# You must also generate keys and provide the appropriate key and trust store locations and passwords. +# +# The available internode options are : all, none, dc, rack +# If set to dc scylla will encrypt the traffic between the DCs +# If set to rack scylla will encrypt the traffic between the racks +# +# SSL/TLS algorithm and ciphers used can be controlled by +# the priority_string parameter. Info on priority string +# syntax and values is available at: +# https://gnutls.org/manual/html_node/Priority-Strings.html +# +# The require_client_auth parameter allows you to +# restrict access to service based on certificate +# validation. Client must provide a certificate +# accepted by the used trust store to connect. +# +# server_encryption_options: +# internode_encryption: none +# certificate: conf/scylla.crt +# keyfile: conf/scylla.key +# truststore: +# certficate_revocation_list: +# require_client_auth: False +# priority_string: + +# enable or disable client/server encryption. +# client_encryption_options: +# enabled: false +# certificate: conf/scylla.crt +# keyfile: conf/scylla.key +# truststore: +# certficate_revocation_list: +# require_client_auth: False +# priority_string: + +# internode_compression controls whether traffic between nodes is +# compressed. +# can be: all - all traffic is compressed +# dc - traffic between different datacenters is compressed +# none - nothing is compressed. +# internode_compression: none + +# Enable or disable tcp_nodelay for inter-dc communication. +# Disabling it will result in larger (but fewer) network packets being sent, +# reducing overhead from the TCP protocol itself, at the cost of increasing +# latency if you block for cross-datacenter responses. +# inter_dc_tcp_nodelay: false + +# Relaxation of environment checks. +# +# Scylla places certain requirements on its environment. If these requirements are +# not met, performance and reliability can be degraded. +# +# These requirements include: +# - A filesystem with good support for asynchronous I/O (AIO). Currently, +# this means XFS. +# +# false: strict environment checks are in place; do not start if they are not met. +# true: relaxed environment checks; performance and reliability may degraade. +# +# developer_mode: false + +# Idle-time background processing +# +# Scylla can perform certain jobs in the background while the system is otherwise idle, +# freeing processor resources when there is other work to be done. +# +# defragment_memory_on_idle: true +# +# prometheus port +# By default, Scylla opens prometheus API port on port 9180 +# setting the port to 0 will disable the prometheus API. +# prometheus_port: 9180 +# +# prometheus address +# Leaving this blank will set it to the same value as listen_address. +# This means that by default, Scylla listens to the prometheus API on the same +# listening address (and therefore network interface) used to listen for +# internal communication. If the monitoring node is not in this internal +# network, you can override prometheus_address explicitly - e.g., setting +# it to 0.0.0.0 to listen on all interfaces. +# prometheus_address: 1.2.3.4 + +# Distribution of data among cores (shards) within a node +# +# Scylla distributes data within a node among shards, using a round-robin +# strategy: +# [shard0] [shard1] ... [shardN-1] [shard0] [shard1] ... [shardN-1] ... +# +# Scylla versions 1.6 and below used just one repetition of the pattern; +# this interfered with data placement among nodes (vnodes). +# +# Scylla versions 1.7 and above use 4096 repetitions of the pattern; this +# provides for better data distribution. +# +# the value below is log (base 2) of the number of repetitions. +# +# Set to 0 to avoid rewriting all data when upgrading from Scylla 1.6 and +# below. +# +# Keep at 12 for new clusters. +murmur3_partitioner_ignore_msb_bits: 12 + +# Bypass in-memory data cache (the row cache) when performing reversed queries. +# reversed_reads_auto_bypass_cache: false + +# Use a new optimized algorithm for performing reversed reads. +# Set to `false` to fall-back to the old algorithm. +# enable_optimized_reversed_reads: true + +# Use on a new, parallel algorithm for performing aggregate queries. +# Set to `false` to fall-back to the old algorithm. +# enable_parallelized_aggregation: true + +# Time for which task manager task is kept in memory after it completes. +# task_ttl_in_seconds: 0 + +# In materialized views, restrictions are allowed only on the view's primary key columns. +# In old versions Scylla mistakenly allowed IS NOT NULL restrictions on columns which were not part +# of the view's primary key. These invalid restrictions were ignored. +# This option controls the behavior when someone tries to create a view with such invalid IS NOT NULL restrictions. +# +# Can be true, false, or warn. +# * `true`: IS NOT NULL is allowed only on the view's primary key columns, +# trying to use it on other columns will cause an error, as it should. +# * `false`: Scylla accepts IS NOT NULL restrictions on regular columns, but they're silently ignored. +# It's useful for backwards compatibility. +# * `warn`: The same as false, but there's a warning about invalid view restrictions. +# +# To preserve backwards compatibility on old clusters, Scylla's default setting is `warn`. +# New clusters have this option set to `true` by scylla.yaml (which overrides the default `warn`) +# to make sure that trying to create an invalid view causes an error. +strict_is_not_null_in_views: true + +# The Unix Domain Socket the node uses for maintenance socket. +# The possible options are: +# * ignore: the node will not open the maintenance socket, +# * workdir: the node will open the maintenance socket on the path /cql.m, +# where is a path defined by the workdir configuration option, +# * : the node will open the maintenance socket on the path . +maintenance_socket: ignore + +# If set to true, configuration parameters defined with LiveUpdate option can be updated in runtime with CQL +# by updating system.config virtual table. If we don't want any configuration parameter to be changed in runtime +# via CQL, this option should be set to false. This parameter doesn't impose any limits on other mechanisms updating +# configuration parameters in runtime, e.g. sending SIGHUP or using API. This option should be set to false +# e.g. for cloud users, for whom scylla's configuration should be changed only by support engineers. +# live_updatable_config_params_changeable_via_cql: true + +# **************** +# * GUARDRAILS * +# **************** + +# Guardrails to warn or fail when Replication Factor is smaller/greater than the threshold. +# Please note that the value of 0 is always allowed, +# which means that having no replication at all, i.e. RF = 0, is always valid. +# A guardrail value smaller than 0, e.g. -1, means that the guardrail is disabled. +# Commenting out a guardrail also means it is disabled. +# minimum_replication_factor_fail_threshold: -1 +# minimum_replication_factor_warn_threshold: 3 +# maximum_replication_factor_warn_threshold: -1 +# maximum_replication_factor_fail_threshold: -1 + +# Guardrails to warn about or disallow creating a keyspace with specific replication strategy. +# Each of these 2 settings is a list storing replication strategies considered harmful. +# The replication strategies to choose from are: +# 1) SimpleStrategy, +# 2) NetworkTopologyStrategy, +# 3) LocalStrategy, +# 4) EverywhereStrategy +# +# replication_strategy_warn_list: +# - SimpleStrategy +# replication_strategy_fail_list: + +# This enables tablets on newly created keyspaces +enable_tablets: true +api_ui_dir: /opt/scylladb/swagger-ui/dist/ +api_doc_dir: /opt/scylladb/api/api-doc/ diff --git a/catalyst-gateway/tests/schemathesis_tests/node3-scylla.yaml b/catalyst-gateway/tests/schemathesis_tests/node3-scylla.yaml new file mode 100644 index 00000000000..aec5e3d224f --- /dev/null +++ b/catalyst-gateway/tests/schemathesis_tests/node3-scylla.yaml @@ -0,0 +1,624 @@ +# Scylla storage config YAML + +# cspell: words fsyncs rackdc partitioner mbean certficate degraade defragment + +####################################### +# This file is split to two sections: +# 1. Supported parameters +# 2. Unsupported parameters: reserved for future use or backwards +# compatibility. +# Scylla will only read and use the first segment +####################################### + +### Supported Parameters + +# The name of the cluster. This is mainly used to prevent machines in +# one logical cluster from joining another. +# It is recommended to change the default value when creating a new cluster. +# You can NOT modify this value for an existing cluster +#cluster_name: 'Test Cluster' + +# This defines the number of tokens randomly assigned to this node on the ring +# The more tokens, relative to other nodes, the larger the proportion of data +# that this node will store. You probably want all nodes to have the same number +# of tokens assuming they have equal hardware capability. +num_tokens: 256 + +# Directory where Scylla should store all its files, which are commitlog, +# data, hints, view_hints and saved_caches subdirectories. All of these +# subs can be overridden by the respective options below. +# If unset, the value defaults to /var/lib/scylla +# workdir: /var/lib/scylla + +# Directory where Scylla should store data on disk. +# data_file_directories: +# - /var/lib/scylla/data + +# commit log. when running on magnetic HDD, this should be a +# separate spindle than the data directories. +# commitlog_directory: /var/lib/scylla/commitlog + +# schema commit log. A special commitlog instance +# used for schema and system tables. +# When running on magnetic HDD, this should be a +# separate spindle than the data directories. +# schema_commitlog_directory: /var/lib/scylla/commitlog/schema + +# commitlog_sync may be either "periodic" or "batch." +# +# When in batch mode, Scylla won't ack writes until the commit log +# has been fsynced to disk. It will wait +# commitlog_sync_batch_window_in_ms milliseconds between fsyncs. +# This window should be kept short because the writer threads will +# be unable to do extra work while waiting. (You may need to increase +# concurrent_writes for the same reason.) +# +# commitlog_sync: batch +# commitlog_sync_batch_window_in_ms: 2 +# +# the other option is "periodic" where writes may be acked immediately +# and the CommitLog is simply synced every commitlog_sync_period_in_ms +# milliseconds. +commitlog_sync: periodic +commitlog_sync_period_in_ms: 10000 + +# The size of the individual commitlog file segments. A commitlog +# segment may be archived, deleted, or recycled once all the data +# in it (potentially from each columnfamily in the system) has been +# flushed to sstables. +# +# The default size is 32, which is almost always fine, but if you are +# archiving commitlog segments (see commitlog_archiving.properties), +# then you probably want a finer granularity of archiving; 8 or 16 MB +# is reasonable. +commitlog_segment_size_in_mb: 32 + +# The size of the individual schema commitlog file segments. +# +# The default size is 128, which is 4 times larger than the default +# size of the data commitlog. It's because the segment size puts +# a limit on the mutation size that can be written at once, and some +# schema mutation writes are much larger than average. +schema_commitlog_segment_size_in_mb: 128 + +# seed_provider class_name is saved for future use. +# A seed address is mandatory. +seed_provider: + # The addresses of hosts that will serve as contact points for the joining node. + # It allows the node to discover the cluster ring topology on startup (when + # joining the cluster). + # Once the node has joined the cluster, the seed list has no function. + - class_name: org.apache.cassandra.locator.SimpleSeedProvider + parameters: + # In a new cluster, provide the address of the first node. + # In an existing cluster, specify the address of at least one existing node. + # If you specify addresses of more than one node, use a comma to separate them. + # For example: ",," + - seeds: "127.0.0.1" + +# Address to bind to and tell other Scylla nodes to connect to. +# You _must_ change this if you want multiple nodes to be able to communicate! +# +# If you leave broadcast_address (below) empty, then setting listen_address +# to 0.0.0.0 is wrong as other nodes will not know how to reach this node. +# If you set broadcast_address, then you can set listen_address to 0.0.0.0. +listen_address: localhost + +# Address to broadcast to other Scylla nodes +# Leaving this blank will set it to the same value as listen_address +# broadcast_address: 1.2.3.4 + +# When using multiple physical network interfaces, set this to true to listen on broadcast_address +# in addition to the listen_address, allowing nodes to communicate in both interfaces. +# Ignore this property if the network configuration automatically routes between the public and private networks such as EC2. +# +# listen_on_broadcast_address: false + +# port for the CQL native transport to listen for clients on +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# To disable the CQL native transport, remove this option and configure native_transport_port_ssl. +native_transport_port: 9044 + +# Like native_transport_port, but clients are forwarded to specific shards, based on the +# client-side port numbers. +native_shard_aware_transport_port: 19044 + +# Enabling native transport encryption in client_encryption_options allows you to either use +# encryption for the standard port or to use a dedicated, additional port along with the unencrypted +# standard native_transport_port. +# Enabling client encryption and keeping native_transport_port_ssl disabled will use encryption +# for native_transport_port. Setting native_transport_port_ssl to a different value +# from native_transport_port will use encryption for native_transport_port_ssl while +# keeping native_transport_port unencrypted. +#native_transport_port_ssl: 9142 + +# Like native_transport_port_ssl, but clients are forwarded to specific shards, based on the +# client-side port numbers. +#native_shard_aware_transport_port_ssl: 19142 + +# How long the coordinator should wait for read operations to complete +read_request_timeout_in_ms: 5000 + +# How long the coordinator should wait for writes to complete +write_request_timeout_in_ms: 2000 +# how long a coordinator should continue to retry a CAS operation +# that contends with other proposals for the same row +cas_contention_timeout_in_ms: 1000 + +# phi value that must be reached for a host to be marked down. +# most users should never need to adjust this. +# phi_convict_threshold: 8 + +# IEndpointSnitch. The snitch has two functions: +# - it teaches Scylla enough about your network topology to route +# requests efficiently +# - it allows Scylla to spread replicas around your cluster to avoid +# correlated failures. It does this by grouping machines into +# "datacenters" and "racks." Scylla will do its best not to have +# more than one replica on the same "rack" (which may not actually +# be a physical location) +# +# IF YOU CHANGE THE SNITCH AFTER DATA IS INSERTED INTO THE CLUSTER, +# YOU MUST RUN A FULL REPAIR, SINCE THE SNITCH AFFECTS WHERE REPLICAS +# ARE PLACED. +# +# Out of the box, Scylla provides +# - SimpleSnitch: +# Treats Strategy order as proximity. This can improve cache +# locality when disabling read repair. Only appropriate for +# single-datacenter deployments. +# - GossipingPropertyFileSnitch +# This should be your go-to snitch for production use. The rack +# and datacenter for the local node are defined in +# cassandra-rackdc.properties and propagated to other nodes via +# gossip. If cassandra-topology.properties exists, it is used as a +# fallback, allowing migration from the PropertyFileSnitch. +# - PropertyFileSnitch: +# Proximity is determined by rack and data center, which are +# explicitly configured in cassandra-topology.properties. +# - Ec2Snitch: +# Appropriate for EC2 deployments in a single Region. Loads Region +# and Availability Zone information from the EC2 API. The Region is +# treated as the datacenter, and the Availability Zone as the rack. +# Only private IPs are used, so this will not work across multiple +# Regions. +# - Ec2MultiRegionSnitch: +# Uses public IPs as broadcast_address to allow cross-region +# connectivity. (Thus, you should set seed addresses to the public +# IP as well.) You will need to open the storage_port or +# ssl_storage_port on the public IP firewall. (For intra-Region +# traffic, Scylla will switch to the private IP after +# establishing a connection.) +# - RackInferringSnitch: +# Proximity is determined by rack and data center, which are +# assumed to correspond to the 3rd and 2nd octet of each node's IP +# address, respectively. Unless this happens to match your +# deployment conventions, this is best used as an example of +# writing a custom Snitch class and is provided in that spirit. +# +# You can use a custom Snitch by setting this to the full class name +# of the snitch, which will be assumed to be on your classpath. +endpoint_snitch: SimpleSnitch + +# The address or interface to bind the Thrift RPC service and native transport +# server to. +# +# Set rpc_address OR rpc_interface, not both. Interfaces must correspond +# to a single address, IP aliasing is not supported. +# +# Leaving rpc_address blank has the same effect as on listen_address +# (i.e. it will be based on the configured hostname of the node). +# +# Note that unlike listen_address, you can specify 0.0.0.0, but you must also +# set broadcast_rpc_address to a value other than 0.0.0.0. +# +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# +# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address +# you can specify which should be chosen using rpc_interface_prefer_ipv6. If false the first ipv4 +# address will be used. If true the first ipv6 address will be used. Defaults to false preferring +# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. +rpc_address: localhost +# rpc_interface: eth1 +# rpc_interface_prefer_ipv6: false + +# port for Thrift to listen for clients on +rpc_port: 9160 + +# port for REST API server +api_port: 10000 + +# IP for the REST API server +api_address: 127.0.0.1 + +# Log WARN on any batch size exceeding this value. 128 kiB per batch by default. +# Caution should be taken on increasing the size of this threshold as it can lead to node instability. +batch_size_warn_threshold_in_kb: 128 + +# Fail any multiple-partition batch exceeding this value. 1 MiB (8x warn threshold) by default. +batch_size_fail_threshold_in_kb: 1024 + +# Authentication backend, identifying users +# Out of the box, Scylla provides org.apache.cassandra.auth.{AllowAllAuthenticator, +# PasswordAuthenticator}. +# +# - AllowAllAuthenticator performs no checks - set it to disable authentication. +# - PasswordAuthenticator relies on username/password pairs to authenticate +# users. It keeps usernames and hashed passwords in system_auth.credentials table. +# Please increase system_auth keyspace replication factor if you use this authenticator. +# - com.scylladb.auth.TransitionalAuthenticator requires username/password pair +# to authenticate in the same manner as PasswordAuthenticator, but improper credentials +# result in being logged in as an anonymous user. Use for upgrading clusters' auth. +# authenticator: AllowAllAuthenticator + +# Authorization backend, implementing IAuthorizer; used to limit access/provide permissions +# Out of the box, Scylla provides org.apache.cassandra.auth.{AllowAllAuthorizer, +# CassandraAuthorizer}. +# +# - AllowAllAuthorizer allows any action to any user - set it to disable authorization. +# - CassandraAuthorizer stores permissions in system_auth.permissions table. Please +# increase system_auth keyspace replication factor if you use this authorizer. +# - com.scylladb.auth.TransitionalAuthorizer wraps around the CassandraAuthorizer, using it for +# authorizing permission management. Otherwise, it allows all. Use for upgrading +# clusters' auth. +# authorizer: AllowAllAuthorizer + +# initial_token allows you to specify tokens manually. While you can use # it with +# vnodes (num_tokens > 1, above) -- in which case you should provide a +# comma-separated list -- it's primarily used when adding nodes # to legacy clusters +# that do not have vnodes enabled. +# initial_token: + +# RPC address to broadcast to drivers and other Scylla nodes. This cannot +# be set to 0.0.0.0. If left blank, this will be set to the value of +# rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must +# be set. +# broadcast_rpc_address: 1.2.3.4 + +# Uncomment to enable experimental features +# experimental_features: +# - udf +# - alternator-streams +# - broadcast-tables +# - keyspace-storage-options +# - tablets + +# The directory where hints files are stored if hinted handoff is enabled. +# hints_directory: /var/lib/scylla/hints + +# The directory where hints files are stored for materialized-view updates +# view_hints_directory: /var/lib/scylla/view_hints + +# See https://docs.scylladb.com/architecture/anti-entropy/hinted-handoff +# May either be "true" or "false" to enable globally, or contain a list +# of data centers to enable per-datacenter. +# hinted_handoff_enabled: DC1,DC2 +# hinted_handoff_enabled: true + +# this defines the maximum amount of time a dead host will have hints +# generated. After it has been dead this long, new hints for it will not be +# created until it has been seen alive and gone down again. +# max_hint_window_in_ms: 10800000 # 3 hours + +# Validity period for permissions cache (fetching permissions can be an +# expensive operation depending on the authorizer, CassandraAuthorizer is +# one example). Defaults to 10000, set to 0 to disable. +# Will be disabled automatically for AllowAllAuthorizer. +# permissions_validity_in_ms: 10000 + +# Refresh interval for permissions cache (if enabled). +# After this interval, cache entries become eligible for refresh. Upon next +# access, an async reload is scheduled and the old value returned until it +# completes. If permissions_validity_in_ms is non-zero, then this also must have +# a non-zero value. Defaults to 2000. It's recommended to set this value to +# be at least 3 times smaller than the permissions_validity_in_ms. +# permissions_update_interval_in_ms: 2000 + +# The partitioner is responsible for distributing groups of rows (by +# partition key) across nodes in the cluster. You should leave this +# alone for new clusters. The partitioner can NOT be changed without +# reloading all data, so when upgrading you should set this to the +# same partitioner you were already using. +# +# Murmur3Partitioner is currently the only supported partitioner, +# +partitioner: org.apache.cassandra.dht.Murmur3Partitioner + +# Total space to use for commitlogs. +# +# If space gets above this value (it will round up to the next nearest +# segment multiple), Scylla will flush every dirty CF in the oldest +# segment and remove it. So a small total commitlog space will tend +# to cause more flush activity on less-active columnfamilies. +# +# A value of -1 (default) will automatically equate it to the total amount of memory +# available for Scylla. +commitlog_total_space_in_mb: -1 + +# TCP port, for commands and data +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# storage_port: 7000 + +# SSL port, for encrypted communication. Unused unless enabled in +# encryption_options +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# ssl_storage_port: 7001 + +# listen_interface: eth0 +# listen_interface_prefer_ipv6: false + +# Whether to start the native transport server. +# Please note that the address on which the native transport is bound is the +# same as the rpc_address. The port however is different and specified below. +# start_native_transport: true + +# The maximum size of allowed frame. Frame (requests) larger than this will +# be rejected as invalid. The default is 256MB. +# native_transport_max_frame_size_in_mb: 256 + +# Whether to start the thrift rpc server. +# start_rpc: true + +# enable or disable keepalive on rpc/native connections +# rpc_keepalive: true + +# Set to true to have Scylla create a hard link to each sstable +# flushed or streamed locally in a backups/ subdirectory of the +# keyspace data. Removing these links is the operator's +# responsibility. +# incremental_backups: false + +# Whether or not to take a snapshot before each compaction. Be +# careful using this option, since Scylla won't clean up the +# snapshots for you. Mostly useful if you're paranoid when there +# is a data format change. +# snapshot_before_compaction: false + +# Whether or not a snapshot is taken of the data before keyspace truncation +# or dropping of column families. The STRONGLY advised default of true +# should be used to provide data safety. If you set this flag to false, you will +# lose data on truncation or drop. +# auto_snapshot: true + +# When executing a scan, within or across a partition, we need to keep the +# tombstones seen in memory so we can return them to the coordinator, which +# will use them to make sure other replicas also know about the deleted rows. +# With workloads that generate a lot of tombstones, this can cause performance +# problems and even exhaust the server heap. +# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets) +# Adjust the thresholds here if you understand the dangers and want to +# scan more tombstones anyway. These thresholds may also be adjusted at runtime +# using the StorageService mbean. +# tombstone_warn_threshold: 1000 +# tombstone_failure_threshold: 100000 + +# Granularity of the collation index of rows within a partition. +# Increase if your rows are large, or if you have a very large +# number of rows per partition. The competing goals are these: +# 1) a smaller granularity means more index entries are generated +# and looking up rows within the partition by collation column +# is faster +# 2) but, Scylla will keep the collation index in memory for hot +# rows (as part of the key cache), so a larger granularity means +# you can cache more hot rows +# column_index_size_in_kb: 64 + +# Auto-scaling of the promoted index prevents running out of memory +# when the promoted index grows too large (due to partitions with many rows +# vs. too small column_index_size_in_kb). When the serialized representation +# of the promoted index grows by this threshold, the desired block size +# for this partition (initialized to column_index_size_in_kb) +# is doubled, to decrease the sampling resolution by half. +# +# To disable promoted index auto-scaling, set the threshold to 0. +# column_index_auto_scale_threshold_in_kb: 10240 + +# Log a warning when writing partitions larger than this value +# compaction_large_partition_warning_threshold_mb: 1000 + +# Log a warning when writing rows larger than this value +# compaction_large_row_warning_threshold_mb: 10 + +# Log a warning when writing cells larger than this value +# compaction_large_cell_warning_threshold_mb: 1 + +# Log a warning when row number is larger than this value +# compaction_rows_count_warning_threshold: 100000 + +# Log a warning when writing a collection containing more elements than this value +# compaction_collection_elements_count_warning_threshold: 10000 + +# How long the coordinator should wait for seq or index scans to complete +# range_request_timeout_in_ms: 10000 +# How long the coordinator should wait for writes to complete +# counter_write_request_timeout_in_ms: 5000 +# How long a coordinator should continue to retry a CAS operation +# that contends with other proposals for the same row +# cas_contention_timeout_in_ms: 1000 +# How long the coordinator should wait for truncates to complete +# (This can be much longer, because unless auto_snapshot is disabled +# we need to flush first so we can snapshot before removing the data.) +# truncate_request_timeout_in_ms: 60000 +# The default timeout for other, miscellaneous operations +# request_timeout_in_ms: 10000 + +# Enable or disable inter-node encryption. +# You must also generate keys and provide the appropriate key and trust store locations and passwords. +# +# The available internode options are : all, none, dc, rack +# If set to dc scylla will encrypt the traffic between the DCs +# If set to rack scylla will encrypt the traffic between the racks +# +# SSL/TLS algorithm and ciphers used can be controlled by +# the priority_string parameter. Info on priority string +# syntax and values is available at: +# https://gnutls.org/manual/html_node/Priority-Strings.html +# +# The require_client_auth parameter allows you to +# restrict access to service based on certificate +# validation. Client must provide a certificate +# accepted by the used trust store to connect. +# +# server_encryption_options: +# internode_encryption: none +# certificate: conf/scylla.crt +# keyfile: conf/scylla.key +# truststore: +# certficate_revocation_list: +# require_client_auth: False +# priority_string: + +# enable or disable client/server encryption. +# client_encryption_options: +# enabled: false +# certificate: conf/scylla.crt +# keyfile: conf/scylla.key +# truststore: +# certficate_revocation_list: +# require_client_auth: False +# priority_string: + +# internode_compression controls whether traffic between nodes is +# compressed. +# can be: all - all traffic is compressed +# dc - traffic between different datacenters is compressed +# none - nothing is compressed. +# internode_compression: none + +# Enable or disable tcp_nodelay for inter-dc communication. +# Disabling it will result in larger (but fewer) network packets being sent, +# reducing overhead from the TCP protocol itself, at the cost of increasing +# latency if you block for cross-datacenter responses. +# inter_dc_tcp_nodelay: false + +# Relaxation of environment checks. +# +# Scylla places certain requirements on its environment. If these requirements are +# not met, performance and reliability can be degraded. +# +# These requirements include: +# - A filesystem with good support for asynchronous I/O (AIO). Currently, +# this means XFS. +# +# false: strict environment checks are in place; do not start if they are not met. +# true: relaxed environment checks; performance and reliability may degraade. +# +# developer_mode: false + +# Idle-time background processing +# +# Scylla can perform certain jobs in the background while the system is otherwise idle, +# freeing processor resources when there is other work to be done. +# +# defragment_memory_on_idle: true +# +# prometheus port +# By default, Scylla opens prometheus API port on port 9180 +# setting the port to 0 will disable the prometheus API. +# prometheus_port: 9180 +# +# prometheus address +# Leaving this blank will set it to the same value as listen_address. +# This means that by default, Scylla listens to the prometheus API on the same +# listening address (and therefore network interface) used to listen for +# internal communication. If the monitoring node is not in this internal +# network, you can override prometheus_address explicitly - e.g., setting +# it to 0.0.0.0 to listen on all interfaces. +# prometheus_address: 1.2.3.4 + +# Distribution of data among cores (shards) within a node +# +# Scylla distributes data within a node among shards, using a round-robin +# strategy: +# [shard0] [shard1] ... [shardN-1] [shard0] [shard1] ... [shardN-1] ... +# +# Scylla versions 1.6 and below used just one repetition of the pattern; +# this interfered with data placement among nodes (vnodes). +# +# Scylla versions 1.7 and above use 4096 repetitions of the pattern; this +# provides for better data distribution. +# +# the value below is log (base 2) of the number of repetitions. +# +# Set to 0 to avoid rewriting all data when upgrading from Scylla 1.6 and +# below. +# +# Keep at 12 for new clusters. +murmur3_partitioner_ignore_msb_bits: 12 + +# Bypass in-memory data cache (the row cache) when performing reversed queries. +# reversed_reads_auto_bypass_cache: false + +# Use a new optimized algorithm for performing reversed reads. +# Set to `false` to fall-back to the old algorithm. +# enable_optimized_reversed_reads: true + +# Use on a new, parallel algorithm for performing aggregate queries. +# Set to `false` to fall-back to the old algorithm. +# enable_parallelized_aggregation: true + +# Time for which task manager task is kept in memory after it completes. +# task_ttl_in_seconds: 0 + +# In materialized views, restrictions are allowed only on the view's primary key columns. +# In old versions Scylla mistakenly allowed IS NOT NULL restrictions on columns which were not part +# of the view's primary key. These invalid restrictions were ignored. +# This option controls the behavior when someone tries to create a view with such invalid IS NOT NULL restrictions. +# +# Can be true, false, or warn. +# * `true`: IS NOT NULL is allowed only on the view's primary key columns, +# trying to use it on other columns will cause an error, as it should. +# * `false`: Scylla accepts IS NOT NULL restrictions on regular columns, but they're silently ignored. +# It's useful for backwards compatibility. +# * `warn`: The same as false, but there's a warning about invalid view restrictions. +# +# To preserve backwards compatibility on old clusters, Scylla's default setting is `warn`. +# New clusters have this option set to `true` by scylla.yaml (which overrides the default `warn`) +# to make sure that trying to create an invalid view causes an error. +strict_is_not_null_in_views: true + +# The Unix Domain Socket the node uses for maintenance socket. +# The possible options are: +# * ignore: the node will not open the maintenance socket, +# * workdir: the node will open the maintenance socket on the path /cql.m, +# where is a path defined by the workdir configuration option, +# * : the node will open the maintenance socket on the path . +maintenance_socket: ignore + +# If set to true, configuration parameters defined with LiveUpdate option can be updated in runtime with CQL +# by updating system.config virtual table. If we don't want any configuration parameter to be changed in runtime +# via CQL, this option should be set to false. This parameter doesn't impose any limits on other mechanisms updating +# configuration parameters in runtime, e.g. sending SIGHUP or using API. This option should be set to false +# e.g. for cloud users, for whom scylla's configuration should be changed only by support engineers. +# live_updatable_config_params_changeable_via_cql: true + +# **************** +# * GUARDRAILS * +# **************** + +# Guardrails to warn or fail when Replication Factor is smaller/greater than the threshold. +# Please note that the value of 0 is always allowed, +# which means that having no replication at all, i.e. RF = 0, is always valid. +# A guardrail value smaller than 0, e.g. -1, means that the guardrail is disabled. +# Commenting out a guardrail also means it is disabled. +# minimum_replication_factor_fail_threshold: -1 +# minimum_replication_factor_warn_threshold: 3 +# maximum_replication_factor_warn_threshold: -1 +# maximum_replication_factor_fail_threshold: -1 + +# Guardrails to warn about or disallow creating a keyspace with specific replication strategy. +# Each of these 2 settings is a list storing replication strategies considered harmful. +# The replication strategies to choose from are: +# 1) SimpleStrategy, +# 2) NetworkTopologyStrategy, +# 3) LocalStrategy, +# 4) EverywhereStrategy +# +# replication_strategy_warn_list: +# - SimpleStrategy +# replication_strategy_fail_list: + +# This enables tablets on newly created keyspaces +enable_tablets: true +api_ui_dir: /opt/scylladb/swagger-ui/dist/ +api_doc_dir: /opt/scylladb/api/api-doc/ diff --git a/catalyst-gateway/tests/schemathesis_tests/node4-scylla.yaml b/catalyst-gateway/tests/schemathesis_tests/node4-scylla.yaml new file mode 100644 index 00000000000..4bf06b2f620 --- /dev/null +++ b/catalyst-gateway/tests/schemathesis_tests/node4-scylla.yaml @@ -0,0 +1,624 @@ +# Scylla storage config YAML + +# cspell: words fsyncs rackdc partitioner mbean certficate degraade defragment + +####################################### +# This file is split to two sections: +# 1. Supported parameters +# 2. Unsupported parameters: reserved for future use or backwards +# compatibility. +# Scylla will only read and use the first segment +####################################### + +### Supported Parameters + +# The name of the cluster. This is mainly used to prevent machines in +# one logical cluster from joining another. +# It is recommended to change the default value when creating a new cluster. +# You can NOT modify this value for an existing cluster +#cluster_name: 'Test Cluster' + +# This defines the number of tokens randomly assigned to this node on the ring +# The more tokens, relative to other nodes, the larger the proportion of data +# that this node will store. You probably want all nodes to have the same number +# of tokens assuming they have equal hardware capability. +num_tokens: 256 + +# Directory where Scylla should store all its files, which are commitlog, +# data, hints, view_hints and saved_caches subdirectories. All of these +# subs can be overridden by the respective options below. +# If unset, the value defaults to /var/lib/scylla +# workdir: /var/lib/scylla + +# Directory where Scylla should store data on disk. +# data_file_directories: +# - /var/lib/scylla/data + +# commit log. when running on magnetic HDD, this should be a +# separate spindle than the data directories. +# commitlog_directory: /var/lib/scylla/commitlog + +# schema commit log. A special commitlog instance +# used for schema and system tables. +# When running on magnetic HDD, this should be a +# separate spindle than the data directories. +# schema_commitlog_directory: /var/lib/scylla/commitlog/schema + +# commitlog_sync may be either "periodic" or "batch." +# +# When in batch mode, Scylla won't ack writes until the commit log +# has been fsynced to disk. It will wait +# commitlog_sync_batch_window_in_ms milliseconds between fsyncs. +# This window should be kept short because the writer threads will +# be unable to do extra work while waiting. (You may need to increase +# concurrent_writes for the same reason.) +# +# commitlog_sync: batch +# commitlog_sync_batch_window_in_ms: 2 +# +# the other option is "periodic" where writes may be acked immediately +# and the CommitLog is simply synced every commitlog_sync_period_in_ms +# milliseconds. +commitlog_sync: periodic +commitlog_sync_period_in_ms: 10000 + +# The size of the individual commitlog file segments. A commitlog +# segment may be archived, deleted, or recycled once all the data +# in it (potentially from each columnfamily in the system) has been +# flushed to sstables. +# +# The default size is 32, which is almost always fine, but if you are +# archiving commitlog segments (see commitlog_archiving.properties), +# then you probably want a finer granularity of archiving; 8 or 16 MB +# is reasonable. +commitlog_segment_size_in_mb: 32 + +# The size of the individual schema commitlog file segments. +# +# The default size is 128, which is 4 times larger than the default +# size of the data commitlog. It's because the segment size puts +# a limit on the mutation size that can be written at once, and some +# schema mutation writes are much larger than average. +schema_commitlog_segment_size_in_mb: 128 + +# seed_provider class_name is saved for future use. +# A seed address is mandatory. +seed_provider: + # The addresses of hosts that will serve as contact points for the joining node. + # It allows the node to discover the cluster ring topology on startup (when + # joining the cluster). + # Once the node has joined the cluster, the seed list has no function. + - class_name: org.apache.cassandra.locator.SimpleSeedProvider + parameters: + # In a new cluster, provide the address of the first node. + # In an existing cluster, specify the address of at least one existing node. + # If you specify addresses of more than one node, use a comma to separate them. + # For example: ",," + - seeds: "127.0.0.1" + +# Address to bind to and tell other Scylla nodes to connect to. +# You _must_ change this if you want multiple nodes to be able to communicate! +# +# If you leave broadcast_address (below) empty, then setting listen_address +# to 0.0.0.0 is wrong as other nodes will not know how to reach this node. +# If you set broadcast_address, then you can set listen_address to 0.0.0.0. +listen_address: localhost + +# Address to broadcast to other Scylla nodes +# Leaving this blank will set it to the same value as listen_address +# broadcast_address: 1.2.3.4 + +# When using multiple physical network interfaces, set this to true to listen on broadcast_address +# in addition to the listen_address, allowing nodes to communicate in both interfaces. +# Ignore this property if the network configuration automatically routes between the public and private networks such as EC2. +# +# listen_on_broadcast_address: false + +# port for the CQL native transport to listen for clients on +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# To disable the CQL native transport, remove this option and configure native_transport_port_ssl. +native_transport_port: 9045 + +# Like native_transport_port, but clients are forwarded to specific shards, based on the +# client-side port numbers. +native_shard_aware_transport_port: 19045 + +# Enabling native transport encryption in client_encryption_options allows you to either use +# encryption for the standard port or to use a dedicated, additional port along with the unencrypted +# standard native_transport_port. +# Enabling client encryption and keeping native_transport_port_ssl disabled will use encryption +# for native_transport_port. Setting native_transport_port_ssl to a different value +# from native_transport_port will use encryption for native_transport_port_ssl while +# keeping native_transport_port unencrypted. +#native_transport_port_ssl: 9142 + +# Like native_transport_port_ssl, but clients are forwarded to specific shards, based on the +# client-side port numbers. +#native_shard_aware_transport_port_ssl: 19142 + +# How long the coordinator should wait for read operations to complete +read_request_timeout_in_ms: 5000 + +# How long the coordinator should wait for writes to complete +write_request_timeout_in_ms: 2000 +# how long a coordinator should continue to retry a CAS operation +# that contends with other proposals for the same row +cas_contention_timeout_in_ms: 1000 + +# phi value that must be reached for a host to be marked down. +# most users should never need to adjust this. +# phi_convict_threshold: 8 + +# IEndpointSnitch. The snitch has two functions: +# - it teaches Scylla enough about your network topology to route +# requests efficiently +# - it allows Scylla to spread replicas around your cluster to avoid +# correlated failures. It does this by grouping machines into +# "datacenters" and "racks." Scylla will do its best not to have +# more than one replica on the same "rack" (which may not actually +# be a physical location) +# +# IF YOU CHANGE THE SNITCH AFTER DATA IS INSERTED INTO THE CLUSTER, +# YOU MUST RUN A FULL REPAIR, SINCE THE SNITCH AFFECTS WHERE REPLICAS +# ARE PLACED. +# +# Out of the box, Scylla provides +# - SimpleSnitch: +# Treats Strategy order as proximity. This can improve cache +# locality when disabling read repair. Only appropriate for +# single-datacenter deployments. +# - GossipingPropertyFileSnitch +# This should be your go-to snitch for production use. The rack +# and datacenter for the local node are defined in +# cassandra-rackdc.properties and propagated to other nodes via +# gossip. If cassandra-topology.properties exists, it is used as a +# fallback, allowing migration from the PropertyFileSnitch. +# - PropertyFileSnitch: +# Proximity is determined by rack and data center, which are +# explicitly configured in cassandra-topology.properties. +# - Ec2Snitch: +# Appropriate for EC2 deployments in a single Region. Loads Region +# and Availability Zone information from the EC2 API. The Region is +# treated as the datacenter, and the Availability Zone as the rack. +# Only private IPs are used, so this will not work across multiple +# Regions. +# - Ec2MultiRegionSnitch: +# Uses public IPs as broadcast_address to allow cross-region +# connectivity. (Thus, you should set seed addresses to the public +# IP as well.) You will need to open the storage_port or +# ssl_storage_port on the public IP firewall. (For intra-Region +# traffic, Scylla will switch to the private IP after +# establishing a connection.) +# - RackInferringSnitch: +# Proximity is determined by rack and data center, which are +# assumed to correspond to the 3rd and 2nd octet of each node's IP +# address, respectively. Unless this happens to match your +# deployment conventions, this is best used as an example of +# writing a custom Snitch class and is provided in that spirit. +# +# You can use a custom Snitch by setting this to the full class name +# of the snitch, which will be assumed to be on your classpath. +endpoint_snitch: SimpleSnitch + +# The address or interface to bind the Thrift RPC service and native transport +# server to. +# +# Set rpc_address OR rpc_interface, not both. Interfaces must correspond +# to a single address, IP aliasing is not supported. +# +# Leaving rpc_address blank has the same effect as on listen_address +# (i.e. it will be based on the configured hostname of the node). +# +# Note that unlike listen_address, you can specify 0.0.0.0, but you must also +# set broadcast_rpc_address to a value other than 0.0.0.0. +# +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# +# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address +# you can specify which should be chosen using rpc_interface_prefer_ipv6. If false the first ipv4 +# address will be used. If true the first ipv6 address will be used. Defaults to false preferring +# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. +rpc_address: localhost +# rpc_interface: eth1 +# rpc_interface_prefer_ipv6: false + +# port for Thrift to listen for clients on +rpc_port: 9160 + +# port for REST API server +api_port: 10000 + +# IP for the REST API server +api_address: 127.0.0.1 + +# Log WARN on any batch size exceeding this value. 128 kiB per batch by default. +# Caution should be taken on increasing the size of this threshold as it can lead to node instability. +batch_size_warn_threshold_in_kb: 128 + +# Fail any multiple-partition batch exceeding this value. 1 MiB (8x warn threshold) by default. +batch_size_fail_threshold_in_kb: 1024 + +# Authentication backend, identifying users +# Out of the box, Scylla provides org.apache.cassandra.auth.{AllowAllAuthenticator, +# PasswordAuthenticator}. +# +# - AllowAllAuthenticator performs no checks - set it to disable authentication. +# - PasswordAuthenticator relies on username/password pairs to authenticate +# users. It keeps usernames and hashed passwords in system_auth.credentials table. +# Please increase system_auth keyspace replication factor if you use this authenticator. +# - com.scylladb.auth.TransitionalAuthenticator requires username/password pair +# to authenticate in the same manner as PasswordAuthenticator, but improper credentials +# result in being logged in as an anonymous user. Use for upgrading clusters' auth. +# authenticator: AllowAllAuthenticator + +# Authorization backend, implementing IAuthorizer; used to limit access/provide permissions +# Out of the box, Scylla provides org.apache.cassandra.auth.{AllowAllAuthorizer, +# CassandraAuthorizer}. +# +# - AllowAllAuthorizer allows any action to any user - set it to disable authorization. +# - CassandraAuthorizer stores permissions in system_auth.permissions table. Please +# increase system_auth keyspace replication factor if you use this authorizer. +# - com.scylladb.auth.TransitionalAuthorizer wraps around the CassandraAuthorizer, using it for +# authorizing permission management. Otherwise, it allows all. Use for upgrading +# clusters' auth. +# authorizer: AllowAllAuthorizer + +# initial_token allows you to specify tokens manually. While you can use # it with +# vnodes (num_tokens > 1, above) -- in which case you should provide a +# comma-separated list -- it's primarily used when adding nodes # to legacy clusters +# that do not have vnodes enabled. +# initial_token: + +# RPC address to broadcast to drivers and other Scylla nodes. This cannot +# be set to 0.0.0.0. If left blank, this will be set to the value of +# rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must +# be set. +# broadcast_rpc_address: 1.2.3.4 + +# Uncomment to enable experimental features +# experimental_features: +# - udf +# - alternator-streams +# - broadcast-tables +# - keyspace-storage-options +# - tablets + +# The directory where hints files are stored if hinted handoff is enabled. +# hints_directory: /var/lib/scylla/hints + +# The directory where hints files are stored for materialized-view updates +# view_hints_directory: /var/lib/scylla/view_hints + +# See https://docs.scylladb.com/architecture/anti-entropy/hinted-handoff +# May either be "true" or "false" to enable globally, or contain a list +# of data centers to enable per-datacenter. +# hinted_handoff_enabled: DC1,DC2 +# hinted_handoff_enabled: true + +# this defines the maximum amount of time a dead host will have hints +# generated. After it has been dead this long, new hints for it will not be +# created until it has been seen alive and gone down again. +# max_hint_window_in_ms: 10800000 # 3 hours + +# Validity period for permissions cache (fetching permissions can be an +# expensive operation depending on the authorizer, CassandraAuthorizer is +# one example). Defaults to 10000, set to 0 to disable. +# Will be disabled automatically for AllowAllAuthorizer. +# permissions_validity_in_ms: 10000 + +# Refresh interval for permissions cache (if enabled). +# After this interval, cache entries become eligible for refresh. Upon next +# access, an async reload is scheduled and the old value returned until it +# completes. If permissions_validity_in_ms is non-zero, then this also must have +# a non-zero value. Defaults to 2000. It's recommended to set this value to +# be at least 3 times smaller than the permissions_validity_in_ms. +# permissions_update_interval_in_ms: 2000 + +# The partitioner is responsible for distributing groups of rows (by +# partition key) across nodes in the cluster. You should leave this +# alone for new clusters. The partitioner can NOT be changed without +# reloading all data, so when upgrading you should set this to the +# same partitioner you were already using. +# +# Murmur3Partitioner is currently the only supported partitioner, +# +partitioner: org.apache.cassandra.dht.Murmur3Partitioner + +# Total space to use for commitlogs. +# +# If space gets above this value (it will round up to the next nearest +# segment multiple), Scylla will flush every dirty CF in the oldest +# segment and remove it. So a small total commitlog space will tend +# to cause more flush activity on less-active columnfamilies. +# +# A value of -1 (default) will automatically equate it to the total amount of memory +# available for Scylla. +commitlog_total_space_in_mb: -1 + +# TCP port, for commands and data +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# storage_port: 7000 + +# SSL port, for encrypted communication. Unused unless enabled in +# encryption_options +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# ssl_storage_port: 7001 + +# listen_interface: eth0 +# listen_interface_prefer_ipv6: false + +# Whether to start the native transport server. +# Please note that the address on which the native transport is bound is the +# same as the rpc_address. The port however is different and specified below. +# start_native_transport: true + +# The maximum size of allowed frame. Frame (requests) larger than this will +# be rejected as invalid. The default is 256MB. +# native_transport_max_frame_size_in_mb: 256 + +# Whether to start the thrift rpc server. +# start_rpc: true + +# enable or disable keepalive on rpc/native connections +# rpc_keepalive: true + +# Set to true to have Scylla create a hard link to each sstable +# flushed or streamed locally in a backups/ subdirectory of the +# keyspace data. Removing these links is the operator's +# responsibility. +# incremental_backups: false + +# Whether or not to take a snapshot before each compaction. Be +# careful using this option, since Scylla won't clean up the +# snapshots for you. Mostly useful if you're paranoid when there +# is a data format change. +# snapshot_before_compaction: false + +# Whether or not a snapshot is taken of the data before keyspace truncation +# or dropping of column families. The STRONGLY advised default of true +# should be used to provide data safety. If you set this flag to false, you will +# lose data on truncation or drop. +# auto_snapshot: true + +# When executing a scan, within or across a partition, we need to keep the +# tombstones seen in memory so we can return them to the coordinator, which +# will use them to make sure other replicas also know about the deleted rows. +# With workloads that generate a lot of tombstones, this can cause performance +# problems and even exhaust the server heap. +# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets) +# Adjust the thresholds here if you understand the dangers and want to +# scan more tombstones anyway. These thresholds may also be adjusted at runtime +# using the StorageService mbean. +# tombstone_warn_threshold: 1000 +# tombstone_failure_threshold: 100000 + +# Granularity of the collation index of rows within a partition. +# Increase if your rows are large, or if you have a very large +# number of rows per partition. The competing goals are these: +# 1) a smaller granularity means more index entries are generated +# and looking up rows within the partition by collation column +# is faster +# 2) but, Scylla will keep the collation index in memory for hot +# rows (as part of the key cache), so a larger granularity means +# you can cache more hot rows +# column_index_size_in_kb: 64 + +# Auto-scaling of the promoted index prevents running out of memory +# when the promoted index grows too large (due to partitions with many rows +# vs. too small column_index_size_in_kb). When the serialized representation +# of the promoted index grows by this threshold, the desired block size +# for this partition (initialized to column_index_size_in_kb) +# is doubled, to decrease the sampling resolution by half. +# +# To disable promoted index auto-scaling, set the threshold to 0. +# column_index_auto_scale_threshold_in_kb: 10240 + +# Log a warning when writing partitions larger than this value +# compaction_large_partition_warning_threshold_mb: 1000 + +# Log a warning when writing rows larger than this value +# compaction_large_row_warning_threshold_mb: 10 + +# Log a warning when writing cells larger than this value +# compaction_large_cell_warning_threshold_mb: 1 + +# Log a warning when row number is larger than this value +# compaction_rows_count_warning_threshold: 100000 + +# Log a warning when writing a collection containing more elements than this value +# compaction_collection_elements_count_warning_threshold: 10000 + +# How long the coordinator should wait for seq or index scans to complete +# range_request_timeout_in_ms: 10000 +# How long the coordinator should wait for writes to complete +# counter_write_request_timeout_in_ms: 5000 +# How long a coordinator should continue to retry a CAS operation +# that contends with other proposals for the same row +# cas_contention_timeout_in_ms: 1000 +# How long the coordinator should wait for truncates to complete +# (This can be much longer, because unless auto_snapshot is disabled +# we need to flush first so we can snapshot before removing the data.) +# truncate_request_timeout_in_ms: 60000 +# The default timeout for other, miscellaneous operations +# request_timeout_in_ms: 10000 + +# Enable or disable inter-node encryption. +# You must also generate keys and provide the appropriate key and trust store locations and passwords. +# +# The available internode options are : all, none, dc, rack +# If set to dc scylla will encrypt the traffic between the DCs +# If set to rack scylla will encrypt the traffic between the racks +# +# SSL/TLS algorithm and ciphers used can be controlled by +# the priority_string parameter. Info on priority string +# syntax and values is available at: +# https://gnutls.org/manual/html_node/Priority-Strings.html +# +# The require_client_auth parameter allows you to +# restrict access to service based on certificate +# validation. Client must provide a certificate +# accepted by the used trust store to connect. +# +# server_encryption_options: +# internode_encryption: none +# certificate: conf/scylla.crt +# keyfile: conf/scylla.key +# truststore: +# certficate_revocation_list: +# require_client_auth: False +# priority_string: + +# enable or disable client/server encryption. +# client_encryption_options: +# enabled: false +# certificate: conf/scylla.crt +# keyfile: conf/scylla.key +# truststore: +# certficate_revocation_list: +# require_client_auth: False +# priority_string: + +# internode_compression controls whether traffic between nodes is +# compressed. +# can be: all - all traffic is compressed +# dc - traffic between different datacenters is compressed +# none - nothing is compressed. +# internode_compression: none + +# Enable or disable tcp_nodelay for inter-dc communication. +# Disabling it will result in larger (but fewer) network packets being sent, +# reducing overhead from the TCP protocol itself, at the cost of increasing +# latency if you block for cross-datacenter responses. +# inter_dc_tcp_nodelay: false + +# Relaxation of environment checks. +# +# Scylla places certain requirements on its environment. If these requirements are +# not met, performance and reliability can be degraded. +# +# These requirements include: +# - A filesystem with good support for asynchronous I/O (AIO). Currently, +# this means XFS. +# +# false: strict environment checks are in place; do not start if they are not met. +# true: relaxed environment checks; performance and reliability may degraade. +# +# developer_mode: false + +# Idle-time background processing +# +# Scylla can perform certain jobs in the background while the system is otherwise idle, +# freeing processor resources when there is other work to be done. +# +# defragment_memory_on_idle: true +# +# prometheus port +# By default, Scylla opens prometheus API port on port 9180 +# setting the port to 0 will disable the prometheus API. +# prometheus_port: 9180 +# +# prometheus address +# Leaving this blank will set it to the same value as listen_address. +# This means that by default, Scylla listens to the prometheus API on the same +# listening address (and therefore network interface) used to listen for +# internal communication. If the monitoring node is not in this internal +# network, you can override prometheus_address explicitly - e.g., setting +# it to 0.0.0.0 to listen on all interfaces. +# prometheus_address: 1.2.3.4 + +# Distribution of data among cores (shards) within a node +# +# Scylla distributes data within a node among shards, using a round-robin +# strategy: +# [shard0] [shard1] ... [shardN-1] [shard0] [shard1] ... [shardN-1] ... +# +# Scylla versions 1.6 and below used just one repetition of the pattern; +# this interfered with data placement among nodes (vnodes). +# +# Scylla versions 1.7 and above use 4096 repetitions of the pattern; this +# provides for better data distribution. +# +# the value below is log (base 2) of the number of repetitions. +# +# Set to 0 to avoid rewriting all data when upgrading from Scylla 1.6 and +# below. +# +# Keep at 12 for new clusters. +murmur3_partitioner_ignore_msb_bits: 12 + +# Bypass in-memory data cache (the row cache) when performing reversed queries. +# reversed_reads_auto_bypass_cache: false + +# Use a new optimized algorithm for performing reversed reads. +# Set to `false` to fall-back to the old algorithm. +# enable_optimized_reversed_reads: true + +# Use on a new, parallel algorithm for performing aggregate queries. +# Set to `false` to fall-back to the old algorithm. +# enable_parallelized_aggregation: true + +# Time for which task manager task is kept in memory after it completes. +# task_ttl_in_seconds: 0 + +# In materialized views, restrictions are allowed only on the view's primary key columns. +# In old versions Scylla mistakenly allowed IS NOT NULL restrictions on columns which were not part +# of the view's primary key. These invalid restrictions were ignored. +# This option controls the behavior when someone tries to create a view with such invalid IS NOT NULL restrictions. +# +# Can be true, false, or warn. +# * `true`: IS NOT NULL is allowed only on the view's primary key columns, +# trying to use it on other columns will cause an error, as it should. +# * `false`: Scylla accepts IS NOT NULL restrictions on regular columns, but they're silently ignored. +# It's useful for backwards compatibility. +# * `warn`: The same as false, but there's a warning about invalid view restrictions. +# +# To preserve backwards compatibility on old clusters, Scylla's default setting is `warn`. +# New clusters have this option set to `true` by scylla.yaml (which overrides the default `warn`) +# to make sure that trying to create an invalid view causes an error. +strict_is_not_null_in_views: true + +# The Unix Domain Socket the node uses for maintenance socket. +# The possible options are: +# * ignore: the node will not open the maintenance socket, +# * workdir: the node will open the maintenance socket on the path /cql.m, +# where is a path defined by the workdir configuration option, +# * : the node will open the maintenance socket on the path . +maintenance_socket: ignore + +# If set to true, configuration parameters defined with LiveUpdate option can be updated in runtime with CQL +# by updating system.config virtual table. If we don't want any configuration parameter to be changed in runtime +# via CQL, this option should be set to false. This parameter doesn't impose any limits on other mechanisms updating +# configuration parameters in runtime, e.g. sending SIGHUP or using API. This option should be set to false +# e.g. for cloud users, for whom scylla's configuration should be changed only by support engineers. +# live_updatable_config_params_changeable_via_cql: true + +# **************** +# * GUARDRAILS * +# **************** + +# Guardrails to warn or fail when Replication Factor is smaller/greater than the threshold. +# Please note that the value of 0 is always allowed, +# which means that having no replication at all, i.e. RF = 0, is always valid. +# A guardrail value smaller than 0, e.g. -1, means that the guardrail is disabled. +# Commenting out a guardrail also means it is disabled. +# minimum_replication_factor_fail_threshold: -1 +# minimum_replication_factor_warn_threshold: 3 +# maximum_replication_factor_warn_threshold: -1 +# maximum_replication_factor_fail_threshold: -1 + +# Guardrails to warn about or disallow creating a keyspace with specific replication strategy. +# Each of these 2 settings is a list storing replication strategies considered harmful. +# The replication strategies to choose from are: +# 1) SimpleStrategy, +# 2) NetworkTopologyStrategy, +# 3) LocalStrategy, +# 4) EverywhereStrategy +# +# replication_strategy_warn_list: +# - SimpleStrategy +# replication_strategy_fail_list: + +# This enables tablets on newly created keyspaces +enable_tablets: true +api_ui_dir: /opt/scylladb/swagger-ui/dist/ +api_doc_dir: /opt/scylladb/api/api-doc/ diff --git a/catalyst-gateway/tests/schemathesis_tests/schemathesis-docker-compose.yml b/catalyst-gateway/tests/schemathesis_tests/schemathesis-docker-compose.yml index 01acbeab551..6ad468f8953 100644 --- a/catalyst-gateway/tests/schemathesis_tests/schemathesis-docker-compose.yml +++ b/catalyst-gateway/tests/schemathesis_tests/schemathesis-docker-compose.yml @@ -1,6 +1,58 @@ version: "3" -name: myproject +name: myproject services: + scylla-node1: + container_name: scylla-node1 + image: scylladb/scylla:latest + restart: unless-stopped + command: --seeds=scylla-node1 --smp 2 --memory 1G --overprovisioned 1 --api-address 0.0.0.0 --broadcast-rpc-address ${HOST_IP} + ports: + - "9042:9042" + volumes: + - "/var/lib/scylla/1:/var/lib/scylla" + - "./node1-scylla.yaml:/etc/scylla/scylla.yaml" + networks: + cluster: + + scylla-node2: + container_name: scylla-node2 + image: scylladb/scylla:latest + restart: unless-stopped + command: --seeds=scylla-node1 --smp 2 --memory 1G --overprovisioned 1 --api-address 0.0.0.0 --broadcast-rpc-address ${HOST_IP} + ports: + - "9043:9043" + volumes: + - "/var/lib/scylla/2:/var/lib/scylla" + - "./node2-scylla.yaml:/etc/scylla/scylla.yaml" + networks: + cluster: + + scylla-node3: + container_name: scylla-node3 + image: scylladb/scylla:latest + restart: unless-stopped + command: --seeds=scylla-node1 --smp 2 --memory 1G --overprovisioned 1 --api-address 0.0.0.0 --broadcast-rpc-address ${HOST_IP} + ports: + - "9044:9044" + volumes: + - "/var/lib/scylla/3:/var/lib/scylla" + - "./node3-scylla.yaml:/etc/scylla/scylla.yaml" + networks: + cluster: + + scylla-node4: + container_name: scylla-node4 + image: scylladb/scylla:latest + restart: unless-stopped + command: --seeds=scylla-node1 --smp 2 --memory 1G --overprovisioned 1 --api-address 0.0.0.0 --broadcast-rpc-address ${HOST_IP} + ports: + - "9045:9045" + volumes: + - "/var/lib/scylla/4:/var/lib/scylla" + - "./node4-scylla.yaml:/etc/scylla/scylla.yaml" + networks: + cluster: + event-db: image: event-db:latest environment: @@ -34,8 +86,15 @@ services: - DATA_REFRESH_TICK=5 - CHECK_CONFIG_TICK=5 - MACHINE_ID="UID" + - CHAIN_FOLLOWER_SYNC_TASKS="16" + - RUST_LOG="error,cat_gateway=debug,cardano_chain_follower=debug,mithril-client=debug" + - CHAIN_NETWORK="Preprod" ports: - 3030:3030 depends_on: event-db: condition: service_healthy + +networks: + cluster: + driver: bridge diff --git a/utilities/local-scylla/Earthfile b/utilities/local-scylla/Earthfile new file mode 100644 index 00000000000..6c9e4755e37 --- /dev/null +++ b/utilities/local-scylla/Earthfile @@ -0,0 +1,22 @@ +VERSION 0.8 + +test: + FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 + COPY docker-compose.yml . + COPY node1-scylla.yaml . + COPY node2-scylla.yaml . + COPY node3-scylla.yaml . + COPY node4-scylla.yaml . + ENV HOST_IP + RUN export HOST_IP="$(ifconfig $(route -n | grep ^0.0.0.0 | awk '{print $NF}') | grep inet | grep -v inet6 | awk '{print $2}')" + WITH DOCKER \ + --compose docker-compose.yml \ + --service scylla-node1 \ + --service scylla-node2 \ + --service scylla-node3 \ + --service scylla-node4 \ + --allow-privileged + + RUN echo 'lalala' && echo $HOST_IP + + END \ No newline at end of file From 2d6f9a7ab1ddce9c7ca7ae61c970cb49b34452d2 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Wed, 18 Dec 2024 15:46:38 +0100 Subject: [PATCH 02/33] wip --- .../tests/schemathesis_tests/Earthfile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 469692f4af7..291b8255e2c 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -74,3 +74,17 @@ fuzzer-api: # SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis.junit-report.xml # SAVE ARTIFACT cassette.yaml AS LOCAL cassette.yaml # END + +test-fuzzer-api: + FROM debian:12.7-slim + + RUN apt-get update && apt install -y gnupg2 wget + RUN mkdir -p /etc/apt/keyrings + RUN gpg --homedir /tmp --no-default-keyring --keyring /etc/apt/keyrings/scylladb.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys a43e06657bac99e3 + RUN wget -O /etc/apt/sources.list.d/scylla.list http://downloads.scylladb.com/deb/debian/scylla-6.2.list + RUN apt-get update && apt-get install -y scylla + + COPY node1-scylla.yaml /etc/scylla/scylla.yaml + #RUN scylla_setup -y + #https://github.com/scylladb/scylladb/issues/3142 + RUN scylla --options-file /etc/scylla/scylla.yaml --developer-mode 1 \ No newline at end of file From 1b162b58db318a44fb06a4a6cd0c993773548169 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:58:06 +0100 Subject: [PATCH 03/33] wip --- catalyst-gateway/tests/schemathesis_tests/Earthfile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 291b8255e2c..c7c584822c8 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -77,6 +77,13 @@ fuzzer-api: test-fuzzer-api: FROM debian:12.7-slim + ENV LOG_LEVEL=error + ENV DATA_REFRESH_TICK=5 + ENV CHECK_CONFIG_TICK=5 + ENV MACHINE_ID="UID" + ENV CHAIN_FOLLOWER_SYNC_TASKS="16" + ENV RUST_LOG="error,cat_gateway=debug,cardano_chain_follower=debug,mithril-client=debug" + ENV CHAIN_NETWORK="Preprod" RUN apt-get update && apt install -y gnupg2 wget RUN mkdir -p /etc/apt/keyrings @@ -87,4 +94,5 @@ test-fuzzer-api: COPY node1-scylla.yaml /etc/scylla/scylla.yaml #RUN scylla_setup -y #https://github.com/scylladb/scylladb/issues/3142 - RUN scylla --options-file /etc/scylla/scylla.yaml --developer-mode 1 \ No newline at end of file + COPY ../../+build/cat-gateway . + RUN ( scylla --options-file /etc/scylla/scylla.yaml --developer-mode 1 & ) && ./cat-gateway run \ No newline at end of file From e25e84cf475575a82b0faef5bc76fa3cb06aed61 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Fri, 20 Dec 2024 13:50:24 +0100 Subject: [PATCH 04/33] wip --- catalyst-gateway/bin/Cargo.toml | 2 +- .../tests/schemathesis_tests/Earthfile | 68 ++++++++++--------- .../schemathesis-docker-compose.yml | 62 +---------------- 3 files changed, 39 insertions(+), 93 deletions(-) diff --git a/catalyst-gateway/bin/Cargo.toml b/catalyst-gateway/bin/Cargo.toml index 4f664ec101d..75c578f6d92 100644 --- a/catalyst-gateway/bin/Cargo.toml +++ b/catalyst-gateway/bin/Cargo.toml @@ -15,7 +15,7 @@ repository.workspace = true workspace = true [dependencies] -cardano-chain-follower = { version = "0.0.5", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "v0.0.9" } +cardano-chain-follower = { version = "0.0.6", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "v0.0.10" } c509-certificate = { version = "0.0.3", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "v0.0.3" } rbac-registration = { version = "0.0.2", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "v0.0.8" } diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index c7c584822c8..974aec2f5c6 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -12,7 +12,7 @@ package-schemathesis: ENTRYPOINT export SCHEMATHESIS_HOOKS=hooks \ && st run --checks all $openapi_spec \ --workers=2 \ - --wait-for-schema=15 \ + --wait-for-schema=120 \ --max-response-time=5000 \ --hypothesis-max-examples=1000 \ --data-generation-method=all \ @@ -33,50 +33,42 @@ fuzzer-api: RUN apk update && apk add iptables-legacy # workaround for https://github.com/earthly/earthly/issues/3784 RUN apk add yq zstd curl COPY schemathesis-docker-compose.yml . - COPY node1-scylla.yaml . - COPY node2-scylla.yaml . - COPY node3-scylla.yaml . - COPY node4-scylla.yaml . - ENV HOST_IP - RUN export HOST_IP="$(ifconfig $(route -n | grep ^0.0.0.0 | awk '{print $NF}') | grep inet | grep -v inet6 | awk '{print $2}')" + # COPY node1-scylla.yaml . + # COPY node2-scylla.yaml . + # COPY node3-scylla.yaml . + # COPY node4-scylla.yaml . + #ENV HOST_IP + #RUN export HOST_IP="$(ifconfig $(route -n | grep ^0.0.0.0 | awk '{print $NF}') | grep inet | grep -v inet6 | awk '{print $2}')" LET OPENAPI_SPEC="http://127.0.0.1:3030/docs/cat-gateway.json" - ENV LOG_LEVEL=error - ENV DATA_REFRESH_TICK=5 - ENV CHECK_CONFIG_TICK=5 - ENV MACHINE_ID="UID" - ENV CHAIN_FOLLOWER_SYNC_TASKS="16" - ENV RUST_LOG="error,cat_gateway=debug,cardano_chain_follower=debug,mithril-client=debug" - ENV CHAIN_NETWORK="Preprod" + #ENV LOG_LEVEL=error + #ENV DATA_REFRESH_TICK=5 + #ENV CHECK_CONFIG_TICK=5 + #ENV MACHINE_ID="UID" + #ENV CHAIN_FOLLOWER_SYNC_TASKS="16" + #ENV RUST_LOG="error,cat_gateway=debug,cardano_chain_follower=debug,mithril-client=debug" + #ENV CHAIN_NETWORK="Preprod" WITH DOCKER \ --compose schemathesis-docker-compose.yml \ - #--load schemathesis:latest=(+package-schemathesis --openapi_spec=$OPENAPI_SPEC) \ + --load schemathesis:latest=(+package-schemathesis --openapi_spec=$OPENAPI_SPEC) \ #--load event-db:latest=(../../event-db+build) \ - --load cat-gateway:latest=(../../+package) \ + --load cat-gateway:latest=(+package-cat-gateway-integration) \ #--service event-db \ - --service scylla-node1 \ - --service scylla-node2 \ - --service scylla-node3 \ - --service scylla-node4 \ - #--service cat-gateway \ + #--service scylla-node1 \ + #--service scylla-node2 \ + #--service scylla-node3 \ + #--service scylla-node4 \ + --service cat-gateway \ --allow-privileged - RUN docker run --net=default_cluster \ - -e LOG_LEVEL \ - -e DATA_REFRESH_TICK \ - -e CHECK_CONFIG_TICK \ - -e MACHINE_ID \ - -e CHAIN_FOLLOWER_SYNC_TASKS \ - -e RUST_LOG="error,cat_gateway=debug,cardano_chain_follower=debug,mithril-client=debug" \ - -e CHAIN_NETWORK="Preprod" \ - cat-gateway:latest + RUN sleep 120 && wget http://127.0.0.1:3030/docs/cat-gateway.json && docker run schemathesis:latest --net=host END # WAIT # SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis.junit-report.xml # SAVE ARTIFACT cassette.yaml AS LOCAL cassette.yaml # END -test-fuzzer-api: - FROM debian:12.7-slim +package-cat-gateway-integration: + FROM python:3.14.0a3-slim-bookworm ENV LOG_LEVEL=error ENV DATA_REFRESH_TICK=5 ENV CHECK_CONFIG_TICK=5 @@ -84,6 +76,8 @@ test-fuzzer-api: ENV CHAIN_FOLLOWER_SYNC_TASKS="16" ENV RUST_LOG="error,cat_gateway=debug,cardano_chain_follower=debug,mithril-client=debug" ENV CHAIN_NETWORK="Preprod" + ENV CAT_ADDRESS=0.0.0.0:3030 + ENV INTERNAL_API_KEY=123 RUN apt-get update && apt install -y gnupg2 wget RUN mkdir -p /etc/apt/keyrings @@ -95,4 +89,12 @@ test-fuzzer-api: #RUN scylla_setup -y #https://github.com/scylladb/scylladb/issues/3142 COPY ../../+build/cat-gateway . - RUN ( scylla --options-file /etc/scylla/scylla.yaml --developer-mode 1 & ) && ./cat-gateway run \ No newline at end of file + + #RUN --no-cache wget -qO cargo_install https://sh.rustup.rs/ && chmod +x cargo_install && ./cargo_install -y && . "$HOME/.cargo/env" && cargo --version && pip install schemathesis==3.27.1 + #RUN mkdir /results + #COPY ./hooks/hooks.py . + #RUN export SCHEMATHESIS_HOOKS=hooks + + ENTRYPOINT ( scylla --options-file /etc/scylla/scylla.yaml --developer-mode 1 --default-log-level error & ) && \ + ./cat-gateway run + SAVE IMAGE cat-gateway:latest diff --git a/catalyst-gateway/tests/schemathesis_tests/schemathesis-docker-compose.yml b/catalyst-gateway/tests/schemathesis_tests/schemathesis-docker-compose.yml index 6ad468f8953..8b51a90f09a 100644 --- a/catalyst-gateway/tests/schemathesis_tests/schemathesis-docker-compose.yml +++ b/catalyst-gateway/tests/schemathesis_tests/schemathesis-docker-compose.yml @@ -1,58 +1,6 @@ version: "3" name: myproject services: - scylla-node1: - container_name: scylla-node1 - image: scylladb/scylla:latest - restart: unless-stopped - command: --seeds=scylla-node1 --smp 2 --memory 1G --overprovisioned 1 --api-address 0.0.0.0 --broadcast-rpc-address ${HOST_IP} - ports: - - "9042:9042" - volumes: - - "/var/lib/scylla/1:/var/lib/scylla" - - "./node1-scylla.yaml:/etc/scylla/scylla.yaml" - networks: - cluster: - - scylla-node2: - container_name: scylla-node2 - image: scylladb/scylla:latest - restart: unless-stopped - command: --seeds=scylla-node1 --smp 2 --memory 1G --overprovisioned 1 --api-address 0.0.0.0 --broadcast-rpc-address ${HOST_IP} - ports: - - "9043:9043" - volumes: - - "/var/lib/scylla/2:/var/lib/scylla" - - "./node2-scylla.yaml:/etc/scylla/scylla.yaml" - networks: - cluster: - - scylla-node3: - container_name: scylla-node3 - image: scylladb/scylla:latest - restart: unless-stopped - command: --seeds=scylla-node1 --smp 2 --memory 1G --overprovisioned 1 --api-address 0.0.0.0 --broadcast-rpc-address ${HOST_IP} - ports: - - "9044:9044" - volumes: - - "/var/lib/scylla/3:/var/lib/scylla" - - "./node3-scylla.yaml:/etc/scylla/scylla.yaml" - networks: - cluster: - - scylla-node4: - container_name: scylla-node4 - image: scylladb/scylla:latest - restart: unless-stopped - command: --seeds=scylla-node1 --smp 2 --memory 1G --overprovisioned 1 --api-address 0.0.0.0 --broadcast-rpc-address ${HOST_IP} - ports: - - "9045:9045" - volumes: - - "/var/lib/scylla/4:/var/lib/scylla" - - "./node4-scylla.yaml:/etc/scylla/scylla.yaml" - networks: - cluster: - event-db: image: event-db:latest environment: @@ -91,10 +39,6 @@ services: - CHAIN_NETWORK="Preprod" ports: - 3030:3030 - depends_on: - event-db: - condition: service_healthy - -networks: - cluster: - driver: bridge + #depends_on: + # event-db: + # condition: service_healthy From cc8f968dfe9794002fd6dd20157237801038be33 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Mon, 23 Dec 2024 19:05:37 +0100 Subject: [PATCH 05/33] wip --- .../tests/schemathesis_tests/Earthfile | 26 +- .../schemathesis_tests/node2-scylla.yaml | 624 ------------------ .../schemathesis_tests/node3-scylla.yaml | 624 ------------------ .../schemathesis_tests/node4-scylla.yaml | 624 ------------------ .../schemathesis-docker-compose.yml | 10 - 5 files changed, 5 insertions(+), 1903 deletions(-) delete mode 100644 catalyst-gateway/tests/schemathesis_tests/node2-scylla.yaml delete mode 100644 catalyst-gateway/tests/schemathesis_tests/node3-scylla.yaml delete mode 100644 catalyst-gateway/tests/schemathesis_tests/node4-scylla.yaml diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 974aec2f5c6..0909f30aafc 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -10,7 +10,7 @@ package-schemathesis: VOLUME /results ARG openapi_spec ENTRYPOINT export SCHEMATHESIS_HOOKS=hooks \ - && st run --checks all $openapi_spec \ + && st run --checks all http://0.0.0.0:3030/docs/cat-gateway.json \ --workers=2 \ --wait-for-schema=120 \ --max-response-time=5000 \ @@ -33,34 +33,18 @@ fuzzer-api: RUN apk update && apk add iptables-legacy # workaround for https://github.com/earthly/earthly/issues/3784 RUN apk add yq zstd curl COPY schemathesis-docker-compose.yml . - # COPY node1-scylla.yaml . - # COPY node2-scylla.yaml . - # COPY node3-scylla.yaml . - # COPY node4-scylla.yaml . - #ENV HOST_IP - #RUN export HOST_IP="$(ifconfig $(route -n | grep ^0.0.0.0 | awk '{print $NF}') | grep inet | grep -v inet6 | awk '{print $2}')" - LET OPENAPI_SPEC="http://127.0.0.1:3030/docs/cat-gateway.json" - #ENV LOG_LEVEL=error - #ENV DATA_REFRESH_TICK=5 - #ENV CHECK_CONFIG_TICK=5 - #ENV MACHINE_ID="UID" - #ENV CHAIN_FOLLOWER_SYNC_TASKS="16" - #ENV RUST_LOG="error,cat_gateway=debug,cardano_chain_follower=debug,mithril-client=debug" - #ENV CHAIN_NETWORK="Preprod" + #LET OPENAPI_SPEC="http://cat-gateway:3030/docs/cat-gateway.json" + WITH DOCKER \ --compose schemathesis-docker-compose.yml \ - --load schemathesis:latest=(+package-schemathesis --openapi_spec=$OPENAPI_SPEC) \ + --load schemathesis:latest=(+package-schemathesis) \ #--load event-db:latest=(../../event-db+build) \ --load cat-gateway:latest=(+package-cat-gateway-integration) \ #--service event-db \ - #--service scylla-node1 \ - #--service scylla-node2 \ - #--service scylla-node3 \ - #--service scylla-node4 \ --service cat-gateway \ --allow-privileged - RUN sleep 120 && wget http://127.0.0.1:3030/docs/cat-gateway.json && docker run schemathesis:latest --net=host + RUN sleep 120 && docker run --net=host schemathesis:latest END # WAIT # SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis.junit-report.xml diff --git a/catalyst-gateway/tests/schemathesis_tests/node2-scylla.yaml b/catalyst-gateway/tests/schemathesis_tests/node2-scylla.yaml deleted file mode 100644 index d0f15402524..00000000000 --- a/catalyst-gateway/tests/schemathesis_tests/node2-scylla.yaml +++ /dev/null @@ -1,624 +0,0 @@ -# Scylla storage config YAML - -# cspell: words fsyncs rackdc partitioner mbean certficate degraade defragment - -####################################### -# This file is split to two sections: -# 1. Supported parameters -# 2. Unsupported parameters: reserved for future use or backwards -# compatibility. -# Scylla will only read and use the first segment -####################################### - -### Supported Parameters - -# The name of the cluster. This is mainly used to prevent machines in -# one logical cluster from joining another. -# It is recommended to change the default value when creating a new cluster. -# You can NOT modify this value for an existing cluster -#cluster_name: 'Test Cluster' - -# This defines the number of tokens randomly assigned to this node on the ring -# The more tokens, relative to other nodes, the larger the proportion of data -# that this node will store. You probably want all nodes to have the same number -# of tokens assuming they have equal hardware capability. -num_tokens: 256 - -# Directory where Scylla should store all its files, which are commitlog, -# data, hints, view_hints and saved_caches subdirectories. All of these -# subs can be overridden by the respective options below. -# If unset, the value defaults to /var/lib/scylla -# workdir: /var/lib/scylla - -# Directory where Scylla should store data on disk. -# data_file_directories: -# - /var/lib/scylla/data - -# commit log. when running on magnetic HDD, this should be a -# separate spindle than the data directories. -# commitlog_directory: /var/lib/scylla/commitlog - -# schema commit log. A special commitlog instance -# used for schema and system tables. -# When running on magnetic HDD, this should be a -# separate spindle than the data directories. -# schema_commitlog_directory: /var/lib/scylla/commitlog/schema - -# commitlog_sync may be either "periodic" or "batch." -# -# When in batch mode, Scylla won't ack writes until the commit log -# has been fsynced to disk. It will wait -# commitlog_sync_batch_window_in_ms milliseconds between fsyncs. -# This window should be kept short because the writer threads will -# be unable to do extra work while waiting. (You may need to increase -# concurrent_writes for the same reason.) -# -# commitlog_sync: batch -# commitlog_sync_batch_window_in_ms: 2 -# -# the other option is "periodic" where writes may be acked immediately -# and the CommitLog is simply synced every commitlog_sync_period_in_ms -# milliseconds. -commitlog_sync: periodic -commitlog_sync_period_in_ms: 10000 - -# The size of the individual commitlog file segments. A commitlog -# segment may be archived, deleted, or recycled once all the data -# in it (potentially from each columnfamily in the system) has been -# flushed to sstables. -# -# The default size is 32, which is almost always fine, but if you are -# archiving commitlog segments (see commitlog_archiving.properties), -# then you probably want a finer granularity of archiving; 8 or 16 MB -# is reasonable. -commitlog_segment_size_in_mb: 32 - -# The size of the individual schema commitlog file segments. -# -# The default size is 128, which is 4 times larger than the default -# size of the data commitlog. It's because the segment size puts -# a limit on the mutation size that can be written at once, and some -# schema mutation writes are much larger than average. -schema_commitlog_segment_size_in_mb: 128 - -# seed_provider class_name is saved for future use. -# A seed address is mandatory. -seed_provider: - # The addresses of hosts that will serve as contact points for the joining node. - # It allows the node to discover the cluster ring topology on startup (when - # joining the cluster). - # Once the node has joined the cluster, the seed list has no function. - - class_name: org.apache.cassandra.locator.SimpleSeedProvider - parameters: - # In a new cluster, provide the address of the first node. - # In an existing cluster, specify the address of at least one existing node. - # If you specify addresses of more than one node, use a comma to separate them. - # For example: ",," - - seeds: "127.0.0.1" - -# Address to bind to and tell other Scylla nodes to connect to. -# You _must_ change this if you want multiple nodes to be able to communicate! -# -# If you leave broadcast_address (below) empty, then setting listen_address -# to 0.0.0.0 is wrong as other nodes will not know how to reach this node. -# If you set broadcast_address, then you can set listen_address to 0.0.0.0. -listen_address: localhost - -# Address to broadcast to other Scylla nodes -# Leaving this blank will set it to the same value as listen_address -# broadcast_address: 1.2.3.4 - -# When using multiple physical network interfaces, set this to true to listen on broadcast_address -# in addition to the listen_address, allowing nodes to communicate in both interfaces. -# Ignore this property if the network configuration automatically routes between the public and private networks such as EC2. -# -# listen_on_broadcast_address: false - -# port for the CQL native transport to listen for clients on -# For security reasons, you should not expose this port to the internet. Firewall it if needed. -# To disable the CQL native transport, remove this option and configure native_transport_port_ssl. -native_transport_port: 9043 - -# Like native_transport_port, but clients are forwarded to specific shards, based on the -# client-side port numbers. -native_shard_aware_transport_port: 19043 - -# Enabling native transport encryption in client_encryption_options allows you to either use -# encryption for the standard port or to use a dedicated, additional port along with the unencrypted -# standard native_transport_port. -# Enabling client encryption and keeping native_transport_port_ssl disabled will use encryption -# for native_transport_port. Setting native_transport_port_ssl to a different value -# from native_transport_port will use encryption for native_transport_port_ssl while -# keeping native_transport_port unencrypted. -#native_transport_port_ssl: 9142 - -# Like native_transport_port_ssl, but clients are forwarded to specific shards, based on the -# client-side port numbers. -#native_shard_aware_transport_port_ssl: 19142 - -# How long the coordinator should wait for read operations to complete -read_request_timeout_in_ms: 5000 - -# How long the coordinator should wait for writes to complete -write_request_timeout_in_ms: 2000 -# how long a coordinator should continue to retry a CAS operation -# that contends with other proposals for the same row -cas_contention_timeout_in_ms: 1000 - -# phi value that must be reached for a host to be marked down. -# most users should never need to adjust this. -# phi_convict_threshold: 8 - -# IEndpointSnitch. The snitch has two functions: -# - it teaches Scylla enough about your network topology to route -# requests efficiently -# - it allows Scylla to spread replicas around your cluster to avoid -# correlated failures. It does this by grouping machines into -# "datacenters" and "racks." Scylla will do its best not to have -# more than one replica on the same "rack" (which may not actually -# be a physical location) -# -# IF YOU CHANGE THE SNITCH AFTER DATA IS INSERTED INTO THE CLUSTER, -# YOU MUST RUN A FULL REPAIR, SINCE THE SNITCH AFFECTS WHERE REPLICAS -# ARE PLACED. -# -# Out of the box, Scylla provides -# - SimpleSnitch: -# Treats Strategy order as proximity. This can improve cache -# locality when disabling read repair. Only appropriate for -# single-datacenter deployments. -# - GossipingPropertyFileSnitch -# This should be your go-to snitch for production use. The rack -# and datacenter for the local node are defined in -# cassandra-rackdc.properties and propagated to other nodes via -# gossip. If cassandra-topology.properties exists, it is used as a -# fallback, allowing migration from the PropertyFileSnitch. -# - PropertyFileSnitch: -# Proximity is determined by rack and data center, which are -# explicitly configured in cassandra-topology.properties. -# - Ec2Snitch: -# Appropriate for EC2 deployments in a single Region. Loads Region -# and Availability Zone information from the EC2 API. The Region is -# treated as the datacenter, and the Availability Zone as the rack. -# Only private IPs are used, so this will not work across multiple -# Regions. -# - Ec2MultiRegionSnitch: -# Uses public IPs as broadcast_address to allow cross-region -# connectivity. (Thus, you should set seed addresses to the public -# IP as well.) You will need to open the storage_port or -# ssl_storage_port on the public IP firewall. (For intra-Region -# traffic, Scylla will switch to the private IP after -# establishing a connection.) -# - RackInferringSnitch: -# Proximity is determined by rack and data center, which are -# assumed to correspond to the 3rd and 2nd octet of each node's IP -# address, respectively. Unless this happens to match your -# deployment conventions, this is best used as an example of -# writing a custom Snitch class and is provided in that spirit. -# -# You can use a custom Snitch by setting this to the full class name -# of the snitch, which will be assumed to be on your classpath. -endpoint_snitch: SimpleSnitch - -# The address or interface to bind the Thrift RPC service and native transport -# server to. -# -# Set rpc_address OR rpc_interface, not both. Interfaces must correspond -# to a single address, IP aliasing is not supported. -# -# Leaving rpc_address blank has the same effect as on listen_address -# (i.e. it will be based on the configured hostname of the node). -# -# Note that unlike listen_address, you can specify 0.0.0.0, but you must also -# set broadcast_rpc_address to a value other than 0.0.0.0. -# -# For security reasons, you should not expose this port to the internet. Firewall it if needed. -# -# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address -# you can specify which should be chosen using rpc_interface_prefer_ipv6. If false the first ipv4 -# address will be used. If true the first ipv6 address will be used. Defaults to false preferring -# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. -rpc_address: localhost -# rpc_interface: eth1 -# rpc_interface_prefer_ipv6: false - -# port for Thrift to listen for clients on -rpc_port: 9160 - -# port for REST API server -api_port: 10000 - -# IP for the REST API server -api_address: 127.0.0.1 - -# Log WARN on any batch size exceeding this value. 128 kiB per batch by default. -# Caution should be taken on increasing the size of this threshold as it can lead to node instability. -batch_size_warn_threshold_in_kb: 128 - -# Fail any multiple-partition batch exceeding this value. 1 MiB (8x warn threshold) by default. -batch_size_fail_threshold_in_kb: 1024 - -# Authentication backend, identifying users -# Out of the box, Scylla provides org.apache.cassandra.auth.{AllowAllAuthenticator, -# PasswordAuthenticator}. -# -# - AllowAllAuthenticator performs no checks - set it to disable authentication. -# - PasswordAuthenticator relies on username/password pairs to authenticate -# users. It keeps usernames and hashed passwords in system_auth.credentials table. -# Please increase system_auth keyspace replication factor if you use this authenticator. -# - com.scylladb.auth.TransitionalAuthenticator requires username/password pair -# to authenticate in the same manner as PasswordAuthenticator, but improper credentials -# result in being logged in as an anonymous user. Use for upgrading clusters' auth. -# authenticator: AllowAllAuthenticator - -# Authorization backend, implementing IAuthorizer; used to limit access/provide permissions -# Out of the box, Scylla provides org.apache.cassandra.auth.{AllowAllAuthorizer, -# CassandraAuthorizer}. -# -# - AllowAllAuthorizer allows any action to any user - set it to disable authorization. -# - CassandraAuthorizer stores permissions in system_auth.permissions table. Please -# increase system_auth keyspace replication factor if you use this authorizer. -# - com.scylladb.auth.TransitionalAuthorizer wraps around the CassandraAuthorizer, using it for -# authorizing permission management. Otherwise, it allows all. Use for upgrading -# clusters' auth. -# authorizer: AllowAllAuthorizer - -# initial_token allows you to specify tokens manually. While you can use # it with -# vnodes (num_tokens > 1, above) -- in which case you should provide a -# comma-separated list -- it's primarily used when adding nodes # to legacy clusters -# that do not have vnodes enabled. -# initial_token: - -# RPC address to broadcast to drivers and other Scylla nodes. This cannot -# be set to 0.0.0.0. If left blank, this will be set to the value of -# rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must -# be set. -# broadcast_rpc_address: 1.2.3.4 - -# Uncomment to enable experimental features -# experimental_features: -# - udf -# - alternator-streams -# - broadcast-tables -# - keyspace-storage-options -# - tablets - -# The directory where hints files are stored if hinted handoff is enabled. -# hints_directory: /var/lib/scylla/hints - -# The directory where hints files are stored for materialized-view updates -# view_hints_directory: /var/lib/scylla/view_hints - -# See https://docs.scylladb.com/architecture/anti-entropy/hinted-handoff -# May either be "true" or "false" to enable globally, or contain a list -# of data centers to enable per-datacenter. -# hinted_handoff_enabled: DC1,DC2 -# hinted_handoff_enabled: true - -# this defines the maximum amount of time a dead host will have hints -# generated. After it has been dead this long, new hints for it will not be -# created until it has been seen alive and gone down again. -# max_hint_window_in_ms: 10800000 # 3 hours - -# Validity period for permissions cache (fetching permissions can be an -# expensive operation depending on the authorizer, CassandraAuthorizer is -# one example). Defaults to 10000, set to 0 to disable. -# Will be disabled automatically for AllowAllAuthorizer. -# permissions_validity_in_ms: 10000 - -# Refresh interval for permissions cache (if enabled). -# After this interval, cache entries become eligible for refresh. Upon next -# access, an async reload is scheduled and the old value returned until it -# completes. If permissions_validity_in_ms is non-zero, then this also must have -# a non-zero value. Defaults to 2000. It's recommended to set this value to -# be at least 3 times smaller than the permissions_validity_in_ms. -# permissions_update_interval_in_ms: 2000 - -# The partitioner is responsible for distributing groups of rows (by -# partition key) across nodes in the cluster. You should leave this -# alone for new clusters. The partitioner can NOT be changed without -# reloading all data, so when upgrading you should set this to the -# same partitioner you were already using. -# -# Murmur3Partitioner is currently the only supported partitioner, -# -partitioner: org.apache.cassandra.dht.Murmur3Partitioner - -# Total space to use for commitlogs. -# -# If space gets above this value (it will round up to the next nearest -# segment multiple), Scylla will flush every dirty CF in the oldest -# segment and remove it. So a small total commitlog space will tend -# to cause more flush activity on less-active columnfamilies. -# -# A value of -1 (default) will automatically equate it to the total amount of memory -# available for Scylla. -commitlog_total_space_in_mb: -1 - -# TCP port, for commands and data -# For security reasons, you should not expose this port to the internet. Firewall it if needed. -# storage_port: 7000 - -# SSL port, for encrypted communication. Unused unless enabled in -# encryption_options -# For security reasons, you should not expose this port to the internet. Firewall it if needed. -# ssl_storage_port: 7001 - -# listen_interface: eth0 -# listen_interface_prefer_ipv6: false - -# Whether to start the native transport server. -# Please note that the address on which the native transport is bound is the -# same as the rpc_address. The port however is different and specified below. -# start_native_transport: true - -# The maximum size of allowed frame. Frame (requests) larger than this will -# be rejected as invalid. The default is 256MB. -# native_transport_max_frame_size_in_mb: 256 - -# Whether to start the thrift rpc server. -# start_rpc: true - -# enable or disable keepalive on rpc/native connections -# rpc_keepalive: true - -# Set to true to have Scylla create a hard link to each sstable -# flushed or streamed locally in a backups/ subdirectory of the -# keyspace data. Removing these links is the operator's -# responsibility. -# incremental_backups: false - -# Whether or not to take a snapshot before each compaction. Be -# careful using this option, since Scylla won't clean up the -# snapshots for you. Mostly useful if you're paranoid when there -# is a data format change. -# snapshot_before_compaction: false - -# Whether or not a snapshot is taken of the data before keyspace truncation -# or dropping of column families. The STRONGLY advised default of true -# should be used to provide data safety. If you set this flag to false, you will -# lose data on truncation or drop. -# auto_snapshot: true - -# When executing a scan, within or across a partition, we need to keep the -# tombstones seen in memory so we can return them to the coordinator, which -# will use them to make sure other replicas also know about the deleted rows. -# With workloads that generate a lot of tombstones, this can cause performance -# problems and even exhaust the server heap. -# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets) -# Adjust the thresholds here if you understand the dangers and want to -# scan more tombstones anyway. These thresholds may also be adjusted at runtime -# using the StorageService mbean. -# tombstone_warn_threshold: 1000 -# tombstone_failure_threshold: 100000 - -# Granularity of the collation index of rows within a partition. -# Increase if your rows are large, or if you have a very large -# number of rows per partition. The competing goals are these: -# 1) a smaller granularity means more index entries are generated -# and looking up rows within the partition by collation column -# is faster -# 2) but, Scylla will keep the collation index in memory for hot -# rows (as part of the key cache), so a larger granularity means -# you can cache more hot rows -# column_index_size_in_kb: 64 - -# Auto-scaling of the promoted index prevents running out of memory -# when the promoted index grows too large (due to partitions with many rows -# vs. too small column_index_size_in_kb). When the serialized representation -# of the promoted index grows by this threshold, the desired block size -# for this partition (initialized to column_index_size_in_kb) -# is doubled, to decrease the sampling resolution by half. -# -# To disable promoted index auto-scaling, set the threshold to 0. -# column_index_auto_scale_threshold_in_kb: 10240 - -# Log a warning when writing partitions larger than this value -# compaction_large_partition_warning_threshold_mb: 1000 - -# Log a warning when writing rows larger than this value -# compaction_large_row_warning_threshold_mb: 10 - -# Log a warning when writing cells larger than this value -# compaction_large_cell_warning_threshold_mb: 1 - -# Log a warning when row number is larger than this value -# compaction_rows_count_warning_threshold: 100000 - -# Log a warning when writing a collection containing more elements than this value -# compaction_collection_elements_count_warning_threshold: 10000 - -# How long the coordinator should wait for seq or index scans to complete -# range_request_timeout_in_ms: 10000 -# How long the coordinator should wait for writes to complete -# counter_write_request_timeout_in_ms: 5000 -# How long a coordinator should continue to retry a CAS operation -# that contends with other proposals for the same row -# cas_contention_timeout_in_ms: 1000 -# How long the coordinator should wait for truncates to complete -# (This can be much longer, because unless auto_snapshot is disabled -# we need to flush first so we can snapshot before removing the data.) -# truncate_request_timeout_in_ms: 60000 -# The default timeout for other, miscellaneous operations -# request_timeout_in_ms: 10000 - -# Enable or disable inter-node encryption. -# You must also generate keys and provide the appropriate key and trust store locations and passwords. -# -# The available internode options are : all, none, dc, rack -# If set to dc scylla will encrypt the traffic between the DCs -# If set to rack scylla will encrypt the traffic between the racks -# -# SSL/TLS algorithm and ciphers used can be controlled by -# the priority_string parameter. Info on priority string -# syntax and values is available at: -# https://gnutls.org/manual/html_node/Priority-Strings.html -# -# The require_client_auth parameter allows you to -# restrict access to service based on certificate -# validation. Client must provide a certificate -# accepted by the used trust store to connect. -# -# server_encryption_options: -# internode_encryption: none -# certificate: conf/scylla.crt -# keyfile: conf/scylla.key -# truststore: -# certficate_revocation_list: -# require_client_auth: False -# priority_string: - -# enable or disable client/server encryption. -# client_encryption_options: -# enabled: false -# certificate: conf/scylla.crt -# keyfile: conf/scylla.key -# truststore: -# certficate_revocation_list: -# require_client_auth: False -# priority_string: - -# internode_compression controls whether traffic between nodes is -# compressed. -# can be: all - all traffic is compressed -# dc - traffic between different datacenters is compressed -# none - nothing is compressed. -# internode_compression: none - -# Enable or disable tcp_nodelay for inter-dc communication. -# Disabling it will result in larger (but fewer) network packets being sent, -# reducing overhead from the TCP protocol itself, at the cost of increasing -# latency if you block for cross-datacenter responses. -# inter_dc_tcp_nodelay: false - -# Relaxation of environment checks. -# -# Scylla places certain requirements on its environment. If these requirements are -# not met, performance and reliability can be degraded. -# -# These requirements include: -# - A filesystem with good support for asynchronous I/O (AIO). Currently, -# this means XFS. -# -# false: strict environment checks are in place; do not start if they are not met. -# true: relaxed environment checks; performance and reliability may degraade. -# -# developer_mode: false - -# Idle-time background processing -# -# Scylla can perform certain jobs in the background while the system is otherwise idle, -# freeing processor resources when there is other work to be done. -# -# defragment_memory_on_idle: true -# -# prometheus port -# By default, Scylla opens prometheus API port on port 9180 -# setting the port to 0 will disable the prometheus API. -# prometheus_port: 9180 -# -# prometheus address -# Leaving this blank will set it to the same value as listen_address. -# This means that by default, Scylla listens to the prometheus API on the same -# listening address (and therefore network interface) used to listen for -# internal communication. If the monitoring node is not in this internal -# network, you can override prometheus_address explicitly - e.g., setting -# it to 0.0.0.0 to listen on all interfaces. -# prometheus_address: 1.2.3.4 - -# Distribution of data among cores (shards) within a node -# -# Scylla distributes data within a node among shards, using a round-robin -# strategy: -# [shard0] [shard1] ... [shardN-1] [shard0] [shard1] ... [shardN-1] ... -# -# Scylla versions 1.6 and below used just one repetition of the pattern; -# this interfered with data placement among nodes (vnodes). -# -# Scylla versions 1.7 and above use 4096 repetitions of the pattern; this -# provides for better data distribution. -# -# the value below is log (base 2) of the number of repetitions. -# -# Set to 0 to avoid rewriting all data when upgrading from Scylla 1.6 and -# below. -# -# Keep at 12 for new clusters. -murmur3_partitioner_ignore_msb_bits: 12 - -# Bypass in-memory data cache (the row cache) when performing reversed queries. -# reversed_reads_auto_bypass_cache: false - -# Use a new optimized algorithm for performing reversed reads. -# Set to `false` to fall-back to the old algorithm. -# enable_optimized_reversed_reads: true - -# Use on a new, parallel algorithm for performing aggregate queries. -# Set to `false` to fall-back to the old algorithm. -# enable_parallelized_aggregation: true - -# Time for which task manager task is kept in memory after it completes. -# task_ttl_in_seconds: 0 - -# In materialized views, restrictions are allowed only on the view's primary key columns. -# In old versions Scylla mistakenly allowed IS NOT NULL restrictions on columns which were not part -# of the view's primary key. These invalid restrictions were ignored. -# This option controls the behavior when someone tries to create a view with such invalid IS NOT NULL restrictions. -# -# Can be true, false, or warn. -# * `true`: IS NOT NULL is allowed only on the view's primary key columns, -# trying to use it on other columns will cause an error, as it should. -# * `false`: Scylla accepts IS NOT NULL restrictions on regular columns, but they're silently ignored. -# It's useful for backwards compatibility. -# * `warn`: The same as false, but there's a warning about invalid view restrictions. -# -# To preserve backwards compatibility on old clusters, Scylla's default setting is `warn`. -# New clusters have this option set to `true` by scylla.yaml (which overrides the default `warn`) -# to make sure that trying to create an invalid view causes an error. -strict_is_not_null_in_views: true - -# The Unix Domain Socket the node uses for maintenance socket. -# The possible options are: -# * ignore: the node will not open the maintenance socket, -# * workdir: the node will open the maintenance socket on the path /cql.m, -# where is a path defined by the workdir configuration option, -# * : the node will open the maintenance socket on the path . -maintenance_socket: ignore - -# If set to true, configuration parameters defined with LiveUpdate option can be updated in runtime with CQL -# by updating system.config virtual table. If we don't want any configuration parameter to be changed in runtime -# via CQL, this option should be set to false. This parameter doesn't impose any limits on other mechanisms updating -# configuration parameters in runtime, e.g. sending SIGHUP or using API. This option should be set to false -# e.g. for cloud users, for whom scylla's configuration should be changed only by support engineers. -# live_updatable_config_params_changeable_via_cql: true - -# **************** -# * GUARDRAILS * -# **************** - -# Guardrails to warn or fail when Replication Factor is smaller/greater than the threshold. -# Please note that the value of 0 is always allowed, -# which means that having no replication at all, i.e. RF = 0, is always valid. -# A guardrail value smaller than 0, e.g. -1, means that the guardrail is disabled. -# Commenting out a guardrail also means it is disabled. -# minimum_replication_factor_fail_threshold: -1 -# minimum_replication_factor_warn_threshold: 3 -# maximum_replication_factor_warn_threshold: -1 -# maximum_replication_factor_fail_threshold: -1 - -# Guardrails to warn about or disallow creating a keyspace with specific replication strategy. -# Each of these 2 settings is a list storing replication strategies considered harmful. -# The replication strategies to choose from are: -# 1) SimpleStrategy, -# 2) NetworkTopologyStrategy, -# 3) LocalStrategy, -# 4) EverywhereStrategy -# -# replication_strategy_warn_list: -# - SimpleStrategy -# replication_strategy_fail_list: - -# This enables tablets on newly created keyspaces -enable_tablets: true -api_ui_dir: /opt/scylladb/swagger-ui/dist/ -api_doc_dir: /opt/scylladb/api/api-doc/ diff --git a/catalyst-gateway/tests/schemathesis_tests/node3-scylla.yaml b/catalyst-gateway/tests/schemathesis_tests/node3-scylla.yaml deleted file mode 100644 index aec5e3d224f..00000000000 --- a/catalyst-gateway/tests/schemathesis_tests/node3-scylla.yaml +++ /dev/null @@ -1,624 +0,0 @@ -# Scylla storage config YAML - -# cspell: words fsyncs rackdc partitioner mbean certficate degraade defragment - -####################################### -# This file is split to two sections: -# 1. Supported parameters -# 2. Unsupported parameters: reserved for future use or backwards -# compatibility. -# Scylla will only read and use the first segment -####################################### - -### Supported Parameters - -# The name of the cluster. This is mainly used to prevent machines in -# one logical cluster from joining another. -# It is recommended to change the default value when creating a new cluster. -# You can NOT modify this value for an existing cluster -#cluster_name: 'Test Cluster' - -# This defines the number of tokens randomly assigned to this node on the ring -# The more tokens, relative to other nodes, the larger the proportion of data -# that this node will store. You probably want all nodes to have the same number -# of tokens assuming they have equal hardware capability. -num_tokens: 256 - -# Directory where Scylla should store all its files, which are commitlog, -# data, hints, view_hints and saved_caches subdirectories. All of these -# subs can be overridden by the respective options below. -# If unset, the value defaults to /var/lib/scylla -# workdir: /var/lib/scylla - -# Directory where Scylla should store data on disk. -# data_file_directories: -# - /var/lib/scylla/data - -# commit log. when running on magnetic HDD, this should be a -# separate spindle than the data directories. -# commitlog_directory: /var/lib/scylla/commitlog - -# schema commit log. A special commitlog instance -# used for schema and system tables. -# When running on magnetic HDD, this should be a -# separate spindle than the data directories. -# schema_commitlog_directory: /var/lib/scylla/commitlog/schema - -# commitlog_sync may be either "periodic" or "batch." -# -# When in batch mode, Scylla won't ack writes until the commit log -# has been fsynced to disk. It will wait -# commitlog_sync_batch_window_in_ms milliseconds between fsyncs. -# This window should be kept short because the writer threads will -# be unable to do extra work while waiting. (You may need to increase -# concurrent_writes for the same reason.) -# -# commitlog_sync: batch -# commitlog_sync_batch_window_in_ms: 2 -# -# the other option is "periodic" where writes may be acked immediately -# and the CommitLog is simply synced every commitlog_sync_period_in_ms -# milliseconds. -commitlog_sync: periodic -commitlog_sync_period_in_ms: 10000 - -# The size of the individual commitlog file segments. A commitlog -# segment may be archived, deleted, or recycled once all the data -# in it (potentially from each columnfamily in the system) has been -# flushed to sstables. -# -# The default size is 32, which is almost always fine, but if you are -# archiving commitlog segments (see commitlog_archiving.properties), -# then you probably want a finer granularity of archiving; 8 or 16 MB -# is reasonable. -commitlog_segment_size_in_mb: 32 - -# The size of the individual schema commitlog file segments. -# -# The default size is 128, which is 4 times larger than the default -# size of the data commitlog. It's because the segment size puts -# a limit on the mutation size that can be written at once, and some -# schema mutation writes are much larger than average. -schema_commitlog_segment_size_in_mb: 128 - -# seed_provider class_name is saved for future use. -# A seed address is mandatory. -seed_provider: - # The addresses of hosts that will serve as contact points for the joining node. - # It allows the node to discover the cluster ring topology on startup (when - # joining the cluster). - # Once the node has joined the cluster, the seed list has no function. - - class_name: org.apache.cassandra.locator.SimpleSeedProvider - parameters: - # In a new cluster, provide the address of the first node. - # In an existing cluster, specify the address of at least one existing node. - # If you specify addresses of more than one node, use a comma to separate them. - # For example: ",," - - seeds: "127.0.0.1" - -# Address to bind to and tell other Scylla nodes to connect to. -# You _must_ change this if you want multiple nodes to be able to communicate! -# -# If you leave broadcast_address (below) empty, then setting listen_address -# to 0.0.0.0 is wrong as other nodes will not know how to reach this node. -# If you set broadcast_address, then you can set listen_address to 0.0.0.0. -listen_address: localhost - -# Address to broadcast to other Scylla nodes -# Leaving this blank will set it to the same value as listen_address -# broadcast_address: 1.2.3.4 - -# When using multiple physical network interfaces, set this to true to listen on broadcast_address -# in addition to the listen_address, allowing nodes to communicate in both interfaces. -# Ignore this property if the network configuration automatically routes between the public and private networks such as EC2. -# -# listen_on_broadcast_address: false - -# port for the CQL native transport to listen for clients on -# For security reasons, you should not expose this port to the internet. Firewall it if needed. -# To disable the CQL native transport, remove this option and configure native_transport_port_ssl. -native_transport_port: 9044 - -# Like native_transport_port, but clients are forwarded to specific shards, based on the -# client-side port numbers. -native_shard_aware_transport_port: 19044 - -# Enabling native transport encryption in client_encryption_options allows you to either use -# encryption for the standard port or to use a dedicated, additional port along with the unencrypted -# standard native_transport_port. -# Enabling client encryption and keeping native_transport_port_ssl disabled will use encryption -# for native_transport_port. Setting native_transport_port_ssl to a different value -# from native_transport_port will use encryption for native_transport_port_ssl while -# keeping native_transport_port unencrypted. -#native_transport_port_ssl: 9142 - -# Like native_transport_port_ssl, but clients are forwarded to specific shards, based on the -# client-side port numbers. -#native_shard_aware_transport_port_ssl: 19142 - -# How long the coordinator should wait for read operations to complete -read_request_timeout_in_ms: 5000 - -# How long the coordinator should wait for writes to complete -write_request_timeout_in_ms: 2000 -# how long a coordinator should continue to retry a CAS operation -# that contends with other proposals for the same row -cas_contention_timeout_in_ms: 1000 - -# phi value that must be reached for a host to be marked down. -# most users should never need to adjust this. -# phi_convict_threshold: 8 - -# IEndpointSnitch. The snitch has two functions: -# - it teaches Scylla enough about your network topology to route -# requests efficiently -# - it allows Scylla to spread replicas around your cluster to avoid -# correlated failures. It does this by grouping machines into -# "datacenters" and "racks." Scylla will do its best not to have -# more than one replica on the same "rack" (which may not actually -# be a physical location) -# -# IF YOU CHANGE THE SNITCH AFTER DATA IS INSERTED INTO THE CLUSTER, -# YOU MUST RUN A FULL REPAIR, SINCE THE SNITCH AFFECTS WHERE REPLICAS -# ARE PLACED. -# -# Out of the box, Scylla provides -# - SimpleSnitch: -# Treats Strategy order as proximity. This can improve cache -# locality when disabling read repair. Only appropriate for -# single-datacenter deployments. -# - GossipingPropertyFileSnitch -# This should be your go-to snitch for production use. The rack -# and datacenter for the local node are defined in -# cassandra-rackdc.properties and propagated to other nodes via -# gossip. If cassandra-topology.properties exists, it is used as a -# fallback, allowing migration from the PropertyFileSnitch. -# - PropertyFileSnitch: -# Proximity is determined by rack and data center, which are -# explicitly configured in cassandra-topology.properties. -# - Ec2Snitch: -# Appropriate for EC2 deployments in a single Region. Loads Region -# and Availability Zone information from the EC2 API. The Region is -# treated as the datacenter, and the Availability Zone as the rack. -# Only private IPs are used, so this will not work across multiple -# Regions. -# - Ec2MultiRegionSnitch: -# Uses public IPs as broadcast_address to allow cross-region -# connectivity. (Thus, you should set seed addresses to the public -# IP as well.) You will need to open the storage_port or -# ssl_storage_port on the public IP firewall. (For intra-Region -# traffic, Scylla will switch to the private IP after -# establishing a connection.) -# - RackInferringSnitch: -# Proximity is determined by rack and data center, which are -# assumed to correspond to the 3rd and 2nd octet of each node's IP -# address, respectively. Unless this happens to match your -# deployment conventions, this is best used as an example of -# writing a custom Snitch class and is provided in that spirit. -# -# You can use a custom Snitch by setting this to the full class name -# of the snitch, which will be assumed to be on your classpath. -endpoint_snitch: SimpleSnitch - -# The address or interface to bind the Thrift RPC service and native transport -# server to. -# -# Set rpc_address OR rpc_interface, not both. Interfaces must correspond -# to a single address, IP aliasing is not supported. -# -# Leaving rpc_address blank has the same effect as on listen_address -# (i.e. it will be based on the configured hostname of the node). -# -# Note that unlike listen_address, you can specify 0.0.0.0, but you must also -# set broadcast_rpc_address to a value other than 0.0.0.0. -# -# For security reasons, you should not expose this port to the internet. Firewall it if needed. -# -# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address -# you can specify which should be chosen using rpc_interface_prefer_ipv6. If false the first ipv4 -# address will be used. If true the first ipv6 address will be used. Defaults to false preferring -# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. -rpc_address: localhost -# rpc_interface: eth1 -# rpc_interface_prefer_ipv6: false - -# port for Thrift to listen for clients on -rpc_port: 9160 - -# port for REST API server -api_port: 10000 - -# IP for the REST API server -api_address: 127.0.0.1 - -# Log WARN on any batch size exceeding this value. 128 kiB per batch by default. -# Caution should be taken on increasing the size of this threshold as it can lead to node instability. -batch_size_warn_threshold_in_kb: 128 - -# Fail any multiple-partition batch exceeding this value. 1 MiB (8x warn threshold) by default. -batch_size_fail_threshold_in_kb: 1024 - -# Authentication backend, identifying users -# Out of the box, Scylla provides org.apache.cassandra.auth.{AllowAllAuthenticator, -# PasswordAuthenticator}. -# -# - AllowAllAuthenticator performs no checks - set it to disable authentication. -# - PasswordAuthenticator relies on username/password pairs to authenticate -# users. It keeps usernames and hashed passwords in system_auth.credentials table. -# Please increase system_auth keyspace replication factor if you use this authenticator. -# - com.scylladb.auth.TransitionalAuthenticator requires username/password pair -# to authenticate in the same manner as PasswordAuthenticator, but improper credentials -# result in being logged in as an anonymous user. Use for upgrading clusters' auth. -# authenticator: AllowAllAuthenticator - -# Authorization backend, implementing IAuthorizer; used to limit access/provide permissions -# Out of the box, Scylla provides org.apache.cassandra.auth.{AllowAllAuthorizer, -# CassandraAuthorizer}. -# -# - AllowAllAuthorizer allows any action to any user - set it to disable authorization. -# - CassandraAuthorizer stores permissions in system_auth.permissions table. Please -# increase system_auth keyspace replication factor if you use this authorizer. -# - com.scylladb.auth.TransitionalAuthorizer wraps around the CassandraAuthorizer, using it for -# authorizing permission management. Otherwise, it allows all. Use for upgrading -# clusters' auth. -# authorizer: AllowAllAuthorizer - -# initial_token allows you to specify tokens manually. While you can use # it with -# vnodes (num_tokens > 1, above) -- in which case you should provide a -# comma-separated list -- it's primarily used when adding nodes # to legacy clusters -# that do not have vnodes enabled. -# initial_token: - -# RPC address to broadcast to drivers and other Scylla nodes. This cannot -# be set to 0.0.0.0. If left blank, this will be set to the value of -# rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must -# be set. -# broadcast_rpc_address: 1.2.3.4 - -# Uncomment to enable experimental features -# experimental_features: -# - udf -# - alternator-streams -# - broadcast-tables -# - keyspace-storage-options -# - tablets - -# The directory where hints files are stored if hinted handoff is enabled. -# hints_directory: /var/lib/scylla/hints - -# The directory where hints files are stored for materialized-view updates -# view_hints_directory: /var/lib/scylla/view_hints - -# See https://docs.scylladb.com/architecture/anti-entropy/hinted-handoff -# May either be "true" or "false" to enable globally, or contain a list -# of data centers to enable per-datacenter. -# hinted_handoff_enabled: DC1,DC2 -# hinted_handoff_enabled: true - -# this defines the maximum amount of time a dead host will have hints -# generated. After it has been dead this long, new hints for it will not be -# created until it has been seen alive and gone down again. -# max_hint_window_in_ms: 10800000 # 3 hours - -# Validity period for permissions cache (fetching permissions can be an -# expensive operation depending on the authorizer, CassandraAuthorizer is -# one example). Defaults to 10000, set to 0 to disable. -# Will be disabled automatically for AllowAllAuthorizer. -# permissions_validity_in_ms: 10000 - -# Refresh interval for permissions cache (if enabled). -# After this interval, cache entries become eligible for refresh. Upon next -# access, an async reload is scheduled and the old value returned until it -# completes. If permissions_validity_in_ms is non-zero, then this also must have -# a non-zero value. Defaults to 2000. It's recommended to set this value to -# be at least 3 times smaller than the permissions_validity_in_ms. -# permissions_update_interval_in_ms: 2000 - -# The partitioner is responsible for distributing groups of rows (by -# partition key) across nodes in the cluster. You should leave this -# alone for new clusters. The partitioner can NOT be changed without -# reloading all data, so when upgrading you should set this to the -# same partitioner you were already using. -# -# Murmur3Partitioner is currently the only supported partitioner, -# -partitioner: org.apache.cassandra.dht.Murmur3Partitioner - -# Total space to use for commitlogs. -# -# If space gets above this value (it will round up to the next nearest -# segment multiple), Scylla will flush every dirty CF in the oldest -# segment and remove it. So a small total commitlog space will tend -# to cause more flush activity on less-active columnfamilies. -# -# A value of -1 (default) will automatically equate it to the total amount of memory -# available for Scylla. -commitlog_total_space_in_mb: -1 - -# TCP port, for commands and data -# For security reasons, you should not expose this port to the internet. Firewall it if needed. -# storage_port: 7000 - -# SSL port, for encrypted communication. Unused unless enabled in -# encryption_options -# For security reasons, you should not expose this port to the internet. Firewall it if needed. -# ssl_storage_port: 7001 - -# listen_interface: eth0 -# listen_interface_prefer_ipv6: false - -# Whether to start the native transport server. -# Please note that the address on which the native transport is bound is the -# same as the rpc_address. The port however is different and specified below. -# start_native_transport: true - -# The maximum size of allowed frame. Frame (requests) larger than this will -# be rejected as invalid. The default is 256MB. -# native_transport_max_frame_size_in_mb: 256 - -# Whether to start the thrift rpc server. -# start_rpc: true - -# enable or disable keepalive on rpc/native connections -# rpc_keepalive: true - -# Set to true to have Scylla create a hard link to each sstable -# flushed or streamed locally in a backups/ subdirectory of the -# keyspace data. Removing these links is the operator's -# responsibility. -# incremental_backups: false - -# Whether or not to take a snapshot before each compaction. Be -# careful using this option, since Scylla won't clean up the -# snapshots for you. Mostly useful if you're paranoid when there -# is a data format change. -# snapshot_before_compaction: false - -# Whether or not a snapshot is taken of the data before keyspace truncation -# or dropping of column families. The STRONGLY advised default of true -# should be used to provide data safety. If you set this flag to false, you will -# lose data on truncation or drop. -# auto_snapshot: true - -# When executing a scan, within or across a partition, we need to keep the -# tombstones seen in memory so we can return them to the coordinator, which -# will use them to make sure other replicas also know about the deleted rows. -# With workloads that generate a lot of tombstones, this can cause performance -# problems and even exhaust the server heap. -# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets) -# Adjust the thresholds here if you understand the dangers and want to -# scan more tombstones anyway. These thresholds may also be adjusted at runtime -# using the StorageService mbean. -# tombstone_warn_threshold: 1000 -# tombstone_failure_threshold: 100000 - -# Granularity of the collation index of rows within a partition. -# Increase if your rows are large, or if you have a very large -# number of rows per partition. The competing goals are these: -# 1) a smaller granularity means more index entries are generated -# and looking up rows within the partition by collation column -# is faster -# 2) but, Scylla will keep the collation index in memory for hot -# rows (as part of the key cache), so a larger granularity means -# you can cache more hot rows -# column_index_size_in_kb: 64 - -# Auto-scaling of the promoted index prevents running out of memory -# when the promoted index grows too large (due to partitions with many rows -# vs. too small column_index_size_in_kb). When the serialized representation -# of the promoted index grows by this threshold, the desired block size -# for this partition (initialized to column_index_size_in_kb) -# is doubled, to decrease the sampling resolution by half. -# -# To disable promoted index auto-scaling, set the threshold to 0. -# column_index_auto_scale_threshold_in_kb: 10240 - -# Log a warning when writing partitions larger than this value -# compaction_large_partition_warning_threshold_mb: 1000 - -# Log a warning when writing rows larger than this value -# compaction_large_row_warning_threshold_mb: 10 - -# Log a warning when writing cells larger than this value -# compaction_large_cell_warning_threshold_mb: 1 - -# Log a warning when row number is larger than this value -# compaction_rows_count_warning_threshold: 100000 - -# Log a warning when writing a collection containing more elements than this value -# compaction_collection_elements_count_warning_threshold: 10000 - -# How long the coordinator should wait for seq or index scans to complete -# range_request_timeout_in_ms: 10000 -# How long the coordinator should wait for writes to complete -# counter_write_request_timeout_in_ms: 5000 -# How long a coordinator should continue to retry a CAS operation -# that contends with other proposals for the same row -# cas_contention_timeout_in_ms: 1000 -# How long the coordinator should wait for truncates to complete -# (This can be much longer, because unless auto_snapshot is disabled -# we need to flush first so we can snapshot before removing the data.) -# truncate_request_timeout_in_ms: 60000 -# The default timeout for other, miscellaneous operations -# request_timeout_in_ms: 10000 - -# Enable or disable inter-node encryption. -# You must also generate keys and provide the appropriate key and trust store locations and passwords. -# -# The available internode options are : all, none, dc, rack -# If set to dc scylla will encrypt the traffic between the DCs -# If set to rack scylla will encrypt the traffic between the racks -# -# SSL/TLS algorithm and ciphers used can be controlled by -# the priority_string parameter. Info on priority string -# syntax and values is available at: -# https://gnutls.org/manual/html_node/Priority-Strings.html -# -# The require_client_auth parameter allows you to -# restrict access to service based on certificate -# validation. Client must provide a certificate -# accepted by the used trust store to connect. -# -# server_encryption_options: -# internode_encryption: none -# certificate: conf/scylla.crt -# keyfile: conf/scylla.key -# truststore: -# certficate_revocation_list: -# require_client_auth: False -# priority_string: - -# enable or disable client/server encryption. -# client_encryption_options: -# enabled: false -# certificate: conf/scylla.crt -# keyfile: conf/scylla.key -# truststore: -# certficate_revocation_list: -# require_client_auth: False -# priority_string: - -# internode_compression controls whether traffic between nodes is -# compressed. -# can be: all - all traffic is compressed -# dc - traffic between different datacenters is compressed -# none - nothing is compressed. -# internode_compression: none - -# Enable or disable tcp_nodelay for inter-dc communication. -# Disabling it will result in larger (but fewer) network packets being sent, -# reducing overhead from the TCP protocol itself, at the cost of increasing -# latency if you block for cross-datacenter responses. -# inter_dc_tcp_nodelay: false - -# Relaxation of environment checks. -# -# Scylla places certain requirements on its environment. If these requirements are -# not met, performance and reliability can be degraded. -# -# These requirements include: -# - A filesystem with good support for asynchronous I/O (AIO). Currently, -# this means XFS. -# -# false: strict environment checks are in place; do not start if they are not met. -# true: relaxed environment checks; performance and reliability may degraade. -# -# developer_mode: false - -# Idle-time background processing -# -# Scylla can perform certain jobs in the background while the system is otherwise idle, -# freeing processor resources when there is other work to be done. -# -# defragment_memory_on_idle: true -# -# prometheus port -# By default, Scylla opens prometheus API port on port 9180 -# setting the port to 0 will disable the prometheus API. -# prometheus_port: 9180 -# -# prometheus address -# Leaving this blank will set it to the same value as listen_address. -# This means that by default, Scylla listens to the prometheus API on the same -# listening address (and therefore network interface) used to listen for -# internal communication. If the monitoring node is not in this internal -# network, you can override prometheus_address explicitly - e.g., setting -# it to 0.0.0.0 to listen on all interfaces. -# prometheus_address: 1.2.3.4 - -# Distribution of data among cores (shards) within a node -# -# Scylla distributes data within a node among shards, using a round-robin -# strategy: -# [shard0] [shard1] ... [shardN-1] [shard0] [shard1] ... [shardN-1] ... -# -# Scylla versions 1.6 and below used just one repetition of the pattern; -# this interfered with data placement among nodes (vnodes). -# -# Scylla versions 1.7 and above use 4096 repetitions of the pattern; this -# provides for better data distribution. -# -# the value below is log (base 2) of the number of repetitions. -# -# Set to 0 to avoid rewriting all data when upgrading from Scylla 1.6 and -# below. -# -# Keep at 12 for new clusters. -murmur3_partitioner_ignore_msb_bits: 12 - -# Bypass in-memory data cache (the row cache) when performing reversed queries. -# reversed_reads_auto_bypass_cache: false - -# Use a new optimized algorithm for performing reversed reads. -# Set to `false` to fall-back to the old algorithm. -# enable_optimized_reversed_reads: true - -# Use on a new, parallel algorithm for performing aggregate queries. -# Set to `false` to fall-back to the old algorithm. -# enable_parallelized_aggregation: true - -# Time for which task manager task is kept in memory after it completes. -# task_ttl_in_seconds: 0 - -# In materialized views, restrictions are allowed only on the view's primary key columns. -# In old versions Scylla mistakenly allowed IS NOT NULL restrictions on columns which were not part -# of the view's primary key. These invalid restrictions were ignored. -# This option controls the behavior when someone tries to create a view with such invalid IS NOT NULL restrictions. -# -# Can be true, false, or warn. -# * `true`: IS NOT NULL is allowed only on the view's primary key columns, -# trying to use it on other columns will cause an error, as it should. -# * `false`: Scylla accepts IS NOT NULL restrictions on regular columns, but they're silently ignored. -# It's useful for backwards compatibility. -# * `warn`: The same as false, but there's a warning about invalid view restrictions. -# -# To preserve backwards compatibility on old clusters, Scylla's default setting is `warn`. -# New clusters have this option set to `true` by scylla.yaml (which overrides the default `warn`) -# to make sure that trying to create an invalid view causes an error. -strict_is_not_null_in_views: true - -# The Unix Domain Socket the node uses for maintenance socket. -# The possible options are: -# * ignore: the node will not open the maintenance socket, -# * workdir: the node will open the maintenance socket on the path /cql.m, -# where is a path defined by the workdir configuration option, -# * : the node will open the maintenance socket on the path . -maintenance_socket: ignore - -# If set to true, configuration parameters defined with LiveUpdate option can be updated in runtime with CQL -# by updating system.config virtual table. If we don't want any configuration parameter to be changed in runtime -# via CQL, this option should be set to false. This parameter doesn't impose any limits on other mechanisms updating -# configuration parameters in runtime, e.g. sending SIGHUP or using API. This option should be set to false -# e.g. for cloud users, for whom scylla's configuration should be changed only by support engineers. -# live_updatable_config_params_changeable_via_cql: true - -# **************** -# * GUARDRAILS * -# **************** - -# Guardrails to warn or fail when Replication Factor is smaller/greater than the threshold. -# Please note that the value of 0 is always allowed, -# which means that having no replication at all, i.e. RF = 0, is always valid. -# A guardrail value smaller than 0, e.g. -1, means that the guardrail is disabled. -# Commenting out a guardrail also means it is disabled. -# minimum_replication_factor_fail_threshold: -1 -# minimum_replication_factor_warn_threshold: 3 -# maximum_replication_factor_warn_threshold: -1 -# maximum_replication_factor_fail_threshold: -1 - -# Guardrails to warn about or disallow creating a keyspace with specific replication strategy. -# Each of these 2 settings is a list storing replication strategies considered harmful. -# The replication strategies to choose from are: -# 1) SimpleStrategy, -# 2) NetworkTopologyStrategy, -# 3) LocalStrategy, -# 4) EverywhereStrategy -# -# replication_strategy_warn_list: -# - SimpleStrategy -# replication_strategy_fail_list: - -# This enables tablets on newly created keyspaces -enable_tablets: true -api_ui_dir: /opt/scylladb/swagger-ui/dist/ -api_doc_dir: /opt/scylladb/api/api-doc/ diff --git a/catalyst-gateway/tests/schemathesis_tests/node4-scylla.yaml b/catalyst-gateway/tests/schemathesis_tests/node4-scylla.yaml deleted file mode 100644 index 4bf06b2f620..00000000000 --- a/catalyst-gateway/tests/schemathesis_tests/node4-scylla.yaml +++ /dev/null @@ -1,624 +0,0 @@ -# Scylla storage config YAML - -# cspell: words fsyncs rackdc partitioner mbean certficate degraade defragment - -####################################### -# This file is split to two sections: -# 1. Supported parameters -# 2. Unsupported parameters: reserved for future use or backwards -# compatibility. -# Scylla will only read and use the first segment -####################################### - -### Supported Parameters - -# The name of the cluster. This is mainly used to prevent machines in -# one logical cluster from joining another. -# It is recommended to change the default value when creating a new cluster. -# You can NOT modify this value for an existing cluster -#cluster_name: 'Test Cluster' - -# This defines the number of tokens randomly assigned to this node on the ring -# The more tokens, relative to other nodes, the larger the proportion of data -# that this node will store. You probably want all nodes to have the same number -# of tokens assuming they have equal hardware capability. -num_tokens: 256 - -# Directory where Scylla should store all its files, which are commitlog, -# data, hints, view_hints and saved_caches subdirectories. All of these -# subs can be overridden by the respective options below. -# If unset, the value defaults to /var/lib/scylla -# workdir: /var/lib/scylla - -# Directory where Scylla should store data on disk. -# data_file_directories: -# - /var/lib/scylla/data - -# commit log. when running on magnetic HDD, this should be a -# separate spindle than the data directories. -# commitlog_directory: /var/lib/scylla/commitlog - -# schema commit log. A special commitlog instance -# used for schema and system tables. -# When running on magnetic HDD, this should be a -# separate spindle than the data directories. -# schema_commitlog_directory: /var/lib/scylla/commitlog/schema - -# commitlog_sync may be either "periodic" or "batch." -# -# When in batch mode, Scylla won't ack writes until the commit log -# has been fsynced to disk. It will wait -# commitlog_sync_batch_window_in_ms milliseconds between fsyncs. -# This window should be kept short because the writer threads will -# be unable to do extra work while waiting. (You may need to increase -# concurrent_writes for the same reason.) -# -# commitlog_sync: batch -# commitlog_sync_batch_window_in_ms: 2 -# -# the other option is "periodic" where writes may be acked immediately -# and the CommitLog is simply synced every commitlog_sync_period_in_ms -# milliseconds. -commitlog_sync: periodic -commitlog_sync_period_in_ms: 10000 - -# The size of the individual commitlog file segments. A commitlog -# segment may be archived, deleted, or recycled once all the data -# in it (potentially from each columnfamily in the system) has been -# flushed to sstables. -# -# The default size is 32, which is almost always fine, but if you are -# archiving commitlog segments (see commitlog_archiving.properties), -# then you probably want a finer granularity of archiving; 8 or 16 MB -# is reasonable. -commitlog_segment_size_in_mb: 32 - -# The size of the individual schema commitlog file segments. -# -# The default size is 128, which is 4 times larger than the default -# size of the data commitlog. It's because the segment size puts -# a limit on the mutation size that can be written at once, and some -# schema mutation writes are much larger than average. -schema_commitlog_segment_size_in_mb: 128 - -# seed_provider class_name is saved for future use. -# A seed address is mandatory. -seed_provider: - # The addresses of hosts that will serve as contact points for the joining node. - # It allows the node to discover the cluster ring topology on startup (when - # joining the cluster). - # Once the node has joined the cluster, the seed list has no function. - - class_name: org.apache.cassandra.locator.SimpleSeedProvider - parameters: - # In a new cluster, provide the address of the first node. - # In an existing cluster, specify the address of at least one existing node. - # If you specify addresses of more than one node, use a comma to separate them. - # For example: ",," - - seeds: "127.0.0.1" - -# Address to bind to and tell other Scylla nodes to connect to. -# You _must_ change this if you want multiple nodes to be able to communicate! -# -# If you leave broadcast_address (below) empty, then setting listen_address -# to 0.0.0.0 is wrong as other nodes will not know how to reach this node. -# If you set broadcast_address, then you can set listen_address to 0.0.0.0. -listen_address: localhost - -# Address to broadcast to other Scylla nodes -# Leaving this blank will set it to the same value as listen_address -# broadcast_address: 1.2.3.4 - -# When using multiple physical network interfaces, set this to true to listen on broadcast_address -# in addition to the listen_address, allowing nodes to communicate in both interfaces. -# Ignore this property if the network configuration automatically routes between the public and private networks such as EC2. -# -# listen_on_broadcast_address: false - -# port for the CQL native transport to listen for clients on -# For security reasons, you should not expose this port to the internet. Firewall it if needed. -# To disable the CQL native transport, remove this option and configure native_transport_port_ssl. -native_transport_port: 9045 - -# Like native_transport_port, but clients are forwarded to specific shards, based on the -# client-side port numbers. -native_shard_aware_transport_port: 19045 - -# Enabling native transport encryption in client_encryption_options allows you to either use -# encryption for the standard port or to use a dedicated, additional port along with the unencrypted -# standard native_transport_port. -# Enabling client encryption and keeping native_transport_port_ssl disabled will use encryption -# for native_transport_port. Setting native_transport_port_ssl to a different value -# from native_transport_port will use encryption for native_transport_port_ssl while -# keeping native_transport_port unencrypted. -#native_transport_port_ssl: 9142 - -# Like native_transport_port_ssl, but clients are forwarded to specific shards, based on the -# client-side port numbers. -#native_shard_aware_transport_port_ssl: 19142 - -# How long the coordinator should wait for read operations to complete -read_request_timeout_in_ms: 5000 - -# How long the coordinator should wait for writes to complete -write_request_timeout_in_ms: 2000 -# how long a coordinator should continue to retry a CAS operation -# that contends with other proposals for the same row -cas_contention_timeout_in_ms: 1000 - -# phi value that must be reached for a host to be marked down. -# most users should never need to adjust this. -# phi_convict_threshold: 8 - -# IEndpointSnitch. The snitch has two functions: -# - it teaches Scylla enough about your network topology to route -# requests efficiently -# - it allows Scylla to spread replicas around your cluster to avoid -# correlated failures. It does this by grouping machines into -# "datacenters" and "racks." Scylla will do its best not to have -# more than one replica on the same "rack" (which may not actually -# be a physical location) -# -# IF YOU CHANGE THE SNITCH AFTER DATA IS INSERTED INTO THE CLUSTER, -# YOU MUST RUN A FULL REPAIR, SINCE THE SNITCH AFFECTS WHERE REPLICAS -# ARE PLACED. -# -# Out of the box, Scylla provides -# - SimpleSnitch: -# Treats Strategy order as proximity. This can improve cache -# locality when disabling read repair. Only appropriate for -# single-datacenter deployments. -# - GossipingPropertyFileSnitch -# This should be your go-to snitch for production use. The rack -# and datacenter for the local node are defined in -# cassandra-rackdc.properties and propagated to other nodes via -# gossip. If cassandra-topology.properties exists, it is used as a -# fallback, allowing migration from the PropertyFileSnitch. -# - PropertyFileSnitch: -# Proximity is determined by rack and data center, which are -# explicitly configured in cassandra-topology.properties. -# - Ec2Snitch: -# Appropriate for EC2 deployments in a single Region. Loads Region -# and Availability Zone information from the EC2 API. The Region is -# treated as the datacenter, and the Availability Zone as the rack. -# Only private IPs are used, so this will not work across multiple -# Regions. -# - Ec2MultiRegionSnitch: -# Uses public IPs as broadcast_address to allow cross-region -# connectivity. (Thus, you should set seed addresses to the public -# IP as well.) You will need to open the storage_port or -# ssl_storage_port on the public IP firewall. (For intra-Region -# traffic, Scylla will switch to the private IP after -# establishing a connection.) -# - RackInferringSnitch: -# Proximity is determined by rack and data center, which are -# assumed to correspond to the 3rd and 2nd octet of each node's IP -# address, respectively. Unless this happens to match your -# deployment conventions, this is best used as an example of -# writing a custom Snitch class and is provided in that spirit. -# -# You can use a custom Snitch by setting this to the full class name -# of the snitch, which will be assumed to be on your classpath. -endpoint_snitch: SimpleSnitch - -# The address or interface to bind the Thrift RPC service and native transport -# server to. -# -# Set rpc_address OR rpc_interface, not both. Interfaces must correspond -# to a single address, IP aliasing is not supported. -# -# Leaving rpc_address blank has the same effect as on listen_address -# (i.e. it will be based on the configured hostname of the node). -# -# Note that unlike listen_address, you can specify 0.0.0.0, but you must also -# set broadcast_rpc_address to a value other than 0.0.0.0. -# -# For security reasons, you should not expose this port to the internet. Firewall it if needed. -# -# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address -# you can specify which should be chosen using rpc_interface_prefer_ipv6. If false the first ipv4 -# address will be used. If true the first ipv6 address will be used. Defaults to false preferring -# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. -rpc_address: localhost -# rpc_interface: eth1 -# rpc_interface_prefer_ipv6: false - -# port for Thrift to listen for clients on -rpc_port: 9160 - -# port for REST API server -api_port: 10000 - -# IP for the REST API server -api_address: 127.0.0.1 - -# Log WARN on any batch size exceeding this value. 128 kiB per batch by default. -# Caution should be taken on increasing the size of this threshold as it can lead to node instability. -batch_size_warn_threshold_in_kb: 128 - -# Fail any multiple-partition batch exceeding this value. 1 MiB (8x warn threshold) by default. -batch_size_fail_threshold_in_kb: 1024 - -# Authentication backend, identifying users -# Out of the box, Scylla provides org.apache.cassandra.auth.{AllowAllAuthenticator, -# PasswordAuthenticator}. -# -# - AllowAllAuthenticator performs no checks - set it to disable authentication. -# - PasswordAuthenticator relies on username/password pairs to authenticate -# users. It keeps usernames and hashed passwords in system_auth.credentials table. -# Please increase system_auth keyspace replication factor if you use this authenticator. -# - com.scylladb.auth.TransitionalAuthenticator requires username/password pair -# to authenticate in the same manner as PasswordAuthenticator, but improper credentials -# result in being logged in as an anonymous user. Use for upgrading clusters' auth. -# authenticator: AllowAllAuthenticator - -# Authorization backend, implementing IAuthorizer; used to limit access/provide permissions -# Out of the box, Scylla provides org.apache.cassandra.auth.{AllowAllAuthorizer, -# CassandraAuthorizer}. -# -# - AllowAllAuthorizer allows any action to any user - set it to disable authorization. -# - CassandraAuthorizer stores permissions in system_auth.permissions table. Please -# increase system_auth keyspace replication factor if you use this authorizer. -# - com.scylladb.auth.TransitionalAuthorizer wraps around the CassandraAuthorizer, using it for -# authorizing permission management. Otherwise, it allows all. Use for upgrading -# clusters' auth. -# authorizer: AllowAllAuthorizer - -# initial_token allows you to specify tokens manually. While you can use # it with -# vnodes (num_tokens > 1, above) -- in which case you should provide a -# comma-separated list -- it's primarily used when adding nodes # to legacy clusters -# that do not have vnodes enabled. -# initial_token: - -# RPC address to broadcast to drivers and other Scylla nodes. This cannot -# be set to 0.0.0.0. If left blank, this will be set to the value of -# rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must -# be set. -# broadcast_rpc_address: 1.2.3.4 - -# Uncomment to enable experimental features -# experimental_features: -# - udf -# - alternator-streams -# - broadcast-tables -# - keyspace-storage-options -# - tablets - -# The directory where hints files are stored if hinted handoff is enabled. -# hints_directory: /var/lib/scylla/hints - -# The directory where hints files are stored for materialized-view updates -# view_hints_directory: /var/lib/scylla/view_hints - -# See https://docs.scylladb.com/architecture/anti-entropy/hinted-handoff -# May either be "true" or "false" to enable globally, or contain a list -# of data centers to enable per-datacenter. -# hinted_handoff_enabled: DC1,DC2 -# hinted_handoff_enabled: true - -# this defines the maximum amount of time a dead host will have hints -# generated. After it has been dead this long, new hints for it will not be -# created until it has been seen alive and gone down again. -# max_hint_window_in_ms: 10800000 # 3 hours - -# Validity period for permissions cache (fetching permissions can be an -# expensive operation depending on the authorizer, CassandraAuthorizer is -# one example). Defaults to 10000, set to 0 to disable. -# Will be disabled automatically for AllowAllAuthorizer. -# permissions_validity_in_ms: 10000 - -# Refresh interval for permissions cache (if enabled). -# After this interval, cache entries become eligible for refresh. Upon next -# access, an async reload is scheduled and the old value returned until it -# completes. If permissions_validity_in_ms is non-zero, then this also must have -# a non-zero value. Defaults to 2000. It's recommended to set this value to -# be at least 3 times smaller than the permissions_validity_in_ms. -# permissions_update_interval_in_ms: 2000 - -# The partitioner is responsible for distributing groups of rows (by -# partition key) across nodes in the cluster. You should leave this -# alone for new clusters. The partitioner can NOT be changed without -# reloading all data, so when upgrading you should set this to the -# same partitioner you were already using. -# -# Murmur3Partitioner is currently the only supported partitioner, -# -partitioner: org.apache.cassandra.dht.Murmur3Partitioner - -# Total space to use for commitlogs. -# -# If space gets above this value (it will round up to the next nearest -# segment multiple), Scylla will flush every dirty CF in the oldest -# segment and remove it. So a small total commitlog space will tend -# to cause more flush activity on less-active columnfamilies. -# -# A value of -1 (default) will automatically equate it to the total amount of memory -# available for Scylla. -commitlog_total_space_in_mb: -1 - -# TCP port, for commands and data -# For security reasons, you should not expose this port to the internet. Firewall it if needed. -# storage_port: 7000 - -# SSL port, for encrypted communication. Unused unless enabled in -# encryption_options -# For security reasons, you should not expose this port to the internet. Firewall it if needed. -# ssl_storage_port: 7001 - -# listen_interface: eth0 -# listen_interface_prefer_ipv6: false - -# Whether to start the native transport server. -# Please note that the address on which the native transport is bound is the -# same as the rpc_address. The port however is different and specified below. -# start_native_transport: true - -# The maximum size of allowed frame. Frame (requests) larger than this will -# be rejected as invalid. The default is 256MB. -# native_transport_max_frame_size_in_mb: 256 - -# Whether to start the thrift rpc server. -# start_rpc: true - -# enable or disable keepalive on rpc/native connections -# rpc_keepalive: true - -# Set to true to have Scylla create a hard link to each sstable -# flushed or streamed locally in a backups/ subdirectory of the -# keyspace data. Removing these links is the operator's -# responsibility. -# incremental_backups: false - -# Whether or not to take a snapshot before each compaction. Be -# careful using this option, since Scylla won't clean up the -# snapshots for you. Mostly useful if you're paranoid when there -# is a data format change. -# snapshot_before_compaction: false - -# Whether or not a snapshot is taken of the data before keyspace truncation -# or dropping of column families. The STRONGLY advised default of true -# should be used to provide data safety. If you set this flag to false, you will -# lose data on truncation or drop. -# auto_snapshot: true - -# When executing a scan, within or across a partition, we need to keep the -# tombstones seen in memory so we can return them to the coordinator, which -# will use them to make sure other replicas also know about the deleted rows. -# With workloads that generate a lot of tombstones, this can cause performance -# problems and even exhaust the server heap. -# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets) -# Adjust the thresholds here if you understand the dangers and want to -# scan more tombstones anyway. These thresholds may also be adjusted at runtime -# using the StorageService mbean. -# tombstone_warn_threshold: 1000 -# tombstone_failure_threshold: 100000 - -# Granularity of the collation index of rows within a partition. -# Increase if your rows are large, or if you have a very large -# number of rows per partition. The competing goals are these: -# 1) a smaller granularity means more index entries are generated -# and looking up rows within the partition by collation column -# is faster -# 2) but, Scylla will keep the collation index in memory for hot -# rows (as part of the key cache), so a larger granularity means -# you can cache more hot rows -# column_index_size_in_kb: 64 - -# Auto-scaling of the promoted index prevents running out of memory -# when the promoted index grows too large (due to partitions with many rows -# vs. too small column_index_size_in_kb). When the serialized representation -# of the promoted index grows by this threshold, the desired block size -# for this partition (initialized to column_index_size_in_kb) -# is doubled, to decrease the sampling resolution by half. -# -# To disable promoted index auto-scaling, set the threshold to 0. -# column_index_auto_scale_threshold_in_kb: 10240 - -# Log a warning when writing partitions larger than this value -# compaction_large_partition_warning_threshold_mb: 1000 - -# Log a warning when writing rows larger than this value -# compaction_large_row_warning_threshold_mb: 10 - -# Log a warning when writing cells larger than this value -# compaction_large_cell_warning_threshold_mb: 1 - -# Log a warning when row number is larger than this value -# compaction_rows_count_warning_threshold: 100000 - -# Log a warning when writing a collection containing more elements than this value -# compaction_collection_elements_count_warning_threshold: 10000 - -# How long the coordinator should wait for seq or index scans to complete -# range_request_timeout_in_ms: 10000 -# How long the coordinator should wait for writes to complete -# counter_write_request_timeout_in_ms: 5000 -# How long a coordinator should continue to retry a CAS operation -# that contends with other proposals for the same row -# cas_contention_timeout_in_ms: 1000 -# How long the coordinator should wait for truncates to complete -# (This can be much longer, because unless auto_snapshot is disabled -# we need to flush first so we can snapshot before removing the data.) -# truncate_request_timeout_in_ms: 60000 -# The default timeout for other, miscellaneous operations -# request_timeout_in_ms: 10000 - -# Enable or disable inter-node encryption. -# You must also generate keys and provide the appropriate key and trust store locations and passwords. -# -# The available internode options are : all, none, dc, rack -# If set to dc scylla will encrypt the traffic between the DCs -# If set to rack scylla will encrypt the traffic between the racks -# -# SSL/TLS algorithm and ciphers used can be controlled by -# the priority_string parameter. Info on priority string -# syntax and values is available at: -# https://gnutls.org/manual/html_node/Priority-Strings.html -# -# The require_client_auth parameter allows you to -# restrict access to service based on certificate -# validation. Client must provide a certificate -# accepted by the used trust store to connect. -# -# server_encryption_options: -# internode_encryption: none -# certificate: conf/scylla.crt -# keyfile: conf/scylla.key -# truststore: -# certficate_revocation_list: -# require_client_auth: False -# priority_string: - -# enable or disable client/server encryption. -# client_encryption_options: -# enabled: false -# certificate: conf/scylla.crt -# keyfile: conf/scylla.key -# truststore: -# certficate_revocation_list: -# require_client_auth: False -# priority_string: - -# internode_compression controls whether traffic between nodes is -# compressed. -# can be: all - all traffic is compressed -# dc - traffic between different datacenters is compressed -# none - nothing is compressed. -# internode_compression: none - -# Enable or disable tcp_nodelay for inter-dc communication. -# Disabling it will result in larger (but fewer) network packets being sent, -# reducing overhead from the TCP protocol itself, at the cost of increasing -# latency if you block for cross-datacenter responses. -# inter_dc_tcp_nodelay: false - -# Relaxation of environment checks. -# -# Scylla places certain requirements on its environment. If these requirements are -# not met, performance and reliability can be degraded. -# -# These requirements include: -# - A filesystem with good support for asynchronous I/O (AIO). Currently, -# this means XFS. -# -# false: strict environment checks are in place; do not start if they are not met. -# true: relaxed environment checks; performance and reliability may degraade. -# -# developer_mode: false - -# Idle-time background processing -# -# Scylla can perform certain jobs in the background while the system is otherwise idle, -# freeing processor resources when there is other work to be done. -# -# defragment_memory_on_idle: true -# -# prometheus port -# By default, Scylla opens prometheus API port on port 9180 -# setting the port to 0 will disable the prometheus API. -# prometheus_port: 9180 -# -# prometheus address -# Leaving this blank will set it to the same value as listen_address. -# This means that by default, Scylla listens to the prometheus API on the same -# listening address (and therefore network interface) used to listen for -# internal communication. If the monitoring node is not in this internal -# network, you can override prometheus_address explicitly - e.g., setting -# it to 0.0.0.0 to listen on all interfaces. -# prometheus_address: 1.2.3.4 - -# Distribution of data among cores (shards) within a node -# -# Scylla distributes data within a node among shards, using a round-robin -# strategy: -# [shard0] [shard1] ... [shardN-1] [shard0] [shard1] ... [shardN-1] ... -# -# Scylla versions 1.6 and below used just one repetition of the pattern; -# this interfered with data placement among nodes (vnodes). -# -# Scylla versions 1.7 and above use 4096 repetitions of the pattern; this -# provides for better data distribution. -# -# the value below is log (base 2) of the number of repetitions. -# -# Set to 0 to avoid rewriting all data when upgrading from Scylla 1.6 and -# below. -# -# Keep at 12 for new clusters. -murmur3_partitioner_ignore_msb_bits: 12 - -# Bypass in-memory data cache (the row cache) when performing reversed queries. -# reversed_reads_auto_bypass_cache: false - -# Use a new optimized algorithm for performing reversed reads. -# Set to `false` to fall-back to the old algorithm. -# enable_optimized_reversed_reads: true - -# Use on a new, parallel algorithm for performing aggregate queries. -# Set to `false` to fall-back to the old algorithm. -# enable_parallelized_aggregation: true - -# Time for which task manager task is kept in memory after it completes. -# task_ttl_in_seconds: 0 - -# In materialized views, restrictions are allowed only on the view's primary key columns. -# In old versions Scylla mistakenly allowed IS NOT NULL restrictions on columns which were not part -# of the view's primary key. These invalid restrictions were ignored. -# This option controls the behavior when someone tries to create a view with such invalid IS NOT NULL restrictions. -# -# Can be true, false, or warn. -# * `true`: IS NOT NULL is allowed only on the view's primary key columns, -# trying to use it on other columns will cause an error, as it should. -# * `false`: Scylla accepts IS NOT NULL restrictions on regular columns, but they're silently ignored. -# It's useful for backwards compatibility. -# * `warn`: The same as false, but there's a warning about invalid view restrictions. -# -# To preserve backwards compatibility on old clusters, Scylla's default setting is `warn`. -# New clusters have this option set to `true` by scylla.yaml (which overrides the default `warn`) -# to make sure that trying to create an invalid view causes an error. -strict_is_not_null_in_views: true - -# The Unix Domain Socket the node uses for maintenance socket. -# The possible options are: -# * ignore: the node will not open the maintenance socket, -# * workdir: the node will open the maintenance socket on the path /cql.m, -# where is a path defined by the workdir configuration option, -# * : the node will open the maintenance socket on the path . -maintenance_socket: ignore - -# If set to true, configuration parameters defined with LiveUpdate option can be updated in runtime with CQL -# by updating system.config virtual table. If we don't want any configuration parameter to be changed in runtime -# via CQL, this option should be set to false. This parameter doesn't impose any limits on other mechanisms updating -# configuration parameters in runtime, e.g. sending SIGHUP or using API. This option should be set to false -# e.g. for cloud users, for whom scylla's configuration should be changed only by support engineers. -# live_updatable_config_params_changeable_via_cql: true - -# **************** -# * GUARDRAILS * -# **************** - -# Guardrails to warn or fail when Replication Factor is smaller/greater than the threshold. -# Please note that the value of 0 is always allowed, -# which means that having no replication at all, i.e. RF = 0, is always valid. -# A guardrail value smaller than 0, e.g. -1, means that the guardrail is disabled. -# Commenting out a guardrail also means it is disabled. -# minimum_replication_factor_fail_threshold: -1 -# minimum_replication_factor_warn_threshold: 3 -# maximum_replication_factor_warn_threshold: -1 -# maximum_replication_factor_fail_threshold: -1 - -# Guardrails to warn about or disallow creating a keyspace with specific replication strategy. -# Each of these 2 settings is a list storing replication strategies considered harmful. -# The replication strategies to choose from are: -# 1) SimpleStrategy, -# 2) NetworkTopologyStrategy, -# 3) LocalStrategy, -# 4) EverywhereStrategy -# -# replication_strategy_warn_list: -# - SimpleStrategy -# replication_strategy_fail_list: - -# This enables tablets on newly created keyspaces -enable_tablets: true -api_ui_dir: /opt/scylladb/swagger-ui/dist/ -api_doc_dir: /opt/scylladb/api/api-doc/ diff --git a/catalyst-gateway/tests/schemathesis_tests/schemathesis-docker-compose.yml b/catalyst-gateway/tests/schemathesis_tests/schemathesis-docker-compose.yml index 8b51a90f09a..61e06672c7d 100644 --- a/catalyst-gateway/tests/schemathesis_tests/schemathesis-docker-compose.yml +++ b/catalyst-gateway/tests/schemathesis_tests/schemathesis-docker-compose.yml @@ -27,16 +27,6 @@ services: cat-gateway: image: cat-gateway:latest - environment: - - DB_URL=postgres://catalyst-event-dev:CHANGE_ME@event-db/CatalystEventDev - - CAT_ADDRESS=0.0.0.0:3030 - - LOG_LEVEL=error - - DATA_REFRESH_TICK=5 - - CHECK_CONFIG_TICK=5 - - MACHINE_ID="UID" - - CHAIN_FOLLOWER_SYNC_TASKS="16" - - RUST_LOG="error,cat_gateway=debug,cardano_chain_follower=debug,mithril-client=debug" - - CHAIN_NETWORK="Preprod" ports: - 3030:3030 #depends_on: From 4870e10d11c87d0dbf48d64ad047068dcc6d049f Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Tue, 24 Dec 2024 10:01:34 +0100 Subject: [PATCH 06/33] wip --- catalyst-gateway/tests/Earthfile | 27 ++++++++++ .../node1-scylla.yaml | 0 .../tests/schemathesis_tests/Earthfile | 53 ++++--------------- 3 files changed, 38 insertions(+), 42 deletions(-) rename catalyst-gateway/tests/{schemathesis_tests => }/node1-scylla.yaml (100%) diff --git a/catalyst-gateway/tests/Earthfile b/catalyst-gateway/tests/Earthfile index 85383a20403..236094c6ccd 100644 --- a/catalyst-gateway/tests/Earthfile +++ b/catalyst-gateway/tests/Earthfile @@ -1,6 +1,33 @@ VERSION 0.8 IMPORT github.com/input-output-hk/catalyst-ci/earthly/spectral:v3.2.24 AS spectral-ci +package-cat-gateway-integration: + FROM python:3.14.0a3-slim-bookworm + ENV LOG_LEVEL=error + ENV DATA_REFRESH_TICK=5 + ENV CHECK_CONFIG_TICK=5 + ENV MACHINE_ID="UID" + ENV CHAIN_FOLLOWER_SYNC_TASKS="16" + ENV RUST_LOG="error,cat_gateway=debug,cardano_chain_follower=debug,mithril-client=debug" + ENV CHAIN_NETWORK="Preprod" + ENV CAT_ADDRESS=0.0.0.0:3030 + ENV INTERNAL_API_KEY=123 + + RUN apt-get update && apt install -y gnupg2 wget + RUN mkdir -p /etc/apt/keyrings + RUN gpg --homedir /tmp --no-default-keyring --keyring /etc/apt/keyrings/scylladb.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys a43e06657bac99e3 + RUN wget -O /etc/apt/sources.list.d/scylla.list http://downloads.scylladb.com/deb/debian/scylla-6.2.list + RUN apt-get update && apt-get install -y scylla + + COPY node1-scylla.yaml /etc/scylla/scylla.yaml + #RUN scylla_setup -y + #https://github.com/scylladb/scylladb/issues/3142 + COPY ../+build/cat-gateway . + + ENTRYPOINT ( scylla --options-file /etc/scylla/scylla.yaml --developer-mode 1 --default-log-level error & ) && \ + ./cat-gateway run + SAVE IMAGE cat-gateway-integration:latest + # cspell: words oapi # test-lint-openapi - OpenAPI linting from an artifact # testing whether the OpenAPI generated during build stage follows good practice. diff --git a/catalyst-gateway/tests/schemathesis_tests/node1-scylla.yaml b/catalyst-gateway/tests/node1-scylla.yaml similarity index 100% rename from catalyst-gateway/tests/schemathesis_tests/node1-scylla.yaml rename to catalyst-gateway/tests/node1-scylla.yaml diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 0909f30aafc..8c1185b8c26 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -11,11 +11,11 @@ package-schemathesis: ARG openapi_spec ENTRYPOINT export SCHEMATHESIS_HOOKS=hooks \ && st run --checks all http://0.0.0.0:3030/docs/cat-gateway.json \ - --workers=2 \ + #--workers=2 \ --wait-for-schema=120 \ - --max-response-time=5000 \ - --hypothesis-max-examples=1000 \ - --data-generation-method=all \ + #--max-response-time=5000 \ + #--hypothesis-max-examples=1000 \ + #--data-generation-method=all \ --skip-deprecated-operations \ --force-schema-version=30 \ --show-trace \ @@ -27,58 +27,27 @@ package-schemathesis: SAVE IMAGE schemathesis:$tag # test-fuzzer-api - Fuzzy test cat-gateway using openapi specs. -# Disabled in CI, to enable it rename to `test-fuzzer-api`. -fuzzer-api: +test-fuzzer-api: FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 RUN apk update && apk add iptables-legacy # workaround for https://github.com/earthly/earthly/issues/3784 RUN apk add yq zstd curl COPY schemathesis-docker-compose.yml . - #LET OPENAPI_SPEC="http://cat-gateway:3030/docs/cat-gateway.json" + #LET OPENAPI_SPEC="http://0.0.0.0:3030/docs/cat-gateway.json" WITH DOCKER \ --compose schemathesis-docker-compose.yml \ --load schemathesis:latest=(+package-schemathesis) \ #--load event-db:latest=(../../event-db+build) \ - --load cat-gateway:latest=(+package-cat-gateway-integration) \ + --load cat-gateway:latest=(../+package-cat-gateway-integration) \ #--service event-db \ --service cat-gateway \ --allow-privileged - RUN sleep 120 && docker run --net=host schemathesis:latest + RUN docker run --net=host --name=st schemathesis:latest && \ + docker cp st:/results/junit-report.xml junit-report.xml && \ + docker cp st:/results/cassette.yaml cassette.yaml END # WAIT # SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis.junit-report.xml # SAVE ARTIFACT cassette.yaml AS LOCAL cassette.yaml - # END - -package-cat-gateway-integration: - FROM python:3.14.0a3-slim-bookworm - ENV LOG_LEVEL=error - ENV DATA_REFRESH_TICK=5 - ENV CHECK_CONFIG_TICK=5 - ENV MACHINE_ID="UID" - ENV CHAIN_FOLLOWER_SYNC_TASKS="16" - ENV RUST_LOG="error,cat_gateway=debug,cardano_chain_follower=debug,mithril-client=debug" - ENV CHAIN_NETWORK="Preprod" - ENV CAT_ADDRESS=0.0.0.0:3030 - ENV INTERNAL_API_KEY=123 - - RUN apt-get update && apt install -y gnupg2 wget - RUN mkdir -p /etc/apt/keyrings - RUN gpg --homedir /tmp --no-default-keyring --keyring /etc/apt/keyrings/scylladb.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys a43e06657bac99e3 - RUN wget -O /etc/apt/sources.list.d/scylla.list http://downloads.scylladb.com/deb/debian/scylla-6.2.list - RUN apt-get update && apt-get install -y scylla - - COPY node1-scylla.yaml /etc/scylla/scylla.yaml - #RUN scylla_setup -y - #https://github.com/scylladb/scylladb/issues/3142 - COPY ../../+build/cat-gateway . - - #RUN --no-cache wget -qO cargo_install https://sh.rustup.rs/ && chmod +x cargo_install && ./cargo_install -y && . "$HOME/.cargo/env" && cargo --version && pip install schemathesis==3.27.1 - #RUN mkdir /results - #COPY ./hooks/hooks.py . - #RUN export SCHEMATHESIS_HOOKS=hooks - - ENTRYPOINT ( scylla --options-file /etc/scylla/scylla.yaml --developer-mode 1 --default-log-level error & ) && \ - ./cat-gateway run - SAVE IMAGE cat-gateway:latest + # END \ No newline at end of file From e44abaf0e02f1c4bb9dad8202958b0d402f0f5a7 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Tue, 24 Dec 2024 10:16:27 +0100 Subject: [PATCH 07/33] fix: remove scylla earthfile --- utilities/local-scylla/Earthfile | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 utilities/local-scylla/Earthfile diff --git a/utilities/local-scylla/Earthfile b/utilities/local-scylla/Earthfile deleted file mode 100644 index 6c9e4755e37..00000000000 --- a/utilities/local-scylla/Earthfile +++ /dev/null @@ -1,22 +0,0 @@ -VERSION 0.8 - -test: - FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 - COPY docker-compose.yml . - COPY node1-scylla.yaml . - COPY node2-scylla.yaml . - COPY node3-scylla.yaml . - COPY node4-scylla.yaml . - ENV HOST_IP - RUN export HOST_IP="$(ifconfig $(route -n | grep ^0.0.0.0 | awk '{print $NF}') | grep inet | grep -v inet6 | awk '{print $2}')" - WITH DOCKER \ - --compose docker-compose.yml \ - --service scylla-node1 \ - --service scylla-node2 \ - --service scylla-node3 \ - --service scylla-node4 \ - --allow-privileged - - RUN echo 'lalala' && echo $HOST_IP - - END \ No newline at end of file From 32204d523e4ab4716d74ea7cdda8aaaa378dfdb1 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Fri, 27 Dec 2024 14:49:01 +0100 Subject: [PATCH 08/33] feat: runs schemathesis only on api/started endpoint --- catalyst-gateway/tests/Earthfile | 28 +++++++++++++------ .../tests/schemathesis_tests/Earthfile | 22 +++++++-------- .../schemathesis-docker-compose.yml | 6 ++-- 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/catalyst-gateway/tests/Earthfile b/catalyst-gateway/tests/Earthfile index 236094c6ccd..90455403f97 100644 --- a/catalyst-gateway/tests/Earthfile +++ b/catalyst-gateway/tests/Earthfile @@ -1,6 +1,23 @@ VERSION 0.8 IMPORT github.com/input-output-hk/catalyst-ci/earthly/spectral:v3.2.24 AS spectral-ci +# INSTALL_SCYLLA - Installs scylla for bookworm-slim/debian +INSTALL_SCYLLA: + FUNCTION + + ARG --required VERSION + + RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + wget gnupg ca-certificates && \ + mkdir -p /etc/apt/keyrings && \ + gpg --homedir /tmp --no-default-keyring --keyring /etc/apt/keyrings/scylladb.gpg \ + --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys a43e06657bac99e3 && \ + wget -O /etc/apt/sources.list.d/scylla.list http://downloads.scylladb.com/deb/debian/scylla-$VERSION.list && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + scylla + package-cat-gateway-integration: FROM python:3.14.0a3-slim-bookworm ENV LOG_LEVEL=error @@ -12,18 +29,13 @@ package-cat-gateway-integration: ENV CHAIN_NETWORK="Preprod" ENV CAT_ADDRESS=0.0.0.0:3030 ENV INTERNAL_API_KEY=123 + ENV EVENT_DB_URL "postgres://catalyst-event-dev:CHANGE_ME@localhost/CatalystEventDev" - RUN apt-get update && apt install -y gnupg2 wget - RUN mkdir -p /etc/apt/keyrings - RUN gpg --homedir /tmp --no-default-keyring --keyring /etc/apt/keyrings/scylladb.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys a43e06657bac99e3 - RUN wget -O /etc/apt/sources.list.d/scylla.list http://downloads.scylladb.com/deb/debian/scylla-6.2.list - RUN apt-get update && apt-get install -y scylla + DO +INSTALL_SCYLLA --VERSION=6.2 COPY node1-scylla.yaml /etc/scylla/scylla.yaml - #RUN scylla_setup -y - #https://github.com/scylladb/scylladb/issues/3142 COPY ../+build/cat-gateway . - + # https://github.com/scylladb/scylladb/issues/3142 ENTRYPOINT ( scylla --options-file /etc/scylla/scylla.yaml --developer-mode 1 --default-log-level error & ) && \ ./cat-gateway run SAVE IMAGE cat-gateway-integration:latest diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 8c1185b8c26..142e6319340 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -10,12 +10,12 @@ package-schemathesis: VOLUME /results ARG openapi_spec ENTRYPOINT export SCHEMATHESIS_HOOKS=hooks \ - && st run --checks all http://0.0.0.0:3030/docs/cat-gateway.json \ - #--workers=2 \ + && st run --endpoint '^/api/v1/health/started' http://0.0.0.0:3030/docs/cat-gateway.json \ + --workers=2 \ --wait-for-schema=120 \ - #--max-response-time=5000 \ - #--hypothesis-max-examples=1000 \ - #--data-generation-method=all \ + --max-response-time=5000 \ + --hypothesis-max-examples=1000 \ + --data-generation-method=all \ --skip-deprecated-operations \ --force-schema-version=30 \ --show-trace \ @@ -37,9 +37,9 @@ test-fuzzer-api: WITH DOCKER \ --compose schemathesis-docker-compose.yml \ --load schemathesis:latest=(+package-schemathesis) \ - #--load event-db:latest=(../../event-db+build) \ + --load event-db:latest=(../../event-db+build) \ --load cat-gateway:latest=(../+package-cat-gateway-integration) \ - #--service event-db \ + --service event-db \ --service cat-gateway \ --allow-privileged @@ -47,7 +47,7 @@ test-fuzzer-api: docker cp st:/results/junit-report.xml junit-report.xml && \ docker cp st:/results/cassette.yaml cassette.yaml END - # WAIT - # SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis.junit-report.xml - # SAVE ARTIFACT cassette.yaml AS LOCAL cassette.yaml - # END \ No newline at end of file + WAIT + SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis.junit-report.xml + SAVE ARTIFACT cassette.yaml AS LOCAL cassette.yaml + END \ No newline at end of file diff --git a/catalyst-gateway/tests/schemathesis_tests/schemathesis-docker-compose.yml b/catalyst-gateway/tests/schemathesis_tests/schemathesis-docker-compose.yml index 61e06672c7d..e896951ef4d 100644 --- a/catalyst-gateway/tests/schemathesis_tests/schemathesis-docker-compose.yml +++ b/catalyst-gateway/tests/schemathesis_tests/schemathesis-docker-compose.yml @@ -29,6 +29,6 @@ services: image: cat-gateway:latest ports: - 3030:3030 - #depends_on: - # event-db: - # condition: service_healthy + depends_on: + event-db: + condition: service_healthy From 8eafea3f610575da0962f191247cfe9e12a8b727 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Thu, 2 Jan 2025 13:41:31 +0100 Subject: [PATCH 09/33] clean up --- catalyst-gateway/tests/schemathesis_tests/Earthfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 142e6319340..88c85b29034 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -10,7 +10,7 @@ package-schemathesis: VOLUME /results ARG openapi_spec ENTRYPOINT export SCHEMATHESIS_HOOKS=hooks \ - && st run --endpoint '^/api/v1/health/started' http://0.0.0.0:3030/docs/cat-gateway.json \ + && st run --endpoint '^/api/v1/health/started' $openapi_spec \ --workers=2 \ --wait-for-schema=120 \ --max-response-time=5000 \ @@ -30,13 +30,12 @@ package-schemathesis: test-fuzzer-api: FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 RUN apk update && apk add iptables-legacy # workaround for https://github.com/earthly/earthly/issues/3784 - RUN apk add yq zstd curl COPY schemathesis-docker-compose.yml . - #LET OPENAPI_SPEC="http://0.0.0.0:3030/docs/cat-gateway.json" + LET OPENAPI_SPEC="http://0.0.0.0:3030/docs/cat-gateway.json" WITH DOCKER \ --compose schemathesis-docker-compose.yml \ - --load schemathesis:latest=(+package-schemathesis) \ + --load schemathesis:latest=(+package-schemathesis --openapi_spec=$OPENAPI_SPEC) \ --load event-db:latest=(../../event-db+build) \ --load cat-gateway:latest=(../+package-cat-gateway-integration) \ --service event-db \ From 8fc85c4ec320e509ae8f7ef013476a6a6a13933e Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Thu, 2 Jan 2025 14:20:50 +0100 Subject: [PATCH 10/33] fix: blueprint --- catalyst-gateway/tests/schemathesis_tests/blueprint.cue | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/catalyst-gateway/tests/schemathesis_tests/blueprint.cue b/catalyst-gateway/tests/schemathesis_tests/blueprint.cue index 4a26b5160d6..0bb32590070 100644 --- a/catalyst-gateway/tests/schemathesis_tests/blueprint.cue +++ b/catalyst-gateway/tests/schemathesis_tests/blueprint.cue @@ -1,2 +1,6 @@ version: "1.0.0" project: name: "gateway-schema-tests" +ci: { + targets: { + test-fuzzer-api: privileged: true + } From 6f2152da72a05e3aa57156512bf2eb03fcbde666 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Thu, 2 Jan 2025 14:38:56 +0100 Subject: [PATCH 11/33] fix: blueprint --- .../tests/schemathesis_tests/blueprint.cue | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/blueprint.cue b/catalyst-gateway/tests/schemathesis_tests/blueprint.cue index 0bb32590070..5f74bd8f759 100644 --- a/catalyst-gateway/tests/schemathesis_tests/blueprint.cue +++ b/catalyst-gateway/tests/schemathesis_tests/blueprint.cue @@ -1,6 +1,8 @@ version: "1.0.0" -project: name: "gateway-schema-tests" -ci: { - targets: { - test-fuzzer-api: privileged: true - } +project: { + name: "gateway-schema-tests" + ci: { + targets: { + test-fuzzer-api: privileged: true + } +} \ No newline at end of file From ea359721296a796225b0ccaf90dbf2b9025207d6 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Thu, 2 Jan 2025 15:12:29 +0100 Subject: [PATCH 12/33] fix: blueprint --- .../tests/schemathesis_tests/blueprint.cue | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/blueprint.cue b/catalyst-gateway/tests/schemathesis_tests/blueprint.cue index 5f74bd8f759..2dd1ee10f4e 100644 --- a/catalyst-gateway/tests/schemathesis_tests/blueprint.cue +++ b/catalyst-gateway/tests/schemathesis_tests/blueprint.cue @@ -1,8 +1,9 @@ version: "1.0.0" project: { - name: "gateway-schema-tests" - ci: { - targets: { - test-fuzzer-api: privileged: true - } -} \ No newline at end of file + name: "gateway-schema-tests" + ci: { + targets: { + test-fuzzer-api: privileged: true + } + } +} From e5030ffe84739967255b19269936069d51aa0972 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Thu, 2 Jan 2025 15:49:51 +0100 Subject: [PATCH 13/33] fix: blueprint --- catalyst-gateway/tests/schemathesis_tests/blueprint.cue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/blueprint.cue b/catalyst-gateway/tests/schemathesis_tests/blueprint.cue index 2dd1ee10f4e..ab7bb81baf9 100644 --- a/catalyst-gateway/tests/schemathesis_tests/blueprint.cue +++ b/catalyst-gateway/tests/schemathesis_tests/blueprint.cue @@ -3,7 +3,7 @@ project: { name: "gateway-schema-tests" ci: { targets: { - test-fuzzer-api: privileged: true + "test-fuzzer-api": privileged: true } } } From 9e9488fc5982fb43a0fa5248aa495afd4bfd5ba5 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Thu, 2 Jan 2025 16:32:48 +0100 Subject: [PATCH 14/33] fix: remove hooks --- catalyst-gateway/tests/schemathesis_tests/Earthfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 88c85b29034..b2877335857 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -9,8 +9,7 @@ package-schemathesis: COPY ./hooks/hooks.py . VOLUME /results ARG openapi_spec - ENTRYPOINT export SCHEMATHESIS_HOOKS=hooks \ - && st run --endpoint '^/api/v1/health/started' $openapi_spec \ + ENTRYPOINT st run --endpoint '^/api/v1/health/started' --checks all $openapi_spec \ --workers=2 \ --wait-for-schema=120 \ --max-response-time=5000 \ From d124756d885b95c5fb1d612b160925432b1faced Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Fri, 3 Jan 2025 15:25:28 +0100 Subject: [PATCH 15/33] fix: remove some checks --- catalyst-gateway/tests/schemathesis_tests/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index b2877335857..1b82b495324 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -9,7 +9,7 @@ package-schemathesis: COPY ./hooks/hooks.py . VOLUME /results ARG openapi_spec - ENTRYPOINT st run --endpoint '^/api/v1/health/started' --checks all $openapi_spec \ + ENTRYPOINT st run --endpoint '^/api/v1/health/started' $openapi_spec \ --workers=2 \ --wait-for-schema=120 \ --max-response-time=5000 \ From a30a4d2e5dc87e3e8d849b94cc30405a54b0cbb9 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Tue, 7 Jan 2025 13:13:46 +0100 Subject: [PATCH 16/33] feat: adds seed param to chemathesis package --- catalyst-gateway/tests/schemathesis_tests/Earthfile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 1b82b495324..3955feab18f 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -9,7 +9,9 @@ package-schemathesis: COPY ./hooks/hooks.py . VOLUME /results ARG openapi_spec - ENTRYPOINT st run --endpoint '^/api/v1/health/started' $openapi_spec \ + # optional argument that can be used to pass a --hypothesis-seed to replicate specific test failures + ARG seed + ENTRYPOINT st run --endpoint '^/api/v1/health/started' --checks all $openapi_spec \ --workers=2 \ --wait-for-schema=120 \ --max-response-time=5000 \ @@ -20,7 +22,8 @@ package-schemathesis: --show-trace \ --force-color \ --junit-xml=/results/junit-report.xml \ - --cassette-path=/results/cassette.yaml + --cassette-path=/results/cassette.yaml \ + $seed ARG tag="latest" SAVE IMAGE schemathesis:$tag @@ -31,10 +34,11 @@ test-fuzzer-api: RUN apk update && apk add iptables-legacy # workaround for https://github.com/earthly/earthly/issues/3784 COPY schemathesis-docker-compose.yml . LET OPENAPI_SPEC="http://0.0.0.0:3030/docs/cat-gateway.json" + ARG seed WITH DOCKER \ --compose schemathesis-docker-compose.yml \ - --load schemathesis:latest=(+package-schemathesis --openapi_spec=$OPENAPI_SPEC) \ + --load schemathesis:latest=(+package-schemathesis --openapi_spec=$OPENAPI_SPEC --seed=$seed) \ --load event-db:latest=(../../event-db+build) \ --load cat-gateway:latest=(../+package-cat-gateway-integration) \ --service event-db \ From 9f2632e6c4a91eb22a38039cb79edf4affd434bb Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Tue, 7 Jan 2025 16:13:50 +0100 Subject: [PATCH 17/33] fix: poetry --- catalyst-gateway/event-db/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/catalyst-gateway/event-db/pyproject.toml b/catalyst-gateway/event-db/pyproject.toml index d7447218181..4f8f75a1b2d 100644 --- a/catalyst-gateway/event-db/pyproject.toml +++ b/catalyst-gateway/event-db/pyproject.toml @@ -7,6 +7,7 @@ description = "" authors = [] readme = "Readme.md" license = "MIT or Apache-2.0" +package-mode = false [tool.poetry.dependencies] python = "^3.11" From f79cdec2f6aeffc0b0a9756211fccec479fdd8f6 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Tue, 7 Jan 2025 16:33:30 +0100 Subject: [PATCH 18/33] fix: earthly dockerd bug --- catalyst-gateway/event-db/Earthfile | 1 + 1 file changed, 1 insertion(+) diff --git a/catalyst-gateway/event-db/Earthfile b/catalyst-gateway/event-db/Earthfile index 47dc36a839b..0ac503b4712 100644 --- a/catalyst-gateway/event-db/Earthfile +++ b/catalyst-gateway/event-db/Earthfile @@ -47,6 +47,7 @@ test: FROM python-ci+python-base DO python-ci+BUILDER + RUN apk update && apk add iptables-legacy # workaround for https://github.com/earthly/earthly/issues/3784 COPY --dir tests . COPY --dir queries . From 50137c7aa63102bb53f6499dfe18112a81385efe Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Tue, 7 Jan 2025 17:04:22 +0100 Subject: [PATCH 19/33] fix: remove checks all param --- catalyst-gateway/tests/schemathesis_tests/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 3955feab18f..3d4a799136b 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -11,7 +11,7 @@ package-schemathesis: ARG openapi_spec # optional argument that can be used to pass a --hypothesis-seed to replicate specific test failures ARG seed - ENTRYPOINT st run --endpoint '^/api/v1/health/started' --checks all $openapi_spec \ + ENTRYPOINT st run --endpoint '^/api/v1/health/started' $openapi_spec \ --workers=2 \ --wait-for-schema=120 \ --max-response-time=5000 \ From f6366f8a52d1898e45f9789e89ecc40d3b5814eb Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Wed, 8 Jan 2025 10:52:07 +0100 Subject: [PATCH 20/33] fix: remove duplicated function --- catalyst-gateway/tests/Earthfile | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/catalyst-gateway/tests/Earthfile b/catalyst-gateway/tests/Earthfile index a7729309ae4..e16a77ec043 100644 --- a/catalyst-gateway/tests/Earthfile +++ b/catalyst-gateway/tests/Earthfile @@ -81,20 +81,3 @@ test-scylla: RUN --mount=$EARTHLY_RUST_CARGO_HOME_CACHE --mount=$EARTHLY_RUST_TARGET_CACHE \ scylla --options-file /etc/scylla/scylla.yaml --smp=2 --memory=4G --overprovisioned --developer-mode=1 & \ cargo nextest run --release --run-ignored=only scylla_session scylla_queries - -# INSTALL_SCYLLA - Installs scylla for bookworm-slim/debian -INSTALL_SCYLLA: - FUNCTION - - ARG --required VERSION - - RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - wget gnupg ca-certificates && \ - mkdir -p /etc/apt/keyrings && \ - gpg --homedir /tmp --no-default-keyring --keyring /etc/apt/keyrings/scylladb.gpg \ - --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys a43e06657bac99e3 && \ - wget -O /etc/apt/sources.list.d/scylla.list http://downloads.scylladb.com/deb/debian/scylla-$VERSION.list && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - scylla \ No newline at end of file From 20051b389e131e3dc18302c909c505d938487368 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Wed, 8 Jan 2025 11:06:17 +0100 Subject: [PATCH 21/33] fix: regex --- catalyst-gateway/tests/schemathesis_tests/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 3d4a799136b..97e8ed756f1 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -11,7 +11,7 @@ package-schemathesis: ARG openapi_spec # optional argument that can be used to pass a --hypothesis-seed to replicate specific test failures ARG seed - ENTRYPOINT st run --endpoint '^/api/v1/health/started' $openapi_spec \ + ENTRYPOINT st run --endpoint '^/api/v1/health' $openapi_spec \ --workers=2 \ --wait-for-schema=120 \ --max-response-time=5000 \ From 63da9aa5757fd636d5efd6cdf2dc9907a14898b0 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Wed, 8 Jan 2025 17:36:14 +0100 Subject: [PATCH 22/33] adds all check --- catalyst-gateway/tests/schemathesis_tests/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 97e8ed756f1..be913a197e5 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -11,7 +11,7 @@ package-schemathesis: ARG openapi_spec # optional argument that can be used to pass a --hypothesis-seed to replicate specific test failures ARG seed - ENTRYPOINT st run --endpoint '^/api/v1/health' $openapi_spec \ + ENTRYPOINT st run --endpoint '^/api/v1/health' --checks=all $openapi_spec \ --workers=2 \ --wait-for-schema=120 \ --max-response-time=5000 \ From fd9b9f20ef137310e643f1014577f1d8fdc8601c Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Thu, 9 Jan 2025 13:57:59 +0100 Subject: [PATCH 23/33] chore: clean up --- .gitignore | 6 +++++- Overall, | 0 catalyst-gateway/tests/schemathesis_tests/Earthfile | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) delete mode 100644 Overall, diff --git a/.gitignore b/.gitignore index 406b997792a..2ffb8743992 100644 --- a/.gitignore +++ b/.gitignore @@ -123,4 +123,8 @@ test-results/ /playwright/.cache/ extensions/ browser_extensions/ -!catalyst_voices/packages/internal/catalyst_voices_driver/lib/src/extensions/ \ No newline at end of file +!catalyst_voices/packages/internal/catalyst_voices_driver/lib/src/extensions/ + +#Test report and coverage +.junit-report.xml +.coverage.info \ No newline at end of file diff --git a/Overall, b/Overall, deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index be913a197e5..1b06aae2f8e 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -31,7 +31,7 @@ package-schemathesis: # test-fuzzer-api - Fuzzy test cat-gateway using openapi specs. test-fuzzer-api: FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 - RUN apk update && apk add iptables-legacy # workaround for https://github.com/earthly/earthly/issues/3784 + RUN apk update && apk add iptables-legacy curl # workaround for https://github.com/earthly/earthly/issues/3784 COPY schemathesis-docker-compose.yml . LET OPENAPI_SPEC="http://0.0.0.0:3030/docs/cat-gateway.json" ARG seed From ebd5d40cd0d07b214c3a7ad65ba8cd14c158356b Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Thu, 9 Jan 2025 14:23:50 +0100 Subject: [PATCH 24/33] clean up --- catalyst-gateway/tests/schemathesis_tests/Earthfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 1b06aae2f8e..328228037c7 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -3,14 +3,15 @@ VERSION 0.8 package-schemathesis: FROM python:3.12-alpine3.20 # TODO: https://github.com/input-output-hk/catalyst-voices/issues/465 + ARG openapi_spec + # optional argument that can be used to pass a --hypothesis-seed to replicate specific test failures + ARG seed + RUN apk add --no-cache gcc musl-dev RUN python -m pip install schemathesis==3.27.1 RUN mkdir /results COPY ./hooks/hooks.py . VOLUME /results - ARG openapi_spec - # optional argument that can be used to pass a --hypothesis-seed to replicate specific test failures - ARG seed ENTRYPOINT st run --endpoint '^/api/v1/health' --checks=all $openapi_spec \ --workers=2 \ --wait-for-schema=120 \ From 185c60262f4e0de85df2fe85f8b5d83887b1c5d6 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Thu, 9 Jan 2025 15:07:14 +0100 Subject: [PATCH 25/33] chore: update schemathesis version --- catalyst-gateway/tests/schemathesis_tests/Earthfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 328228037c7..6cbc3796da2 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -6,9 +6,10 @@ package-schemathesis: ARG openapi_spec # optional argument that can be used to pass a --hypothesis-seed to replicate specific test failures ARG seed + ARG version=3.39.5 RUN apk add --no-cache gcc musl-dev - RUN python -m pip install schemathesis==3.27.1 + RUN python -m pip install schemathesis==$version RUN mkdir /results COPY ./hooks/hooks.py . VOLUME /results From 813827a74c290bf6a5115faea5263f8bb144d8d1 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Thu, 9 Jan 2025 15:49:12 +0100 Subject: [PATCH 26/33] fix: update schemathesis option to latest version --- catalyst-gateway/tests/schemathesis_tests/Earthfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 6cbc3796da2..9c8ee08d073 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -13,13 +13,15 @@ package-schemathesis: RUN mkdir /results COPY ./hooks/hooks.py . VOLUME /results - ENTRYPOINT st run --endpoint '^/api/v1/health' --checks=all $openapi_spec \ + ENTRYPOINT st run --include-path-regex '^/api/v1/health/' \ + --exclude-path '/api/v1/health/inspection' #excluding since this is a internal debug endpoint \ + --checks=all $openapi_spec \ --workers=2 \ --wait-for-schema=120 \ --max-response-time=5000 \ --hypothesis-max-examples=1000 \ --data-generation-method=all \ - --skip-deprecated-operations \ + --exclude-deprecated \ --force-schema-version=30 \ --show-trace \ --force-color \ From 20e5fbd92f6699a0bb3b1a8f0adeac2133b1375d Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Thu, 9 Jan 2025 16:14:35 +0100 Subject: [PATCH 27/33] fixes --- .gitignore | 4 ++-- catalyst-gateway/tests/schemathesis_tests/.gitignore | 4 ++-- catalyst-gateway/tests/schemathesis_tests/Earthfile | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 2ffb8743992..9d15200d205 100644 --- a/.gitignore +++ b/.gitignore @@ -126,5 +126,5 @@ browser_extensions/ !catalyst_voices/packages/internal/catalyst_voices_driver/lib/src/extensions/ #Test report and coverage -.junit-report.xml -.coverage.info \ No newline at end of file +*.junit-report.xml +*.coverage.info \ No newline at end of file diff --git a/catalyst-gateway/tests/schemathesis_tests/.gitignore b/catalyst-gateway/tests/schemathesis_tests/.gitignore index 3b9825d2f9c..12e88d03ad5 100644 --- a/catalyst-gateway/tests/schemathesis_tests/.gitignore +++ b/catalyst-gateway/tests/schemathesis_tests/.gitignore @@ -1,3 +1,3 @@ __pycache__ -junit-report.xml -.hypothesis/ \ No newline at end of file +.hypothesis/ +cassette.yaml \ No newline at end of file diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 9c8ee08d073..895c8239a9f 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -14,7 +14,7 @@ package-schemathesis: COPY ./hooks/hooks.py . VOLUME /results ENTRYPOINT st run --include-path-regex '^/api/v1/health/' \ - --exclude-path '/api/v1/health/inspection' #excluding since this is a internal debug endpoint \ + --exclude-path '/api/v1/health/inspection' \ #excluding since this is a internal debug endpoint --checks=all $openapi_spec \ --workers=2 \ --wait-for-schema=120 \ @@ -49,7 +49,7 @@ test-fuzzer-api: --service cat-gateway \ --allow-privileged - RUN docker run --net=host --name=st schemathesis:latest && \ + RUN docker run --net=host --name=st schemathesis:latest; \ docker cp st:/results/junit-report.xml junit-report.xml && \ docker cp st:/results/cassette.yaml cassette.yaml END From 7558708052798a4ec1e9ad1dbeedf89d698b09e1 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Fri, 10 Jan 2025 10:47:14 +0100 Subject: [PATCH 28/33] fix: st target failing statment --- catalyst-gateway/tests/schemathesis_tests/Earthfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 895c8239a9f..4e29e62855d 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -49,11 +49,15 @@ test-fuzzer-api: --service cat-gateway \ --allow-privileged - RUN docker run --net=host --name=st schemathesis:latest; \ + RUN docker run --net=host --name=st schemathesis:latest || echo fail > fail; \ docker cp st:/results/junit-report.xml junit-report.xml && \ docker cp st:/results/cassette.yaml cassette.yaml END WAIT SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis.junit-report.xml SAVE ARTIFACT cassette.yaml AS LOCAL cassette.yaml + END + IF [ -f fail ] + RUN --no-cache echo "Schemathesis test failed. Check the logs for more details" && \ + exit 1 END \ No newline at end of file From 665b887d15705cf159f61e1805d45a9c248ba75b Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Fri, 10 Jan 2025 18:17:14 +0100 Subject: [PATCH 29/33] feat: adds schemathesis test target --- .../tests/schemathesis_tests/Earthfile | 73 +++++++++++++++++-- 1 file changed, 66 insertions(+), 7 deletions(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 4e29e62855d..383d3c7351d 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -3,9 +3,8 @@ VERSION 0.8 package-schemathesis: FROM python:3.12-alpine3.20 # TODO: https://github.com/input-output-hk/catalyst-voices/issues/465 - ARG openapi_spec - # optional argument that can be used to pass a --hypothesis-seed to replicate specific test failures - ARG seed + ARG api_spec + ARG seed # optional argument that can be used to pass a --hypothesis-seed to replicate specific test failures ARG version=3.39.5 RUN apk add --no-cache gcc musl-dev @@ -13,9 +12,9 @@ package-schemathesis: RUN mkdir /results COPY ./hooks/hooks.py . VOLUME /results - ENTRYPOINT st run --include-path-regex '^/api/v1/health/' \ + ENTRYPOINT st run --exclude-path-regex 'draft' \ --exclude-path '/api/v1/health/inspection' \ #excluding since this is a internal debug endpoint - --checks=all $openapi_spec \ + $api_spec \ --workers=2 \ --wait-for-schema=120 \ --max-response-time=5000 \ @@ -37,12 +36,12 @@ test-fuzzer-api: FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 RUN apk update && apk add iptables-legacy curl # workaround for https://github.com/earthly/earthly/issues/3784 COPY schemathesis-docker-compose.yml . - LET OPENAPI_SPEC="http://0.0.0.0:3030/docs/cat-gateway.json" + LET api_spec="http://0.0.0.0:3030/docs/cat-gateway.json" ARG seed WITH DOCKER \ --compose schemathesis-docker-compose.yml \ - --load schemathesis:latest=(+package-schemathesis --openapi_spec=$OPENAPI_SPEC --seed=$seed) \ + --load schemathesis:latest=(+package-schemathesis --api_spec=$api_spec --seed=$seed) \ --load event-db:latest=(../../event-db+build) \ --load cat-gateway:latest=(../+package-cat-gateway-integration) \ --service event-db \ @@ -60,4 +59,64 @@ test-fuzzer-api: IF [ -f fail ] RUN --no-cache echo "Schemathesis test failed. Check the logs for more details" && \ exit 1 + END + +nightly-package-schemathesis: + FROM python:3.12-alpine3.20 + # TODO: https://github.com/input-output-hk/catalyst-voices/issues/465 + ARG api_spec + # optional argument that can be used to pass a --hypothesis-seed to replicate specific test failures + ARG seed + ARG version=3.39.5 + + RUN apk add --no-cache gcc musl-dev + RUN python -m pip install schemathesis==$version + RUN mkdir /results + COPY ./hooks/hooks.py . + VOLUME /results + ENTRYPOINT st run --checks=all $api_spec \ + --workers=2 \ + --wait-for-schema=120 \ + --max-response-time=300 \ + --hypothesis-max-examples=1000 \ + --data-generation-method=all \ + --exclude-deprecated \ + --force-schema-version=30 \ + --show-trace \ + --force-color \ + --junit-xml=/results/junit-report.xml \ + --cassette-path=/results/cassette.yaml \ + $seed + + ARG tag="latest" + SAVE IMAGE schemathesis:$tag + +# test-fuzzer-api - Fuzzy test cat-gateway using openapi specs. +nightly-test-fuzzer-api: + FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 + RUN apk update && apk add iptables-legacy curl # workaround for https://github.com/earthly/earthly/issues/3784 + COPY schemathesis-docker-compose.yml . + LET api_spec="http://0.0.0.0:3030/docs/cat-gateway.json" + ARG seed + + WITH DOCKER \ + --compose schemathesis-docker-compose.yml \ + --load schemathesis:latest=(+nightly-package-schemathesis --api_spec=$api_spec --seed=$seed) \ + --load event-db:latest=(../../event-db+build) \ + --load cat-gateway:latest=(../+package-cat-gateway-integration) \ + --service event-db \ + --service cat-gateway \ + --allow-privileged + + RUN --no-cache docker run --net=host --name=st schemathesis:latest || echo fail > fail; \ + docker cp st:/results/junit-report.xml junit-report.xml && \ + docker cp st:/results/cassette.yaml cassette.yaml + END + WAIT + SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis-nightly.junit-report.xml + SAVE ARTIFACT cassette.yaml AS LOCAL cassette.yaml + END + IF [ -f fail ] + RUN --no-cache echo "Nightly schemathesis test failed. Check the logs for more details" && \ + exit 1 END \ No newline at end of file From 90f18236628a647bb727fabb4dbcc8e4d5797912 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Fri, 10 Jan 2025 19:47:43 +0100 Subject: [PATCH 30/33] test run --- catalyst-gateway/tests/schemathesis_tests/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 383d3c7351d..a024145b0ad 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -92,7 +92,7 @@ nightly-package-schemathesis: SAVE IMAGE schemathesis:$tag # test-fuzzer-api - Fuzzy test cat-gateway using openapi specs. -nightly-test-fuzzer-api: +test-nightly-test-fuzzer-api: FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 RUN apk update && apk add iptables-legacy curl # workaround for https://github.com/earthly/earthly/issues/3784 COPY schemathesis-docker-compose.yml . From 067eb2e26895390773d4dabccf458b65dd6b0eb9 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Fri, 10 Jan 2025 23:37:23 +0100 Subject: [PATCH 31/33] test run --- catalyst-gateway/tests/schemathesis_tests/blueprint.cue | 1 + 1 file changed, 1 insertion(+) diff --git a/catalyst-gateway/tests/schemathesis_tests/blueprint.cue b/catalyst-gateway/tests/schemathesis_tests/blueprint.cue index ab7bb81baf9..9f3255970d7 100644 --- a/catalyst-gateway/tests/schemathesis_tests/blueprint.cue +++ b/catalyst-gateway/tests/schemathesis_tests/blueprint.cue @@ -4,6 +4,7 @@ project: { ci: { targets: { "test-fuzzer-api": privileged: true + "test-nightly-test-fuzzer-api": privileged: true } } } From d6d6f03e6bec7a2be6f916fae273dd8a167e6d38 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Sat, 11 Jan 2025 00:11:31 +0100 Subject: [PATCH 32/33] remove test values --- catalyst-gateway/tests/schemathesis_tests/Earthfile | 4 ++-- catalyst-gateway/tests/schemathesis_tests/blueprint.cue | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index a024145b0ad..b57809dc7ee 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -91,8 +91,8 @@ nightly-package-schemathesis: ARG tag="latest" SAVE IMAGE schemathesis:$tag -# test-fuzzer-api - Fuzzy test cat-gateway using openapi specs. -test-nightly-test-fuzzer-api: +# nightly-test-fuzzer-api - Fuzzy test cat-gateway using openapi specs. +nightly-test-fuzzer-api: FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 RUN apk update && apk add iptables-legacy curl # workaround for https://github.com/earthly/earthly/issues/3784 COPY schemathesis-docker-compose.yml . diff --git a/catalyst-gateway/tests/schemathesis_tests/blueprint.cue b/catalyst-gateway/tests/schemathesis_tests/blueprint.cue index 9f3255970d7..134e32aaa5e 100644 --- a/catalyst-gateway/tests/schemathesis_tests/blueprint.cue +++ b/catalyst-gateway/tests/schemathesis_tests/blueprint.cue @@ -4,7 +4,7 @@ project: { ci: { targets: { "test-fuzzer-api": privileged: true - "test-nightly-test-fuzzer-api": privileged: true + "nightly-test-fuzzer-api": privileged: true } } } From d65e0378198624a92e5860176d7e95763d052ee8 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Wed, 15 Jan 2025 09:38:24 +0100 Subject: [PATCH 33/33] test run --- catalyst-gateway/tests/schemathesis_tests/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index b980f9cb6ab..f5cdb0f9a4c 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -17,7 +17,7 @@ package-schemathesis: --exclude-path '/api/v1/health/inspection' \ #excluding since this is a internal debug endpoint $api_spec \ --workers=2 \ - --wait-for-schema=120 \ + --wait-for-schema=500 \ --max-response-time=5000 \ --hypothesis-max-examples=1000 \ --data-generation-method=all \