Skip to content

Commit

Permalink
undo test changes
Browse files Browse the repository at this point in the history
  • Loading branch information
Tishj committed Jan 13, 2025
1 parent 36bd8d8 commit 04a6f34
Show file tree
Hide file tree
Showing 40 changed files with 426 additions and 30 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ storage persistent

load
DROP TABLE IF EXISTS test;
PRAGMA force_compression='dict_fsst';
PRAGMA force_compression='dictionary';
CREATE TABLE test AS SELECT (100 + (i%1000))::VARCHAR AS i FROM range(0, 200000000) tbl(i);
checkpoint;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ storage persistent

load
DROP TABLE IF EXISTS test;
PRAGMA force_compression='dict_fsst';
PRAGMA force_compression='dictionary';
CREATE TABLE test AS SELECT (100 + (i%2))::VARCHAR AS i FROM range(0, 200000000) tbl(i);
checkpoint;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ storage persistent

load
DROP TABLE IF EXISTS test;
PRAGMA force_compression='dict_fsst';
PRAGMA force_compression='dictionary';
CREATE TABLE test AS SELECT i::VARCHAR AS i FROM range(0, 200000000) tbl(i);
checkpoint;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ storage persistent
require_reinit

load
PRAGMA force_compression='dict_fsst';
PRAGMA force_compression='dictionary';
DROP TABLE IF EXISTS test;

run
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ storage persistent
require_reinit

load
PRAGMA force_compression='dict_fsst';
PRAGMA force_compression='dictionary';
DROP TABLE IF EXISTS test;

run
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ storage persistent

load
DROP TABLE IF EXISTS test;
PRAGMA force_compression='dict_fsst';
PRAGMA force_compression='fsst';
CREATE TABLE test AS SELECT i as id, (100 + (i%2))::VARCHAR AS value FROM range(0, 50000000) tbl(i);
checkpoint;
SET enable_fsst_vectors=false;
Expand Down
2 changes: 1 addition & 1 deletion benchmark/micro/compression/fsst/fsst_read.benchmark
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ storage persistent

load
DROP TABLE IF EXISTS test;
PRAGMA force_compression='dict_fsst';
PRAGMA force_compression='fsst';
CREATE TABLE test AS SELECT (100 + (i%1000))::VARCHAR AS i FROM range(0, 50000000) tbl(i);
checkpoint;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ storage persistent

load
DROP TABLE IF EXISTS test;
PRAGMA force_compression='dict_fsst';
PRAGMA force_compression='fsst';
CREATE TABLE test AS SELECT gen_random_uuid()::VARCHAR AS i FROM range(0, 20000000) tbl(i);
checkpoint;

Expand Down
2 changes: 1 addition & 1 deletion benchmark/micro/compression/fsst/fsst_store.benchmark
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ storage persistent
require_reinit

load
PRAGMA force_compression='dict_fsst';
PRAGMA force_compression='fsst';

run
CREATE TABLE test_compressed AS SELECT (100 + (i%1000))::VARCHAR AS i FROM range(0, 2500000) tbl(i);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ storage persistent
require_reinit

load
PRAGMA force_compression='dict_fsst';
PRAGMA force_compression='fsst';
DROP TABLE IF EXISTS test;

run
Expand Down
2 changes: 1 addition & 1 deletion test/issues/general/test_3611.test
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ statement ok
PRAGMA enable_verification

statement ok
PRAGMA force_compression='dict_fsst'
PRAGMA force_compression='dictionary'

statement ok
CREATE TABLE all_types AS SELECT varchar FROM test_all_types();
Expand Down
2 changes: 1 addition & 1 deletion test/issues/general/test_5488.test
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
load __TEST_DIR__/issue_5488.db

statement ok
pragma force_compression='dict_fsst'
pragma force_compression='dictionary'

statement ok
CREATE TABLE test ( col_a TEXT);
Expand Down
2 changes: 1 addition & 1 deletion test/sql/pragma/test_force_compression.test
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# description: Test PRAGMA force_compression
# group: [pragma]

foreach compression none uncompressed rle dict_fsst pfor bitpacking
foreach compression none uncompressed rle dictionary pfor bitpacking fsst dict_fsst

statement ok
PRAGMA force_compression='${compression}'
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# name: test/sql/storage/compression/dictionary/dictionary_compression_ratio.test_slow
# description: Assert dictionary compression ratio is within reasonable margins
# group: [dictionary]

load __TEST_DIR__/test_dictionary.db

# First test: detailed compression ratio
statement ok
PRAGMA force_compression='dictionary';

# Assuming 10 chars at 1 byte, with a 4byte offset and a 2byte length per string uncompressed:
# Ratio absolute max at 3 bits per value (ignoring dict size) = (16/(3/8)) = 42.6666666667
statement ok
CREATE TABLE test_dictionary AS SELECT concat('BEEPBOOP-', (i%3)::VARCHAR) AS i FROM range(0, 1250000) tbl(i);

statement ok
CHECKPOINT;

statement ok
PRAGMA force_compression='uncompressed';

statement ok
CREATE TABLE test_uncompressed AS SELECT concat('BEEPBOOP-', (i%3)::VARCHAR) AS i FROM range(0, 1250000) tbl(i);

statement ok
CHECKPOINT;

# keep a wide margin for the compression ratio to account for changes (like the block size) that
# influence the compression ratio

query I
SELECT uncompressed::FLOAT / dictionary::FLOAT > 30 AND uncompressed::FLOAT / dictionary::FLOAT < 55 FROM
(SELECT count(DISTINCT block_id) AS dictionary FROM pragma_storage_info('test_dictionary') WHERE segment_type IN ('VARCHAR')) AS dictionary,
(SELECT count(DISTINCT block_id) AS uncompressed FROM pragma_storage_info('test_uncompressed') WHERE segment_type IN ('VARCHAR')) AS uncompressed;
----
True
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
# name: test/sql/storage/compression/dictionary/dictionary_covers_validity.test
# group: [dictionary]

load __TEST_DIR__/dictionary_covers_validity

statement ok
set checkpoint_threshold='10mb';

statement ok
CREATE TABLE tbl AS SELECT
{
'a': i,
'b': NULL::VARCHAR
} col
FROM range(5000) t(i);

statement ok
set force_compression='dictionary';

statement ok
INSERT INTO tbl VALUES (
{
'a': 10000,
'b': 'hello'
}
);

statement ok
force checkpoint;

# Dictionary covers the validity, so the validity gets replaced with "Empty Validity"
query II
select segment_type, compression from pragma_storage_info('tbl');
----
VALIDITY Constant
BIGINT BitPacking
VALIDITY Constant
VARCHAR Dictionary
VALIDITY Empty Validity

# Now force a different compression method, that doesn't cover the validity
statement ok
set force_compression='fsst';

statement ok
INSERT INTO tbl VALUES (
{
'a': 10000,
'b': 'hello'
}
);

statement ok
force checkpoint;

# During checkpoint this will scan the dictionary compressed segments to get the validity
# this then gets compressed as normal (since FSST does not cover the validity)
query II
select segment_type, compression from pragma_storage_info('tbl');
----
VALIDITY Constant
BIGINT BitPacking
VALIDITY Constant
VARCHAR FSST
VALIDITY Roaring

query I
SELECT col FROM tbl ORDER BY col.a DESC LIMIT 3;
----
{'a': 10000, 'b': hello}
{'a': 10000, 'b': hello}
{'a': 4999, 'b': NULL}
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# name: test/sql/storage/compression/dictionary/dictionary_storage_info.test
# description: Test storage with Dictionary compression
# group: [dictionary]

# load the DB from disk
load __TEST_DIR__/test_dictionary.db

statement ok
PRAGMA force_compression = 'dictionary'

statement ok
CREATE TABLE test (a VARCHAR, b VARCHAR);

statement ok
INSERT INTO test VALUES ('11', '22'), ('11', '22'), ('12', '21'), (NULL, NULL)

statement ok
CHECKPOINT

query I
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'VARCHAR' LIMIT 1
----
Dictionary
52 changes: 52 additions & 0 deletions test/sql/storage/compression/dictionary/fetch_row.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
# name: test/sql/storage/compression/dictionary/fetch_row.test
# description: Test storage with Dictionary compression
# group: [dictionary]

require block_size 262144

# load the DB from disk
load __TEST_DIR__/test_dictionary_fetchrow.db

statement ok
PRAGMA force_compression = 'dictionary'

statement ok
CREATE TABLE test (
a INTEGER,
b VARCHAR
);

statement ok
INSERT INTO test (a, b)
SELECT
x AS a,
CASE x % 5
WHEN 0 THEN 'aaaa'
WHEN 1 THEN 'bbbb'
WHEN 2 THEN 'cccc'
WHEN 3 THEN 'dddd'
WHEN 4 THEN NULL
END AS b
FROM range(10_000) t(x);

statement ok
CHECKPOINT

restart

query I
SELECT compression FROM pragma_storage_info('test') WHERE segment_type ILIKE 'VARCHAR' LIMIT 1
----
Dictionary

statement ok
pragma verify_fetch_row;

query I
select distinct b from test order by a % 5;
----
aaaa
bbbb
cccc
dddd
NULL
24 changes: 24 additions & 0 deletions test/sql/storage/compression/dictionary/force_dictionary.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# name: test/sql/storage/compression/dictionary/force_dictionary.test
# description: Test forcing dictionary encoding as the compression scheme
# group: [dictionary]

require vector_size 2048

load __TEST_DIR__/force_dictionary.db

statement ok
PRAGMA force_compression = 'dictionary'

statement ok
CREATE TABLE test_dict (a VARCHAR);

statement ok
INSERT INTO test_dict SELECT i::VARCHAR FROM range(0, 2000) tbl(i);

statement ok
CHECKPOINT

query I
SELECT compression FROM pragma_storage_info('test_dict') WHERE segment_type ILIKE 'VARCHAR' LIMIT 1
----
Dictionary
Loading

0 comments on commit 04a6f34

Please sign in to comment.