diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..e69de29bb diff --git a/Cargo.lock b/Cargo.lock index 4593928c9..8b79ba3b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] @@ -147,6 +147,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -203,9 +212,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.7" +version = "1.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8191fb3091fa0561d1379ef80333c3c7191c6f0435d986e85821bcf7acbd1126" +checksum = "7198e6f03240fdceba36656d8be440297b6b82270325908c7381f37d826a74f6" dependencies = [ "aws-credential-types", "aws-runtime", @@ -245,9 +254,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f95446d919226d587817a7d21379e6eb099b97b45110a7f272a444ca5c54070" +checksum = "cdd82dba44d209fddb11c190e0a94b78651f95299598e472215667417a03ff1d" dependencies = [ "aws-lc-sys", "mirai-annotations", @@ -257,9 +266,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.21.2" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3ddc4a5b231dd6958b140ff3151b6412b3f4321fab354f399eec8f14b06df62" +checksum = "df7a4168111d7eb622a31b214057b8509c0a7e1794f44c546d742330dc793972" dependencies = [ "bindgen", "cc", @@ -298,9 +307,9 @@ dependencies = [ [[package]] name = "aws-sdk-kms" -version = "1.45.0" +version = "1.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0caf20b8855dbeb458552e6c8f8f9eb92b95e4a131725b93540ec73d60c38eb3" +checksum = "564a597a3c71a957d60a2e4c62c93d78ee5a0d636531e15b760acad983a5c18e" dependencies = [ "aws-credential-types", "aws-runtime", @@ -320,9 +329,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.53.0" +version = "1.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43fad71130014e11f42fadbdcce5df12ee61866f8ab9bad773b138d4b3c11087" +checksum = "cecd672c8d4265fd4fbecacd4a479180e616881bbe639250cf81ddb604e4c301" dependencies = [ "ahash", "aws-credential-types", @@ -355,9 +364,9 @@ dependencies = [ [[package]] name = "aws-sdk-secretsmanager" -version = "1.48.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34f0ed64c48d8d703d93791e5315f9c12789f7280e8ee7c2c8fe8c1bc5d91907" +checksum = "05a5d3faceba815f3a81039e0c6952e7afad1467c0e2378d28f642c2a5fb299b" dependencies = [ "aws-credential-types", "aws-runtime", @@ -378,9 +387,9 @@ dependencies = [ [[package]] name = "aws-sdk-sns" -version = "1.45.0" +version = "1.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30768bb93c6f1e5b466e9d477d1172c02a55bc5697075f702e4935980695e935" +checksum = "93d16dcbb2991fb5e00997cb4ceabb80b53a9b8bf7c3755da112c8c5f011b4ef" dependencies = [ "aws-credential-types", "aws-runtime", @@ -401,9 +410,9 @@ dependencies = [ [[package]] name = "aws-sdk-sqs" -version = "1.44.0" +version = "1.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e88af26c1a077a59e1146c4bbb55d64cb84cb1a0dd14c7d40cc273e9292b43" +checksum = "657982a9e70b8aa1b903c84f8e76c36202358c48f119330d5f4b74d7e6cf27b7" dependencies = [ "aws-credential-types", "aws-runtime", @@ -423,9 +432,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.44.0" +version = "1.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b90cfe6504115e13c41d3ea90286ede5aa14da294f3fe077027a6e83850843c" +checksum = "0dc2faec3205d496c7e57eff685dd944203df7ce16a4116d0281c44021788a7b" dependencies = [ "aws-credential-types", "aws-runtime", @@ -445,9 +454,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.45.0" +version = "1.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167c0fad1f212952084137308359e8e4c4724d1c643038ce163f06de9662c1d0" +checksum = "c93c241f52bc5e0476e259c953234dab7e2a35ee207ee202e86c0095ec4951dc" dependencies = [ "aws-credential-types", "aws-runtime", @@ -467,9 +476,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.44.0" +version = "1.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb5f98188ec1435b68097daa2a37d74b9d17c9caa799466338a8d1544e71b9d" +checksum = "b259429be94a3459fa1b00c5684faee118d74f9577cc50aebadc36e507c63b5f" dependencies = [ "aws-credential-types", "aws-runtime", @@ -602,9 +611,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1ce695746394772e7000b39fe073095db6d45a862d0767dd5ad0ac0d7f8eb87" +checksum = "a065c0fe6fdbdf9f11817eb68582b2ab4aff9e9c39e986ae48f7ec576c6322db" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -814,9 +823,9 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.69.4" +version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ "bitflags 2.6.0", "cexpr", @@ -900,18 +909,18 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytemuck" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae" +checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" dependencies = [ "bytemuck_derive", ] [[package]] name = "bytemuck_derive" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc8b54b395f2fcfbb3d90c47b01c7f444d94d05bdeb775811dec868ac3bbc26" +checksum = "bcfcc3cd946cb52f0bbfdbbcfa2f4e24f75ebb6c0e1002f7c25904fada18b9ec" dependencies = [ "proc-macro2", "quote", @@ -940,6 +949,27 @@ dependencies = [ "either", ] +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cadence" version = "1.5.0" @@ -957,9 +987,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.1.24" +version = "1.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938" +checksum = "b16803a61b81d9eabb7eae2588776c4c1e584b738ede45fdbb4c972cec1e9945" dependencies = [ "jobserver", "libc", @@ -1045,9 +1075,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.19" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7be5744db7978a28d9df86a214130d106a89ce49644cbc4e3f0c22c3fba30615" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ "clap_builder", "clap_derive", @@ -1055,9 +1085,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.19" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5fbc17d3ef8278f55b282b2a2e75ae6f6c7d4bb70ed3d0382375104bfafdb4b" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" dependencies = [ "anstream", "anstyle", @@ -1166,6 +1196,12 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "convert_case" version = "0.4.0" @@ -1440,6 +1476,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" +[[package]] +name = "deflate64" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" + [[package]] name = "der" version = "0.6.1" @@ -1481,6 +1523,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "derive_more" version = "0.99.18" @@ -1533,6 +1586,17 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "dlv-list" version = "0.5.2" @@ -1720,6 +1784,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "flate2" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "float_eq" version = "1.0.1" @@ -1743,6 +1817,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -1781,9 +1861,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -1807,9 +1887,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -1832,15 +1912,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -1860,9 +1940,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" @@ -1881,9 +1961,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -1892,21 +1972,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -1943,9 +2023,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" @@ -2033,6 +2113,11 @@ name = "hashbrown" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "hashlink" @@ -2046,7 +2131,7 @@ dependencies = [ [[package]] name = "hawk-pack" version = "0.1.0" -source = "git+https://github.com/Inversed-Tech/hawk-pack.git?rev=d34a1b3#d34a1b31489a505e29cdb56ecddbd64e3f5c4b7e" +source = "git+https://github.com/therealyingtong/hawk-pack.git?branch=new-with-params#7e6137b6e4253d709427aacc531f188da795fa20" dependencies = [ "aes-prng 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "criterion", @@ -2291,7 +2376,7 @@ dependencies = [ "http 1.1.0", "hyper 1.4.1", "hyper-util", - "rustls 0.23.13", + "rustls 0.23.14", "rustls-pki-types", "tokio", "tokio-rustls 0.26.0", @@ -2463,9 +2548,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" [[package]] name = "iris-mpc" @@ -2564,10 +2649,12 @@ dependencies = [ "num-traits", "rand", "serde", + "sqlx", "static_assertions", "tokio", "tracing", "tracing-test", + "zip", ] [[package]] @@ -2639,7 +2726,7 @@ dependencies = [ "rand", "rand_chacha", "rcgen", - "rustls 0.23.13", + "rustls 0.23.14", "rustls-pemfile 2.2.0", "serde", "serde-big-array", @@ -2722,9 +2809,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" dependencies = [ "wasm-bindgen", ] @@ -2832,6 +2919,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.22" @@ -2840,11 +2933,11 @@ checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "lru" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.15.0", ] [[package]] @@ -2856,6 +2949,16 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -3046,7 +3149,7 @@ dependencies = [ "hmac", "lazy_static", "md-5", - "pbkdf2", + "pbkdf2 0.11.0", "percent-encoding", "rand", "rustc_version_runtime", @@ -3246,21 +3349,18 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.36.4" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.20.1" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" -dependencies = [ - "portable-atomic", -] +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "oorandom" @@ -3478,9 +3578,9 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "pathdiff" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" +checksum = "d61c5ce1153ab5b689d0c074c4e7fc613e942dfb7dd9eea5ab202d2ad91fe361" [[package]] name = "pbkdf2" @@ -3491,6 +3591,16 @@ dependencies = [ "digest", ] +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + [[package]] name = "pem" version = "3.0.4" @@ -3563,18 +3673,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" dependencies = [ "proc-macro2", "quote", @@ -3700,9 +3810,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" dependencies = [ "unicode-ident", ] @@ -4131,9 +4241,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.13" +version = "0.23.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" +checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" dependencies = [ "aws-lc-rs", "log", @@ -4225,9 +4335,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.24" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" +checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" dependencies = [ "windows-sys 0.59.0", ] @@ -4487,6 +4597,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "sketches-ddsketch" version = "0.2.2" @@ -5106,7 +5222,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.13", + "rustls 0.23.14", "rustls-pki-types", "tokio", ] @@ -5545,9 +5661,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", "once_cell", @@ -5556,9 +5672,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", @@ -5571,9 +5687,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.43" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" +checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" dependencies = [ "cfg-if", "js-sys", @@ -5583,9 +5699,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5593,9 +5709,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", @@ -5606,15 +5722,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "web-sys" -version = "0.3.70" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" dependencies = [ "js-sys", "wasm-bindgen", @@ -5993,3 +6109,88 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + +[[package]] +name = "zip" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494" +dependencies = [ + "aes", + "arbitrary", + "bzip2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "deflate64", + "displaydoc", + "flate2", + "hmac", + "indexmap", + "lzma-rs", + "memchr", + "pbkdf2 0.12.2", + "rand", + "sha1", + "thiserror", + "time", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", +] + +[[package]] +name = "zstd" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.13+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/iris-mpc-common/src/iris_db/iris.rs b/iris-mpc-common/src/iris_db/iris.rs index b8acc9e88..74ceb6359 100644 --- a/iris-mpc-common/src/iris_db/iris.rs +++ b/iris-mpc-common/src/iris_db/iris.rs @@ -4,12 +4,14 @@ use rand::{ distributions::{Bernoulli, Distribution}, Rng, }; +use serde::{Deserialize, Serialize}; +use serde_big_array::BigArray; pub const MATCH_THRESHOLD_RATIO: f64 = 0.375; #[repr(transparent)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct IrisCodeArray(pub [u64; Self::IRIS_CODE_SIZE_U64]); +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Hash)] +pub struct IrisCodeArray(#[serde(with = "BigArray")] pub [u64; Self::IRIS_CODE_SIZE_U64]); impl Default for IrisCodeArray { fn default() -> Self { Self::ZERO diff --git a/iris-mpc-cpu/Cargo.toml b/iris-mpc-cpu/Cargo.toml index 3ec874db8..915d12907 100644 --- a/iris-mpc-cpu/Cargo.toml +++ b/iris-mpc-cpu/Cargo.toml @@ -15,12 +15,13 @@ bytemuck.workspace = true dashmap = "6.1.0" eyre.workspace = true futures.workspace = true -hawk-pack = { git = "https://github.com/Inversed-Tech/hawk-pack.git", rev = "d34a1b3" } +hawk-pack = { git = "https://github.com/therealyingtong/hawk-pack.git", branch = "new-with-params" } iris-mpc-common = { path = "../iris-mpc-common" } itertools.workspace = true num-traits.workspace = true rand.workspace = true serde.workspace = true +sqlx.workspace = true static_assertions.workspace = true tokio.workspace = true tracing.workspace = true @@ -28,7 +29,15 @@ tracing-test = "0.2.5" [dev-dependencies] criterion = { version = "0.5.1", features = ["async_tokio"] } +zip = "2.2.0" + +[features] +db_dependent = [] [[bench]] name = "hnsw" harness = false + +[[bench]] +name = "hnsw_db" +harness = false diff --git a/iris-mpc-cpu/benches/assets/.gitattributes b/iris-mpc-cpu/benches/assets/.gitattributes new file mode 100644 index 000000000..1b82ce12d --- /dev/null +++ b/iris-mpc-cpu/benches/assets/.gitattributes @@ -0,0 +1,17 @@ +hnsw_db_1000000_hawk_graph_links.csv.zip filter=lfs diff=lfs merge=lfs -text +hnsw_db_1000000_hawk_vectors.csv.zip filter=lfs diff=lfs merge=lfs -text +hnsw_db_100000_hawk_graph_links.csv.zip filter=lfs diff=lfs merge=lfs -text +hnsw_db_100000_hawk_vectors.csv.zip filter=lfs diff=lfs merge=lfs -text +hnsw_db_200000_hawk_graph_links.csv.zip filter=lfs diff=lfs merge=lfs -text +hnsw_db_200000_hawk_vectors.csv.zip filter=lfs diff=lfs merge=lfs -text +100K_rust_format_synthetic_data.dat.zip filter=lfs diff=lfs merge=lfs -text +processed_masked_irises_chunk_8 filter=lfs diff=lfs merge=lfs -text +processed_masked_irises_chunk_9 filter=lfs diff=lfs merge=lfs -text +processed_masked_irises_chunk_3 filter=lfs diff=lfs merge=lfs -text +processed_masked_irises_chunk_4 filter=lfs diff=lfs merge=lfs -text +processed_masked_irises_chunk_2 filter=lfs diff=lfs merge=lfs -text +processed_masked_irises_chunk_5 filter=lfs diff=lfs merge=lfs -text +processed_masked_irises_chunk_6 filter=lfs diff=lfs merge=lfs -text +processed_masked_irises_chunk_7 filter=lfs diff=lfs merge=lfs -text +processed_masked_irises_chunk_0 filter=lfs diff=lfs merge=lfs -text +processed_masked_irises_chunk_1 filter=lfs diff=lfs merge=lfs -text diff --git a/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_0 b/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_0 new file mode 100644 index 000000000..9923f72ad --- /dev/null +++ b/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cac4b339df2cde28e2de1776837cc8824fe3b40138c2b479f1220e12ecfda7f +size 1280000000 diff --git a/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_1 b/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_1 new file mode 100644 index 000000000..9d84c8f3e --- /dev/null +++ b/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa8ba22a085755bacd30bbacfb2d8d90738e586dd3d5ffc165a310b4e8ed6433 +size 1280000000 diff --git a/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_2 b/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_2 new file mode 100644 index 000000000..35aa26205 --- /dev/null +++ b/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:030a4eea8c5e066fedc6f3496a79b6b73fb9892d4fe9b5e4852800cb87c43b62 +size 1280000000 diff --git a/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_3 b/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_3 new file mode 100644 index 000000000..2b2cfa757 --- /dev/null +++ b/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_3 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7ab16e2f0d9a629693ff08b6263d6d1f157dd6f2ccfa2446628dbb09ced75b8 +size 1280000000 diff --git a/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_4 b/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_4 new file mode 100644 index 000000000..6d8649aea --- /dev/null +++ b/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d97d03932c9e34c496880eec1e5387ecd4f9fa18ba4663ee9e613c5fb07f3b4 +size 1280000000 diff --git a/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_5 b/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_5 new file mode 100644 index 000000000..44c78d36d --- /dev/null +++ b/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d598e1baca7d0493497d7a6078c43ceee90ee47759a28144724114896f352a2 +size 1280000000 diff --git a/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_6 b/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_6 new file mode 100644 index 000000000..7a5e913f9 --- /dev/null +++ b/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_6 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0146318477420af7aa3dd805630e6c16b89c96cd2f73b66d1ba613b413626b93 +size 1280000000 diff --git a/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_7 b/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_7 new file mode 100644 index 000000000..2d99063e9 --- /dev/null +++ b/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_7 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e2441f599517d41af205373ab4333df8eb36512039a32344dff7734c6aff59b +size 1280000000 diff --git a/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_8 b/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_8 new file mode 100644 index 000000000..4a8079b80 --- /dev/null +++ b/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_8 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ace6c6681ac8dea80788a3534ba0759b294f9b39ef50111d315667093543abeb +size 1280000000 diff --git a/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_9 b/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_9 new file mode 100644 index 000000000..429a7594d --- /dev/null +++ b/iris-mpc-cpu/benches/assets/processed_masked_irises_chunk_9 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ff6e33ad767d7bbf71564ccfdcf928ba9dc46e15bc7470e05347ac96b6af4e0 +size 1280000000 diff --git a/iris-mpc-cpu/benches/hnsw.rs b/iris-mpc-cpu/benches/hnsw.rs index db71a3981..174bbeed8 100644 --- a/iris-mpc-cpu/benches/hnsw.rs +++ b/iris-mpc-cpu/benches/hnsw.rs @@ -15,7 +15,7 @@ fn bench_plaintext_hnsw(c: &mut Criterion) { group.sample_size(10); group.sampling_mode(SamplingMode::Flat); - for database_size in [100_usize, 1000, 10000] { + for database_size in [10000] { let rt = tokio::runtime::Builder::new_multi_thread() .enable_all() .build() @@ -29,7 +29,9 @@ fn bench_plaintext_hnsw(c: &mut Criterion) { for _ in 0..database_size { let raw_query = IrisCode::random_rng(&mut rng); - let query = plain_searcher.vector_store.prepare_query(raw_query.clone()); + let query = plain_searcher + .vector_store + .prepare_query(raw_query.clone().into()); let neighbors = plain_searcher.search_to_insert(&query).await; let inserted = plain_searcher.vector_store.insert(&query).await; plain_searcher @@ -45,7 +47,7 @@ fn bench_plaintext_hnsw(c: &mut Criterion) { |mut my_db| async move { let mut rng = AesRng::seed_from_u64(0_u64); let on_the_fly_query = IrisDB::new_random_rng(1, &mut rng).db[0].clone(); - let query = my_db.vector_store.prepare_query(on_the_fly_query); + let query = my_db.vector_store.prepare_query(on_the_fly_query.into()); let neighbors = my_db.search_to_insert(&query).await; my_db.insert_from_search_results(query, neighbors).await; }, diff --git a/iris-mpc-cpu/benches/hnsw_db.rs b/iris-mpc-cpu/benches/hnsw_db.rs new file mode 100644 index 000000000..9a5b3d898 --- /dev/null +++ b/iris-mpc-cpu/benches/hnsw_db.rs @@ -0,0 +1,132 @@ +use aes_prng::AesRng; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, SamplingMode}; +use hawk_pack::{ + graph_store::{GraphMem, GraphPg}, + hnsw_db::HawkSearcher, + DbStore, VectorStore, +}; +use iris_mpc_common::iris_db::db::IrisDB; +use iris_mpc_cpu::hawkers::{ + plaintext_store::PlaintextStore, plaintext_store_db::PlaintextStoreDb, +}; +use rand::{RngCore, SeedableRng}; +use std::vec; + +const HAWK_DATABASE_URL: &str = "postgres://postgres:postgres@localhost/postgres"; + +/// Table names +const HAWK_GRAPH_ENTRY: &str = "hawk_graph_entry"; +const HAWK_GRAPH_LINKS: &str = "hawk_graph_links"; +const HAWK_VECTORS: &str = "hawk_vectors"; + +fn csv_filename(db_size: usize, table_name: String) -> String { + format!("hnsw_db_{}_{}.csv", db_size, table_name) +} + +fn zip_filename(db_size: usize, table_name: String) -> String { + format!("{}.zip", csv_filename(db_size, table_name)) +} + +fn to_path(file: &str) -> String { + format!("./benches/assets/{}", file.to_string()) +} + +fn unzip(zip_file: &str) { + let zip_path = to_path(zip_file); + + let path = std::path::Path::new(&zip_path); + let zip = std::fs::File::open(path).unwrap(); + let mut archive: zip::ZipArchive = zip::ZipArchive::new(zip).unwrap(); + archive + .extract("./benches/assets") + .expect(&format!("Could not extract {}", zip_file)); +} + +async fn hawk_searcher_from_csv( + mut rng: impl RngCore, + db_size: usize, + graph_store: GraphPg, + vector_store: PlaintextStoreDb, +) -> HawkSearcher> { + // Unzip hawk_graph_links and hawk_vectors files + unzip(&zip_filename(db_size, HAWK_VECTORS.to_string())); + unzip(&zip_filename(db_size, HAWK_GRAPH_LINKS.to_string())); + + let hawk_graph_entry_path = to_path(&csv_filename(db_size, HAWK_GRAPH_ENTRY.to_string())); + let hawk_graph_links_path = to_path(&csv_filename(db_size, HAWK_GRAPH_LINKS.to_string())); + let hawk_vectors_path = to_path(&csv_filename(db_size, HAWK_VECTORS.to_string())); + + graph_store + .copy_in(vec![ + (HAWK_GRAPH_ENTRY.to_string(), hawk_graph_entry_path), + (HAWK_GRAPH_LINKS.to_string(), hawk_graph_links_path.clone()), + ]) + .await + .unwrap(); + std::fs::remove_file(hawk_graph_links_path).unwrap(); + let graph_mem = graph_store.to_graph_mem().await; + graph_store.cleanup().await.unwrap(); + + vector_store + .copy_in(vec![(HAWK_VECTORS.to_string(), hawk_vectors_path.clone())]) + .await + .unwrap(); + std::fs::remove_file(hawk_vectors_path).unwrap(); + let vector_mem = vector_store.to_plaintext_store().await; + vector_store.cleanup().await.unwrap(); + + HawkSearcher::new(vector_mem, graph_mem, &mut rng) +} + +fn bench_hnsw_db(c: &mut Criterion) { + let mut group = c.benchmark_group("hnsw_db"); + group.sample_size(10); + group.sampling_mode(SamplingMode::Flat); + + for database_size in [100000, 200000, 1000000] { + let schema_name = format!("hnsw_db_{}", database_size.to_string()); + let temporary_name = || format!("{}_{}", schema_name, rand::random::()); + + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + + let plain_searcher = rt.block_on(async move { + let rng = AesRng::seed_from_u64(0_u64); + let vector_store = PlaintextStoreDb::new(HAWK_DATABASE_URL, &temporary_name()) + .await + .unwrap(); + let graph_store = GraphPg::new(HAWK_DATABASE_URL, &temporary_name()) + .await + .unwrap(); + let plain_searcher = + hawk_searcher_from_csv(rng, database_size, graph_store, vector_store).await; + + plain_searcher + }); + + group.bench_function(BenchmarkId::new("insert", database_size), |b| { + b.to_async(&rt).iter_batched( + || plain_searcher.clone(), + |mut my_db| async move { + let mut rng = AesRng::seed_from_u64(0_u64); + let on_the_fly_query = IrisDB::new_random_rng(1, &mut rng).db[0].clone(); + let query = my_db.vector_store.prepare_query(on_the_fly_query.into()); + let neighbors = my_db.search_to_insert(&query).await; + my_db.insert_from_search_results(query, neighbors).await; + }, + criterion::BatchSize::SmallInput, + ) + }); + } + + group.finish(); +} + +criterion_group! { + hnsw, + bench_hnsw_db, +} + +criterion_main!(hnsw); diff --git a/iris-mpc-cpu/migrations/20240909105323_init.down.sql b/iris-mpc-cpu/migrations/20240909105323_init.down.sql new file mode 100644 index 000000000..99af295b2 --- /dev/null +++ b/iris-mpc-cpu/migrations/20240909105323_init.down.sql @@ -0,0 +1 @@ +DROP TABLE hawk_vectors; diff --git a/iris-mpc-cpu/migrations/20240909105323_init.up.sql b/iris-mpc-cpu/migrations/20240909105323_init.up.sql new file mode 100644 index 000000000..163e2e86a --- /dev/null +++ b/iris-mpc-cpu/migrations/20240909105323_init.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS hawk_vectors ( + id integer NOT NULL, + point jsonb NOT NULL, + CONSTRAINT hawk_vectors_pkey PRIMARY KEY (id) +); diff --git a/iris-mpc-cpu/src/database_generators.rs b/iris-mpc-cpu/src/database_generators.rs index 2c0f5d0c7..a74360816 100644 --- a/iris-mpc-cpu/src/database_generators.rs +++ b/iris-mpc-cpu/src/database_generators.rs @@ -1,6 +1,7 @@ use crate::shares::{ring_impl::RingElement, share::Share, vecshare::VecShare}; use iris_mpc_common::iris_db::iris::{IrisCode, IrisCodeArray}; use rand::{Rng, RngCore}; +use serde::{Deserialize, Serialize}; use std::sync::Arc; type ShareRing = u16; @@ -9,13 +10,13 @@ type VecShareType = VecShare; type ShareRingPlain = RingElement; // type ShareType = Share; -#[derive(PartialEq, Eq, Debug, Default, Clone)] +#[derive(PartialEq, Eq, Debug, Default, Clone, Serialize, Deserialize, Hash)] pub struct SharedIris { pub shares: VecShareType, pub mask: IrisCodeArray, } -#[derive(PartialEq, Eq, Debug, Default, Clone)] +#[derive(PartialEq, Eq, Debug, Default, Clone, Serialize, Deserialize, Hash)] pub struct NgSharedIris { pub code: VecShareType, pub mask: VecShareType, diff --git a/iris-mpc-cpu/src/hawkers/mod.rs b/iris-mpc-cpu/src/hawkers/mod.rs index 036fedecc..6dc7336c9 100644 --- a/iris-mpc-cpu/src/hawkers/mod.rs +++ b/iris-mpc-cpu/src/hawkers/mod.rs @@ -1,2 +1,4 @@ pub mod ng_aby3_store; pub mod plaintext_store; +// #[cfg(feature = "db_dependent")] +pub mod plaintext_store_db; diff --git a/iris-mpc-cpu/src/hawkers/ng_aby3_store.rs b/iris-mpc-cpu/src/hawkers/ng_aby3_store.rs index 1279a6297..7bf61d0a1 100644 --- a/iris-mpc-cpu/src/hawkers/ng_aby3_store.rs +++ b/iris-mpc-cpu/src/hawkers/ng_aby3_store.rs @@ -105,8 +105,13 @@ pub fn setup_local_store_aby3_players() -> eyre::Result) -> PointId { +impl VectorStore for LocalNetAby3NgStoreProtocol { + type QueryRef = PointId; // Vector ID, pending insertion. + type VectorRef = PointId; // Vector ID, inserted. + type DistanceRef = (PointId, PointId); // Lazy distance representation. + type Data = Vec; + + fn prepare_query(&mut self, code: Vec) -> PointId { assert_eq!(code.len(), 3); assert_eq!(self.players.len(), 3); let pid0 = self @@ -128,12 +133,6 @@ impl LocalNetAby3NgStoreProtocol { assert_eq!(pid1, pid2); pid0 } -} - -impl VectorStore for LocalNetAby3NgStoreProtocol { - type QueryRef = PointId; // Vector ID, pending insertion. - type VectorRef = PointId; // Vector ID, inserted. - type DistanceRef = (PointId, PointId); // Lazy distance representation. async fn insert(&mut self, query: &Self::QueryRef) -> Self::VectorRef { // The query is now accepted in the store. It keeps the same ID. @@ -243,7 +242,7 @@ pub async fn ng_create_ready_made_hawk_searcher( for raw_query in cleartext_database.iter() { let query = cleartext_searcher .vector_store - .prepare_query(raw_query.clone()); + .prepare_query(raw_query.clone().into()); let neighbors = cleartext_searcher.search_to_insert(&query).await; let inserted = cleartext_searcher.vector_store.insert(&query).await; cleartext_searcher @@ -315,10 +314,9 @@ mod tests { let queries = (0..database_size) .map(|id| { - db.vector_store.prepare_query(ng_generate_iris_shares( - &mut rng, - cleartext_database[id].clone(), - )) + db.vector_store.prepare_query( + ng_generate_iris_shares(&mut rng, cleartext_database[id].clone()).into(), + ) }) .collect::>(); @@ -437,7 +435,7 @@ mod tests { // Now do the work for the plaintext store let mut plaintext_store = PlaintextStore::default(); let plaintext_preps: Vec<_> = (0..db_dim) - .map(|id| plaintext_store.prepare_query(cleartext_database[id].clone())) + .map(|id| plaintext_store.prepare_query(cleartext_database[id].clone().into())) .collect(); let mut plaintext_inserts = Vec::new(); for p in plaintext_preps.iter() { diff --git a/iris-mpc-cpu/src/hawkers/plaintext_store.rs b/iris-mpc-cpu/src/hawkers/plaintext_store.rs index dd679aea5..d9265f9e3 100644 --- a/iris-mpc-cpu/src/hawkers/plaintext_store.rs +++ b/iris-mpc-cpu/src/hawkers/plaintext_store.rs @@ -1,16 +1,17 @@ use hawk_pack::VectorStore; use iris_mpc_common::iris_db::iris::{IrisCode, IrisCodeArray, MATCH_THRESHOLD_RATIO}; use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; #[derive(Default, Debug, Clone)] pub struct PlaintextStore { - pub points: Vec, + pub points: BTreeMap, } -#[derive(Default, Debug, Clone)] +#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)] pub struct FormattedIris { - data: Vec, - mask: IrisCodeArray, + pub data: Vec, + pub mask: IrisCodeArray, } impl From for FormattedIris { @@ -39,13 +40,22 @@ impl FormattedIris { } } -#[derive(Clone, Default, Debug)] +#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)] pub struct PlaintextPoint { /// Whatever encoding of a vector. - data: FormattedIris, + pub data: FormattedIris, /// Distinguish between queries that are pending, and those that were /// ultimately accepted into the vector store. - is_persistent: bool, + pub is_persistent: bool, +} + +impl From for PlaintextPoint { + fn from(value: IrisCode) -> Self { + Self { + data: FormattedIris::from(value), + is_persistent: false, + } + } } impl FormattedIris { @@ -58,13 +68,13 @@ impl FormattedIris { } impl PlaintextPoint { - fn compute_distance(&self, other: &PlaintextPoint) -> (i16, usize) { + pub fn compute_distance(&self, other: &PlaintextPoint) -> (i16, usize) { let combined_mask = self.data.mask & other.data.mask; let dot = self.data.dot_on_code(&other.data) as i16; (dot, combined_mask.count_ones()) } - fn is_close(&self, other: &PlaintextPoint) -> bool { + pub fn is_close(&self, other: &PlaintextPoint) -> bool { let hd = self.data.dot_on_code(&other.data); let mask_ones = (self.data.mask & other.data.mask).count_ones(); let threshold = (mask_ones as f64) * (1. - 2. * MATCH_THRESHOLD_RATIO); @@ -72,7 +82,7 @@ impl PlaintextPoint { } } -#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize, PartialOrd, Ord)] pub struct PointId(pub usize); impl PointId { @@ -82,28 +92,18 @@ impl PointId { } impl PlaintextStore { - pub fn prepare_query(&mut self, raw_query: IrisCode) -> ::QueryRef { - self.points.push(PlaintextPoint { - data: FormattedIris::from(raw_query), - is_persistent: false, - }); - - let point_id = self.points.len() - 1; - PointId(point_id) - } - pub fn distance_computation( &self, distance1: &(PointId, PointId), distance2: &(PointId, PointId), ) -> (i32, i32) { let (x1, y1) = ( - &self.points[distance1.0.val()], - &self.points[distance1.1.val()], + &self.points.get(&distance1.0).unwrap(), + &self.points.get(&distance1.1).unwrap(), ); let (x2, y2) = ( - &self.points[distance2.0.val()], - &self.points[distance2.1.val()], + &self.points.get(&distance2.0).unwrap(), + &self.points.get(&distance2.1).unwrap(), ); let (d1, t1) = x1.compute_distance(y1); let (d2, t2) = x2.compute_distance(y2); @@ -118,10 +118,18 @@ impl VectorStore for PlaintextStore { type QueryRef = PointId; // Vector ID, pending insertion. type VectorRef = PointId; // Vector ID, inserted. type DistanceRef = (PointId, PointId); // Lazy distance representation. + type Data = PlaintextPoint; + + fn prepare_query(&mut self, raw_query: PlaintextPoint) -> PointId { + let point_id = PointId(self.points.len()); + self.points.insert(point_id, raw_query); + + point_id + } async fn insert(&mut self, query: &Self::QueryRef) -> Self::VectorRef { // The query is now accepted in the store. It keeps the same ID. - self.points[query.0].is_persistent = true; + self.points.get_mut(query).unwrap().is_persistent = true; *query } @@ -135,8 +143,8 @@ impl VectorStore for PlaintextStore { } async fn is_match(&self, distance: &Self::DistanceRef) -> bool { - let x = &self.points[distance.0 .0]; - let y = &self.points[distance.1 .0]; + let x = &self.points.get(&distance.0).unwrap(); + let y = &self.points.get(&distance.1).unwrap(); x.is_close(y) } @@ -170,10 +178,10 @@ mod tests { .collect(); let mut plaintext_store = PlaintextStore::default(); - let pid0 = plaintext_store.prepare_query(cleartext_database[0].clone()); - let pid1 = plaintext_store.prepare_query(cleartext_database[1].clone()); - let pid2 = plaintext_store.prepare_query(cleartext_database[2].clone()); - let pid3 = plaintext_store.prepare_query(cleartext_database[3].clone()); + let pid0 = plaintext_store.prepare_query(cleartext_database[0].clone().into()); + let pid1 = plaintext_store.prepare_query(cleartext_database[1].clone().into()); + let pid2 = plaintext_store.prepare_query(cleartext_database[2].clone().into()); + let pid3 = plaintext_store.prepare_query(cleartext_database[3].clone().into()); let q0 = plaintext_store.insert(&pid0).await; let q1 = plaintext_store.insert(&pid1).await; diff --git a/iris-mpc-cpu/src/hawkers/plaintext_store_db.rs b/iris-mpc-cpu/src/hawkers/plaintext_store_db.rs new file mode 100644 index 000000000..fc46b4268 --- /dev/null +++ b/iris-mpc-cpu/src/hawkers/plaintext_store_db.rs @@ -0,0 +1,453 @@ +use super::plaintext_store::{PlaintextPoint, PlaintextStore, PointId}; +use eyre::{eyre, Result}; +use futures::stream::TryStreamExt; +use hawk_pack::{DbStore, VectorStore}; +use sqlx::{ + migrate::Migrator, + postgres::{PgPoolOptions, PgRow}, + Executor, PgPool, Row, +}; +use std::{collections::BTreeMap, path}; +use tokio::io::AsyncWriteExt; + +const MAX_CONNECTIONS: u32 = 5; + +static MIGRATOR: Migrator = sqlx::migrate!("./migrations"); + +#[derive(Debug, Clone)] +pub struct PlaintextStoreDb { + cache: PlaintextStore, + schema_name: String, + pool: sqlx::PgPool, +} + +impl VectorStore for PlaintextStoreDb { + type QueryRef = PointId; // Vector ID, pending insertion. + type VectorRef = PointId; // Vector ID, inserted. + type DistanceRef = (PointId, PointId); // Lazy distance representation. + type Data = PlaintextPoint; + + fn prepare_query(&mut self, raw_query: PlaintextPoint) -> PointId { + self.cache.prepare_query(raw_query) + } + + async fn insert(&mut self, query: &Self::QueryRef) -> Self::VectorRef { + let point = self.get_point(*query).await.unwrap(); + + sqlx::query( + " + INSERT INTO hawk_vectors (id, point) + VALUES ($1, $2) + ", + ) + .bind(query.val() as i32) + .bind(sqlx::types::Json(point)) + .execute(&self.pool) + .await + .expect(&format!( + "Failed to insert query {} into vector store", + query.val() + )); + + *query + } + + async fn eval_distance( + &self, + query: &Self::QueryRef, + vector: &Self::VectorRef, + ) -> Self::DistanceRef { + // Do not compute the distance yet, just forward the IDs. + (*query, *vector) + } + + async fn is_match(&self, distance: &Self::DistanceRef) -> bool { + let x = &self.get_point(distance.0).await.unwrap(); + let y = &self.get_point(distance.1).await.unwrap(); + x.is_close(y) + } + + async fn less_than( + &self, + distance1: &Self::DistanceRef, + distance2: &Self::DistanceRef, + ) -> bool { + let (d2t1, d1t2) = self.distance_computation(distance1, distance2).await; + (d2t1 - d1t2) < 0 + } +} + +impl DbStore for PlaintextStoreDb { + async fn new(url: &str, schema_name: &str) -> Result { + let connect_sql = sql_switch_schema(schema_name)?; + + let pool = PgPoolOptions::new() + .max_connections(MAX_CONNECTIONS) + .after_connect(move |conn, _meta| { + // Switch to the given schema in every connection. + let connect_sql = connect_sql.clone(); + Box::pin(async move { + conn.execute(connect_sql.as_ref()).await.inspect_err(|e| { + eprintln!("error in after_connect: {:?}", e); + })?; + Ok(()) + }) + }) + .connect(url) + .await?; + + // Create the schema on the first startup. + MIGRATOR.run(&pool).await?; + + Ok(PlaintextStoreDb { + cache: PlaintextStore { + points: BTreeMap::new(), + }, + schema_name: schema_name.to_owned(), + pool, + }) + } + + fn pool(&self) -> &PgPool { + &self.pool + } + + fn schema_name(&self) -> String { + self.schema_name.to_string() + } + + async fn copy_out(&self) -> Result> { + let file_name = format!("{}_vectors.csv", self.schema_name.clone()); + self.copy_out_with_filename(file_name).await + } +} + +impl PlaintextStoreDb { + pub async fn to_plaintext_store(&self) -> PlaintextStore { + let points = sqlx::query( + " + SELECT * FROM hawk_vectors + ", + ) + .fetch_all(&self.pool) + .await + .unwrap() + .iter() + .map(|row| { + let id: i32 = row.get("id"); + let point: sqlx::types::Json = row.get("point"); + (PointId(id as usize), point.as_ref().clone()) + }) + .collect(); + + PlaintextStore { points } + } + + pub async fn get_point(&self, point: PointId) -> Option { + let mut res = self.cache.points.get(&point).map(|p| p.clone()); + if res.is_none() { + res = sqlx::query( + " + SELECT point FROM hawk_vectors WHERE id = $1 + ", + ) + .bind(point.0 as i32) + .fetch_optional(&self.pool) + .await + .expect(&format!("Failed to fetch point {}", point.0)) + .map(|row: PgRow| { + let x: sqlx::types::Json = row.get("point"); + x.as_ref().clone() + }); + } + res + } + + pub async fn distance_computation( + &self, + distance1: &(PointId, PointId), + distance2: &(PointId, PointId), + ) -> (i32, i32) { + let (x1, y1) = ( + &self.get_point(distance1.0).await.unwrap(), + &self.get_point(distance1.1).await.unwrap(), + ); + let (x2, y2) = ( + &self.get_point(distance2.0).await.unwrap(), + &self.get_point(distance2.1).await.unwrap(), + ); + let (d1, t1) = x1.compute_distance(y1); + let (d2, t2) = x2.compute_distance(y2); + + let cross_1 = d2 as i32 * t1 as i32; + let cross_2 = d1 as i32 * t2 as i32; + (cross_1, cross_2) + } + + async fn copy_out_with_filename(&self, file_name: String) -> Result> { + let table_name = "hawk_vectors"; + + let path = path::absolute(file_name.clone())? + .as_os_str() + .to_str() + .unwrap() + .to_owned(); + + let mut file = tokio::fs::File::create(path.clone()).await?; + let mut conn = self.pool.acquire().await?; + + let mut copy_stream = conn + .copy_out_raw(&format!( + "COPY {} TO STDOUT (FORMAT CSV, HEADER)", + table_name + )) + .await?; + + while let Some(chunk) = copy_stream.try_next().await? { + file.write_all(&chunk).await?; + } + + Ok(vec![(table_name.to_string(), path)]) + } +} + +fn sql_switch_schema(schema_name: &str) -> Result { + sanitize_identifier(schema_name)?; + Ok(format!( + " + CREATE SCHEMA IF NOT EXISTS \"{}\"; + SET search_path TO \"{}\"; + ", + schema_name, schema_name + )) +} + +fn sanitize_identifier(input: &str) -> Result<()> { + if input.chars().all(|c| c.is_alphanumeric() || c == '_') { + Ok(()) + } else { + Err(eyre!("Invalid SQL identifier")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::hawkers::plaintext_store::FormattedIris; + use aes_prng::AesRng; + use hawk_pack::{ + graph_store::GraphPg, + hnsw_db::{HawkSearcher, Params}, + }; + use iris_mpc_common::iris_db::iris::{IrisCode, IrisCodeArray}; + use rand::SeedableRng; + + struct PlaintextPointReader { + inner: R, + } + + impl PlaintextPointReader { + pub fn new(inner: R) -> Self { + Self { inner } + } + } + + impl Iterator for PlaintextPointReader { + type Item = Vec; + + fn next(&mut self) -> Option { + let mut buf: [u8; 12800] = [0; 12800]; + self.inner.read_exact(&mut buf).ok()?; + + Some(buf.iter().map(|&byte| byte as i8).collect()) + } + } + + fn plaintext_point_from_data(data: Vec) -> PlaintextPoint { + let mut mask = IrisCodeArray::ZERO; + + for (bit, &val) in data.iter().enumerate() { + if val != 0 { + mask.set_bit(bit, true); + } + } + let iris_code = FormattedIris { data, mask }; + PlaintextPoint { + data: iris_code, + is_persistent: false, + } + } + + #[tokio::test] + async fn hawk_searcher_from_db() { + let database_size = 100; + let schema_name = format!("hnsw_db_{}", database_size.to_string()); + let temporary_name = || format!("{}_{}", schema_name, rand::random::()); + let hawk_database_url: &str = "postgres://postgres:postgres@localhost/postgres"; + + let mut rng = AesRng::seed_from_u64(0_u64); + let graph_store = GraphPg::::new(hawk_database_url, &temporary_name()) + .await + .unwrap(); + let vector_store = PlaintextStoreDb::new(hawk_database_url, &temporary_name()) + .await + .unwrap(); + let mut plain_searcher = HawkSearcher::new(vector_store, graph_store.clone(), &mut rng); + + let queries = (0..database_size) + .map(|_| { + let raw_query = IrisCode::random_rng(&mut rng); + plain_searcher.vector_store.prepare_query(raw_query.into()) + }) + .collect::>(); + + for query in queries.iter() { + let neighbors = plain_searcher.search_to_insert(query).await; + let inserted = plain_searcher.vector_store.insert(query).await; + plain_searcher + .insert_from_search_results(inserted, neighbors) + .await; + } + let graph_path = graph_store.copy_out().await.unwrap(); + let vectors_path = plain_searcher.vector_store.copy_out().await.unwrap(); + + graph_store.cleanup().await.unwrap(); + plain_searcher.vector_store.cleanup().await.unwrap(); + + // Copy in to memory + { + let graph_store = + GraphPg::::new(hawk_database_url, &temporary_name()) + .await + .unwrap(); + let vector_store = PlaintextStoreDb::new(hawk_database_url, &temporary_name()) + .await + .unwrap(); + + graph_store.copy_in(graph_path).await.unwrap(); + let graph_mem = graph_store.to_graph_mem().await; + + vector_store.copy_in(vectors_path).await.unwrap(); + let vector_mem = vector_store.to_plaintext_store().await; + vector_store.cleanup().await.unwrap(); + + let plain_searcher = HawkSearcher::new(vector_mem, graph_mem.clone(), &mut rng); + + for query in queries.iter() { + let neighbors = plain_searcher.search_to_insert(query).await; + assert!(plain_searcher.is_match(&neighbors).await); + } + graph_store.cleanup().await.unwrap(); + } + } + + #[tokio::test] + async fn checkpoint_from_data() { + use std::io::BufReader; + + let step_size = 50000; + let database_size = 1000000; + let m_values = [128]; + let params = m_values.iter().map(|m| Params::new_with_m(*m as usize)); + + let mut rng = AesRng::seed_from_u64(0_u64); + let hawk_database_url: &str = "postgres://postgres:postgres@localhost/postgres"; + + let mut queries = vec![]; + for chunk in 0..10 { + let dat_filename = format!("benches/assets/processed_masked_irises_chunk_{}", chunk); + let dat_path = path::absolute(dat_filename).unwrap(); + + let input = + BufReader::new(std::fs::File::open(dat_path.clone()).expect("Failed to open file")); + let values: Vec> = PlaintextPointReader::new(input).collect(); + let mut values: Vec<_> = values + .into_iter() + .map(|data| plaintext_point_from_data(data)) + .collect(); + queries.append(&mut values); + } + + for params in params { + let mut prev_checkpoint_name = None; + for checkpoint in 1..=(database_size / step_size) { + let checkpoint_time = std::time::Instant::now(); + println!( + "M: {:?}, checkpoint: {:?}", + params.M(), + checkpoint * step_size + ); + + // Copy in vectors + let vector_mem = PlaintextStore { + points: queries + .iter() + .enumerate() + .map(|(id, val)| (PointId(id), val.clone())) + .collect(), + }; + + println!("vector_mem.points.len(): {:?}", vector_mem.points.len()); + + // Copy in graph + let graph_mem = { + let graph_store = + GraphPg::::new(hawk_database_url, &"hnsw_1M") + .await + .unwrap(); + if let Some(prev_checkpoint_name) = prev_checkpoint_name { + graph_store.copy_in(prev_checkpoint_name).await.unwrap(); + } + let graph_mem = graph_store.to_graph_mem().await; + graph_store.cleanup().await.unwrap(); + graph_mem + }; + + println!( + "graph_mem.layers: {:?}", + graph_mem + .layers + .iter() + .map(|layer| layer.links.iter()) + .fold(0, |acc, link| { acc + link.len() }) + ); + + let mut hawk_searcher = + HawkSearcher::new_with_params(params.clone(), vector_mem, graph_mem, &mut rng); + + let queries = hawk_searcher.vector_store.points.clone(); + for (query, _) in queries.range(PointId((checkpoint - 1) * step_size)..) { + if query.val() >= 1000 && (query.val() % 1000) == 0 { + println!( + "{:?}s: Inserting vector {}", + checkpoint_time.elapsed().as_secs(), + query.val() + ); + } + let neighbors = hawk_searcher.search_to_insert(&query).await; + hawk_searcher + .insert_from_search_results(*query, neighbors) + .await; + } + println!( + "{:?}s: Done searching and inserting vectors", + checkpoint_time.elapsed().as_secs() + ); + + let checkpoint_name = format!( + "1M_{}_M{}_checkpoint", + (checkpoint * step_size).to_string(), + params.M().to_string() + ); + prev_checkpoint_name = Some( + hawk_searcher + .graph_store + // .copy_out_with_filename(checkpoint_name) + .write_to_db(hawk_database_url, &checkpoint_name) + .await + .unwrap(), + ); + } + } + } +} diff --git a/iris-mpc-cpu/src/shares/ring_impl.rs b/iris-mpc-cpu/src/shares/ring_impl.rs index 9ec197ab8..3a834f59f 100644 --- a/iris-mpc-cpu/src/shares/ring_impl.rs +++ b/iris-mpc-cpu/src/shares/ring_impl.rs @@ -14,7 +14,9 @@ use std::{ }, }; -#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize, PartialOrd, Eq, Ord)] +#[derive( + Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize, PartialOrd, Eq, Ord, Hash, +)] #[serde(bound = "")] #[repr(transparent)] pub struct RingElement(pub T); diff --git a/iris-mpc-cpu/src/shares/share.rs b/iris-mpc-cpu/src/shares/share.rs index 950eaaf1d..4092760b4 100644 --- a/iris-mpc-cpu/src/shares/share.rs +++ b/iris-mpc-cpu/src/shares/share.rs @@ -8,7 +8,7 @@ use std::ops::{ SubAssign, }; -#[derive(Clone, Debug, PartialEq, Default, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Default, Eq, PartialOrd, Ord, Serialize, Deserialize, Hash)] #[serde(bound = "")] pub struct Share { pub a: RingElement, diff --git a/iris-mpc-cpu/src/shares/vecshare.rs b/iris-mpc-cpu/src/shares/vecshare.rs index 9076bd23f..04727199a 100644 --- a/iris-mpc-cpu/src/shares/vecshare.rs +++ b/iris-mpc-cpu/src/shares/vecshare.rs @@ -92,7 +92,7 @@ impl<'a, T: IntRing2k> SliceShareMut<'a, T> { } } -#[derive(Clone, Debug, PartialEq, Default, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Default, Eq, PartialOrd, Ord, Serialize, Deserialize, Hash)] #[serde(bound = "")] #[repr(transparent)] pub struct VecShare {