diff --git a/package-lock.json b/package-lock.json index 3ca6ec8..e0f9f84 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,6 +22,7 @@ "@ucanto/principal": "^9.0.1", "@ucanto/transport": "^9.1.1", "@web3-storage/filecoin-client": "^3.3.3", + "@web3-storage/upload-api": "17.0.0", "ava": "^5.3.0", "aws-cdk-lib": "^2.84.0", "constructs": "10.1.156", @@ -3893,6 +3894,40 @@ "dev": true, "license": "MIT" }, + "node_modules/@scure/base": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@scure/base/-/base-1.1.6.tgz", + "integrity": "sha512-ok9AWwhcgYuGG3Zfhyqg+zwl+Wn5uE+dwC0NV/2qQkx4dABbb/bx96vWu8NSj+BNjjSjno+JRYRjle1jV08k3g==", + "dev": true, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip39": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@scure/bip39/-/bip39-1.3.0.tgz", + "integrity": "sha512-disdg7gHuTDZtY+ZdkmLpPCk7fxZSu3gBiEGuoC1XYxv9cGx3Z6cpTggCgW6odSOOIXCiDjuGejW+aJKCY/pIQ==", + "dev": true, + "dependencies": { + "@noble/hashes": "~1.4.0", + "@scure/base": "~1.1.6" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip39/node_modules/@noble/hashes": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.4.0.tgz", + "integrity": "sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg==", + "dev": true, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@sentry-internal/tracing": { "version": "7.61.1", "dev": true, @@ -5622,6 +5657,298 @@ "web-streams-polyfill": "^3.1.1" } }, + "node_modules/@web3-storage/access": { + "version": "20.0.0", + "resolved": "https://registry.npmjs.org/@web3-storage/access/-/access-20.0.0.tgz", + "integrity": "sha512-kl2b3ZuN3NvAxDM6K8GlPgu1o67JtA3EGId8Bg+tFRqZiQlERQmMMOfLl8zKVOUvlq4bDFtWHl6EViLO4BwSJw==", + "dev": true, + "dependencies": { + "@ipld/car": "^5.1.1", + "@ipld/dag-ucan": "^3.4.0", + "@scure/bip39": "^1.2.1", + "@ucanto/client": "^9.0.1", + "@ucanto/core": "^10.0.1", + "@ucanto/interface": "^10.0.1", + "@ucanto/principal": "^9.0.1", + "@ucanto/transport": "^9.1.1", + "@ucanto/validator": "^9.0.2", + "@web3-storage/capabilities": "^17.1.1", + "@web3-storage/did-mailto": "^2.1.0", + "bigint-mod-arith": "^3.1.2", + "conf": "11.0.2", + "multiformats": "^12.1.2", + "one-webcrypto": "git+https://github.com/web3-storage/one-webcrypto.git", + "p-defer": "^4.0.0", + "type-fest": "^4.9.0", + "uint8arrays": "^4.0.6" + } + }, + "node_modules/@web3-storage/access/node_modules/@ucanto/interface": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/@ucanto/interface/-/interface-10.0.1.tgz", + "integrity": "sha512-+Vr/N4mLsdynV9/bqtdFiq7WsUf3265/Qx2aHJmPtXo9/QvWKthJtpe0g8U4NWkWpVfqIFvyAO2db6D9zWQfQw==", + "dev": true, + "dependencies": { + "@ipld/dag-ucan": "^3.4.0", + "multiformats": "^11.0.2" + } + }, + "node_modules/@web3-storage/access/node_modules/@ucanto/interface/node_modules/multiformats": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", + "dev": true, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@web3-storage/access/node_modules/@web3-storage/capabilities": { + "version": "17.1.1", + "resolved": "https://registry.npmjs.org/@web3-storage/capabilities/-/capabilities-17.1.1.tgz", + "integrity": "sha512-zmDGBN7/HMt8FUZhg+hdc7CHrYBzV2PaRJToPN0mA496EH1rbNY7c1a8eYxqhM1OugoWohCKH6YOdS3V+Eyxig==", + "dev": true, + "dependencies": { + "@ucanto/core": "^10.0.1", + "@ucanto/interface": "^10.0.1", + "@ucanto/principal": "^9.0.1", + "@ucanto/transport": "^9.1.1", + "@ucanto/validator": "^9.0.2", + "@web3-storage/data-segment": "^3.2.0", + "uint8arrays": "^5.0.3" + } + }, + "node_modules/@web3-storage/access/node_modules/@web3-storage/capabilities/node_modules/multiformats": { + "version": "13.1.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.1.0.tgz", + "integrity": "sha512-HzdtdBwxsIkzpeXzhQ5mAhhuxcHbjEHH+JQoxt7hG/2HGFjjwyolLo7hbaexcnhoEuV4e0TNJ8kkpMjiEYY4VQ==", + "dev": true + }, + "node_modules/@web3-storage/access/node_modules/@web3-storage/capabilities/node_modules/uint8arrays": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-5.1.0.tgz", + "integrity": "sha512-vA6nFepEmlSKkMBnLBaUMVvAC4G3CTmO58C12y4sq6WPDOR7mOFYOi7GlrQ4djeSbP6JG9Pv9tJDM97PedRSww==", + "dev": true, + "dependencies": { + "multiformats": "^13.0.0" + } + }, + "node_modules/@web3-storage/access/node_modules/ajv": { + "version": "8.14.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.14.0.tgz", + "integrity": "sha512-oYs1UUtO97ZO2lJ4bwnWeQW8/zvOIQLGKcvPTsWmvc2SYgBb+upuNS5NxoLaMU4h8Ju3Nbj6Cq8mD2LQoqVKFA==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.3", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.4.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@web3-storage/access/node_modules/atomically": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/atomically/-/atomically-2.0.3.tgz", + "integrity": "sha512-kU6FmrwZ3Lx7/7y3hPS5QnbJfaohcIul5fGqf7ok+4KklIEk9tJ0C2IQPdacSbVUWv6zVHXEBWoWd6NrVMT7Cw==", + "dev": true, + "dependencies": { + "stubborn-fs": "^1.2.5", + "when-exit": "^2.1.1" + } + }, + "node_modules/@web3-storage/access/node_modules/conf": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/conf/-/conf-11.0.2.tgz", + "integrity": "sha512-jjyhlQ0ew/iwmtwsS2RaB6s8DBifcE2GYBEaw2SJDUY/slJJbNfY4GlDVzOs/ff8cM/Wua5CikqXgbFl5eu85A==", + "dev": true, + "dependencies": { + "ajv": "^8.12.0", + "ajv-formats": "^2.1.1", + "atomically": "^2.0.0", + "debounce-fn": "^5.1.2", + "dot-prop": "^7.2.0", + "env-paths": "^3.0.0", + "json-schema-typed": "^8.0.1", + "semver": "^7.3.8" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@web3-storage/access/node_modules/debounce-fn": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/debounce-fn/-/debounce-fn-5.1.2.tgz", + "integrity": "sha512-Sr4SdOZ4vw6eQDvPYNxHogvrxmCIld/VenC5JbNrFwMiwd7lY/Z18ZFfo+EWNG4DD9nFlAujWAo/wGuOPHmy5A==", + "dev": true, + "dependencies": { + "mimic-fn": "^4.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@web3-storage/access/node_modules/dot-prop": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-7.2.0.tgz", + "integrity": "sha512-Ol/IPXUARn9CSbkrdV4VJo7uCy1I3VuSiWCaFSg+8BdUOzF9n3jefIpcgAydvUZbTdEBZs2vEiTiS9m61ssiDA==", + "dev": true, + "dependencies": { + "type-fest": "^2.11.2" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@web3-storage/access/node_modules/dot-prop/node_modules/type-fest": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.19.0.tgz", + "integrity": "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==", + "dev": true, + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@web3-storage/access/node_modules/env-paths": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-3.0.0.tgz", + "integrity": "sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==", + "dev": true, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@web3-storage/access/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true + }, + "node_modules/@web3-storage/access/node_modules/json-schema-typed": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.1.tgz", + "integrity": "sha512-XQmWYj2Sm4kn4WeTYvmpKEbyPsL7nBsb647c7pMe6l02/yx2+Jfc4dT6UZkEXnIUb5LhD55r2HPsJ1milQ4rDg==", + "dev": true + }, + "node_modules/@web3-storage/access/node_modules/multiformats": { + "version": "12.1.3", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.3.tgz", + "integrity": "sha512-eajQ/ZH7qXZQR2AgtfpmSMizQzmyYVmCql7pdhldPuYQi4atACekbJaQplk6dWyIi10jCaFnd6pqvcEFXjbaJw==", + "dev": true, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@web3-storage/access/node_modules/p-defer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-4.0.1.tgz", + "integrity": "sha512-Mr5KC5efvAK5VUptYEIopP1bakB85k2IWXaRC0rsh1uwn1L6M0LVml8OIQ4Gudg4oyZakf7FmeRLkMMtZW1i5A==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@web3-storage/access/node_modules/type-fest": { + "version": "4.18.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.18.3.tgz", + "integrity": "sha512-Q08/0IrpvM+NMY9PA2rti9Jb+JejTddwmwmVQGskAlhtcrw1wsRzoR6ode6mR+OAabNa75w/dxedSUY2mlphaQ==", + "dev": true, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@web3-storage/blob-index": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@web3-storage/blob-index/-/blob-index-1.0.2.tgz", + "integrity": "sha512-N+yMIk2cmgaGYVy9EewsRx1sxSDv67i2IBlZ4y72a/+lVIAmb3ZP0IwZ+Med0xrNZShA4blxIGJm1LVF7Q4mSg==", + "dev": true, + "dependencies": { + "@ipld/dag-cbor": "^9.0.6", + "@ucanto/core": "^10.0.1", + "@ucanto/interface": "^10.0.1", + "@web3-storage/capabilities": "^17.1.0", + "carstream": "^2.1.0", + "multiformats": "^13.0.1", + "uint8arrays": "^5.0.3" + }, + "engines": { + "node": ">=16.15" + } + }, + "node_modules/@web3-storage/blob-index/node_modules/@ucanto/interface": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/@ucanto/interface/-/interface-10.0.1.tgz", + "integrity": "sha512-+Vr/N4mLsdynV9/bqtdFiq7WsUf3265/Qx2aHJmPtXo9/QvWKthJtpe0g8U4NWkWpVfqIFvyAO2db6D9zWQfQw==", + "dev": true, + "dependencies": { + "@ipld/dag-ucan": "^3.4.0", + "multiformats": "^11.0.2" + } + }, + "node_modules/@web3-storage/blob-index/node_modules/@ucanto/interface/node_modules/multiformats": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", + "dev": true, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@web3-storage/blob-index/node_modules/@web3-storage/capabilities": { + "version": "17.1.1", + "resolved": "https://registry.npmjs.org/@web3-storage/capabilities/-/capabilities-17.1.1.tgz", + "integrity": "sha512-zmDGBN7/HMt8FUZhg+hdc7CHrYBzV2PaRJToPN0mA496EH1rbNY7c1a8eYxqhM1OugoWohCKH6YOdS3V+Eyxig==", + "dev": true, + "dependencies": { + "@ucanto/core": "^10.0.1", + "@ucanto/interface": "^10.0.1", + "@ucanto/principal": "^9.0.1", + "@ucanto/transport": "^9.1.1", + "@ucanto/validator": "^9.0.2", + "@web3-storage/data-segment": "^3.2.0", + "uint8arrays": "^5.0.3" + } + }, + "node_modules/@web3-storage/blob-index/node_modules/multiformats": { + "version": "13.1.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.1.0.tgz", + "integrity": "sha512-HzdtdBwxsIkzpeXzhQ5mAhhuxcHbjEHH+JQoxt7hG/2HGFjjwyolLo7hbaexcnhoEuV4e0TNJ8kkpMjiEYY4VQ==", + "dev": true + }, + "node_modules/@web3-storage/blob-index/node_modules/uint8arrays": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-5.1.0.tgz", + "integrity": "sha512-vA6nFepEmlSKkMBnLBaUMVvAC4G3CTmO58C12y4sq6WPDOR7mOFYOi7GlrQ4djeSbP6JG9Pv9tJDM97PedRSww==", + "dev": true, + "dependencies": { + "multiformats": "^13.0.0" + } + }, "node_modules/@web3-storage/capabilities": { "version": "16.0.0", "resolved": "https://registry.npmjs.org/@web3-storage/capabilities/-/capabilities-16.0.0.tgz", @@ -5659,16 +5986,16 @@ "integrity": "sha512-HzdtdBwxsIkzpeXzhQ5mAhhuxcHbjEHH+JQoxt7hG/2HGFjjwyolLo7hbaexcnhoEuV4e0TNJ8kkpMjiEYY4VQ==" }, "node_modules/@web3-storage/content-claims": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/@web3-storage/content-claims/-/content-claims-4.0.5.tgz", - "integrity": "sha512-+WpCkTN8aRfUCrCm0kOMZad+FRnFymVDFvS6/+PJMPGP17cci1/c5lqYdrjFV+5MkhL+BkUJVtRTx02G31FHmQ==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@web3-storage/content-claims/-/content-claims-5.0.0.tgz", + "integrity": "sha512-HJFRFsR0qHCe0cOERsb3AjAxxzohYMMoIWaGJgrShDycnl6yqXHrGcdua1BWUDu5pmvKzwD9D7VmI8aSfrCcRA==", "dependencies": { "@ucanto/client": "^9.0.1", "@ucanto/interface": "^10.0.0", "@ucanto/server": "^10.0.0", "@ucanto/transport": "^9.1.1", - "carstream": "^1.0.2", - "multiformats": "^12.0.1" + "carstream": "^2.0.0", + "multiformats": "^13.1.0" } }, "node_modules/@web3-storage/content-claims/node_modules/@ucanto/interface": { @@ -5690,13 +6017,9 @@ } }, "node_modules/@web3-storage/content-claims/node_modules/multiformats": { - "version": "12.1.3", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.3.tgz", - "integrity": "sha512-eajQ/ZH7qXZQR2AgtfpmSMizQzmyYVmCql7pdhldPuYQi4atACekbJaQplk6dWyIi10jCaFnd6pqvcEFXjbaJw==", - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - } + "version": "13.1.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.1.0.tgz", + "integrity": "sha512-HzdtdBwxsIkzpeXzhQ5mAhhuxcHbjEHH+JQoxt7hG/2HGFjjwyolLo7hbaexcnhoEuV4e0TNJ8kkpMjiEYY4VQ==" }, "node_modules/@web3-storage/data-segment": { "version": "3.2.0", @@ -5708,10 +6031,19 @@ "sync-multihash-sha2": "^1.0.0" } }, + "node_modules/@web3-storage/did-mailto": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@web3-storage/did-mailto/-/did-mailto-2.1.0.tgz", + "integrity": "sha512-TRmfSXj1IhtX3ESurSNOylZSBKi0z/VJNoMLpof+AVRdovgZjjocpiePQTs2pfHKqHTHfJXc9AboWyK4IKTWMw==", + "dev": true, + "engines": { + "node": ">=16.15" + } + }, "node_modules/@web3-storage/filecoin-api": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/@web3-storage/filecoin-api/-/filecoin-api-7.0.0.tgz", - "integrity": "sha512-WKZwIQRMrihj/6kNctjEyWEfgdUq3Qub4CISEmfH+YcrO+pnzkGfbN3mCxNVDXbnn1OuBo0netbxulnqOjS5Vg==", + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/@web3-storage/filecoin-api/-/filecoin-api-7.1.0.tgz", + "integrity": "sha512-rf3DhKWv8MGpKj84lSb1lzxAJRIpkNrPFpzs6iP5Es4LVxiRTm7k2L6CxUYMhtfgUegWP0O21xmDh0Y+Yquz9Q==", "dependencies": { "@ipld/dag-ucan": "^3.4.0", "@ucanto/client": "^9.0.1", @@ -5719,8 +6051,8 @@ "@ucanto/interface": "^10.0.1", "@ucanto/server": "^10.0.0", "@ucanto/transport": "^9.1.1", - "@web3-storage/capabilities": "^17.1.0", - "@web3-storage/content-claims": "^4.0.4", + "@web3-storage/capabilities": "^17.1.1", + "@web3-storage/content-claims": "^5.0.0", "@web3-storage/data-segment": "^4.0.0", "fr32-sha2-256-trunc254-padded-binary-tree-multihash": "^3.3.0", "p-map": "^6.0.0" @@ -5739,9 +6071,9 @@ } }, "node_modules/@web3-storage/filecoin-api/node_modules/@web3-storage/capabilities": { - "version": "17.1.0", - "resolved": "https://registry.npmjs.org/@web3-storage/capabilities/-/capabilities-17.1.0.tgz", - "integrity": "sha512-p5Wn2O3TSEZ7JFSph2KY9OuFnofbkhKi7Tp+1zcPEYAUsEvDWGabd1NvSPDDMpFBE74UX4ZljE8aQzDAtI3qRw==", + "version": "17.1.1", + "resolved": "https://registry.npmjs.org/@web3-storage/capabilities/-/capabilities-17.1.1.tgz", + "integrity": "sha512-zmDGBN7/HMt8FUZhg+hdc7CHrYBzV2PaRJToPN0mA496EH1rbNY7c1a8eYxqhM1OugoWohCKH6YOdS3V+Eyxig==", "dependencies": { "@ucanto/core": "^10.0.1", "@ucanto/interface": "^10.0.1", @@ -5818,6 +6150,92 @@ "multiformats": "^11.0.2" } }, + "node_modules/@web3-storage/upload-api": { + "version": "17.0.0", + "resolved": "https://registry.npmjs.org/@web3-storage/upload-api/-/upload-api-17.0.0.tgz", + "integrity": "sha512-LXV5Atvyfwvfe0G6g1ym2bn6M1LR/DudCMiJOMpSZxwFtjDxneMaWrE+Nrfns+NIFuqrEhz3MNyRABGuBaPSFg==", + "dev": true, + "dependencies": { + "@ucanto/client": "^9.0.1", + "@ucanto/interface": "^10.0.1", + "@ucanto/principal": "^9.0.1", + "@ucanto/server": "^10.0.0", + "@ucanto/transport": "^9.1.1", + "@ucanto/validator": "^9.0.2", + "@web3-storage/access": "^20.0.0", + "@web3-storage/blob-index": "^1.0.2", + "@web3-storage/capabilities": "^17.1.1", + "@web3-storage/content-claims": "^5.0.0", + "@web3-storage/did-mailto": "^2.1.0", + "@web3-storage/filecoin-api": "^7.1.0", + "multiformats": "^12.1.2", + "p-retry": "^5.1.2", + "uint8arrays": "^5.0.3" + }, + "engines": { + "node": ">=16.15" + } + }, + "node_modules/@web3-storage/upload-api/node_modules/@ucanto/interface": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/@ucanto/interface/-/interface-10.0.1.tgz", + "integrity": "sha512-+Vr/N4mLsdynV9/bqtdFiq7WsUf3265/Qx2aHJmPtXo9/QvWKthJtpe0g8U4NWkWpVfqIFvyAO2db6D9zWQfQw==", + "dev": true, + "dependencies": { + "@ipld/dag-ucan": "^3.4.0", + "multiformats": "^11.0.2" + } + }, + "node_modules/@web3-storage/upload-api/node_modules/@ucanto/interface/node_modules/multiformats": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", + "dev": true, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@web3-storage/upload-api/node_modules/@web3-storage/capabilities": { + "version": "17.1.1", + "resolved": "https://registry.npmjs.org/@web3-storage/capabilities/-/capabilities-17.1.1.tgz", + "integrity": "sha512-zmDGBN7/HMt8FUZhg+hdc7CHrYBzV2PaRJToPN0mA496EH1rbNY7c1a8eYxqhM1OugoWohCKH6YOdS3V+Eyxig==", + "dev": true, + "dependencies": { + "@ucanto/core": "^10.0.1", + "@ucanto/interface": "^10.0.1", + "@ucanto/principal": "^9.0.1", + "@ucanto/transport": "^9.1.1", + "@ucanto/validator": "^9.0.2", + "@web3-storage/data-segment": "^3.2.0", + "uint8arrays": "^5.0.3" + } + }, + "node_modules/@web3-storage/upload-api/node_modules/multiformats": { + "version": "12.1.3", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.3.tgz", + "integrity": "sha512-eajQ/ZH7qXZQR2AgtfpmSMizQzmyYVmCql7pdhldPuYQi4atACekbJaQplk6dWyIi10jCaFnd6pqvcEFXjbaJw==", + "dev": true, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@web3-storage/upload-api/node_modules/uint8arrays": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-5.1.0.tgz", + "integrity": "sha512-vA6nFepEmlSKkMBnLBaUMVvAC4G3CTmO58C12y4sq6WPDOR7mOFYOi7GlrQ4djeSbP6JG9Pv9tJDM97PedRSww==", + "dev": true, + "dependencies": { + "multiformats": "^13.0.0" + } + }, + "node_modules/@web3-storage/upload-api/node_modules/uint8arrays/node_modules/multiformats": { + "version": "13.1.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.1.0.tgz", + "integrity": "sha512-HzdtdBwxsIkzpeXzhQ5mAhhuxcHbjEHH+JQoxt7hG/2HGFjjwyolLo7hbaexcnhoEuV4e0TNJ8kkpMjiEYY4VQ==", + "dev": true + }, "node_modules/@whatwg-node/events": { "version": "0.0.2", "dev": true, @@ -6989,6 +7407,15 @@ "tweetnacl": "^0.14.3" } }, + "node_modules/bigint-mod-arith": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/bigint-mod-arith/-/bigint-mod-arith-3.3.1.tgz", + "integrity": "sha512-pX/cYW3dCa87Jrzv6DAr8ivbbJRzEX5yGhdt8IutnX/PCIXfpx+mabWNK/M8qqh+zQ0J3thftUBHW0ByuUlG0w==", + "dev": true, + "engines": { + "node": ">=10.4.0" + } + }, "node_modules/binary-extensions": { "version": "2.2.0", "dev": true, @@ -7293,23 +7720,19 @@ "license": "CC-BY-4.0" }, "node_modules/carstream": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/carstream/-/carstream-1.1.1.tgz", - "integrity": "sha512-cgn3TqHo6SPsHBTfM5QgXngv6HtwgO1bKCHcdS35vBrweLcYrIG/+UboCbvnIGA0k8NtAYl/DvDdej/9pZGZxQ==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/carstream/-/carstream-2.2.0.tgz", + "integrity": "sha512-/gHkK0lQjmGM45fhdx8JD+x7a1XS1qUk3T9xWWSt3oZiWPLq4u/lnDstp+N55K7hqTKKlb0CCr43EHTrlbmJSQ==", "dependencies": { "@ipld/dag-cbor": "^9.0.3", - "multiformats": "^12.0.1", + "multiformats": "^13.0.1", "uint8arraylist": "^2.4.3" } }, "node_modules/carstream/node_modules/multiformats": { - "version": "12.1.3", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.3.tgz", - "integrity": "sha512-eajQ/ZH7qXZQR2AgtfpmSMizQzmyYVmCql7pdhldPuYQi4atACekbJaQplk6dWyIi10jCaFnd6pqvcEFXjbaJw==", - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - } + "version": "13.1.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.1.0.tgz", + "integrity": "sha512-HzdtdBwxsIkzpeXzhQ5mAhhuxcHbjEHH+JQoxt7hG/2HGFjjwyolLo7hbaexcnhoEuV4e0TNJ8kkpMjiEYY4VQ==" }, "node_modules/cbor": { "version": "8.1.0", @@ -12179,6 +12602,7 @@ }, "node_modules/one-webcrypto": { "version": "1.0.3", + "resolved": "git+ssh://git@github.com/web3-storage/one-webcrypto.git#5148cd14d5489a8ac4cd38223870e02db15a2382", "license": "MIT" }, "node_modules/onetime": { @@ -14234,6 +14658,12 @@ "version": "1.0.5", "license": "MIT" }, + "node_modules/stubborn-fs": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/stubborn-fs/-/stubborn-fs-1.2.5.tgz", + "integrity": "sha512-H2N9c26eXjzL/S/K+i/RHHcFanE74dptvvjM8iwzwbVcWY/zjBbgRqF3K0DY4+OD+uTTASTBvDoxPDaPN02D7g==", + "dev": true + }, "node_modules/supertap": { "version": "3.0.1", "dev": true, @@ -15037,6 +15467,12 @@ "webidl-conversions": "^3.0.0" } }, + "node_modules/when-exit": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/when-exit/-/when-exit-2.1.2.tgz", + "integrity": "sha512-u9J+toaf3CCxCAzM/484qNAxQE75rFdVgiFEEV8Xps2gzYhf0tx73s1WXDQhkwV17E3MxRMz40m7Ekd2/121Lg==", + "dev": true + }, "node_modules/which": { "version": "2.0.2", "dev": true, diff --git a/package.json b/package.json index ccf95d3..6bef1c7 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "@ucanto/principal": "^9.0.1", "@ucanto/transport": "^9.1.1", "@web3-storage/filecoin-client": "^3.3.3", + "@web3-storage/upload-api": "17.0.0", "ava": "^5.3.0", "aws-cdk-lib": "^2.84.0", "constructs": "10.1.156", diff --git a/test/helpers/agent/store.js b/test/helpers/agent/store.js new file mode 100644 index 0000000..23c13d1 --- /dev/null +++ b/test/helpers/agent/store.js @@ -0,0 +1,333 @@ +import { + S3Client, + GetObjectCommand, + ListObjectsV2Command, +} from '@aws-sdk/client-s3' +import { + RecordNotFound, + StorageOperationFailed, +} from '@web3-storage/upload-api/errors' +import * as CAR from '@ucanto/transport/car' +import { parseLink, Receipt } from '@ucanto/core' + +/** + * @typedef {import('@aws-sdk/client-s3').S3ClientConfig} Address + * @typedef {S3Client} Channel + * + * @typedef {{ + * channel: Channel + * address: Address + * }} Connection + * + * @typedef {object} Buckets + * @property {{name:string}} index + * @property {{name:string}} message + * + * @typedef {object} Options + * @property {Connection} connection + * @property {string} region + * @property {Buckets} buckets + * + * @typedef {object} Store + * @property {Channel} channel + * @property {string} region + * @property {Buckets} buckets + */ + +/** + * @param {Options} options + * @returns {Store} + */ +export const open = ({ connection, region, buckets }) => ({ + channel: connection.channel ?? new S3Client({ ...connection.address, region }), + region, + buckets, +}) + +/** + * Gets a receipt corresponding to the given task. + * + * @param {Store} store + * @param {import('@ucanto/interface').UnknownLink} task + * @returns {Promise>} + */ +export const getReceipt = async (store, task) => { + const result = await load(store, { invocation: task }) + + if (result.error) { + return result + } + + // + const invocation = /** @type {import('@ucanto/interface').UCANLink<[import('@ucanto/interface').Capability]>} */ (task) + const { ok: entry, error } = await resolve(store, { + receipt: invocation, + }) + if (error) { + return { error } + } + + const { ok: body, error: readError } = await read(store, entry.message) + if (readError) { + return { error: readError } + } + + if (entry.root) { + const archive = await CAR.codec.decode(body) + const receipt = Receipt.view( + { + root: entry.root, + blocks: archive.blocks, + } + ) + if (receipt) { + // @ts-ignore + return { ok: receipt } + } + } else { + const message = await CAR.request.decode({ + body, + headers: {}, + }) + + // Attempt to find a receipt corresponding to this task + const receipt = message.receipts.get(`${task}`) + if (receipt) { + // @ts-ignore + return { ok: receipt } + } + } + + return { + error: new RecordNotFound( + `agent message ${entry.message} does not contain receipt for ${task} task` + ), + } +} + +/** + * We may want to lookup an agent message by task that either contains + * corresponding invocation or a receipt . This type describes a query + * as variant of `invocation` and `receipt` types signaling which of the + * two messages to lookup. + * + * @typedef {import('@ucanto/interface').Variant<{ + * invocation: import('@ucanto/interface').UnknownLink + * receipt: import('@ucanto/interface').UnknownLink + * }>} AgentMessageQuery + */ + +/** + * Resolves paths to an agent message for the given key. + * + * @typedef {{root?: import('@ucanto/interface').Link, message:import('@ucanto/interface').Link}} IndexEntry + * + * @param {Store} store + * @param {AgentMessageQuery} query + * @returns {Promise>} + */ +export const resolve = async (store, { invocation, receipt }) => { + // If we are resolving an invocation we need to find an INcoming message + // which get `.in` suffix. If we are looking for a receipt we need to find + // an OUTgoing message which get `.out` suffix. + const [prefix, suffix] = invocation + ? [`${invocation}/`, '.in'] + : [`${receipt}/`, '.out'] + + // Previously we used to treat task and invocation as a same and used + // following indexing using following pseudo symlinks + // + // ${invocation.cid}/${message.cid}.in + // ${invocation.cid}/${message.cid}.out + // + // After we started distinguishing between task and invocation, we have + // adopted following indexing instead + // ${task.cid}/${invocation.cid}@${message.cid}.in + // ${task.cid}/${invocation.cid}@${message.cid}.out + // + // Here we could be looking up old receipts or new ones which is why we + // simply list all entries under the link (which is either task or invocation) + // and then filter out by the prefix. + const entries = await list(store, { prefix, suffix }) + if (entries.error) { + return entries + } + + // Prefer an entry containing a root link if non found return the first entry + const [first, ...rest] = entries.ok + const head = toIndexEntry(first) + if (head.root) { + return { ok: head } + } else { + for (const path of rest) { + const entry = toIndexEntry(path) + if (entry.root) { + return { ok: entry } + } + } + return { ok: head } + } +} + +/** + * Loads a view based on the query + * + * + * @param {Store} store + * @param {AgentMessageQuery} query + */ +const load = async (store, query) => { + const { ok: index, error } = await resolve(store, query) + + if (error) { + return { error } + } + + const { message, root } = index + const { ok: bytes, error: readError } = await read(store, message) + if (readError) { + return { error: readError } + } + + return { + ok: { + archive: await CAR.codec.decode(bytes), + root: root ?? undefined, + }, + } +} +/** + * Takes a pseudo symlink and extracts the DAG `root` and DAG `archive` + * identifiers. + * + * @param {string} path + * @returns {IndexEntry} + */ +const toIndexEntry = (path) => { + const start = path.indexOf('/') + const offset = path.indexOf('@') + return offset > 0 + ? { + root: parseLink(path.slice(start + 1, offset)), + message: parseLink(path.slice(offset + 1, path.indexOf('.'))), + } + : { + message: parseLink(path.slice(start + 1, path.indexOf('.'))), + } +} + +/** + * + * @param {Store} connection + * @param {object} key + * @param {string} key.prefix + * @param {string} key.suffix + * @returns {Promise>} + */ +const list = async (connection, { prefix, suffix }) => { + try { + const command = new ListObjectsV2Command({ + Bucket: connection.buckets.index.name, + Prefix: prefix, + }) + + const { Contents } = await connection.channel.send(command) + const entries = + Contents?.map((c) => c.Key ?? '').filter((key) => key.endsWith(suffix)) ?? + [] + + return entries.length > 0 + ? { ok: /** @type {[string, ...string[]]} */ (entries) } + : { + error: new RecordNotFound( + `no pseudo symlink matching query ${prefix}*${suffix} was found` + ), + } + } catch (/** @type {any} */ error) { + if (error?.$metadata?.httpStatusCode === 404) { + return { + error: new RecordNotFound( + `no pseudo symlink matching query ${prefix}*${suffix} was found` + ), + } + } + return { + error: new StorageOperationFailed(error.message), + } + } +} + +/** + * @param {Store} connection + * @param {import('@ucanto/interface').Link} message + * @returns {Promise>} + */ +const read = async ({ buckets, channel }, message) => { + const getCmd = new GetObjectCommand({ + Bucket: buckets.message.name, + Key: toMessagePath({ message }), + }) + + let res + try { + res = await channel.send(getCmd) + } catch (/** @type {any} */ error) { + if (error?.$metadata?.httpStatusCode === 404) { + return { + error: new RecordNotFound( + `agent message archive ${message} not found in store` + ), + } + } + return { + error: new StorageOperationFailed(error.message), + } + } + if (!res || !res.Body) { + return { + error: new RecordNotFound( + `agent message archive ${message} not found in store` + ), + } + } + + const bytes = await res.Body.transformToByteArray() + + return { + ok: bytes, + } +} + +/** + * @param {Store} store + * @param {import('@ucanto/interface').Link} message + */ +export const toMessageURL = (store, message) => + new URL(`https://${store.buckets.message.name}.s3.${store.region}.amazonaws.com/${toMessagePath({message})}`) + + +/** + * @param {object} source + * @param {import('@ucanto/interface').Link} source.message + */ + +export const toMessagePath = ({message}) => + `${message}/${message}` + +/** + * @param {object} source + * @param {import('@ucanto/interface').Link} source.message + * @param {import('@ucanto/interface').Link} source.task + * @param {import('@ucanto/interface').Link} source.invocation + */ +export const toInvocationPath = ({message, task, invocation}) => + `${task}/${invocation}@${message}.in` + +/** + * @param {object} source + * @param {import('@ucanto/interface').Link} source.message + * @param {import('@ucanto/interface').Link} source.task + * @param {import('@ucanto/interface').Link} source.receipt + */ +export const toReceiptPath = ({message, task, receipt}) => + `${task}/${receipt}@${message}.out` diff --git a/test/helpers/receipt-store.js b/test/helpers/receipt-store.js index 206d356..ff14f51 100644 --- a/test/helpers/receipt-store.js +++ b/test/helpers/receipt-store.js @@ -1,10 +1,8 @@ import { S3Client, - GetObjectCommand, - ListObjectsV2Command } from '@aws-sdk/client-s3' -import * as CAR from '@ucanto/transport/car' -import { StoreOperationFailed, RecordNotFound } from '@web3-storage/filecoin-api/errors' +import { StoreOperationFailed } from '@web3-storage/filecoin-api/errors' +import * as Store from './agent/store.js' /** * Abstraction layer with Factory to perform operations on bucket storing @@ -30,98 +28,28 @@ export function createReceiptStore(region, invocationBucketName, workflowBucketN * @returns {import('@web3-storage/filecoin-api/storefront/api').ReceiptStore} */ export const useReceiptStore = (s3client, invocationBucketName, workflowBucketName) => { + const store = Store.open({ + // @ts-ignore + connection: { channel: s3client }, + region: typeof s3client.config.region === 'string' ? s3client.config.region : 'us-west-2', + buckets: { + index: { name: invocationBucketName }, + message: { name: workflowBucketName }, + } + }) + return { put: async (record) => { return { error: new StoreOperationFailed('no new receipt should be put by storefront') } }, - get: async (taskCid) => { - // TODO: When we distinct between TaskCid and InvocationCid, we also need to see this mapping. - const invocationCid = taskCid - - // Find agent message archive CID where this receipt was stored - const encodedInvocationKeyPrefix = `${invocationCid.toString()}/` - const listCmd = new ListObjectsV2Command({ - Bucket: invocationBucketName, - Prefix: encodedInvocationKeyPrefix - }) - let listRes - try { - listRes = await s3client.send(listCmd) - } catch (/** @type {any} */ error) { - if (error?.$metadata?.httpStatusCode === 404) { - return { - error: new RecordNotFound(`any pseudo symlink from invocation ${invocationCid.toString()} was found`) - } - } - return { - error: new StoreOperationFailed(error.message) - } - } - if (!listRes.Contents?.length) { - return { - error: new RecordNotFound(`any pseudo symlink from invocation ${invocationCid.toString()} was found`) - } - } - // Key in format `${invocation.cid}/${agentMessageArchive.cid}.out` - const agentMessageArchiveWithReceipt = listRes.Contents.find(c => c.Key?.endsWith('.out')) - if (!agentMessageArchiveWithReceipt || !agentMessageArchiveWithReceipt.Key) { - return { - error: new RecordNotFound(`any pseudo symlink from invocation ${invocationCid.toString()} was found with a receipt`) - } - } - - // Get Message Archive with receipt - const agentMessageArchiveWithReceiptCid = agentMessageArchiveWithReceipt.Key - .replace(encodedInvocationKeyPrefix, '') - .replace('.out', '') - - const encodedAgentMessageArchiveKey = `${agentMessageArchiveWithReceiptCid}/${agentMessageArchiveWithReceiptCid}` - const getCmd = new GetObjectCommand({ - Bucket: workflowBucketName, - Key: encodedAgentMessageArchiveKey, - }) - - let res - try { - res = await s3client.send(getCmd) - } catch (/** @type {any} */ error) { - if (error?.$metadata?.httpStatusCode === 404) { - return { - error: new RecordNotFound(`agent message archive ${encodedAgentMessageArchiveKey} not found in store`) - } - } - return { - error: new StoreOperationFailed(error.message) - } - } - if (!res || !res.Body) { - return { - error: new RecordNotFound(`agent message archive ${encodedAgentMessageArchiveKey} not found in store`) - } - } - - - // Get receipt from Message Archive - const agentMessageBytes = await res.Body.transformToByteArray() - const agentMessage = await CAR.request.decode({ - body: agentMessageBytes, - headers: {}, - }) - - - // @ts-expect-error unknown link does not mach expectations - const receipt = agentMessage.receipts.get(invocationCid.toString()) - if (!receipt) { - return { - error: new RecordNotFound(`agent message archive ${encodedAgentMessageArchiveKey} does not include receipt for invocation ${invocationCid.toString()}`) - } - } - return { - ok: receipt - } - }, + /** + * @param {import('@ucanto/interface').UnknownLink} taskCid + */ + get: (taskCid) => + // @ts-expect-error - need to align RecordNotFoundError + Store.getReceipt(store, taskCid), has: async (record) => { return { error: new StoreOperationFailed('no receipt should checked by storefront')