diff --git a/config/init.go b/config/init.go index 4a86aa5183e..6099712f4b0 100644 --- a/config/init.go +++ b/config/init.go @@ -138,6 +138,17 @@ func DefaultDatastoreConfig() Datastore { } } +func pebbleSpec() map[string]interface{} { + return map[string]interface{}{ + "type": "measure", + "prefix": "pebble.datastore", + "child": map[string]interface{}{ + "type": "pebbleds", + "path": "pebbleds", + }, + } +} + func badgerSpec() map[string]interface{} { return map[string]interface{}{ "type": "measure", diff --git a/config/profile.go b/config/profile.go index 0ee9225bee0..3a432963938 100644 --- a/config/profile.go +++ b/config/profile.go @@ -135,7 +135,11 @@ You should use this datastore if: * You want to minimize memory usage. * You are ok with the default speed of data import, or prefer to use --nocopy. -This profile may only be applied when first initializing the node. +See configuration documentation at: +https://github.com/ipfs/kubo/blob/master/docs/datastores.md#flatfs + +NOTE: This profile may only be applied when first initializing node at IPFS_PATH + via 'ipfs init --profile flatfs' `, InitOnly: true, @@ -144,6 +148,32 @@ This profile may only be applied when first initializing the node. return nil }, }, + "pebbleds": { + Description: `Configures the node to use the pebble high-performance datastore. + +Pebble is a LevelDB/RocksDB inspired key-value store focused on performance +and internal usage by CockroachDB. +You should use this datastore if: + +- You need a datastore that is focused on performance. +- You need reliability by default, but may choose to disable WAL for maximum performance when reliability is not critical. +- This datastore is good for multi-terabyte data sets. +- May benefit from tuning depending on read/write patterns and throughput. +- Performance is helped significantly by running on a system with plenty of memory. + +See configuration documentation at: +https://github.com/ipfs/kubo/blob/master/docs/datastores.md#pebbleds + +NOTE: This profile may only be applied when first initializing node at IPFS_PATH + via 'ipfs init --profile pebbleds' +`, + + InitOnly: true, + Transform: func(c *Config) error { + c.Datastore.Spec = pebbleSpec() + return nil + }, + }, "badgerds": { Description: `Configures the node to use the legacy badgerv1 datastore. @@ -160,7 +190,12 @@ Other caveats: * Good for medium-size datastores, but may run into performance issues if your dataset is bigger than a terabyte. -This profile may only be applied when first initializing the node.`, +See configuration documentation at: +https://github.com/ipfs/kubo/blob/master/docs/datastores.md#badgerds + +NOTE: This profile may only be applied when first initializing node at IPFS_PATH + via 'ipfs init --profile badgerds' +`, InitOnly: true, Transform: func(c *Config) error { diff --git a/docs/changelogs/v0.31.md b/docs/changelogs/v0.31.md index 1d6501ec98b..0b814ff65d6 100644 --- a/docs/changelogs/v0.31.md +++ b/docs/changelogs/v0.31.md @@ -6,6 +6,7 @@ - [Overview](#overview) - [🔦 Highlights](#-highlights) + - [Experimental Pebble Datastore](#experimental-pebble-datastore) - [New metrics](#new-metrics) - [`lowpower` profile no longer breaks DHT announcements](#lowpower-profile-no-longer-breaks-dht-announcements) - [📝 Changelog](#-changelog) @@ -15,6 +16,15 @@ ### 🔦 Highlights +#### Experimental Pebble Datastore + +[Pebble](https://github.com/ipfs/kubo/blob/master/docs/config.md#pebbleds-profile) provides a high-performance alternative to leveldb as the datastore, and provides a modern replacement for [legacy badgerv1](https://github.com/ipfs/kubo/blob/master/docs/config.md#badgerds-profile). + +A fresh Kubo node can be initialized with [`pebbleds` profile](https://github.com/ipfs/kubo/blob/master/docs/config.md#pebbleds-profile) via `ipfs init --profile pebbleds`. + +There are a number of parameters available for tuning pebble's performance to your specific needs. Default values are used for any parameters that are not configured or are set to their zero-value. +For a description of the available tuning parameters, see [kubo/docs/datastores.md#pebbleds](https://github.com/ipfs/kubo/blob/master/docs/datastores.md#pebbleds). + #### New metrics - Added 3 new go metrics: `go_gc_gogc_percent`, `go_gc_gomemlimit_bytes` and `go_sched_gomaxprocs_threads` as those are [recommended by the Go team](https://github.com/prometheus/client_golang/pull/1559) diff --git a/docs/config.md b/docs/config.md index 2bcf4b0f22f..bbbb94bd85e 100644 --- a/docs/config.md +++ b/docs/config.md @@ -181,6 +181,7 @@ config file at runtime. - [`local-discovery` profile](#local-discovery-profile) - [`default-networking` profile](#default-networking-profile) - [`flatfs` profile](#flatfs-profile) + - [`pebbleds` profile](#pebbleds-profile) - [`badgerds` profile](#badgerds-profile) - [`lowpower` profile](#lowpower-profile) - [`announce-off` profile](#announce-off-profile) @@ -524,13 +525,8 @@ Spec defines the structure of the ipfs datastore. It is a composable structure, where each datastore is represented by a json object. Datastores can wrap other datastores to provide extra functionality (eg metrics, logging, or caching). -This can be changed manually, however, if you make any changes that require a -different on-disk structure, you will need to run the [ipfs-ds-convert -tool](https://github.com/ipfs/ipfs-ds-convert) to migrate data into the new -structures. - -For more information on possible values for this configuration option, see -[docs/datastores.md](datastores.md) +> [!NOTE] +> For more information on possible values for this configuration option, see [`kubo/docs/datastores.md`](datastores.md) Default: ``` @@ -2403,9 +2399,9 @@ Inverse profile of the test profile. ### `flatfs` profile -Configures the node to use the flatfs datastore. Flatfs is the default datastore. +Configures the node to use the flatfs datastore. +Flatfs is the default, most battle-tested and reliable datastore. -This is the most battle-tested and reliable datastore. You should use this datastore if: - You need a very simple and very reliable datastore, and you trust your @@ -2416,7 +2412,30 @@ You should use this datastore if: - You want to minimize memory usage. - You are ok with the default speed of data import, or prefer to use `--nocopy`. -This profile may only be applied when first initializing the node. +> [!WARNING] +> This profile may only be applied when first initializing the node via `ipfs init --profile flatfs` + +> [!NOTE] +> See caveats and configuration options at [`datastores.md#flatfs`](datastores.md#flatfs) + +### `pebbleds` profile + +Configures the node to use the pebble high-performance datastore. + +Pebble is a LevelDB/RocksDB inspired key-value store focused on performance and internal usage by CockroachDB. +You should use this datastore if: + +- You need a datastore that is focused on performance. +- You need reliability by default, but may choose to disable WAL for maximum performance when reliability is not critical. +- This datastore is good for multi-terrabyte data sets. +- May benefit from tuning depending on read/write patterns and throughput. +- Performance is helped significantly by running on a system with plenty of memory. + +> [!WARNING] +> This profile may only be applied when first initializing the node via `ipfs init --profile pebbleds` + +> [!NOTE] +> See other caveats and configuration options at [`datastores.md#pebbleds`](datastores.md#pebbleds) ### `badgerds` profile @@ -2437,7 +2456,11 @@ Also, be aware that: - Good for medium-size datastores, but may run into performance issues if your dataset is bigger than a terabyte. - The current implementation is based on old badger 1.x which is no longer supported by the upstream team. -This profile may only be applied when first initializing the node. +> [!WARNING] +> This profile may only be applied when first initializing the node via `ipfs init --profile badgerds` + +> [!NOTE] +> See other caveats and configuration options at [`datastores.md#pebbleds`](datastores.md#pebbleds) ### `lowpower` profile diff --git a/docs/datastores.md b/docs/datastores.md index c18ecb0c7c5..f2051b601d4 100644 --- a/docs/datastores.md +++ b/docs/datastores.md @@ -3,6 +3,13 @@ This document describes the different possible values for the `Datastore.Spec` field in the ipfs configuration file. +- [flatfs](#flatfs) +- [levelds](#levelds) +- [pebbleds](#pebbleds) +- [badgerds](#badgerds) +- [mount](#mount) +- [measure](#measure) + ## flatfs Stores each key value pair as a file on the filesystem. @@ -35,6 +42,35 @@ Uses a leveldb database to store key value pairs. } ``` +## pebbleds + +Uses [pebble](https://github.com/cockroachdb/pebble) as a key value store. + +```json +{ + "type": "pebbleds", + "path": "", +} +``` + +The following options are availble for tuning pebble. +If they are not configured (or assigned their zero-valued), then default values are used. + +* `bytesPerSync`: int, Sync sstables periodically in order to smooth out writes to disk. (default: 512KB) +* `bisableWAL`: true|false, Disable the write-ahead log (WAL) at expense of prohibiting crash recovery. (default: false) +* `cacheSize`: Size of pebble's shared block cache. (default: 8MB) +* `l0CompactionThreshold`: int, Count of L0 files necessary to trigger an L0 compaction. +* `l0StopWritesThreshold`: int, Limit on L0 read-amplification, computed as the number of L0 sublevels. +* `lBaseMaxBytes`: int, Maximum number of bytes for LBase. The base level is the level which L0 is compacted into. +* `maxConcurrentCompactions`: int, Maximum number of concurrent compactions. (default: 1) +* `memTableSize`: int, Size of a MemTable in steady state. The actual MemTable size starts at min(256KB, MemTableSize) and doubles for each subsequent MemTable up to MemTableSize (default: 4MB) +* `memTableStopWritesThreshold`: int, Limit on the number of queued of MemTables. (default: 2) +* `walBytesPerSync`: int: Sets the number of bytes to write to a WAL before calling Sync on it in the background. (default: 0, no background syncing) +* `walMinSyncSeconds`: int: Sets the minimum duration between syncs of the WAL. (default: 0) + +> [!TIP] +> Start using pebble with only default values and configure tuning items are needed for your needs. For a more complete description of these values, see: `https://pkg.go.dev/github.com/cockroachdb/pebble@vA.B.C#Options` (where `A.B.C` is pebble version from Kubo's `go.mod`). + ## badgerds Uses [badger](https://github.com/dgraph-io/badger) as a key value store. diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 6c0600757dd..6b94d3802ba 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -16,6 +16,7 @@ require ( require ( bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc // indirect github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect + github.com/DataDog/zstd v1.4.5 // indirect github.com/Jorropo/jsync v1.0.1 // indirect github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 // indirect github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect @@ -25,6 +26,12 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/ceramicnetwork/go-dag-jose v0.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cockroachdb/errors v1.11.3 // indirect + github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/pebble v1.1.2 // indirect + github.com/cockroachdb/redact v1.1.5 // indirect + github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 // indirect @@ -43,6 +50,7 @@ require ( github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.4 // indirect + github.com/getsentry/sentry-go v0.27.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect @@ -74,6 +82,7 @@ require ( github.com/ipfs/go-ds-flatfs v0.5.1 // indirect github.com/ipfs/go-ds-leveldb v0.5.0 // indirect github.com/ipfs/go-ds-measure v0.2.0 // indirect + github.com/ipfs/go-ds-pebble v0.4.0 // indirect github.com/ipfs/go-fs-lock v0.0.7 // indirect github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect @@ -103,6 +112,8 @@ require ( github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/koron/go-ssdp v0.0.4 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-doh-resolver v0.4.0 // indirect @@ -172,6 +183,7 @@ require ( github.com/quic-go/quic-go v0.45.2 // indirect github.com/quic-go/webtransport-go v0.8.0 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/samber/lo v1.46.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/stretchr/testify v1.9.0 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 2cd9a72e526..8b4c2ee8cbd 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -29,6 +29,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIo github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= +github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU= github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -74,6 +76,20 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= +github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= +github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= +github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA= +github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= @@ -143,10 +159,14 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.4 h1:QjV6pZ7/XZ7ryI2KuyeEDE8wnh7fHP9YnQy+R0LnH8I= github.com/gabriel-vasile/mimetype v1.4.4/go.mod h1:JwLei5XPtWdGiMFB5Pjle1oEeoSeEuJfJE+TtfvdB/s= +github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= +github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -301,6 +321,8 @@ github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUN github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= github.com/ipfs/go-ds-measure v0.2.0 h1:sG4goQe0KDTccHMyT45CY1XyUbxe5VwTKpg2LjApYyQ= github.com/ipfs/go-ds-measure v0.2.0/go.mod h1:SEUD/rE2PwRa4IQEC5FuNAmjJCyYObZr9UvVh8V3JxE= +github.com/ipfs/go-ds-pebble v0.4.0 h1:88lgFAs2ck8jCQ8lMYRBtksEg18r9BlvTxIMnNJkZaQ= +github.com/ipfs/go-ds-pebble v0.4.0/go.mod h1:ZyYU+weIni+4NG/Yjva+cPkU3ghlsU1HA2R/VLHJ9sM= github.com/ipfs/go-fs-lock v0.0.7 h1:6BR3dajORFrFTkb5EpCUFIAypsoxpGpDSVUdFwzgL9U= github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc= github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ= @@ -585,6 +607,8 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhM github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pion/datachannel v1.5.8 h1:ph1P1NsGkazkjrvyMfhRBUAWMxugJjq2HfQifaOoSNo= github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu8QzbL3tI= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= @@ -627,6 +651,7 @@ github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I= github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -658,6 +683,7 @@ github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtB github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= diff --git a/go.mod b/go.mod index 5f27e27a8b8..3a33543f416 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 github.com/ceramicnetwork/go-dag-jose v0.1.0 github.com/cheggaaa/pb v1.0.29 + github.com/cockroachdb/pebble v1.1.2 github.com/coreos/go-systemd/v22 v22.5.0 github.com/dustin/go-humanize v1.0.1 github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 @@ -30,6 +31,7 @@ require ( github.com/ipfs/go-ds-flatfs v0.5.1 github.com/ipfs/go-ds-leveldb v0.5.0 github.com/ipfs/go-ds-measure v0.2.0 + github.com/ipfs/go-ds-pebble v0.4.0 github.com/ipfs/go-fs-lock v0.0.7 github.com/ipfs/go-ipfs-cmds v0.13.0 github.com/ipfs/go-ipld-cbor v0.2.0 @@ -95,12 +97,18 @@ require ( require ( github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect + github.com/DataDog/zstd v1.4.5 // indirect github.com/Jorropo/jsync v1.0.1 // indirect github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 // indirect github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cockroachdb/errors v1.11.3 // indirect + github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/redact v1.1.5 // indirect + github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 // indirect github.com/cskr/pubsub v1.0.2 // indirect @@ -115,6 +123,7 @@ require ( github.com/flynn/noise v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect github.com/gabriel-vasile/mimetype v1.4.4 // indirect + github.com/getsentry/sentry-go v0.27.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logr/logr v1.4.2 // indirect @@ -151,6 +160,8 @@ require ( github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/koron/go-ssdp v0.0.4 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect @@ -164,7 +175,7 @@ require ( github.com/libp2p/go-yamux/v4 v4.0.1 // indirect github.com/libp2p/zeroconf/v2 v2.2.0 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect - github.com/mattn/go-colorable v0.1.6 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.4 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect @@ -210,6 +221,7 @@ require ( github.com/quic-go/quic-go v0.45.2 // indirect github.com/quic-go/webtransport-go v0.8.0 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/rs/cors v1.10.1 // indirect github.com/samber/lo v1.46.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect diff --git a/go.sum b/go.sum index 0c5e5bbd6e3..09a91f624ae 100644 --- a/go.sum +++ b/go.sum @@ -47,6 +47,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIo github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= +github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU= github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= @@ -104,6 +106,20 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= +github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= +github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= +github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA= +github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= @@ -177,10 +193,14 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gabriel-vasile/mimetype v1.4.4 h1:QjV6pZ7/XZ7ryI2KuyeEDE8wnh7fHP9YnQy+R0LnH8I= github.com/gabriel-vasile/mimetype v1.4.4/go.mod h1:JwLei5XPtWdGiMFB5Pjle1oEeoSeEuJfJE+TtfvdB/s= +github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= +github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -365,6 +385,8 @@ github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUN github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= github.com/ipfs/go-ds-measure v0.2.0 h1:sG4goQe0KDTccHMyT45CY1XyUbxe5VwTKpg2LjApYyQ= github.com/ipfs/go-ds-measure v0.2.0/go.mod h1:SEUD/rE2PwRa4IQEC5FuNAmjJCyYObZr9UvVh8V3JxE= +github.com/ipfs/go-ds-pebble v0.4.0 h1:88lgFAs2ck8jCQ8lMYRBtksEg18r9BlvTxIMnNJkZaQ= +github.com/ipfs/go-ds-pebble v0.4.0/go.mod h1:ZyYU+weIni+4NG/Yjva+cPkU3ghlsU1HA2R/VLHJ9sM= github.com/ipfs/go-fs-lock v0.0.7 h1:6BR3dajORFrFTkb5EpCUFIAypsoxpGpDSVUdFwzgL9U= github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc= github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ= @@ -560,15 +582,15 @@ github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= @@ -685,6 +707,8 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhM github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pion/datachannel v1.5.8 h1:ph1P1NsGkazkjrvyMfhRBUAWMxugJjq2HfQifaOoSNo= github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu8QzbL3tI= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= @@ -727,6 +751,7 @@ github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I= github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -785,6 +810,7 @@ github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtB github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= @@ -1200,6 +1226,7 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/plugin/loader/preload.go b/plugin/loader/preload.go index 2ad84e59489..75e21270c92 100644 --- a/plugin/loader/preload.go +++ b/plugin/loader/preload.go @@ -8,6 +8,7 @@ import ( pluginipldgit "github.com/ipfs/kubo/plugin/plugins/git" pluginlevelds "github.com/ipfs/kubo/plugin/plugins/levelds" pluginnopfs "github.com/ipfs/kubo/plugin/plugins/nopfs" + pluginpebbleds "github.com/ipfs/kubo/plugin/plugins/pebbleds" pluginpeerlog "github.com/ipfs/kubo/plugin/plugins/peerlog" ) @@ -21,6 +22,7 @@ func init() { Preload(pluginbadgerds.Plugins...) Preload(pluginflatfs.Plugins...) Preload(pluginlevelds.Plugins...) + Preload(pluginpebbleds.Plugins...) Preload(pluginpeerlog.Plugins...) Preload(pluginfxtest.Plugins...) Preload(pluginnopfs.Plugins...) diff --git a/plugin/loader/preload_list b/plugin/loader/preload_list index 462a3f39337..190cc65d716 100644 --- a/plugin/loader/preload_list +++ b/plugin/loader/preload_list @@ -9,6 +9,7 @@ iplddagjose github.com/ipfs/kubo/plugin/plugins/dagjose * badgerds github.com/ipfs/kubo/plugin/plugins/badgerds * flatfs github.com/ipfs/kubo/plugin/plugins/flatfs * levelds github.com/ipfs/kubo/plugin/plugins/levelds * +pebbleds github.com/ipfs/kubo/plugin/plugins/pebbleds * peerlog github.com/ipfs/kubo/plugin/plugins/peerlog * fxtest github.com/ipfs/kubo/plugin/plugins/fxtest * -nopfs github.com/ipfs/kubo/plugin/plugins/nopfs * \ No newline at end of file +nopfs github.com/ipfs/kubo/plugin/plugins/nopfs * diff --git a/plugin/plugins/flatfs/flatfs.go b/plugin/plugins/flatfs/flatfs.go index 1a23dfcca9d..397c2656c76 100644 --- a/plugin/plugins/flatfs/flatfs.go +++ b/plugin/plugins/flatfs/flatfs.go @@ -42,7 +42,7 @@ type datastoreConfig struct { syncField bool } -// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore +// BadgerdsDatastoreConfig returns a configuration stub for a flatfs datastore // from the given parameters. func (*flatfsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap { return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) { diff --git a/plugin/plugins/levelds/levelds.go b/plugin/plugins/levelds/levelds.go index b08872de63d..78331730ed3 100644 --- a/plugin/plugins/levelds/levelds.go +++ b/plugin/plugins/levelds/levelds.go @@ -42,7 +42,7 @@ type datastoreConfig struct { compression ldbopts.Compression } -// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore +// DatastoreConfigParser returns a configuration stub for a badger datastore // from the given parameters. func (*leveldsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap { return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) { diff --git a/plugin/plugins/pebbleds/pebbleds.go b/plugin/plugins/pebbleds/pebbleds.go new file mode 100644 index 00000000000..320fb315624 --- /dev/null +++ b/plugin/plugins/pebbleds/pebbleds.go @@ -0,0 +1,246 @@ +package pebbleds + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "time" + + "github.com/cockroachdb/pebble" + pebbleds "github.com/ipfs/go-ds-pebble" + "github.com/ipfs/kubo/plugin" + "github.com/ipfs/kubo/repo" + "github.com/ipfs/kubo/repo/fsrepo" +) + +// Plugins is exported list of plugins that will be loaded. +var Plugins = []plugin.Plugin{ + &pebbledsPlugin{}, +} + +type pebbledsPlugin struct{} + +var _ plugin.PluginDatastore = (*pebbledsPlugin)(nil) + +func (*pebbledsPlugin) Name() string { + return "ds-pebble" +} + +func (*pebbledsPlugin) Version() string { + return "0.1.0" +} + +func (*pebbledsPlugin) Init(_ *plugin.Environment) error { + return nil +} + +func (*pebbledsPlugin) DatastoreTypeName() string { + return "pebbleds" +} + +type datastoreConfig struct { + path string + cacheSize int64 + + // Documentation of these values: https://pkg.go.dev/github.com/cockroachdb/pebble@v1.1.2#Options + pebbleOpts *pebble.Options +} + +// PebbleDatastoreConfig returns a configuration stub for a pebble datastore +// from the given parameters. +func (*pebbledsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap { + return func(params map[string]any) (fsrepo.DatastoreConfig, error) { + var c datastoreConfig + var ok bool + + c.path, ok = params["path"].(string) + if !ok { + return nil, fmt.Errorf("'path' field is missing or not string") + } + + cacheSize, err := getConfigInt("cacheSize", params) + if err != nil { + return nil, err + } + c.cacheSize = int64(cacheSize) + + bytesPerSync, err := getConfigInt("bytesPerSync", params) + if err != nil { + return nil, err + } + disableWAL, err := getConfigBool("disableWAL", params) + if err != nil { + return nil, err + } + l0CompactionThreshold, err := getConfigInt("l0CompactionThreshold", params) + if err != nil { + return nil, err + } + l0StopWritesThreshold, err := getConfigInt("l0StopWritesThreshold", params) + if err != nil { + return nil, err + } + lBaseMaxBytes, err := getConfigInt("lBaseMaxBytes", params) + if err != nil { + return nil, err + } + maxConcurrentCompactions, err := getConfigInt("maxConcurrentCompactions", params) + if err != nil { + return nil, err + } + memTableSize, err := getConfigInt("memTableSize", params) + if err != nil { + return nil, err + } + memTableStopWritesThreshold, err := getConfigInt("memTableStopWritesThreshold", params) + if err != nil { + return nil, err + } + walBytesPerSync, err := getConfigInt("walBytesPerSync", params) + if err != nil { + return nil, err + } + walMinSyncSec, err := getConfigInt("walMinSyncIntervalSeconds", params) + if err != nil { + return nil, err + } + + if bytesPerSync != 0 || disableWAL || l0CompactionThreshold != 0 || l0StopWritesThreshold != 0 || lBaseMaxBytes != 0 || maxConcurrentCompactions != 0 || memTableSize != 0 || memTableStopWritesThreshold != 0 || walBytesPerSync != 0 || walMinSyncSec != 0 { + c.pebbleOpts = &pebble.Options{ + BytesPerSync: bytesPerSync, + DisableWAL: disableWAL, + L0CompactionThreshold: l0CompactionThreshold, + L0StopWritesThreshold: l0StopWritesThreshold, + LBaseMaxBytes: int64(lBaseMaxBytes), + MemTableSize: uint64(memTableSize), + MemTableStopWritesThreshold: memTableStopWritesThreshold, + WALBytesPerSync: walBytesPerSync, + } + if maxConcurrentCompactions != 0 { + c.pebbleOpts.MaxConcurrentCompactions = func() int { return maxConcurrentCompactions } + } + if walMinSyncSec != 0 { + c.pebbleOpts.WALMinSyncInterval = func() time.Duration { return time.Duration(walMinSyncSec) * time.Second } + } + } + + return &c, nil + } +} + +func getConfigBool(name string, params map[string]any) (bool, error) { + val, ok := params[name] + if ok { + bval, ok := val.(bool) + if !ok { + return false, fmt.Errorf("%q field was not a bool", name) + } + return bval, nil + } + return false, nil +} + +func getConfigInt(name string, params map[string]any) (int, error) { + val, ok := params[name] + if ok { + // TODO: see why val may be an int or a float64. + ival, ok := val.(int) + if !ok { + fval, ok := val.(float64) + if !ok { + return 0, fmt.Errorf("%q field was not an integer or a float64", name) + } + return int(fval), nil + } + return ival, nil + } + return 0, nil +} + +func (c *datastoreConfig) DiskSpec() fsrepo.DiskSpec { + return map[string]interface{}{ + "type": "pebbleds", + "path": c.path, + } +} + +func (c *datastoreConfig) Create(path string) (repo.Datastore, error) { + p := c.path + if !filepath.IsAbs(p) { + p = filepath.Join(path, p) + } + + if err := dirWritable(p); err != nil { + return nil, err + } + + return pebbleds.NewDatastore(p, pebbleds.WithCacheSize(c.cacheSize), pebbleds.WithPebbleOpts(c.pebbleOpts)) +} + +// dirWritable checks if a directory is writable. If the directory does +// not exist it is created with writable permission. +func dirWritable(dir string) error { + if dir == "" { + return errors.New("directory not specified") + } + var err error + dir, err = expandHome(dir) + if err != nil { + return err + } + + fi, err := os.Stat(dir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + // Directory does not exist, so create it. + err = os.Mkdir(dir, 0775) + if err == nil { + return nil + } + } + if errors.Is(err, fs.ErrPermission) { + err = fs.ErrPermission + } + return fmt.Errorf("directory not writable: %s: %w", dir, err) + } + if !fi.IsDir() { + return fmt.Errorf("not a directory: %s", dir) + } + + // Directory exists, check that a file can be written. + file, err := os.CreateTemp(dir, "writetest") + if err != nil { + if errors.Is(err, fs.ErrPermission) { + err = fs.ErrPermission + } + return fmt.Errorf("directory not writable: %s: %w", dir, err) + } + file.Close() + return os.Remove(file.Name()) +} + +// expandHome expands the path to include the home directory if the path is +// prefixed with `~`. If it isn't prefixed with `~`, the path is returned +// as-is. +func expandHome(path string) (string, error) { + if path == "" { + return path, nil + } + + if path[0] != '~' { + return path, nil + } + + if len(path) > 1 && path[1] != '/' && path[1] != '\\' { + return "", errors.New("cannot expand user-specific home dir") + } + + dir, err := os.UserHomeDir() + if err != nil { + return "", err + } + + return filepath.Join(dir, path[1:]), nil +} diff --git a/test/sharness/t0021-config.sh b/test/sharness/t0021-config.sh index 5264908c73f..95a8a7d8746 100755 --- a/test/sharness/t0021-config.sh +++ b/test/sharness/t0021-config.sh @@ -281,7 +281,7 @@ test_config_cmd() { # won't work as it changes datastore definition, which makes ipfs not launch # without converting first - # test_profile_apply_revert badgerds + # test_profile_apply_revert pebbleds test_expect_success "cleanup config backups" ' find "$IPFS_PATH" -name "config-*" -exec rm {} \; diff --git a/test/sharness/t0025-datastores.sh b/test/sharness/t0025-datastores.sh index f0ddd4e2e8f..6be9eb3ed48 100755 --- a/test/sharness/t0025-datastores.sh +++ b/test/sharness/t0025-datastores.sh @@ -4,13 +4,20 @@ test_description="Test non-standard datastores" . lib/test-lib.sh -test_expect_success "'ipfs init --empty-repo=false --profile=badgerds' succeeds" ' - BITS="2048" && - ipfs init --empty-repo=false --profile=badgerds -' +profiles=("flatfs" "pebbleds" "badgerds") +proot="$(mktemp -d "${TMPDIR:-/tmp}/t0025.XXXXXX")" -test_expect_success "'ipfs pin ls' works" ' - ipfs pin ls | wc -l | grep 9 -' +for profile in "${profiles[@]}"; do + test_expect_success "'ipfs init --empty-repo=false --profile=$profile' succeeds" ' + BITS="2048" && + IPFS_PATH="$proot/$profile" && + ipfs init --empty-repo=false --profile=$profile + ' + test_expect_success "'ipfs pin add' and 'pin ls' works with $profile" ' + export IPFS_PATH="$proot/$profile" && + echo -n "hello_$profile" | ipfs block put --pin=true > hello_cid && + ipfs pin ls -t recursive "$(cat hello_cid)" + ' +done test_done