Skip to content

Commit

Permalink
add squasher module for cleaning up history
Browse files Browse the repository at this point in the history
  • Loading branch information
@zeerooth
zeerooth committed Nov 10, 2024
1 parent f513978 commit a78191b
Show file tree
Hide file tree
Showing 9 changed files with 652 additions and 22 deletions.
13 changes: 11 additions & 2 deletions yamabiko/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,9 @@ full = ["log", "serde_yaml"]
yaml = ["serde_yaml"]

[dev-dependencies]
criterion = "0.4"
simple_logger = "4.0"
criterion = "0.5.1"
simple_logger = "5.0.0"
tokio = { version = "1.41", features = ["full"] }

[[bench]]
name = "perf"
Expand All @@ -31,5 +32,13 @@ harness = false
name = "queries"
harness = false

[[bench]]
name = "squash"
harness = false

[[bench]]
name = "squash_extreme"
harness = false

[lints]
workspace = true
26 changes: 16 additions & 10 deletions yamabiko/benches/perf.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ fn bench_sets(bench: &mut Criterion) {
format!("key-{}", i).as_str(),
yamabiko::test::SampleDbStruct::new(String::from("test value")),
OperationTarget::Main,
);
)
.unwrap();
i += 1;
})
});
Expand All @@ -25,7 +26,8 @@ fn bench_sets(bench: &mut Criterion) {
format!("key-{}", i).as_str(),
yamabiko::test::SampleDbStruct::new(String::from("test value")),
OperationTarget::Main,
);
)
.unwrap();
i += 1;
})
});
Expand All @@ -36,14 +38,15 @@ fn bench_sets(bench: &mut Criterion) {
let hm2 = hm
.iter()
.map(|x| (format!("key-{}", x), "some value".as_bytes()));
db.set_batch(hm2, OperationTarget::Main);
db.set_batch(hm2, OperationTarget::Main).unwrap();
let mut i = INIT_DB_SIZE;
b.iter(|| {
db.set(
format!("key-{}", i).as_str(),
yamabiko::test::SampleDbStruct::new(String::from("test value")),
OperationTarget::Main,
);
)
.unwrap();
i += 1;
})
});
Expand All @@ -55,14 +58,15 @@ fn bench_sets(bench: &mut Criterion) {
let hm2 = hm
.iter()
.map(|x| (format!("key-{}", x), "some value".as_bytes()));
db.set_batch(hm2, OperationTarget::Main);
db.set_batch(hm2, OperationTarget::Main).unwrap();
let mut i = INIT_DB_SIZE;
b.iter(|| {
db.set(
format!("key-{}", i).as_str(),
yamabiko::test::SampleDbStruct::new(String::from("test value")),
OperationTarget::Main,
);
)
.unwrap();
i += 1;
})
});
Expand All @@ -77,7 +81,7 @@ fn bench_sets(bench: &mut Criterion) {
yamabiko::test::SampleDbStruct::new(String::from("test value")),
);
}
db.set_batch(hm, OperationTarget::Main);
db.set_batch(hm, OperationTarget::Main).unwrap();
i += 100;
});
});
Expand All @@ -92,7 +96,8 @@ fn bench_sets_and_gets(bench: &mut Criterion) {
format!("key-{}", i).as_str(),
yamabiko::test::SampleDbStruct::new(String::from("test value")),
OperationTarget::Main,
);
)
.unwrap();
db.get::<yamabiko::test::SampleDbStruct>(
format!("key-{}", i).as_str(),
OperationTarget::Main,
Expand All @@ -108,14 +113,15 @@ fn bench_sets_and_gets(bench: &mut Criterion) {
let hm2 = hm
.iter()
.map(|x| (format!("key-{}", x), "some value".as_bytes()));
db.set_batch(hm2, OperationTarget::Main);
db.set_batch(hm2, OperationTarget::Main).unwrap();
let mut i = INIT_DB_SIZE;
b.iter(|| {
db.set(
format!("key-{}", i).as_str(),
yamabiko::test::SampleDbStruct::new(String::from("test value")),
OperationTarget::Main,
);
)
.unwrap();
db.get::<yamabiko::test::SampleDbStruct>(
format!("key-{}", i).as_str(),
OperationTarget::Main,
Expand Down
4 changes: 2 additions & 2 deletions yamabiko/benches/queries.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ fn bench_queries(bench: &mut Criterion) {
yamabiko::test::ComplexDbStruct::new(String::from("test value"), *x, *x as f64),
)
});
db.set_batch(hm2, OperationTarget::Main);
db.set_batch(hm2, OperationTarget::Main).unwrap();
bench.bench_function("query large database without indexes", |b| {
b.iter(|| {
assert_eq!(
Expand All @@ -44,7 +44,7 @@ fn bench_queries(bench: &mut Criterion) {
yamabiko::test::ComplexDbStruct::new(String::from("test value"), *x, *x as f64),
)
});
db.set_batch(hm2, OperationTarget::Main);
db.set_batch(hm2, OperationTarget::Main).unwrap();
bench.bench_function("query large database with an index", |b| {
b.iter(|| {
let query_result = QueryBuilder::new()
Expand Down
163 changes: 163 additions & 0 deletions yamabiko/benches/squash.rs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
use std::path::Path;

use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
use git2::Oid;
use yamabiko::{squash::Squasher, test::create_db, OperationTarget};

fn bench_squash(bench: &mut Criterion) {
bench.bench_function("squash with 2 commits", |b| {
b.iter_batched(
|| {
let (db, td) = create_db();
db.set(
"key-1",
yamabiko::test::SampleDbStruct::new(String::from("test value")),
OperationTarget::Main,
)
.unwrap();
db.set(
"key-2",
yamabiko::test::SampleDbStruct::new(String::from("test value")),
OperationTarget::Main,
)
.unwrap();
let squasher = Squasher::initialize(Path::new(td.path())).unwrap();
let head = db
.repository()
.head()
.unwrap()
.peel_to_commit()
.unwrap()
.id();
(db, squasher, head, td)
},
|(_db, squasher, commit, _td)| {
squasher.squash_before_commit(commit).unwrap();
},
BatchSize::SmallInput,
)
});
bench.bench_function("squash with 1k commits", |b| {
b.iter_with_setup(
|| {
let (db, td) = create_db();
let squasher = Squasher::initialize(Path::new(td.path())).unwrap();
for i in 0..1000 {
db.set(
format!("key-{}", i).as_str(),
yamabiko::test::SampleDbStruct::new(String::from("test value")),
OperationTarget::Main,
)
.unwrap();
}
let head = db
.repository()
.head()
.unwrap()
.peel_to_commit()
.unwrap()
.id();
(db, squasher, head, td)
},
|(_db, squasher, commit, _td)| {
squasher.squash_before_commit(commit).unwrap();
},
)
});
bench.bench_function("squash 5 commits in repo with 10k", |b| {
b.iter_with_setup(
|| {
let (db, td) = create_db();
let squasher = Squasher::initialize(Path::new(td.path())).unwrap();
let mut commit_n1k = Oid::zero();
for i in 0..10000 {
db.set(
format!("key-{}", i).as_str(),
yamabiko::test::SampleDbStruct::new(String::from("test value")),
OperationTarget::Main,
)
.unwrap();
if i == 5 {
commit_n1k = db
.repository()
.head()
.unwrap()
.peel_to_commit()
.unwrap()
.id();
}
}
(db, squasher, commit_n1k, td)
},
|(_db, squasher, commit, _td)| {
squasher.squash_before_commit(commit).unwrap();
},
)
});
bench.bench_function("squash 1k commits in repo with 10k", |b| {
b.iter_with_setup(
|| {
let (db, td) = create_db();
let squasher = Squasher::initialize(Path::new(td.path())).unwrap();
let mut commit_n1k = Oid::zero();
for i in 0..10000 {
db.set(
format!("key-{}", i).as_str(),
yamabiko::test::SampleDbStruct::new(String::from("test value")),
OperationTarget::Main,
)
.unwrap();
if i == 1000 {
commit_n1k = db
.repository()
.head()
.unwrap()
.peel_to_commit()
.unwrap()
.id();
}
}
(db, squasher, commit_n1k, td)
},
|(_db, squasher, commit, _td)| {
squasher.squash_before_commit(commit).unwrap();
},
)
});
bench.bench_function("squash 9k commits in repo with 10k", |b| {
b.iter_with_setup(
|| {
let (db, td) = create_db();
let squasher = Squasher::initialize(Path::new(td.path())).unwrap();
let mut commit_n1k = Oid::zero();
for i in 0..10000 {
db.set(
format!("key-{}", i).as_str(),
yamabiko::test::SampleDbStruct::new(String::from("test value")),
OperationTarget::Main,
)
.unwrap();
if i == 9000 {
commit_n1k = db
.repository()
.head()
.unwrap()
.peel_to_commit()
.unwrap()
.id();
}
}
(db, squasher, commit_n1k, td)
},
|(_db, squasher, commit, _td)| {
squasher.squash_before_commit(commit).unwrap();
},
)
});
}

criterion_group! {
name = benches;
config = Criterion::default().sample_size(20);
targets = bench_squash}
criterion_main!(benches);
44 changes: 44 additions & 0 deletions yamabiko/benches/squash_extreme.rs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
use std::path::Path;

use criterion::{criterion_group, criterion_main, Criterion};
use git2::Oid;
use yamabiko::{squash::Squasher, test::create_db, OperationTarget};

fn bench_squash_extreme(bench: &mut Criterion) {
bench.bench_function("squash 10k commits in repo with 100k", |b| {
b.iter_with_setup(
|| {
let (db, td) = create_db();
let squasher = Squasher::initialize(Path::new(td.path())).unwrap();
let mut commit_n1k = Oid::zero();
for i in 0..100_000 {
db.set(
format!("key-{}", i).as_str(),
yamabiko::test::SampleDbStruct::new(String::from("test value")),
OperationTarget::Main,
)
.unwrap();
if i == 10000 {
commit_n1k = db
.repository()
.head()
.unwrap()
.peel_to_commit()
.unwrap()
.id();
}
}
(db, squasher, commit_n1k, td)
},
|(_db, squasher, commit, _td)| {
squasher.squash_before_commit(commit).unwrap();
},
)
});
}

criterion_group! {
name = benches;
config = Criterion::default().sample_size(10);
targets = bench_squash_extreme}
criterion_main!(benches);
5 changes: 4 additions & 1 deletion yamabiko/src/field.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,10 @@ impl PartialOrd<serde_json::Value> for Field {
match self {
Field::Float(f) => other.as_f64().map(|x| x.partial_cmp(f)).unwrap_or(None),
Field::Int(i) => other.as_i64().map(|x| x.partial_cmp(i)).unwrap_or(None),
Field::String(s) => other.as_str().map(|x| x.partial_cmp(s)).unwrap_or(None),
Field::String(s) => other
.as_str()
.map(|x| x.partial_cmp(s.as_str()))
.unwrap_or(None),
}
}
}
Expand Down
1 change: 1 addition & 0 deletions yamabiko/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ pub mod logging;
pub mod query;
pub mod replica;
pub mod serialization;
pub mod squash;

pub enum OperationTarget<'a> {
Main,
Expand Down
Loading

0 comments on commit a78191b

Please sign in to comment.