diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
new file mode 100644
index 00000000..dbf1d0a2
--- /dev/null
+++ b/.github/workflows/maven.yml
@@ -0,0 +1,26 @@
+# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+
+name: Java CI with Maven
+
+on:
+ push:
+ branches: [ "master" ]
+ pull_request:
+ branches: [ "master" ]
+
+jobs:
+ build:
+
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v3
+ - name: Set up JDK 11
+ uses: actions/setup-java@v3
+ with:
+ java-version: '11'
+ distribution: 'temurin'
+ cache: maven
+ - name: Build with Maven
+ run: mvn clean package -P release
diff --git a/leveldb-api/pom.xml b/leveldb-api/pom.xml
index 32cc1715..8901d495 100644
--- a/leveldb-api/pom.xml
+++ b/leveldb-api/pom.xml
@@ -3,9 +3,9 @@
4.0.0
- org.iq80.leveldb
+ com.halibobor
leveldb-project
- 0.13-SNAPSHOT
+ 1.18.2
leveldb-api
diff --git a/leveldb-api/src/license/LICENSE-HEADER.txt b/leveldb-api/src/license/LICENSE-HEADER.txt
new file mode 100644
index 00000000..1de76d19
--- /dev/null
+++ b/leveldb-api/src/license/LICENSE-HEADER.txt
@@ -0,0 +1,15 @@
+Copyright (C) 2011 the original author or authors.
+See the notice.md file distributed with this work for additional
+information regarding copyright ownership.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
\ No newline at end of file
diff --git a/leveldb-api/src/main/java/org/iq80/leveldb/Options.java b/leveldb-api/src/main/java/org/iq80/leveldb/Options.java
index 630753cc..b3bd03f2 100644
--- a/leveldb-api/src/main/java/org/iq80/leveldb/Options.java
+++ b/leveldb-api/src/main/java/org/iq80/leveldb/Options.java
@@ -33,6 +33,8 @@ public class Options
private DBComparator comparator;
private Logger logger;
private long cacheSize;
+ private int maxBatchSize = 80_000;
+ private int maxManifestSize = 0; //M
static void checkArgNotNull(Object value, String name)
{
@@ -173,4 +175,33 @@ public Options paranoidChecks(boolean paranoidChecks)
this.paranoidChecks = paranoidChecks;
return this;
}
+
+ public int maxBatchSize()
+ {
+ return maxBatchSize;
+ }
+
+ public Options maxBatchSize(int maxBatchSize)
+ {
+ if (maxBatchSize < 0) {
+ maxBatchSize = Integer.MAX_VALUE;
+ }
+ this.maxBatchSize = maxBatchSize;
+ return this;
+ }
+
+ public int maxManifestSize()
+ {
+ return maxManifestSize;
+ }
+
+ public Options maxManifestSize(int maxManifestSize)
+ {
+ if (maxManifestSize < 0) {
+ maxManifestSize = -1;
+ maxBatchSize(-1);
+ }
+ this.maxManifestSize = maxManifestSize;
+ return this;
+ }
}
diff --git a/leveldb-benchmark/pom.xml b/leveldb-benchmark/pom.xml
index 0262f0b8..481e2330 100644
--- a/leveldb-benchmark/pom.xml
+++ b/leveldb-benchmark/pom.xml
@@ -19,9 +19,9 @@
4.0.0
+ com.halibobor
leveldb-project
- org.iq80.leveldb
- 0.13-SNAPSHOT
+ 1.18.2
leveldb-benchmark
@@ -34,11 +34,11 @@
- org.iq80.leveldb
+ com.halibobor
leveldb-api
- org.iq80.leveldb
+ com.halibobor
leveldb
diff --git a/leveldb/pom.xml b/leveldb/pom.xml
index ebd55194..62a9999c 100644
--- a/leveldb/pom.xml
+++ b/leveldb/pom.xml
@@ -3,9 +3,9 @@
4.0.0
- org.iq80.leveldb
+ com.halibobor
leveldb-project
- 0.13-SNAPSHOT
+ 1.18.2
leveldb
@@ -20,14 +20,14 @@
- org.iq80.leveldb
+ com.halibobor
leveldb-api
org.xerial.snappy
snappy-java
- 1.1.2.6
+ 1.1.2.6
true
@@ -60,7 +60,13 @@
org.fusesource.leveldbjni
leveldbjni
- 1.1
+ 1.8
+
+
+ org.iq80.leveldb
+ leveldb-api
+
+
test
diff --git a/leveldb/src/main/java/org/iq80/leveldb/impl/DbImpl.java b/leveldb/src/main/java/org/iq80/leveldb/impl/DbImpl.java
index d0cafac4..426fd998 100755
--- a/leveldb/src/main/java/org/iq80/leveldb/impl/DbImpl.java
+++ b/leveldb/src/main/java/org/iq80/leveldb/impl/DbImpl.java
@@ -177,7 +177,7 @@ public void uncaughtException(Thread t, Throwable e)
checkArgument(!options.errorIfExists(), "Database '%s' exists and the error if exists option is enabled", databaseDir);
}
- versions = new VersionSet(databaseDir, tableCache, internalKeyComparator);
+ versions = new VersionSet(databaseDir, tableCache, options, internalKeyComparator);
// load (and recover) current version
versions.recover();
diff --git a/leveldb/src/main/java/org/iq80/leveldb/impl/VersionSet.java b/leveldb/src/main/java/org/iq80/leveldb/impl/VersionSet.java
index 22f9bc57..a7464944 100755
--- a/leveldb/src/main/java/org/iq80/leveldb/impl/VersionSet.java
+++ b/leveldb/src/main/java/org/iq80/leveldb/impl/VersionSet.java
@@ -24,6 +24,7 @@
import com.google.common.collect.MapMaker;
import com.google.common.collect.Maps;
import com.google.common.io.Files;
+import org.iq80.leveldb.Options;
import org.iq80.leveldb.table.UserComparator;
import org.iq80.leveldb.util.InternalIterator;
import org.iq80.leveldb.util.Level0Iterator;
@@ -43,6 +44,7 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.SortedMap;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
@@ -80,12 +82,14 @@ public class VersionSet
private LogWriter descriptorLog;
private final Map compactPointers = new TreeMap<>();
+ private final Options options;
- public VersionSet(File databaseDir, TableCache tableCache, InternalKeyComparator internalKeyComparator)
- throws IOException
+ public VersionSet(File databaseDir, TableCache tableCache, Options options, InternalKeyComparator internalKeyComparator)
+ throws IOException
{
this.databaseDir = databaseDir;
this.tableCache = tableCache;
+ this.options = options;
this.internalKeyComparator = internalKeyComparator;
appendVersion(new Version(this));
@@ -353,9 +357,10 @@ public void recover()
Builder builder = new Builder(this, current);
LogReader reader = new LogReader(fileChannel, throwExceptionMonitor(), true, 0);
+ VersionEdit edit;
for (Slice record = reader.readRecord(); record != null; record = reader.readRecord()) {
// read version edit
- VersionEdit edit = new VersionEdit(record);
+ edit = new VersionEdit(record);
// verify comparator
// todo implement user comparator
@@ -366,7 +371,16 @@ public void recover()
// apply edit
builder.apply(edit);
+ if (builder.batchSize > options.maxBatchSize()) {
+ Version version = new Version(this);
+ builder.saveTo(version);
+
+ // Install recovered version
+ finalizeVersion(version);
+ appendVersion(version);
+ builder = new Builder(this, current);
+ }
// save edit values for verification below
logNumber = coalesce(edit.getLogNumber(), logNumber);
prevLogNumber = coalesce(edit.getPreviousLogNumber(), prevLogNumber);
@@ -687,6 +701,7 @@ private static class Builder
private final VersionSet versionSet;
private final Version baseVersion;
private final List levels;
+ private int batchSize;
private Builder(VersionSet versionSet, Version baseVersion)
{
@@ -716,7 +731,11 @@ public void apply(VersionEdit edit)
Integer level = entry.getKey();
Long fileNumber = entry.getValue();
levels.get(level).deletedFiles.add(fileNumber);
- // todo missing update to addedFiles?
+ batchSize++;
+ // missing update to addedFiles for open db to release resource
+ if (levels.get(level).addedFilesMap.remove(fileNumber) != null) {
+ batchSize--;
+ }
}
// Add new files
@@ -743,8 +762,11 @@ public void apply(VersionEdit edit)
}
fileMetaData.setAllowedSeeks(allowedSeeks);
- levels.get(level).deletedFiles.remove(fileMetaData.getNumber());
- levels.get(level).addedFiles.add(fileMetaData);
+ if (levels.get(level).deletedFiles.remove(fileMetaData.getNumber())) {
+ batchSize--;
+ }
+ levels.get(level).addedFilesMap.put(fileMetaData.getNumber(), fileMetaData);
+ batchSize++;
}
}
@@ -763,6 +785,7 @@ public void saveTo(Version version)
if (baseFiles == null) {
baseFiles = ImmutableList.of();
}
+ levels.get(level).addedFiles.addAll(levels.get(level).addedFilesMap.values());
SortedSet addedFiles = levels.get(level).addedFiles;
if (addedFiles == null) {
addedFiles = ImmutableSortedSet.of();
@@ -833,11 +856,13 @@ public int compare(FileMetaData f1, FileMetaData f2)
private static class LevelState
{
private final SortedSet addedFiles;
+ private final SortedMap addedFilesMap;
private final Set deletedFiles = new HashSet();
public LevelState(InternalKeyComparator internalKeyComparator)
{
addedFiles = new TreeSet(new FileMetaDataBySmallestKey(internalKeyComparator));
+ addedFilesMap = new TreeMap<>();
}
@Override
diff --git a/leveldb/src/test/java/org/iq80/leveldb/impl/DbImplTest.java b/leveldb/src/test/java/org/iq80/leveldb/impl/DbImplTest.java
index cc833cbe..3e04670c 100644
--- a/leveldb/src/test/java/org/iq80/leveldb/impl/DbImplTest.java
+++ b/leveldb/src/test/java/org/iq80/leveldb/impl/DbImplTest.java
@@ -20,6 +20,9 @@
import com.google.common.collect.ImmutableList;
import com.google.common.primitives.Ints;
import com.google.common.primitives.UnsignedBytes;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import org.iq80.leveldb.CompressionType;
import org.iq80.leveldb.DB;
import org.iq80.leveldb.DBComparator;
import org.iq80.leveldb.DBIterator;
@@ -49,6 +52,7 @@
import static com.google.common.collect.Maps.immutableEntry;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Arrays.asList;
+import static org.iq80.leveldb.impl.Iq80DBFactory.factory;
import static org.iq80.leveldb.CompressionType.NONE;
import static org.iq80.leveldb.impl.DbConstants.NUM_LEVELS;
import static org.iq80.leveldb.table.BlockHelper.afterString;
@@ -862,6 +866,36 @@ public void testCustomComparator()
assertFalse(seekingIterator.hasNext());
}
+ @Test
+ // test for local
+ public void testHugeManifest()
+ {
+ DB database;
+ Path db = Paths.get("/tmp/leveldb", "account");
+ File file = db.toFile();
+ org.iq80.leveldb.Options dbOptions = new org.iq80.leveldb.Options();
+ dbOptions.createIfMissing(true);
+ dbOptions.paranoidChecks(true);
+ dbOptions.verifyChecksums(true);
+ dbOptions.maxBatchSize(64_000);
+ dbOptions.compressionType(CompressionType.SNAPPY);
+ dbOptions.blockSize(4 * 1024);
+ dbOptions.writeBufferSize(10 * 1024 * 1024);
+ dbOptions.cacheSize(10 * 1024 * 1024L);
+ dbOptions.maxOpenFiles(100_000);
+ dbOptions.maxManifestSize(256);
+ try {
+ long s = System.currentTimeMillis();
+ database = factory.open(file, dbOptions);
+ long e = System.currentTimeMillis();
+ System.out.println("open cost :" + (e - s));
+ database.close();
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
@SafeVarargs
private final void testDb(DbStringWrapper db, Entry... entries)
throws IOException
diff --git a/pom.xml b/pom.xml
index 223d88e2..e11eecb1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -8,13 +8,13 @@
95
- org.iq80.leveldb
+ com.halibobor
leveldb-project
- 0.13-SNAPSHOT
+ 1.18.2
pom
Port of LevelDB to Java
- http://github.com/dain/leveldb
+ http://github.com/halibobor/leveldb-java
leveldb-api
@@ -45,12 +45,21 @@
http://hiramchirino.com
-5
+
+ halibobo
+ halibobo1205
+ halibobo1205@gmail.com
+
+ Developer
+
+ +8
+
- scm:git:git://github.com/dain/leveldb.git
- scm:git:git@github.com:dain/leveldb.git
- http://github.com/dain/leveldb/tree/master
+ scm:https://github.com/halibobor/leveldb-java.git
+ scm:https://github.com/halibobor/leveldb-java.git
+ https://github.com/halibobor/leveldb-java/tree/master
HEAD
@@ -65,18 +74,20 @@
-missing
src/license/LICENSE-HEADER.txt
${air.main.basedir}/src/checkstyle/checks.xml
+ UTF-8
+ 0xEF7F2D6C
- org.iq80.leveldb
+ com.halibobor
leveldb-api
${project.version}
- org.iq80.leveldb
+ com.halibobor
leveldb
${project.version}
@@ -108,4 +119,121 @@
+
+
+
+
+ release
+
+
+
+
+ org.apache.maven.plugins
+ maven-gpg-plugin
+ 3.0.1
+
+
+ sign-artifacts
+ verify
+
+ sign
+
+
+ ${gpg.keyname}
+ ${gpg.keyname}
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-source-plugin
+
+
+ attach-sources
+
+ jar-no-fork
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-javadoc-plugin
+
+ ${project.build.sourceEncoding}
+
+
+
+ attach-javadocs
+
+ jar
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+
+
+ 1.8
+ true
+ true
+ UTF-8
+ false
+
+
+
+
+ org.sonatype.plugins
+ nexus-staging-maven-plugin
+ 1.6.7
+ true
+
+ ossrh
+ https://s01.oss.sonatype.org/
+ true
+
+
+
+
+
+
+
+
+
+ enforce-no-snapshots
+
+
+
+ org.apache.maven.plugins
+ maven-enforcer-plugin
+ 1.0
+
+
+ enforce-no-snapshots
+ enforce
+
+
+
+ No Snapshots Allowed!
+
+
+ No Snapshots Allowed!
+
+
+
+
+
+
+
+
+
+