implement support for merge join

the-qa-company · Jan 3, 2024 · dc60b71 · dc60b71
1 parent 17b1b2e
commit dc60b71
Show file tree

Hide file tree

Showing 43 changed files with 1,765 additions and 454 deletions.
diff --git a/.gitignore b/.gitignore
@@ -55,3 +55,6 @@ release/RELEASE_OUTPUT.md
 .idea
 data
 .nvmrc
+wikidata
+qendpoint-store/wdbench-indexes
+wdbench-results
diff --git a/pom.xml b/pom.xml
@@ -69,6 +69,19 @@
         <maven.compiler.target>17</maven.compiler.target>
     </properties>
 
+    <repositories>
+        <repository>
+            <id>oss.sonatype.org-snapshot</id>
+            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+            <releases>
+                <enabled>false</enabled>
+            </releases>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
+
     <distributionManagement>
         <snapshotRepository>
             <id>sonatype-nexus-snapshots</id>
@@ -205,4 +218,4 @@
         <system>Github</system>
         <url>https://github.com/the-qa-company/qEndpoint/issues</url>
     </issueManagement>
-</project>
+</project>
diff --git a/qendpoint-backend/pom.xml b/qendpoint-backend/pom.xml
@@ -25,6 +25,19 @@
         </license>
     </licenses>
 
+    <repositories>
+        <repository>
+            <id>oss.sonatype.org-snapshot</id>
+            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+            <releases>
+                <enabled>false</enabled>
+            </releases>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
+
     <properties>
         <common_codec.version>1.15</common_codec.version>
         <json_simple.version>1.1.1</json_simple.version>

diff --git a/qendpoint-core/pom.xml b/qendpoint-core/pom.xml
@@ -41,6 +41,19 @@
         </license>
     </licenses>
 
+    <repositories>
+        <repository>
+            <id>oss.sonatype.org-snapshot</id>
+            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+            <releases>
+                <enabled>false</enabled>
+            </releases>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
+
     <properties>
         <common_codec.version>1.15</common_codec.version>
         <json_simple.version>1.1.1</json_simple.version>
@@ -122,5 +135,15 @@
             <artifactId>RoaringBitmap</artifactId>
             <version>${roaringbitmap.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.eclipse.rdf4j</groupId>
+            <artifactId>rdf4j-common-order</artifactId>
+            <version>${rdf4j.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.eclipse.rdf4j</groupId>
+            <artifactId>rdf4j-common-iterator</artifactId>
+            <version>${rdf4j.version}</version>
+        </dependency>
     </dependencies>
 </project>
diff --git a/...oint-core/src/main/java/com/the_qa_company/qendpoint/core/enums/TripleComponentOrder.java b/...oint-core/src/main/java/com/the_qa_company/qendpoint/core/enums/TripleComponentOrder.java
@@ -19,7 +19,13 @@
 
 package com.the_qa_company.qendpoint.core.enums;
 
+import org.eclipse.rdf4j.common.order.StatementOrder;
+
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * Indicates the order of the triples
@@ -78,6 +84,24 @@ public enum TripleComponentOrder {
 		this.mask = mask;
 	}
 
+	/**
+	 * Search for an acceptable value in a map of orders
+	 *
+	 * @param flags flags to search the value
+	 * @param map   map
+	 * @param <T>   value type
+	 * @return find value, null for no matching value
+	 */
+	public static <T, Z extends TripleComponentOrder> List<Z> fetchAllBestForCfg(int flags, Map<Z, T> map) {
+		ArrayList<Z> ret = new ArrayList<>();
+		for (Map.Entry<Z, T> e : map.entrySet()) {
+			if ((e.getKey().mask & flags) != 0) {
+				ret.add(e.getKey());
+			}
+		}
+		return ret;
+	}
+
 	/**
 	 * Search for an acceptable value in a map of orders
 	 *
@@ -123,4 +147,39 @@ public TripleComponentRole getPredicateMapping() {
 	public TripleComponentRole getObjectMapping() {
 		return objectMapping;
 	}
+
+	public Set<StatementOrder> getStatementOrder(boolean subject, boolean predicate, boolean object) {
+		List<TripleComponentRole> subjectMappings = List.of(subjectMapping, predicateMapping, objectMapping);
+
+		EnumSet<StatementOrder> statementOrders = EnumSet.noneOf(StatementOrder.class);
+		if (subject) {
+			statementOrders.add(StatementOrder.S);
+		}
+		if (predicate) {
+			statementOrders.add(StatementOrder.P);
+		}
+		if (object) {
+			statementOrders.add(StatementOrder.O);
+		}
+
+		for (TripleComponentRole mapping : subjectMappings) {
+			if (mapping == TripleComponentRole.SUBJECT) {
+				if (!subject) {
+					statementOrders.add(StatementOrder.S);
+					break;
+				}
+			} else if (mapping == TripleComponentRole.PREDICATE) {
+				if (!predicate) {
+					statementOrders.add(StatementOrder.P);
+					break;
+				}
+			} else if (mapping == TripleComponentRole.OBJECT) {
+				if (!object) {
+					statementOrders.add(StatementOrder.O);
+					break;
+				}
+			}
+		}
+		return statementOrders;
+	}
 }
diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/hdt/impl/HDTImpl.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/hdt/impl/HDTImpl.java
@@ -206,7 +206,8 @@ public void mapFromHDT(File f, long offset, ProgressListener listener) throws IO
 			String hdtFormat = ci.getFormat();
 			if (!hdtFormat.equals(HDTVocabulary.HDT_CONTAINER) && !hdtFormat.equals(HDTVocabulary.HDT_CONTAINER_2)) {
 				throw new IllegalFormatException("This software (v" + HDTVersion.HDT_VERSION + ".x.x | v"
-						+ HDTVersion.HDT_VERSION_2 + ".x.x) cannot open this version of HDT File (" + hdtFormat + ")");
+						+ HDTVersion.HDT_VERSION_2 + ".x.x) cannot open this version of HDT File hdtFileName:"
+						+ hdtFileName + " format:" + hdtFormat + "");
 			}
 
 			input.printIndex("HDT Header");

diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/storage/QEPComponent.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/storage/QEPComponent.java
@@ -27,19 +27,20 @@ public class QEPComponent implements Cloneable, Serializable {
 	@Serial
 	private static final long serialVersionUID = 6621230579376315429L;
 
-	record SharedElement(long id, DictionarySectionRole role, QEPDataset dataset, String debugMapped) implements Serializable {}
+	record SharedElement(long id, DictionarySectionRole role, QEPDataset dataset, String debugMapped)
+			implements Serializable {}
 
 	record PredicateElement(long id, QEPDataset dataset) implements Serializable {}
 
 	private static final Logger logger = LoggerFactory.getLogger(QEPComponent.class);
 
-     Map<Integer, PredicateElement> predicateIds;
-     Map<Integer, SharedElement> sharedIds;
-     CharSequence value;
-     RDFNodeType rdfNodeType;
-	 Optional<CharSequence> language;
-     CharSequence datatype;
-     transient final QEPCore core;
+	Map<Integer, PredicateElement> predicateIds;
+	Map<Integer, SharedElement> sharedIds;
+	CharSequence value;
+	RDFNodeType rdfNodeType;
+	Optional<CharSequence> language;
+	CharSequence datatype;
+	transient final QEPCore core;
 
 	private QEPComponent(QEPComponent other) {
 		this.predicateIds = new HashMap<>(other.predicateIds);

diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/storage/QEPCore.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/storage/QEPCore.java
@@ -121,7 +121,9 @@ public class QEPCore implements AutoCloseable {
 	 */
 	public static final String FILE_CORE_CONFIG_OPT = "config.opt";
 
-	  private final Map<String, QEPDataset> dataset = new HashMap<>();private final Object datasetLock = new Object() {};private final ReentrantLock insertLock = new ReentrantLock();
+	private final Map<String, QEPDataset> dataset = new HashMap<>();
+	private final Object datasetLock = new Object() {};
+	private final ReentrantLock insertLock = new ReentrantLock();
 	private final Object bindLock = new Object() {};
 	private final Object idBuilderLock = new Object() {};
 	private final ConcurrentMap<Integer, QEPDataset> datasetByUid = new ConcurrentHashMap<>();

diff --git a/...main/java/com/the_qa_company/qendpoint/core/storage/iterator/CloseableAttachIterator.java b/...main/java/com/the_qa_company/qendpoint/core/storage/iterator/CloseableAttachIterator.java
@@ -14,7 +14,7 @@
 public class CloseableAttachIterator<T extends RuntimeException> implements CloseableIterator<T> {
 	@SafeVarargs
 	public static <T> CloseableIterator<T> of(CloseableIterator<T> it,
-                                                                   AutoCloseableGeneric<? extends RuntimeException>... closeables) {
+			AutoCloseableGeneric<? extends RuntimeException>... closeables) {
 		if (closeables.length == 0) {
 			return it;
 		}
@@ -25,13 +25,14 @@ public static <T> CloseableIterator<T> of(CloseableIterator<T> it,
 	private final List<AutoCloseableGeneric<RuntimeException>> closeables;
 
 	@SafeVarargs
-	private CloseableAttachIterator(CloseableIterator<T> handle, AutoCloseableGeneric<RuntimeException>... closeableGenerics) {
+	private CloseableAttachIterator(CloseableIterator<T> handle,
+			AutoCloseableGeneric<RuntimeException>... closeableGenerics) {
 		this.handle = handle;
 		closeables = new ArrayList<>(List.of(closeableGenerics));
 	}
 
 	@Override
-	public void close()  {
+	public void close() {
 		try {
 			handle.close();
 		} catch (Error | Exception t) {