From 192d12dd5621be8c46024913960111c0bf585684 Mon Sep 17 00:00:00 2001 From: Ferdinando Villa Date: Wed, 4 Dec 2024 13:29:01 +0100 Subject: [PATCH] KG query fixes --- .../klab/api/provenance/Activity.java | 7 +++ .../api/provenance/impl/ActivityImpl.java | 10 ++++ .../services/runtime/ExecutionSequence.java | 18 +++++-- .../runtime/neo4j/AbstractKnowledgeGraph.java | 1 + .../runtime/neo4j/KnowledgeGraphNeo4j.java | 47 +++++++++++++++---- 5 files changed, 69 insertions(+), 14 deletions(-) diff --git a/klab.core.api/src/main/java/org/integratedmodelling/klab/api/provenance/Activity.java b/klab.core.api/src/main/java/org/integratedmodelling/klab/api/provenance/Activity.java index 7fabe4ab4..94bc5d557 100644 --- a/klab.core.api/src/main/java/org/integratedmodelling/klab/api/provenance/Activity.java +++ b/klab.core.api/src/main/java/org/integratedmodelling/klab/api/provenance/Activity.java @@ -113,6 +113,13 @@ enum Outcome { */ String getDescription(); + /** + * Non-null for all activities that manipulate an observation, including those that failed. + * + * @return + */ + String getObservationUrn(); + Outcome getOutcome(); String getStackTrace(); diff --git a/klab.core.api/src/main/java/org/integratedmodelling/klab/api/provenance/impl/ActivityImpl.java b/klab.core.api/src/main/java/org/integratedmodelling/klab/api/provenance/impl/ActivityImpl.java index f6b09dc87..09341c137 100644 --- a/klab.core.api/src/main/java/org/integratedmodelling/klab/api/provenance/impl/ActivityImpl.java +++ b/klab.core.api/src/main/java/org/integratedmodelling/klab/api/provenance/impl/ActivityImpl.java @@ -24,6 +24,7 @@ public class ActivityImpl extends ProvenanceNodeImpl implements Activity { private KlabService.Type serviceType; private String dataflow; private String urn; + private String observationUrn; @Override public long getStart() { @@ -94,6 +95,15 @@ public void setId(long id) { this.id = id; } + @Override + public String getObservationUrn() { + return this.observationUrn; + } + + public void setObservationUrn(String observationUrn) { + this.observationUrn = observationUrn; + } + /** * Non-API: the ID path through which the activity hierarchy can be reconstructed. When an activity is * included in a message, the taskId gets in there. diff --git a/klab.services.runtime/src/main/java/org/integratedmodelling/klab/services/runtime/ExecutionSequence.java b/klab.services.runtime/src/main/java/org/integratedmodelling/klab/services/runtime/ExecutionSequence.java index e35c39415..a700bc77c 100644 --- a/klab.services.runtime/src/main/java/org/integratedmodelling/klab/services/runtime/ExecutionSequence.java +++ b/klab.services.runtime/src/main/java/org/integratedmodelling/klab/services/runtime/ExecutionSequence.java @@ -11,6 +11,7 @@ import org.integratedmodelling.klab.api.digitaltwin.DigitalTwin; import org.integratedmodelling.klab.api.geometry.Geometry; import org.integratedmodelling.klab.api.knowledge.Observable; +import org.integratedmodelling.klab.api.knowledge.SemanticType; import org.integratedmodelling.klab.api.knowledge.observation.Observation; import org.integratedmodelling.klab.api.knowledge.observation.scale.Scale; import org.integratedmodelling.klab.api.knowledge.observation.scale.space.Space; @@ -183,6 +184,8 @@ private void compile(Actuator actuator) { } } + // TODO this should return a list of candidates, to match based on the parameters. For numeric there + // should be a float and double version. var descriptor = componentRegistry.getFunctionDescriptor(call); if (descriptor.serviceInfo.getGeometry().isScalar()) { @@ -206,12 +209,19 @@ private void compile(Actuator actuator) { } var scale = Scale.create(observation.getGeometry()); - Storage storage = digitalTwin.stateStorage().getOrCreateStorage(observation, - Storage.class); + + // if we're a quality, we need storage at the discretion of the StorageManager. + Storage storage = observation.getObservable().is(SemanticType.QUALITY) ? + digitalTwin.stateStorage().getOrCreateStorage(observation, + Storage.class) : + null; /* * Create a runnable with matched parameters and have it set the context observation - * TODO allow multiple methods taking different storage implementations, enabling the - * storage manager to be configured for the wanted precision + * TODO allow multiple methods with same annotation, taking different storage + * implementations, enabling the storage manager to be configured for the wanted precision + * + * Should match arguments, check if they all match, and if not move to the next until + * no available implementations remain. */ List runArguments = new ArrayList<>(); if (descriptor.method != null) { diff --git a/klab.services.runtime/src/main/java/org/integratedmodelling/klab/services/runtime/neo4j/AbstractKnowledgeGraph.java b/klab.services.runtime/src/main/java/org/integratedmodelling/klab/services/runtime/neo4j/AbstractKnowledgeGraph.java index 302e6399d..cc9b19e34 100644 --- a/klab.services.runtime/src/main/java/org/integratedmodelling/klab/services/runtime/neo4j/AbstractKnowledgeGraph.java +++ b/klab.services.runtime/src/main/java/org/integratedmodelling/klab/services/runtime/neo4j/AbstractKnowledgeGraph.java @@ -154,6 +154,7 @@ protected Map asParameters(Object asset, Object... additionalPar ret.put("name", activity.getName()); ret.put("id", activity.getId()); ret.put("serviceId", activity.getServiceId()); + ret.put("observationUrn", activity.getObservationUrn()); ret.put("serviceName", activity.getServiceName()); ret.put("serviceType", activity.getServiceType() == null ? null : activity.getServiceType().name()); ret.put("dataflow", activity.getDataflow()); diff --git a/klab.services.runtime/src/main/java/org/integratedmodelling/klab/services/runtime/neo4j/KnowledgeGraphNeo4j.java b/klab.services.runtime/src/main/java/org/integratedmodelling/klab/services/runtime/neo4j/KnowledgeGraphNeo4j.java index aa0039650..37f13915d 100644 --- a/klab.services.runtime/src/main/java/org/integratedmodelling/klab/services/runtime/neo4j/KnowledgeGraphNeo4j.java +++ b/klab.services.runtime/src/main/java/org/integratedmodelling/klab/services/runtime/neo4j/KnowledgeGraphNeo4j.java @@ -224,6 +224,7 @@ public void close() throws IOException { for (var asset : assets) { if (asset instanceof ObservationImpl obs) { observation = obs; + activity.setObservationUrn(obs.getUrn()); } else if (asset instanceof Throwable t) { activity.setStackTrace(ExceptionUtils.getStackTrace(t)); } @@ -542,6 +543,7 @@ protected List adapt(EagerResult query, Class cls, Scope scope) { // TODO instance.setStart(node.get("start").asLong()); instance.setEnd(node.get("end").asLong()); + instance.setObservationUrn(node.get("observationUrn").asString()); instance.setName(node.get("name").asString()); instance.setServiceName(node.get("serviceName").isNull() ? null : node.get("serviceName").asString()); @@ -761,6 +763,7 @@ protected void link(Transaction transaction, RuntimeAsset source, RuntimeAsset d } private String matchAsset(RuntimeAsset asset, String name, String queryVariable) { + var ret = switch (asset) { case Activity activity -> name + ".id = $" + queryVariable; case Observation observation -> name + ".id = $" + queryVariable; @@ -994,30 +997,54 @@ public List get(ContextScope scope, Class resultC private List getActivity(ContextScope scope, Object... queriables) { Map queryParameters = new LinkedHashMap<>(); - var query = new StringBuilder(getScopeQuery(scope, queryParameters) + "-[:HAS_PROVENANCE]->" + - "(:Provenance)-[:HAS_CHILD]->"); + Activity rootActivity = null; if (queriables != null) { for (var parameter : queriables) { - if (parameter instanceof Observable observable) { - // - } else if (parameter instanceof Activity rootActivity) { + if (parameter instanceof Activity root) { + rootActivity = root; } else if (parameter instanceof Long id) { queryParameters.put("id", id); - query = new StringBuilder("MATCH (a:Activity {id: $id}"); } else if (parameter instanceof Observation observation) { - // define start node as the one with the observation URN + queryParameters.put("observationUrn", observation.getUrn()); } else if (parameter instanceof Activity.Type activityType) { - queryParameters.put("name", activityType.name()); - query.append("(a:Activity {name: $name}"); + queryParameters.put("type", activityType.name()); } } } - var result = query(query.append(") return a").toString(), queryParameters, scope); + var query = assetQuery("a", "Activity", queryParameters.keySet()); + if (rootActivity != null) { + query.append("<-[*]-(r:Activity {id: $rootActivityId})"); + queryParameters.put("rootActivityId", rootActivity.getId()); + } else { + query.append("<-[*]-(p:Provenance {id: $provenanceId})"); + queryParameters.put("provenanceId", scope.getId()+".PROVENANCE"); + } + + var result = query(query.append(" return a").toString(), queryParameters, scope); return adapt(result, Activity.class, scope); } + private StringBuilder assetQuery(String variableName, String assetLabel, Collection keys) { + + var ret = new StringBuilder("MATCH (").append(variableName).append(":").append(assetLabel); + + if (keys.isEmpty()) { + ret.append(")"); + } else { + int n = 0; + for (String key : keys) { + ret.append(n == 0 ? " {" : ", "); + ret.append(key).append(": $").append(key); + n++; + } + ret.append("})"); + } + + return ret; + } + private List getAgent(ContextScope scope, Object... queriables) { Map queryParameters = new LinkedHashMap<>();