Skip to content

Commit

Permalink
Merge branch 'main' into feature/406-non-experimental-aggregates
Browse files Browse the repository at this point in the history
  • Loading branch information
rlwww committed Nov 9, 2023
2 parents e72588e + 0f8bf5f commit 4e46c1d
Show file tree
Hide file tree
Showing 225 changed files with 112 additions and 126 deletions.
4 changes: 4 additions & 0 deletions .github/workflows/pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ on:
branches:
- main

concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: true

env:
RUST_BACKTRACE: 1

Expand Down
104 changes: 51 additions & 53 deletions nemo-python/tests/test_blackbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import csv
import os
import math
from os.path import dirname, exists, abspath, isfile
from os.path import dirname, exists, abspath, isfile, isdir

from nmo_python import load_file, NemoEngine, NemoOutputManager

Expand All @@ -18,82 +18,80 @@ def stringify(value):
return str(value)


def generate_test(file):
def run_blackbox_test(self, file):
os.chdir(self.path)

program = load_file(file)
program_name = file.removesuffix(".rls")
test_name = "test_" + program_name
engine = NemoEngine(program)
engine.reason()

for relation in program.output_predicates():
expected_results_file_name = os.path.join(program_name, f"{relation}.csv")

def run_test(self):
os.chdir(self.path)
if not exists(expected_results_file_name):
continue

program = load_file(file)
engine = NemoEngine(program)
engine.reason()
with tempfile.TemporaryDirectory() as tmp_dir:
output_manager = NemoOutputManager(tmp_dir)
engine.write_result(f"{relation}", output_manager)

for relation in program.output_predicates():
expected_results_file_name = os.path.join(
program_name, f"{relation}.csv"
)
with open(os.path.join(tmp_dir, f"{relation}.csv")) as results_file:
results = list(csv.reader(results_file))

if not exists(expected_results_file_name):
continue
with open(expected_results_file_name) as expected_results_file:
expected = list(csv.reader(expected_results_file))

with tempfile.TemporaryDirectory() as tmp_dir:
output_manager = NemoOutputManager(tmp_dir)
engine.write_result(f"{relation}", output_manager)
for result in results:
self.assertTrue(
result in expected,
f"unexpected {result} in {relation}",
)

with open(
os.path.join(tmp_dir, f"{relation}.csv")
) as results_file:
results = list(csv.reader(results_file))
expected.remove(result)

self.assertTrue(
len(expected) == 0,
f"missing results in {relation}: {expected}",
)

with open(
expected_results_file_name
) as expected_results_file:
expected = list(csv.reader(expected_results_file))

for result in results:
self.assertTrue(
result in expected,
f"unexpected {result} in {relation}",
)
def generate_test(file):
program_name = file.removesuffix(".rls")
test_name = "test_" + program_name

expected.remove(result)
return test_name, classmethod(lambda self: run_blackbox_test(self, file))

self.assertTrue(
len(expected) == 0,
f"missing results in {relation}: {expected}",
)

return test_name, classmethod(run_test)
def visit_path(test_path, loader, tests, subgroup_path=[]):
os.chdir(test_path)
dir_entries = os.listdir(test_path)
subgroup_path.append(os.path.basename(test_path))

for subdir in filter(isdir, dir_entries):
visit_path(os.path.join(test_path, subdir), loader, tests, subgroup_path)
os.chdir(test_path)

def end_to_end_test_case(test_path):
class_name = os.path.basename(test_path)
test_case = type(class_name, (unittest.TestCase,), {"path": test_path})
rules_files = list(filter(lambda f: isfile(f) and f.endswith(".rls"), dir_entries))

for file in os.listdir(test_path):
if not isfile(os.path.join(test_path, file)):
continue
if rules_files.count == 0:
return

if not file.endswith(".rls"):
continue
class_name = "_".join(subgroup_path)
test_case = type(class_name, (unittest.TestCase,), {"path": test_path})

for file in rules_files:
name, method = generate_test(file)
setattr(test_case, name, method)

return test_case
tests.addTests(loader.loadTestsFromTestCase(test_case))
subgroup_path.pop()


def load_tests(loader, tests, pattern):
test_cases_dir = os.path.join(
dirname(__file__), "..", "..", "resources", "testcases"
)

for directory in os.listdir(test_cases_dir):
path = abspath(os.path.join(test_cases_dir, directory))
test_case = end_to_end_test_case(path)
tests.addTests(loader.loadTestsFromTestCase(test_case))
test_cases_dir = os.path.join(dirname(__file__), "..", "..", "resources", "testcases")

visit_path(abspath(test_cases_dir), loader, tests)
return tests


Expand Down
4 changes: 2 additions & 2 deletions nemo-wasm/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -164,8 +164,8 @@ struct SyncAccessHandleWriter(web_sys::FileSystemSyncAccessHandle);
#[cfg(web_sys_unstable_apis)]
impl std::io::Write for SyncAccessHandleWriter {
fn write(&mut self, buf: &[u8]) -> Result<usize, std::io::Error> {
let mut buf: Vec<_> = buf.into();
let bytes_written = self.0.write_with_u8_array(&mut buf).map_err(|js_value| {
let buf: Vec<_> = buf.into();
let bytes_written = self.0.write_with_u8_array(&buf).map_err(|js_value| {
std_io_error_from_js_value(
js_value,
"Error while writing to FileSystemSyncAccessHandle",
Expand Down
4 changes: 2 additions & 2 deletions nemo/src/execution/planning/plan_head_datalog.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ impl HeadStrategy for DatalogStrategy {
current_plan: &mut SubtableExecutionPlan,
body: ExecutionNodeRef,
_rule_info: &RuleInfo,
variable_order: VariableOrder,
variable_order: &VariableOrder,
step: usize,
) {
for (predicate, head_instructions) in self.predicate_to_atoms.iter() {
Expand All @@ -72,7 +72,7 @@ impl HeadStrategy for DatalogStrategy {
let mut project_append_nodes =
Vec::<ExecutionNodeRef>::with_capacity(head_instructions.len());
for head_instruction in head_instructions {
let head_binding = atom_binding(&head_instruction.reduced_atom, &variable_order);
let head_binding = atom_binding(&head_instruction.reduced_atom, variable_order);
let head_reordering =
ProjectReordering::from_vector(head_binding.clone(), self.num_body_variables);

Expand Down
2 changes: 1 addition & 1 deletion nemo/src/execution/planning/plan_head_restricted.rs
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ impl HeadStrategy for RestrictedChaseStrategy {
current_plan: &mut SubtableExecutionPlan,
node_matches: ExecutionNodeRef,
rule_info: &RuleInfo,
body_join_order: VariableOrder,
body_join_order: &VariableOrder,
step: usize,
) {
// High-level description of the strategy:
Expand Down
2 changes: 1 addition & 1 deletion nemo/src/execution/planning/strategy_head.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ pub trait HeadStrategy: Debug {
current_plan: &mut SubtableExecutionPlan,
body: ExecutionNodeRef,
rule_info: &RuleInfo,
variable_order: VariableOrder,
variable_order: &VariableOrder,
step_number: usize,
);
}
6 changes: 3 additions & 3 deletions nemo/src/execution/rule_execution.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,14 +59,14 @@ impl RuleExecution {
)
);
// TODO: Just because its the first doesn't mean its the best
let best_variable_order = &mut self.promising_variable_orders[0].clone();
let mut best_variable_order = self.promising_variable_orders[0].clone();

let mut subtable_execution_plan = SubtableExecutionPlan::default();
let body_tree = self.body_strategy.add_plan_body(
table_manager,
&mut subtable_execution_plan,
rule_info,
best_variable_order, // Variable order possibly gets updated
&mut best_variable_order, // Variable order possibly gets updated
step_number,
);

Expand All @@ -75,7 +75,7 @@ impl RuleExecution {
&mut subtable_execution_plan,
body_tree,
rule_info,
best_variable_order.clone(),
&best_variable_order,
step_number,
);

Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
8 changes: 8 additions & 0 deletions resources/testcases/basic/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
= testcases/basic

Tests / documents all basic functionalities of the rule syntax:
- join
- negation
- projection
- union
- assignment
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@

J1(?X, ?Y, ?Z) :- sourceA(?X, ?Z, ?Y), sourceB(?X, ?Y, ?T) .
J2(?X, ?Y, ?Z) :- sourceA(?Z, ?Y, ?X), sourceC(?X, ?Y, ?T) .
J3(?X, ?Y, ?W) :- sourceA(?T, ?Y, ?X), sourceB(?T, ?Y, ?X), sourceC(?X, ?Y, ?W) .
J3(?X, ?Y, ?W) :- sourceA(?T, ?Y, ?X), sourceB(?T, ?Y, ?X), sourceC(?X, ?Y, ?W) .
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,6 @@ A,C,F
A,E,F
A,G,Q
B,B,D
B,D,A
C,D,D
Q,Q,A
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,11 @@ A,C,F
A,E,F
A,G,Q
B,B,D
B,D,A
C,D,D
D,B,Q
D,B,Z
D,B,F
F,E,D
Q,Q,A
Z,Z,Z
10 changes: 0 additions & 10 deletions resources/testcases/basic_join/output.rls

This file was deleted.

2 changes: 0 additions & 2 deletions resources/testcases/basic_join/run/J2.csv

This file was deleted.

3 changes: 0 additions & 3 deletions resources/testcases/basic_join/run/J3.csv

This file was deleted.

5 changes: 0 additions & 5 deletions resources/testcases/basic_union/sources/dataB.csv

This file was deleted.

2 changes: 0 additions & 2 deletions resources/testcases/basic_union/sources/dataC.csv

This file was deleted.

File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
3 changes: 0 additions & 3 deletions resources/testcases/load_empty/run.rls

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
issue: 392
title: Panic when filtering for non-existing constant
tracker: https://github.com/knowsys/nemo/issues/392
type: bugfix
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
@declare c(integer) .
@declare numeric(integer, float64) .

@source c[1]: load-csv("sources/c.csv") .
@source numeric[2]: load-csv("sources/numeric.csv") .
@source c[1]: load-csv("../sources/c.csv") .
@source numeric[2]: load-csv("../sources/numeric.csv") .

a(Hello, ?X) :- c(?X) .
b(?Y, ?X) :- a(?X, ?Y) .
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
@declare a(integer) .

@source a[1]: load-csv("sources/a.csv") .
@source a[1]: load-csv("../sources/a.csv") .

c(?X, ?Y) :- b(?X, ?Y) .
b(?X, ?X) :- a(?X) .
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
@declare b(any) .
@declare c(any) .

@source b[any]: load-csv("sources/b.csv") .
@source b[any]: load-csv("../sources/b.csv") .

a(Some Constant With Spaces).
a(<AnotherConstant>).
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
issue: 355
title: Support equality of two integer variables
tracker: https://github.com/knowsys/nemo/issues/355
type: bugfix
4 changes: 4 additions & 0 deletions resources/testcases/regression/load/empty/regression.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
issue: 341
title: invariant violation when reading empty file
tracker: https://github.com/knowsys/nemo/issues/308
type: bugfix
3 changes: 3 additions & 0 deletions resources/testcases/regression/load/empty/run.rls
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
@source source[3]: load-csv("../sources/empty.csv").

out(?X, ?Y, ?Z) :- source(?X, ?Y, ?Z) .
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
@source source[3]: load-csv("sources/dataA.csv").
@source source[3]: load-csv("sources/dataB.csv").
@source source[3]: load-csv("sources/dataC.csv").
@source source[3]: load-csv("../sources/dataA.csv").
@source source[3]: load-csv("../sources/dataB.csv").
@source source[3]: load-csv("../sources/dataC.csv").

source("A", "B", "C") .
source("B", "C", "D") .
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,5 @@ B,D,A
A,C,F
A,G,Q
A,Q,Q
C,D,D
C,D,D
C,D,A
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
<A>,<B>
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
<B>,<C>
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
<A>,<__Null#9223372036854775809>
<B>,<__Null#9223372036854775810>
<C>,<__Null#9223372036854775811>
4 changes: 4 additions & 0 deletions resources/testcases/regression/stratification/regression.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
pr: 220
title: Stratified Negation
tracker: https://github.com/knowsys/nemo/pull/220
type: feature
4 changes: 0 additions & 4 deletions resources/testcases/repeated_variables/sources/dataA.csv

This file was deleted.

3 changes: 0 additions & 3 deletions resources/testcases/restricted_datalog_head/sources/data.csv

This file was deleted.

3 changes: 0 additions & 3 deletions resources/testcases/restricted_head_constant/sources/data.csv

This file was deleted.

3 changes: 0 additions & 3 deletions resources/testcases/restricted_multihead/sources/data.csv

This file was deleted.

Loading

0 comments on commit 4e46c1d

Please sign in to comment.