Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

reorganize testcases hierarchically #404

Merged
merged 1 commit into from
Nov 9, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 51 additions & 53 deletions nemo-python/tests/test_blackbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import csv
import os
import math
from os.path import dirname, exists, abspath, isfile
from os.path import dirname, exists, abspath, isfile, isdir

from nmo_python import load_file, NemoEngine, NemoOutputManager

Expand All @@ -18,82 +18,80 @@ def stringify(value):
return str(value)


def generate_test(file):
def run_blackbox_test(self, file):
os.chdir(self.path)

program = load_file(file)
program_name = file.removesuffix(".rls")
test_name = "test_" + program_name
engine = NemoEngine(program)
engine.reason()

for relation in program.output_predicates():
expected_results_file_name = os.path.join(program_name, f"{relation}.csv")

def run_test(self):
os.chdir(self.path)
if not exists(expected_results_file_name):
continue

program = load_file(file)
engine = NemoEngine(program)
engine.reason()
with tempfile.TemporaryDirectory() as tmp_dir:
output_manager = NemoOutputManager(tmp_dir)
engine.write_result(f"{relation}", output_manager)

for relation in program.output_predicates():
expected_results_file_name = os.path.join(
program_name, f"{relation}.csv"
)
with open(os.path.join(tmp_dir, f"{relation}.csv")) as results_file:
results = list(csv.reader(results_file))

if not exists(expected_results_file_name):
continue
with open(expected_results_file_name) as expected_results_file:
expected = list(csv.reader(expected_results_file))

with tempfile.TemporaryDirectory() as tmp_dir:
output_manager = NemoOutputManager(tmp_dir)
engine.write_result(f"{relation}", output_manager)
for result in results:
self.assertTrue(
result in expected,
f"unexpected {result} in {relation}",
)

with open(
os.path.join(tmp_dir, f"{relation}.csv")
) as results_file:
results = list(csv.reader(results_file))
expected.remove(result)

self.assertTrue(
len(expected) == 0,
f"missing results in {relation}: {expected}",
)

with open(
expected_results_file_name
) as expected_results_file:
expected = list(csv.reader(expected_results_file))

for result in results:
self.assertTrue(
result in expected,
f"unexpected {result} in {relation}",
)
def generate_test(file):
program_name = file.removesuffix(".rls")
test_name = "test_" + program_name

expected.remove(result)
return test_name, classmethod(lambda self: run_blackbox_test(self, file))

self.assertTrue(
len(expected) == 0,
f"missing results in {relation}: {expected}",
)

return test_name, classmethod(run_test)
def visit_path(test_path, loader, tests, subgroup_path=[]):
os.chdir(test_path)
dir_entries = os.listdir(test_path)
subgroup_path.append(os.path.basename(test_path))

for subdir in filter(isdir, dir_entries):
visit_path(os.path.join(test_path, subdir), loader, tests, subgroup_path)
os.chdir(test_path)

def end_to_end_test_case(test_path):
class_name = os.path.basename(test_path)
test_case = type(class_name, (unittest.TestCase,), {"path": test_path})
rules_files = list(filter(lambda f: isfile(f) and f.endswith(".rls"), dir_entries))

for file in os.listdir(test_path):
if not isfile(os.path.join(test_path, file)):
continue
if rules_files.count == 0:
return

if not file.endswith(".rls"):
continue
class_name = "_".join(subgroup_path)
test_case = type(class_name, (unittest.TestCase,), {"path": test_path})

for file in rules_files:
name, method = generate_test(file)
setattr(test_case, name, method)

return test_case
tests.addTests(loader.loadTestsFromTestCase(test_case))
subgroup_path.pop()


def load_tests(loader, tests, pattern):
test_cases_dir = os.path.join(
dirname(__file__), "..", "..", "resources", "testcases"
)

for directory in os.listdir(test_cases_dir):
path = abspath(os.path.join(test_cases_dir, directory))
test_case = end_to_end_test_case(path)
tests.addTests(loader.loadTestsFromTestCase(test_case))
test_cases_dir = os.path.join(dirname(__file__), "..", "..", "resources", "testcases")

visit_path(abspath(test_cases_dir), loader, tests)
return tests


Expand Down
8 changes: 8 additions & 0 deletions resources/testcases/basic/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
= testcases/basic

Tests / documents all basic functionalities of the rule syntax:
- join
- negation
- projection
- union
- assignment
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@

J1(?X, ?Y, ?Z) :- sourceA(?X, ?Z, ?Y), sourceB(?X, ?Y, ?T) .
J2(?X, ?Y, ?Z) :- sourceA(?Z, ?Y, ?X), sourceC(?X, ?Y, ?T) .
J3(?X, ?Y, ?W) :- sourceA(?T, ?Y, ?X), sourceB(?T, ?Y, ?X), sourceC(?X, ?Y, ?W) .
J3(?X, ?Y, ?W) :- sourceA(?T, ?Y, ?X), sourceB(?T, ?Y, ?X), sourceC(?X, ?Y, ?W) .
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,6 @@ A,C,F
A,E,F
A,G,Q
B,B,D
B,D,A
C,D,D
Q,Q,A
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,11 @@ A,C,F
A,E,F
A,G,Q
B,B,D
B,D,A
C,D,D
D,B,Q
D,B,Z
D,B,F
F,E,D
Q,Q,A
Z,Z,Z
10 changes: 0 additions & 10 deletions resources/testcases/basic_join/output.rls

This file was deleted.

2 changes: 0 additions & 2 deletions resources/testcases/basic_join/run/J2.csv

This file was deleted.

3 changes: 0 additions & 3 deletions resources/testcases/basic_join/run/J3.csv

This file was deleted.

5 changes: 0 additions & 5 deletions resources/testcases/basic_union/sources/dataB.csv

This file was deleted.

2 changes: 0 additions & 2 deletions resources/testcases/basic_union/sources/dataC.csv

This file was deleted.

3 changes: 0 additions & 3 deletions resources/testcases/load_empty/run.rls

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
issue: 392
title: Panic when filtering for non-existing constant
tracker: https://github.com/knowsys/nemo/issues/392
type: bugfix
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
@declare c(integer) .
@declare numeric(integer, float64) .

@source c[1]: load-csv("sources/c.csv") .
@source numeric[2]: load-csv("sources/numeric.csv") .
@source c[1]: load-csv("../sources/c.csv") .
@source numeric[2]: load-csv("../sources/numeric.csv") .

a(Hello, ?X) :- c(?X) .
b(?Y, ?X) :- a(?X, ?Y) .
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
@declare a(integer) .

@source a[1]: load-csv("sources/a.csv") .
@source a[1]: load-csv("../sources/a.csv") .

c(?X, ?Y) :- b(?X, ?Y) .
b(?X, ?X) :- a(?X) .
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
@declare b(any) .
@declare c(any) .

@source b[any]: load-csv("sources/b.csv") .
@source b[any]: load-csv("../sources/b.csv") .

a(Some Constant With Spaces).
a(<AnotherConstant>).
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
issue: 355
title: Support equality of two integer variables
tracker: https://github.com/knowsys/nemo/issues/355
type: bugfix
4 changes: 4 additions & 0 deletions resources/testcases/regression/load/empty/regression.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
issue: 341
title: invariant violation when reading empty file
tracker: https://github.com/knowsys/nemo/issues/308
type: bugfix
3 changes: 3 additions & 0 deletions resources/testcases/regression/load/empty/run.rls
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
@source source[3]: load-csv("../sources/empty.csv").

out(?X, ?Y, ?Z) :- source(?X, ?Y, ?Z) .
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
@source source[3]: load-csv("sources/dataA.csv").
@source source[3]: load-csv("sources/dataB.csv").
@source source[3]: load-csv("sources/dataC.csv").
@source source[3]: load-csv("../sources/dataA.csv").
@source source[3]: load-csv("../sources/dataB.csv").
@source source[3]: load-csv("../sources/dataC.csv").

source("A", "B", "C") .
source("B", "C", "D") .
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,5 @@ B,D,A
A,C,F
A,G,Q
A,Q,Q
C,D,D
C,D,D
C,D,A
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
<A>,<B>
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
<B>,<C>
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
<A>,<__Null#9223372036854775809>
<B>,<__Null#9223372036854775810>
<C>,<__Null#9223372036854775811>
4 changes: 4 additions & 0 deletions resources/testcases/regression/stratification/regression.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
pr: 220
title: Stratified Negation
tracker: https://github.com/knowsys/nemo/pull/220
type: feature
4 changes: 0 additions & 4 deletions resources/testcases/repeated_variables/sources/dataA.csv

This file was deleted.

3 changes: 0 additions & 3 deletions resources/testcases/restricted_datalog_head/sources/data.csv

This file was deleted.

3 changes: 0 additions & 3 deletions resources/testcases/restricted_head_constant/sources/data.csv

This file was deleted.

3 changes: 0 additions & 3 deletions resources/testcases/restricted_multihead/sources/data.csv

This file was deleted.

3 changes: 0 additions & 3 deletions resources/testcases/restricted_multinulls/sources/data.csv

This file was deleted.

3 changes: 0 additions & 3 deletions resources/testcases/restricted_multipieces/sources/data.csv

This file was deleted.

3 changes: 0 additions & 3 deletions resources/testcases/restricted_only_nulls/sources/data.csv

This file was deleted.

1 change: 0 additions & 1 deletion resources/testcases/restricted_recursive/sources/dataA.csv

This file was deleted.

1 change: 0 additions & 1 deletion resources/testcases/restricted_recursive/sources/dataB.csv

This file was deleted.

3 changes: 0 additions & 3 deletions resources/testcases/restricted_simple/run/result.csv

This file was deleted.

3 changes: 0 additions & 3 deletions resources/testcases/restricted_simple/sources/data.csv

This file was deleted.