Merge pull request #70 from tbarbette/PRzlt

Zero-loss throughput experimental design search

.gitignore

@@ -1,5 +1,6 @@
 results/
 .DS_Store
+._*
 *.log
 *.pyc
 .vimhistory
@@ -17,3 +18,4 @@ tmp/
 .vscode/
 
 venv
+

integration/integration.sh

@@ -8,18 +8,18 @@ compare() {
     test=$1
     python=$2
     echo "Executing npf test $test..."
-    $python npf-run.py click-2022 --force-test --no-graph-time --test integration/$test.npf --quiet-build &> res$test
+    $python npf-run.py click-2022 --force-test --no-graph-time --test integration/$test.npf --quiet-build ${@:3} &> res$test
     if [ $? -ne 0 ] ; then
         echo "npf-run.py returned an error for test $test !"
         cat res$test
-        exit 1
+        #exit 1
     fi
     cmp res$test integration/$test.stdout
     if [ $? -eq 0 ] ; then
         echo "$test passed !"
     else
         echo "Error for $test : expected output does not match !"
-        echo "Command : $python npf-run.py click-2022 --force-test --test integration/$test.npf --quiet-build"
+        echo "Command : $python npf-run.py click-2022 --force-test --test integration/$test.npf --quiet-build ${@:3}"
         diff res$test integration/$test.stdout
         ret=1
     fi
@@ -51,8 +51,7 @@ compare_watcher() {
 try() {
     test=$1
     python=$2
-    params=$3
-    $python npf-run.py --force-test --no-graph-time --test $test --quiet --config n_runs=1 --tags fastregression $3
+    $python npf-run.py --force-test --no-graph-time --test $test --quiet --config n_runs=1 --tags fastregression ${@:3}
     if [ $? -ne 0 ] ; then
         echo "npf-run.py returned an error for test $test !"
         exit 1
@@ -90,6 +89,7 @@ compare event $python
 compare math $python
 compare jinja $python
 compare globsync $python
+compare zlt $python --exp-design "zlt(RATE,THROUGHPUT)"
 try integration/cdf.npf $python "--config n_runs=20"
 try integration/heatmap.npf $python
 try tests/tcp/01-iperf.npf $python "--variables TIME=1"

integration/test_unittest.py

@@ -1,4 +1,6 @@
+import unittest
 from npf import repository
+from npf.expdesign.zltexp import ZLTVariableExpander
 from npf.grapher import Grapher
 from npf.test_driver import Comparator
 import npf.npf
@@ -9,10 +11,18 @@
 from npf.repository import Repository
 from npf.test import Test
 from npf.build import Build
-from npf.variable import dtype, numeric_dict
+from npf.variable import RangeVariable, SimpleVariable, dtype, numeric_dict
 from npf.types.dataset import Run
 
 import numpy as np
+import logging
+
+
+logger = logging.getLogger()
+logger.level = logging.DEBUG
+stream_handler = logging.StreamHandler(sys.stdout)
+logger.addHandler(stream_handler)
+
 
 def get_args():
     parser = argparse.ArgumentParser(description='NPF Tester')
@@ -168,3 +178,60 @@ def test_core():
                           filename=filename,
                           options=args
                           )
+
+def test_zlt():
+    vlist = {'RATE' : RangeVariable("RATE",1,10,log=False)}
+    results = OrderedDict()
+    zlt = ZLTVariableExpander(vlist, results, {}, "RATE", "PPS", 1.01)
+    it = iter(zlt)
+    run = next(it)
+    assert run["RATE"] == 10
+    results[Run({'RATE' : 10})] = {'PPS':[3.0]}
+    run = next(it)
+    assert run["RATE"] == 3
+    results[Run({'RATE' : 3})] = {'PPS':[1.5]}
+    run = next(it)
+    assert run["RATE"] == 1
+    results[Run({'RATE' : 1})] = {'PPS':[1]}
+    run = next(it)
+    assert run["RATE"] == 2
+    results[Run({'RATE' : 2})] = {'PPS':[1.5]}
+    try:
+        next(it)
+        assert False
+    except StopIteration:
+        pass
+    logger.error(run)
+
+
+def _test_allzlt(monotonic):
+    vlist = {'RATE' : RangeVariable("RATE",1,10,log=False)}
+    results = OrderedDict()
+    zlt = ZLTVariableExpander(vlist, results, {}, "RATE", "PPS", 1.01,all=True,monotonic=monotonic)
+    it = iter(zlt)
+    run = next(it)
+    assert run["RATE"] == 10
+    results[Run({'RATE' : 10})] = {'PPS':[3.0]}
+    run = next(it)
+    assert run["RATE"] == 3
+    results[Run({'RATE' : 3})] = {'PPS':[3]}
+    run = next(it)
+    assert run["RATE"] == 2
+    results[Run({'RATE' : 2})] = {'PPS':[2]}
+    run = next(it)
+    assert run["RATE"] == 1
+    results[Run({'RATE' : 1})] = {'PPS':[1]}
+    if not monotonic:
+        run = next(it)
+        assert run["RATE"] == 4
+        results[Run({'RATE' : 4})] = {'PPS':[3.1]}
+    try:
+        next(it)
+        assert False
+    except StopIteration:
+        pass
+    logger.error(run)
+
+def test_allzlt():
+    _test_allzlt(monotonic=True)
+    _test_allzlt(monotonic=False)

integration/zlt.npf

@@ -0,0 +1,28 @@
+%config
+graph_fillstyle=full
+graph_filter_by={THROUGHPUT:DROPPEDPC>1}
+var_format={THROUGHPUT:%d}
+var_names={RATE:Input rate (Gbps),THROUGHPUT:Throughput (Gbps),THRESH:System capacity (Gbps)}
+graph_legend=1
+var_lim={result:0-100}
+agg:var_aggregate={RATE:max(where(DROPPEDPC<1))}
+
+%variables
+RATE=[10-100#5]
+THRESH={50,70,90}
+
+%script
+if [ $RATE -lt $THRESH ] ; then
+    d=0
+else
+    d=$(echo "($RATE - $THRESH) / 2" | bc)
+fi
+
+t=$(echo "$RATE*(100-$d)/100" | bc)
+p=$(echo "$t * 100 / $RATE" | bc)
+echo "RESULT-DROPPEDPC $d"
+echo "RESULT-GOODPC $p"
+echo "RESULT-THROUGHPUT $t"
+
+
+%import graph-beautiful

integration/zlt.stdout

@@ -0,0 +1,148 @@
+Executing test zlt.npf
+Executing init scripts...
+THRESH = 50, RATE = 100 [run 1/3 for test 1/~15(max 57)]
+[0] RESULT-DROPPEDPC 25
+[0] RESULT-GOODPC 75
+[0] RESULT-THROUGHPUT 75
+THRESH = 50, RATE = 100 [run 2/3 for test 1/~15(max 57)]
+[0] RESULT-DROPPEDPC 25
+[0] RESULT-GOODPC 75
+[0] RESULT-THROUGHPUT 75
+THRESH = 50, RATE = 100 [run 3/3 for test 1/~15(max 57)]
+[0] RESULT-DROPPEDPC 25
+[0] RESULT-GOODPC 75
+[0] RESULT-THROUGHPUT 75
+DROPPEDPC: [25.0, 25.0, 25.0], GOODPC: [75.0, 75.0, 75.0], THROUGHPUT: [75.0, 75.0, 75.0]
+THRESH = 50, RATE = 75 [run 1/3 for test 2/~15(max 57)]
+[0] RESULT-DROPPEDPC 12
+[0] RESULT-GOODPC 88
+[0] RESULT-THROUGHPUT 66
+THRESH = 50, RATE = 75 [run 2/3 for test 2/~15(max 57)]
+[0] RESULT-DROPPEDPC 12
+[0] RESULT-GOODPC 88
+[0] RESULT-THROUGHPUT 66
+THRESH = 50, RATE = 75 [run 3/3 for test 2/~15(max 57)]
+[0] RESULT-DROPPEDPC 12
+[0] RESULT-GOODPC 88
+[0] RESULT-THROUGHPUT 66
+DROPPEDPC: [12.0, 12.0, 12.0], GOODPC: [88.0, 88.0, 88.0], THROUGHPUT: [66.0, 66.0, 66.0]
+THRESH = 50, RATE = 55 [run 1/3 for test 3/~15(max 57)]
+[0] RESULT-DROPPEDPC 2
+[0] RESULT-GOODPC 96
+[0] RESULT-THROUGHPUT 53
+THRESH = 50, RATE = 55 [run 2/3 for test 3/~15(max 57)]
+[0] RESULT-DROPPEDPC 2
+[0] RESULT-GOODPC 96
+[0] RESULT-THROUGHPUT 53
+THRESH = 50, RATE = 55 [run 3/3 for test 3/~15(max 57)]
+[0] RESULT-DROPPEDPC 2
+[0] RESULT-GOODPC 96
+[0] RESULT-THROUGHPUT 53
+DROPPEDPC: [2.0, 2.0, 2.0], GOODPC: [96.0, 96.0, 96.0], THROUGHPUT: [53.0, 53.0, 53.0]
+THRESH = 50, RATE = 50 [run 1/3 for test 4/~15(max 57)]
+[0] RESULT-DROPPEDPC 0
+[0] RESULT-GOODPC 100
+[0] RESULT-THROUGHPUT 50
+THRESH = 50, RATE = 50 [run 2/3 for test 4/~15(max 57)]
+[0] RESULT-DROPPEDPC 0
+[0] RESULT-GOODPC 100
+[0] RESULT-THROUGHPUT 50
+THRESH = 50, RATE = 50 [run 3/3 for test 4/~15(max 57)]
+[0] RESULT-DROPPEDPC 0
+[0] RESULT-GOODPC 100
+[0] RESULT-THROUGHPUT 50
+DROPPEDPC: [0.0, 0.0, 0.0], GOODPC: [100.0, 100.0, 100.0], THROUGHPUT: [50.0, 50.0, 50.0]
+THRESH = 70, RATE = 100 [run 1/3 for test 5/~12(max 57)]
+[0] RESULT-DROPPEDPC 15
+[0] RESULT-GOODPC 85
+[0] RESULT-THROUGHPUT 85
+THRESH = 70, RATE = 100 [run 2/3 for test 5/~12(max 57)]
+[0] RESULT-DROPPEDPC 15
+[0] RESULT-GOODPC 85
+[0] RESULT-THROUGHPUT 85
+THRESH = 70, RATE = 100 [run 3/3 for test 5/~12(max 57)]
+[0] RESULT-DROPPEDPC 15
+[0] RESULT-GOODPC 85
+[0] RESULT-THROUGHPUT 85
+DROPPEDPC: [15.0, 15.0, 15.0], GOODPC: [85.0, 85.0, 85.0], THROUGHPUT: [85.0, 85.0, 85.0]
+THRESH = 70, RATE = 85 [run 1/3 for test 6/~12(max 57)]
+[0] RESULT-DROPPEDPC 7
+[0] RESULT-GOODPC 92
+[0] RESULT-THROUGHPUT 79
+THRESH = 70, RATE = 85 [run 2/3 for test 6/~12(max 57)]
+[0] RESULT-DROPPEDPC 7
+[0] RESULT-GOODPC 92
+[0] RESULT-THROUGHPUT 79
+THRESH = 70, RATE = 85 [run 3/3 for test 6/~12(max 57)]
+[0] RESULT-DROPPEDPC 7
+[0] RESULT-GOODPC 92
+[0] RESULT-THROUGHPUT 79
+DROPPEDPC: [7.0, 7.0, 7.0], GOODPC: [92.0, 92.0, 92.0], THROUGHPUT: [79.0, 79.0, 79.0]
+THRESH = 70, RATE = 70 [run 1/3 for test 7/~12(max 57)]
+[0] RESULT-DROPPEDPC 0
+[0] RESULT-GOODPC 100
+[0] RESULT-THROUGHPUT 70
+THRESH = 70, RATE = 70 [run 2/3 for test 7/~12(max 57)]
+[0] RESULT-DROPPEDPC 0
+[0] RESULT-GOODPC 100
+[0] RESULT-THROUGHPUT 70
+THRESH = 70, RATE = 70 [run 3/3 for test 7/~12(max 57)]
+[0] RESULT-DROPPEDPC 0
+[0] RESULT-GOODPC 100
+[0] RESULT-THROUGHPUT 70
+DROPPEDPC: [0.0, 0.0, 0.0], GOODPC: [100.0, 100.0, 100.0], THROUGHPUT: [70.0, 70.0, 70.0]
+THRESH = 70, RATE = 75 [run 1/3 for test 8/~12(max 57)]
+[0] RESULT-DROPPEDPC 2
+[0] RESULT-GOODPC 97
+[0] RESULT-THROUGHPUT 73
+THRESH = 70, RATE = 75 [run 2/3 for test 8/~12(max 57)]
+[0] RESULT-DROPPEDPC 2
+[0] RESULT-GOODPC 97
+[0] RESULT-THROUGHPUT 73
+THRESH = 70, RATE = 75 [run 3/3 for test 8/~12(max 57)]
+[0] RESULT-DROPPEDPC 2
+[0] RESULT-GOODPC 97
+[0] RESULT-THROUGHPUT 73
+DROPPEDPC: [2.0, 2.0, 2.0], GOODPC: [97.0, 97.0, 97.0], THROUGHPUT: [73.0, 73.0, 73.0]
+THRESH = 90, RATE = 100 [run 1/3 for test 9/~12(max 57)]
+[0] RESULT-DROPPEDPC 5
+[0] RESULT-GOODPC 95
+[0] RESULT-THROUGHPUT 95
+THRESH = 90, RATE = 100 [run 2/3 for test 9/~12(max 57)]
+[0] RESULT-DROPPEDPC 5
+[0] RESULT-GOODPC 95
+[0] RESULT-THROUGHPUT 95
+THRESH = 90, RATE = 100 [run 3/3 for test 9/~12(max 57)]
+[0] RESULT-DROPPEDPC 5
+[0] RESULT-GOODPC 95
+[0] RESULT-THROUGHPUT 95
+DROPPEDPC: [5.0, 5.0, 5.0], GOODPC: [95.0, 95.0, 95.0], THROUGHPUT: [95.0, 95.0, 95.0]
+THRESH = 90, RATE = 95 [run 1/3 for test 10/~12(max 57)]
+[0] RESULT-DROPPEDPC 2
+[0] RESULT-GOODPC 97
+[0] RESULT-THROUGHPUT 93
+THRESH = 90, RATE = 95 [run 2/3 for test 10/~12(max 57)]
+[0] RESULT-DROPPEDPC 2
+[0] RESULT-GOODPC 97
+[0] RESULT-THROUGHPUT 93
+THRESH = 90, RATE = 95 [run 3/3 for test 10/~12(max 57)]
+[0] RESULT-DROPPEDPC 2
+[0] RESULT-GOODPC 97
+[0] RESULT-THROUGHPUT 93
+DROPPEDPC: [2.0, 2.0, 2.0], GOODPC: [97.0, 97.0, 97.0], THROUGHPUT: [93.0, 93.0, 93.0]
+THRESH = 90, RATE = 90 [run 1/3 for test 11/~12(max 57)]
+[0] RESULT-DROPPEDPC 0
+[0] RESULT-GOODPC 100
+[0] RESULT-THROUGHPUT 90
+THRESH = 90, RATE = 90 [run 2/3 for test 11/~12(max 57)]
+[0] RESULT-DROPPEDPC 0
+[0] RESULT-GOODPC 100
+[0] RESULT-THROUGHPUT 90
+THRESH = 90, RATE = 90 [run 3/3 for test 11/~12(max 57)]
+[0] RESULT-DROPPEDPC 0
+[0] RESULT-GOODPC 100
+[0] RESULT-THROUGHPUT 90
+DROPPEDPC: [0.0, 0.0, 0.0], GOODPC: [100.0, 100.0, 100.0], THROUGHPUT: [90.0, 90.0, 90.0]
+Graph of test written to results/click-2022/9197a59/zlt-DROPPEDPC.pdf
+Graph of test written to results/click-2022/9197a59/zlt-GOODPC.pdf
+Graph of test written to results/click-2022/9197a59/zlt-THROUGHPUT.pdf

npf/expdesign/fullexp.py

@@ -0,0 +1,38 @@
+from npf.variable import OrderedDict
+
+
+from collections import OrderedDict
+
+
+class FullVariableExpander:
+    """Expand all variables building the full
+    matrix first."""
+
+    def __init__(self, vlist, overriden):
+        self.expanded = [OrderedDict()]
+        for k, v in vlist.items():
+            if k in overriden:
+                continue
+            newList = []
+            l = v.makeValues()
+
+            for nvalue in l:
+                for ovalue in self.expanded:
+                    z = ovalue.copy()
+                    z.update(nvalue if type(nvalue) is OrderedDict else {k: nvalue})
+                    newList.append(z)
+
+            self.expanded = newList
+        self.it = self.expanded.__iter__()
+
+    def __iter__(self):
+        return self.expanded.__iter__()
+
+    def __next__(self):
+        return self.it.__next__()
+
+    def __len__(self):
+        return len(self.expanded)
+
+    def strlen(self):
+        return len(self)

npf/expdesign/randomexp.py

@@ -0,0 +1,11 @@
+from random import shuffle
+
+from npf.expdesign.fullexp import FullVariableExpander
+
+
+class RandomVariableExpander(FullVariableExpander):
+    """Same as BruteVariableExpander but shuffle the series to test"""
+
+    def __init__(self, vlist, overriden):
+        super().__init__(vlist, overriden)
+        shuffle(self.expanded)