diff --git a/mega-test-quick.cfg b/mega-test-quick.cfg new file mode 100644 index 0000000000..95f324b6ed --- /dev/null +++ b/mega-test-quick.cfg @@ -0,0 +1,5 @@ +linux-gcc-4.7 1428 ./super-test.sh tmpdir capnp-gcc-4.7 quick +linux-gcc-4.8 1431 ./super-test.sh tmpdir capnp-gcc-4.8 quick gcc-4.8 +linux-clang 1445 ./super-test.sh tmpdir capnp-clang quick clang +mac 704 ./super-test.sh remote beat caffeinate quick +cygwin 709 ./super-test.sh remote Kenton@flashman quick diff --git a/mega-test.cfg b/mega-test.cfg new file mode 100644 index 0000000000..e4f6acf714 --- /dev/null +++ b/mega-test.cfg @@ -0,0 +1,5 @@ +linux-gcc-4.7 12075 ./super-test.sh tmpdir capnp-gcc-4.7 +linux-gcc-4.8 10761 ./super-test.sh tmpdir capnp-gcc-4.8 gcc-4.8 +linux-clang 12174 ./super-test.sh tmpdir capnp-clang clang +mac 5064 ./super-test.sh remote beat caffeinate +cygwin 5817 ./super-test.sh remote Kenton@flashman diff --git a/mega-test.py b/mega-test.py new file mode 100755 index 0000000000..480752e9ed --- /dev/null +++ b/mega-test.py @@ -0,0 +1,144 @@ +#! /usr/bin/env python + +# MEGA TEST +# +# usage: mega-test.py +# +# This runs several tests in parallel and shows progress bars for each, based on a config file. +# +# is a file containing a list of commands to run along with the expected number of lines +# they will output (to stdout and stderr combined), which is how the progress bar is calculated. +# The format of the file is simply one test per line, with the line containing the test name, +# the number of output lines expected, and the test command. Example: +# +# mytest 1523 ./my-test --foo bar +# another 862 ./another-test --baz +# +# Each command is interpreted by `sh -euc`, therefore it is acceptable to use environment +# variables and other shell syntax. +# +# After all tests complete, the config file will be rewritten to update the line counts to the +# actual number of lines seen for all passing tests (failing tests are not updated). + +import sys +import re +import os +from errno import EAGAIN +from fcntl import fcntl, F_GETFL, F_SETFL +from select import poll, POLLIN, POLLHUP +from subprocess import Popen, PIPE, STDOUT + +CONFIG_LINE = re.compile("^([^ ]+) +([0-9]+) +(.*)$") + +if len(sys.argv) != 2: + sys.stderr.write("Wrong number of arguments.\n"); + sys.exit(1) + +if not os.access("/tmp/test-output", os.F_OK): + os.mkdir("/tmp/test-output") + +config = open(sys.argv[1], 'r') + +tests = [] + +class Test: + def __init__(self, name, command, lines): + self.name = name + self.command = command + self.lines = lines + self.count = 0 + self.done = False + + def start(self, poller): + self.proc = Popen(["sh", "-euc", test.command], stdin=dev_null, stdout=PIPE, stderr=STDOUT) + fd = self.proc.stdout.fileno() + flags = fcntl(fd, F_GETFL) + fcntl(fd, F_SETFL, flags | os.O_NONBLOCK) + poller.register(self.proc.stdout, POLLIN) + self.log = open("/tmp/test-output/" + self.name + ".log", "w") + + def update(self): + try: + while True: + text = self.proc.stdout.read() + if text == "": + self.proc.wait() + self.done = True + self.log.close() + return True + self.count += text.count("\n") + self.log.write(text) + except IOError as e: + if e.errno == EAGAIN: + return False + raise + + def print_bar(self): + percent = self.count * 100 / self.lines + status = "(%3d%%)" % percent + + color_on = "" + color_off = "" + + if self.done: + if self.proc.returncode == 0: + color_on = "\033[0;32m" + status = "PASS" + else: + color_on = "\033[0;31m" + status = "FAIL: /tmp/test-output/%s.log" % self.name + color_off = "\033[0m" + + print "%s%-16s |%-25s| %6d/%6d %s%s " % ( + color_on, self.name, '=' * min(percent / 4, 25), self.count, self.lines, status, color_off) + + def passed(self): + return self.proc.returncode == 0 + +for line in config: + if len(line) > 0 and not line.startswith("#"): + match = CONFIG_LINE.match(line) + if not match: + sys.stderr.write("Invalid config syntax: %s\n" % line); + sys.exit(1) + test = Test(match.group(1), match.group(3), int(match.group(2))) + tests.append(test) + +config.close() + +dev_null = open("/dev/null", "rw") +poller = poll() +fd_map = {} + +for test in tests: + test.start(poller) + fd_map[test.proc.stdout.fileno()] = test + +active_count = len(tests) + +def print_bars(): + for test in tests: + test.print_bar() + +print_bars() + +while active_count > 0: + for (fd, event) in poller.poll(): + if fd_map[fd].update(): + active_count -= 1 + poller.unregister(fd) + sys.stdout.write("\033[%dA\r" % len(tests)) + print_bars() + +new_config = open(sys.argv[1], "w") +for test in tests: + if test.passed(): + new_config.write("%-16s %6d %s\n" % (test.name, test.count, test.command)) + else: + new_config.write("%-16s %6d %s\n" % (test.name, test.lines, test.command)) + +for test in tests: + if not test.passed(): + sys.exit(1) + +sys.exit(0) diff --git a/super-test.sh b/super-test.sh index 2c119f583e..da261756df 100755 --- a/super-test.sh +++ b/super-test.sh @@ -16,6 +16,39 @@ while [ $# -gt 0 ]; do quick ) QUICK=quick ;; + caffeinate ) + # Re-run preventing sleep. + shift + exec caffeinate $0 $@ + ;; + tmpdir ) + # Clone to a temp directory. + if [ "$#" -lt 2 ]; then + echo "usage: $0 tmpdir NAME [COMMAND]" >&2 + exit 1 + fi + DIR=/tmp/$2 + shift 2 + if [ -e $DIR ]; then + if [ "${DIR/*..*}" = "" ]; then + echo "NO DO NOT PUT .. IN THERE IT'S GOING TO GO IN /tmp AND I'M GONNA DELETE IT" >&2 + exit 1 + fi + if [ ! -e "$DIR/super-test.sh" ]; then + echo "$DIR exists and it doesn't look like one of mine." >&2 + exit 1 + fi + # make distcheck leaves non-writable files when it fails, so we need to chmod to be safe. + chmod -R +w $DIR + rm -rf $DIR + fi + git clone . $DIR + if [ -e c++/gtest ]; then + cp -r c++/gtest $DIR/c++/gtest + fi + cd $DIR + exec ./super-test.sh $@ + ;; remote ) if [ "$#" -lt 2 ]; then echo "usage: $0 remote HOST [COMMAND]" >&2 @@ -29,8 +62,9 @@ while [ $# -gt 0 ]; do BRANCH=$(git rev-parse --abbrev-ref HEAD) ssh $HOST 'rm -rf tmp-test-capnp && mkdir tmp-test-capnp && git init tmp-test-capnp' git push ssh://$HOST/~/tmp-test-capnp "$BRANCH:test" - ssh $HOST "cd tmp-test-capnp && git checkout test && ./super-test.sh $@ && cd .. && rm -rf tmp-test-capnp" - exit 0 + ssh $HOST "cd tmp-test-capnp && git checkout test" + scp -qr c++/gtest $HOST:~/tmp-test-capnp/c++/gtest + exec ssh $HOST "cd tmp-test-capnp && ./super-test.sh $@ && cd .. && rm -rf tmp-test-capnp" ;; clang ) export CXX=clang++ @@ -65,7 +99,7 @@ __EOF__ ************************************************************************* ========================================================================= __EOF__ - $0 remote beat $QUICK + $0 remote beat caffeinate $QUICK cat << "__EOF__" ========================================================================= *************************************************************************