Merge pull request #2421 from cms-sw/sig_dostack_to_es_2

Send stacktraces from unit tests to ES

es_ibs_log.py

@@ -2,7 +2,7 @@
 from __future__ import print_function
 
 from hashlib import sha1
-import os, json, datetime, sys
+import os, json, datetime, sys, copy, re
 from glob import glob
 from os.path import exists, dirname, getmtime
 from es_utils import send_payload
@@ -43,10 +43,24 @@ def process_unittest_log(logFile):
     release = pathInfo[8]
     week, rel_sec = cmsswIB2Week(release)
     package = pathInfo[-3] + "/" + pathInfo[-2]
-    payload = {"type": "unittest"}
-    payload["release"] = release
-    payload["architecture"] = architecture
-    payload["@timestamp"] = timestp
+    payload_dataset = {"type": "unittest"}
+    payload_dataset["release"] = release
+    release_queue = "_".join(release.split("_", -1)[:-1]).split("_", 3)
+    payload_dataset["release_queue"] = "_".join(release_queue[0:3])
+    flavor = release_queue[-1]
+    if flavor == "X":
+        flavor = "DEFAULT"
+    payload_dataset["flavor"] = flavor
+    payload_dataset["architecture"] = architecture
+    payload_dataset["@timestamp"] = timestp
+
+    payload_utest = copy.deepcopy(payload_dataset)
+    del payload_utest["type"]
+
+    inStacktrace = False
+    stacktrace = []
+    datasets = []
+
     config_list = []
     custom_rule_set = [
         {
@@ -60,32 +74,97 @@ def process_unittest_log(logFile):
             "control_type": ResultTypeEnum.TEST,
         },
     ]
+
+    pkgTestStartRe = re.compile('^===== Test\s+"([^\s]+)" ====')
+    pkgTestEndRe = re.compile(r"^\^\^\^\^ End Test\s+([^\s]+)\s+\^\^\^\^")
+    pkgTestResultRe = re.compile(".*---> test\s+([^\s]+)\s+(had ERRORS|succeeded)")
+
     with open(logFile, encoding="ascii", errors="ignore") as f:
         utname = None
         datasets = []
-        xid = None
+        test_status = 0
         for index, l in enumerate(f):
             l = l.strip()
             config_list = add_exception_to_config(l, index, config_list, custom_rule_set)
-            if l.startswith('===== Test "') and l.endswith('" ===='):
-                if utname:
-                    send_unittest_dataset(
-                        datasets, payload, xid, "ib-dataset-" + week, "unittest-dataset"
-                    )
+            if m := pkgTestStartRe.match(l):
+                utname = m[1]
+                test_status = 0
                 datasets = []
-                utname = l.split('"')[1]
-                payload["name"] = "%s/%s" % (package, utname)
-                xid = sha1hexdigest(release + architecture + package + str(utname))
-            elif " Initiating request to open file " in l:
+                continue
+
+            if m := pkgTestResultRe.match(l):
+                if m[1] != utname:
+                    print(
+                        "ERROR: Unit test name mismatch - expected {0}, got {1}".format(
+                            utname, m[1]
+                        )
+                    )  # TODO: do we want a more visible error (exit 1)? Or maybe skip this file?
+                else:
+                    test_status = 1 if m[2] == "had ERRORS" else 0
+                continue
+
+            if m := pkgTestEndRe.match(l):
+                if m[1] != utname:
+                    print(
+                        "ERROR: Unit test name mismatch - expected {0}, got {1}".format(
+                            utname, m[1]
+                        )
+                    )  # TODO: do we want a more visible error (exit 1)? Or maybe skip this file?
+                    continue
+
+                if test_status == -1:
+                    print("ERROR: test state for UT {0} unknown".format(utname))
+                    continue
+
+                payload_utest["url"] = (
+                    "https://cmssdt.cern.ch/SDT/cgi-bin/buildlogs/"
+                    + architecture
+                    + "/"
+                    + release
+                    + "/unitTestLogs/"
+                    + package
+                )
+                payload_utest["status"] = test_status
+                payload_utest["name"] = utname
+                payload_utest["package"] = package
+                if stacktrace:
+                    payload_utest["stacktrace"] = "\n".join(stacktrace)
+                    stacktrace = []
+                utest_id = sha1hexdigest(release + architecture + utname)
+                print("==> ", json.dumps(payload_utest) + "\n")
+                send_payload(index, "unittests", utest_id, json.dumps(payload_utest))
+
+                payload_dataset["name"] = "%s/%s" % (package, utname)
+                dataset_id = sha1hexdigest(release + architecture + package + utname)
+                print("==> ", json.dumps(payload_dataset) + "\n")
+                send_unittest_dataset(
+                    datasets, payload_dataset, dataset_id, "ib-dataset-" + week, "unittest-dataset"
+                )
+                continue
+
+            if " Initiating request to open file " in l:
                 try:
                     rootfile = l.split(" Initiating request to open file ")[1].split(" ")[0]
                     if (not "file:" in rootfile) and (not rootfile in datasets):
                         datasets.append(rootfile)
                 except Exception as e:
                     print("ERROR: ", logFile, e)
                     traceback.print_exc(file=sys.stdout)
-        if datasets and xid:
-            send_unittest_dataset(datasets, payload, xid, "ib-dataset-" + week, "unittest-dataset")
+                continue
+
+            if "sig_dostack_then_abort" in l:
+                inStacktrace = True
+                continue
+
+            if inStacktrace and not l.startswith("#"):
+                inStacktrace = False
+                continue
+
+            if inStacktrace:
+                if len(stacktrace) < 20:
+                    stacktrace.append(l)
+                continue
+
     transform_and_write_config_file(logFile + "-read_config", config_list)
     return
 
@@ -149,75 +228,6 @@ def process_hlt_log(logFile):
     return
 
 
-def process_ib_utests(logFile):
-    t = getmtime(logFile)
-    timestp = int(t * 1000)
-    payload = {}
-    pathInfo = logFile.split("/")
-    architecture = pathInfo[4]
-    release = pathInfo[8]
-    week, rel_sec = cmsswIB2Week(release)
-    index = "ibs-" + week
-    document = "unittests"
-    payload["release"] = release
-    release_queue = "_".join(release.split("_", -1)[:-1]).split("_", 3)
-    payload["release_queue"] = "_".join(release_queue[0:3])
-    flavor = release_queue[-1]
-    if flavor == "X":
-        flavor = "DEFAULT"
-    payload["flavor"] = flavor
-    payload["architecture"] = architecture
-    payload["@timestamp"] = timestp
-
-    if exists(logFile):
-        with open(logFile, encoding="ascii", errors="ignore") as f:
-            try:
-                it = iter(f)
-                line = next(it)
-                while "--------" not in line:
-                    line = next(it)
-                while True:
-                    line = next(it).strip()
-                    if ":" in line:
-                        pkg = line.split(":")[0].strip()
-                        payload["url"] = (
-                            "https://cmssdt.cern.ch/SDT/cgi-bin/buildlogs/"
-                            + architecture
-                            + "/"
-                            + release
-                            + "/unitTestLogs/"
-                            + pkg
-                        )
-                        line = next(it).strip()
-                        while ":" not in line:
-                            if "had ERRORS" in line:
-                                payload["status"] = 1
-                            else:
-                                payload["status"] = 0
-                            utest = line.split(" ")[0]
-                            payload["package"] = pkg
-                            payload["name"] = utest
-                            id = sha1hexdigest(release + architecture + utest)
-                            print("==> ", json.dumps(payload) + "\n")
-                            send_payload(index, document, id, json.dumps(payload))
-                            line = next(it).strip()
-            except Exception as e:
-                print("ERROR: File processed: %s" % e)
-    else:
-        print("Invalid File Path")
-
-
-# get log files
-logs = run_cmd("find /data/sdt/buildlogs -mindepth 6 -maxdepth 6 -name 'unitTests-summary.log'")
-logs = logs[1].split("\n")
-# process log files
-for logFile in logs:
-    flagFile = logFile + ".checked"
-    if not exists(flagFile):
-        print("Working on ", logFile)
-        process_ib_utests(logFile)
-        os.system('touch "' + flagFile + '"')
-
 logs = run_cmd("find /data/sdt/buildlogs -mindepth 6 -maxdepth 6 -name 'unitTestLogs.zip'")
 logs = logs[1].split("\n")
 # process zip log files