include common functions in utils.py

README.md

@@ -458,7 +458,7 @@ Check out the JSON format they follow in the [test_suites](test_suites)
 directory. Add a new json file within, Fluster will automatically pick it
 up.
 
-There are also a [generator script (H.265, H.266)](scripts/gen_jvet_jct_vc.py) and a [generator script (H.264)](scripts/gen_jvt.py) for the [conformance
+There is also a [generator script (H.265, H.266)](scripts/gen_jvet_jctvc.py) and a [generator script (H.264)](scripts/gen_jvt.py) for the [conformance
 test suites](#test_suites) that you can use as a base to generate automatically
 new ones.
 

fluster/utils.py

@@ -176,3 +176,46 @@ def _win_site_data_dirs(appname: str) -> List[str]:
 else:
     site_data_dirs = _linux_site_data_dirs
     user_data_dir = _linux_user_data_dir
+
+
+def find_by_ext(dest_dir, exts, excludes=None):
+    """Return name by file extension"""
+    excludes = excludes or []
+
+    # Respect the priority for extensions
+    for ext in exts:
+        for subdir, _, files in os.walk(dest_dir):
+            for filename in files:
+                excluded = False
+                filepath = subdir + os.sep + filename
+                if not filepath.endswith(ext) or "__MACOSX" in filepath:
+                    continue
+                for excl in excludes:
+                    if excl in filepath:
+                        excluded = True
+                        break
+                if not excluded:
+                    return filepath
+    return None
+
+
+def find_by_ext_multiple(dest_dir, exts, excludes=None):
+    """Return multiple names by file extension"""
+    excludes = excludes or []
+    found_files = []
+
+    # Respect the priority for extensions
+    for ext in exts:
+        for subdir, _, files in os.walk(dest_dir):
+            for filename in files:
+                excluded = False
+                filepath = subdir + os.sep + filename
+                if not filepath.endswith(ext) or "__MACOSX" in filepath:
+                    continue
+                for excl in excludes:
+                    if excl in filepath:
+                        excluded = True
+                        break
+                if not excluded:
+                    found_files.append(filepath)
+    return found_files

scripts/gen_jvet_jct_vc.py → scripts/gen_jvet_jctvc.py

@@ -72,7 +72,7 @@ def handle_starttag(self, tag, attrs):
                     self.links.append(base_url + value)
 
 
-class JCTVTGenerator:
+class JVET_JCTVTGenerator:
     """Generates a test suite from the conformance bitstreams"""
 
     def __init__(
@@ -136,7 +136,7 @@ def generate(self, download, jobs):
                 test_suite.resources_dir, test_suite.name, test_vector.name
             )
             dest_path = os.path.join(dest_dir, os.path.basename(test_vector.source))
-            test_vector.input_file = self._find_by_ext(dest_dir, BITSTREAM_EXTS)
+            test_vector.input_file = utils.find_by_ext(dest_dir, BITSTREAM_EXTS)
             absolute_input_path = test_vector.input_file
             test_vector.input_file = test_vector.input_file.replace(
                 os.path.join(
@@ -193,7 +193,7 @@ def generate(self, download, jobs):
         print("Generate new test suite: " + test_suite.name + ".json")
 
     def _fill_checksum_h265(self, test_vector, dest_dir):
-        checksum_file = self._find_by_ext(dest_dir, MD5_EXTS, MD5_EXCLUDES)
+        checksum_file = utils.find_by_ext(dest_dir, MD5_EXTS, MD5_EXCLUDES)
         if checksum_file is None:
             raise Exception("MD5 not found")
         with open(checksum_file, "r") as checksum_file:
@@ -238,26 +238,6 @@ def _fill_checksum_h265(self, test_vector, dest_dir):
             assert len(test_vector.result) == 32 and re.search(
                 r"^[a-fA-F0-9]{32}$", test_vector.result) is not None, f"{test_vector.result} is not a valid MD5 hash"
 
-    @staticmethod
-    def _find_by_ext(dest_dir, exts, excludes=None):
-        excludes = excludes or []
-
-        # Respect the priority for extensions
-        for ext in exts:
-            for subdir, _, files in os.walk(dest_dir):
-                for filename in files:
-                    excluded = False
-                    filepath = subdir + os.sep + filename
-                    if not filepath.endswith(ext) or "__MACOSX" in filepath:
-                        continue
-                    for excl in excludes:
-                        if excl in filepath:
-                            excluded = True
-                            break
-                    if not excluded:
-                        return filepath
-        return None
-
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
@@ -275,7 +255,7 @@ def _find_by_ext(dest_dir, exts, excludes=None):
         default=2 * multiprocessing.cpu_count(),
     )
     args = parser.parse_args()
-    generator = JCTVTGenerator(
+    generator = JVET_JCTVTGenerator(
         "HEVC_v1",
         "JCT-VC-HEVC_V1",
         Codec.H265,
@@ -284,7 +264,7 @@ def _find_by_ext(dest_dir, exts, excludes=None):
     )
     generator.generate(not args.skip_download, args.jobs)
 
-    generator = JCTVTGenerator(
+    generator = JVET_JCTVTGenerator(
         "RExt",
         "JCT-VC-RExt",
         Codec.H265,
@@ -294,7 +274,7 @@ def _find_by_ext(dest_dir, exts, excludes=None):
     )
     generator.generate(not args.skip_download, args.jobs)
 
-    generator = JCTVTGenerator(
+    generator = JVET_JCTVTGenerator(
         "SCC",
         "JCT-VC-SCC",
         Codec.H265,
@@ -304,7 +284,7 @@ def _find_by_ext(dest_dir, exts, excludes=None):
     )
     generator.generate(not args.skip_download, args.jobs)
 
-    generator = JCTVTGenerator(
+    generator = JVET_JCTVTGenerator(
         "MV-HEVC",
         "JCT-VC-MV-HEVC",
         Codec.H265,
@@ -314,7 +294,7 @@ def _find_by_ext(dest_dir, exts, excludes=None):
     )
     generator.generate(not args.skip_download, args.jobs)
 
-    generator = JCTVTGenerator(
+    generator = JVET_JCTVTGenerator(
         'draft6',
         'JVET-VVC_draft6',
         Codec.H266,

scripts/gen_jvt.py

@@ -23,7 +23,6 @@
 import sys
 import urllib.request
 import multiprocessing
-import re
 
 # pylint: disable=wrong-import-position
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
@@ -73,7 +72,7 @@ def handle_starttag(self, tag, attrs):
                     self.links.append(base_url + value)
 
 
-class JCTVTGenerator:
+class JVTGenerator:
     """Generates a test suite from the conformance bitstreams"""
 
     def __init__(
@@ -133,12 +132,11 @@ def generate(self, download, jobs):
             )
 
         for test_vector in test_suite.test_vectors.values():
-            print("1111111111111111 " + str(test_vector))
             dest_dir = os.path.join(
                 test_suite.resources_dir, test_suite.name, test_vector.name
             )
             dest_path = os.path.join(dest_dir, os.path.basename(test_vector.source))
-            test_vector.input_file = self._find_by_ext(dest_dir, BITSTREAM_EXTS)
+            test_vector.input_file = utils.find_by_ext(dest_dir, BITSTREAM_EXTS)
             absolute_input_path = test_vector.input_file
             test_vector.input_file = test_vector.input_file.replace(
                 os.path.join(
@@ -197,61 +195,20 @@ def generate(self, download, jobs):
         test_suite.to_json_file(output_filepath)
         print("Generate new test suite: " + test_suite.name + ".json")
 
-    def _fill_checksum_h264(self, test_vector, dest_dir):
-        raw_file = self._find_by_ext(dest_dir, RAW_EXTS)
+    @staticmethod
+    def _fill_checksum_h264(test_vector, dest_dir):
+        raw_file = utils.find_by_ext(dest_dir, RAW_EXTS)
         if raw_file is None or len(raw_file) == 0:
             raise Exception(f"RAW file not found in {dest_dir}")
         test_vector.result = utils.file_checksum(raw_file)
 
-    def _fill_checksum_h264_multiple(self, test_vector, dest_dir):
-        raw_files = self._find_by_ext_multiple(dest_dir, RAW_EXTS)
+    @staticmethod
+    def _fill_checksum_h264_multiple(test_vector, dest_dir):
+        raw_files = utils.find_by_ext_multiple(dest_dir, RAW_EXTS)
         if not raw_files:
             raise Exception(f"RAW file not found in {dest_dir}")
         for raw_file in raw_files:
-            print("aaaaaaaaaaaaaaaa " + str(raw_file))
             test_vector.result = utils.file_checksum(raw_file)
-            print("eeeeeeeeeeeeeeee " + str(test_vector.result))
-
-    @staticmethod
-    def _find_by_ext(dest_dir, exts, excludes=None):
-        excludes = excludes or []
-
-        # Respect the priority for extensions
-        for ext in exts:
-            for subdir, _, files in os.walk(dest_dir):
-                for filename in files:
-                    excluded = False
-                    filepath = subdir + os.sep + filename
-                    if not filepath.endswith(ext) or "__MACOSX" in filepath:
-                        continue
-                    for excl in excludes:
-                        if excl in filepath:
-                            excluded = True
-                            break
-                    if not excluded:
-                        return filepath
-        return None
-
-    @staticmethod
-    def _find_by_ext_multiple(dest_dir, exts, excludes=None):
-        excludes = excludes or []
-        found_files = []
-
-        # Respect the priority for extensions
-        for ext in exts:
-            for subdir, _, files in os.walk(dest_dir):
-                for filename in files:
-                    excluded = False
-                    filepath = subdir + os.sep + filename
-                    if not filepath.endswith(ext) or "__MACOSX" in filepath:
-                        continue
-                    for excl in excludes:
-                        if excl in filepath:
-                            excluded = True
-                            break
-                    if not excluded:
-                        found_files.append(filepath)
-        return found_files
 
 
 if __name__ == "__main__":
@@ -270,16 +227,16 @@ def _find_by_ext_multiple(dest_dir, exts, excludes=None):
         default=2 * multiprocessing.cpu_count(),
     )
     args = parser.parse_args()
-    # generator = JCTVTGenerator(
-    #     "AVCv1",
-    #     "JVT-AVC_V1",
-    #     Codec.H264,
-    #     "JVT AVC version 1",
-    #     H264_URL
-    # )
-    # generator.generate(not args.skip_download, args.jobs)
+    generator = JVTGenerator(
+        "AVCv1",
+        "JVT-AVC_V1",
+        Codec.H264,
+        "JVT AVC version 1",
+        H264_URL
+    )
+    generator.generate(not args.skip_download, args.jobs)
 
-    generator = JCTVTGenerator(
+    generator = JVTGenerator(
         "SVC",
         "JVT-SVC_V1",
         Codec.H264,
@@ -289,12 +246,12 @@ def _find_by_ext_multiple(dest_dir, exts, excludes=None):
     )
     generator.generate(not args.skip_download, args.jobs)
 
-    # generator = JCTVTGenerator(
-    #     "Professional_profiles",
-    #     "JVT-Professional_profiles_V1",
-    #     Codec.H264,
-    #     "JVT professional profiles version 1",
-    #     H264_URL,
-    #     True
-    # )
-    # generator.generate(not args.skip_download, args.jobs)
+    generator = JVTGenerator(
+        "Professional_profiles",
+        "JVT-Professional_profiles_V1",
+        Codec.H264,
+        "JVT professional profiles version 1",
+        H264_URL,
+        True
+    )
+    generator.generate(not args.skip_download, args.jobs)