Merge pull request #36 from martinjankoehler/protobuf-generation-fix

[Build Process]: Reorganize protobuf generated files

.github/workflows/ci-cd.yml

@@ -198,6 +198,13 @@ jobs:
         --build ${{ steps.strings.outputs.build-output-dir }} 
         --config ${{ matrix.build_type }}
 
+    - name: "CMake: Verify python protobuf generation"
+      run: |
+        # check if protobuf python modules were created
+        ls klayout_pex_protobuf/tech_pb2.py
+      shell: bash
+      working-directory: ${{github.workspace}}
+
     - name: "[C++] Test"
       continue-on-error: true
       working-directory: ${{ steps.strings.outputs.build-output-dir }}
@@ -208,12 +215,12 @@ jobs:
 
     - name: "[C++] Generate PB Tech (Windows)"
       if: ${{ matrix.os == 'windows-latest' }}
-      run: build\${{matrix.build_type}}\gen_tech_pb.exe
+      run: build\${{matrix.build_type}}\gen_tech_pb.exe klayout_pex_protobuf
       working-directory: ${{github.workspace}}
 
     - name: "[C++] Generate PB Tech (Unix)"
       if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }}
-      run: build/gen_tech_pb
+      run: build/gen_tech_pb klayout_pex_protobuf
       working-directory: ${{github.workspace}}
 
     - name: "[🐍] Install project"
@@ -224,6 +231,16 @@ jobs:
       run: poetry build --output dist
       shell: bash    # NOTE: required for windows poetry calls
 
+    - name: "[🐍] Verify wheel contains generation tech protobuf json messages"
+      run: |
+        unzip -l dist/*.whl | grep klayout_pex_protobuf | grep .pb.json
+      shell: bash
+
+    - name: "[🐍] Verify wheel contains generation protobuf python modules"
+      run: |
+        unzip -l dist/*.whl | grep klayout_pex_protobuf | grep _pb2.py
+      shell: bash
+
     - name: "[🐍] Store dist artifact"
       uses: actions/upload-artifact@v4
       with:

.github/workflows/integration-tests.yml

@@ -130,9 +130,10 @@ jobs:
         merge-multiple: true
         path: dist   # destination
 
-    - name: "Unpack Source Distribution"
+    - name: "Unpack Python Wheel"
       run: |
-        tar xvfz dist/klayout_pex*.tar.gz --strip-components=1
+        tar xvfz dist/klayout_pex*.tar.gz --strip-components=1   # source dist
+        unzip -o dist/klayout_pex*.whl klayout_pex_protobuf/\*   # generated protobuf files
 
     - name: "[🐍] Run Unit tests"
       run: >

.gitignore

@@ -2,6 +2,10 @@
 testdata/designs/**/*.cir
 output*
 
+# Automatically generated by CMake / gen_tech_pb
+klayout_pex_protobuf/*.pb.json
+klayout_pex_protobuf/*_pb2.py
+
 # Mac
 .DS_Store
 

CMakeLists.txt

@@ -76,7 +76,7 @@ protobuf_generate(
 
 #_____________________________________________________________________________________________
 
-set(PROTOBUF_PYTHON_GENERATED "${CMAKE_CURRENT_LIST_DIR}/build/python/klayout_pex_protobuf")
+set(PROTOBUF_PYTHON_GENERATED "${CMAKE_CURRENT_LIST_DIR}/klayout_pex_protobuf")
 file(MAKE_DIRECTORY "${PROTOBUF_PYTHON_GENERATED}")
 
 protobuf_generate(

cxx/gen_tech_pb/main.cpp

@@ -21,48 +21,55 @@
  * SPDX-License-Identifier: GPL-3.0-or-later
  * --------------------------------------------------------------------------------
  */
+
+#include <iostream>
+#include <filesystem>
+
 #include "protobuf.h"
 #include "pdk/ihp_sg13g2.h"
 #include "pdk/sky130A.h"
 
-void writeTech(const std::string &tech_name,
+void writeTech(const std::filesystem::path &output_directory,
+               const std::string &tech_name,
                const kpex::tech::Technology &tech)
 {
-    const std::string stem = "build/" + tech_name + "_tech";
-    write(tech, stem + ".pb.json", Format::JSON);
-
-    //    write(tech, stem + ".binpb", Format::PROTOBUF_BINARY);
-    //    write(tech, stem + ".txtpb", Format::PROTOBUF_TEXTUAL);
-
-    //    std::cout << "--------------------------------------------" << std::endl;
-    //    convert(stem + ".pb.json", Format::JSON,
-    //            stem + "__from_json.txtpb", Format::PROTOBUF_TEXTUAL);
-    //    std::cout << "--------------------------------------------" << std::endl;
-    //    convert(stem + "_tech.pb.json", Format::JSON,
-    //            stem + "_tech__from_json.binpb", Format::PROTOBUF_BINARY);
-    //    std::cout << "--------------------------------------------" << std::endl;
-    //    convert(stem + "_tech.pb.json", Format::JSON,
-    //            stem + "_tech__from_json.json", Format::JSON);
+    const std::filesystem::path json_pb_path = output_directory / (tech_name + "_tech" + ".pb.json");
+    write(tech, json_pb_path.string(), Format::JSON);
 }
 
 int main(int argc, char **argv) {
     // Verify that the version of the library that we linked against is
     // compatible with the version of the headers we compiled against.
     GOOGLE_PROTOBUF_VERIFY_VERSION;
+
 
+    if (argc < 2) {
+        std::cerr << "Usage: " << argv[0] << " <output-directory>" << std::endl;
+        return 1;
+    }
+
+    std::filesystem::path output_directory(argv[1]);
+    if (std::filesystem::exists(output_directory) && !std::filesystem::is_directory(output_directory)) {
+        std::cerr << "ERROR: Output directory path already exists, but is not a directory" << std::endl;
+        return 2;
+    }
+    std::filesystem::create_directories(output_directory);
+
     {
         kpex::tech::Technology tech;
         sky130A::buildTech(tech);
-        writeTech("sky130A", tech);
+        writeTech(output_directory, "sky130A", tech);
     }
 
     {
         kpex::tech::Technology tech;
         ihp_sg13g2::buildTech(tech);
-        writeTech("ihp_sg13g2", tech);
+        writeTech(output_directory, "ihp_sg13g2", tech);
     }
 
     // Optional:  Delete all global objects allocated by libprotobuf.
     google::protobuf::ShutdownProtobufLibrary();
+
+    return 0;
 }
 

klayout_pex/kpex_cli.py

@@ -96,14 +96,15 @@ class PDK(StrEnum):
 
     @cached_property
     def config(self) -> PDKConfig:
-        # NOTE: installation paths of resources in the distribution wheel differes from source repo
+        # NOTE: installation paths of resources in the distribution wheel differs from source repo
         base_dir = os.path.dirname(os.path.realpath(__file__))
+
         if os.path.isdir(os.path.join(base_dir, '..', '.git')): # in source repo
             base_dir = os.path.dirname(base_dir)
-            tech_pb_json_dir = os.path.join(base_dir, 'build')
+            tech_pb_json_dir = os.path.join(base_dir, 'klayout_pex_protobuf')
         else:  # site-packages/klayout_pex -> site-packages/klayout_pex_protobuf
             tech_pb_json_dir = os.path.join(os.path.dirname(base_dir), 'klayout_pex_protobuf')
-        
+
         match self:
             case PDK.IHP_SG13G2:
                 return PDKConfig(
@@ -279,6 +280,10 @@ def validate_args(args: argparse.Namespace):
             error(f"Can't read technology file at path {args.tech_pbjson_path}")
             found_errors = True
 
+        if not os.path.isfile(args.lvs_script_path):
+            error(f"Can't locate LVS script path at {args.lvs_script_path}")
+            found_errors = True
+
         rule('Input Layout')
 
         # input mode: LVS or existing LVSDB?

klayout_pex/version.py

@@ -21,4 +21,4 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 # --------------------------------------------------------------------------------
 #
-__version__ = "0.1.8"
+__version__ = "0.1.12"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "klayout-pex"
-version = "0.1.8"
+version = "0.1.12"
 description = "Parasitic Extraction Tool for KLayout"
 authors = ["Martin Köhler <[email protected]>"]
 homepage = "https://martinjankoehler.github.io/klayout-pex-website/"
@@ -27,20 +27,19 @@ classifiers = [
 packages = [
     { include = "klayout_pex" },
     { include = "pdk", to = "klayout_pex" },
-    { include = "process_stack_pb2.py", from = "build/python/klayout_pex_protobuf/", to = "klayout_pex_protobuf" },
-    { include = "process_parasitics_pb2.py", from = "build/python/klayout_pex_protobuf/", to = "klayout_pex_protobuf" },
-    { include = "tech_pb2.py", from = "build/python/klayout_pex_protobuf/", to = "klayout_pex_protobuf" },
-    { include = "__init__.py", from = "klayout_pex_protobuf/", to = "klayout_pex_protobuf" },
-    { include = "sky130A_tech.pb.json", from = "build", to = "klayout_pex_protobuf" },
-    { include = "ihp_sg13g2_tech.pb.json", from = "build", to = "klayout_pex_protobuf"  },
+    { include = "klayout_pex_protobuf" }
 ]
 include = [
     "kpex",
-    "klayout_pex_protobuf",
     "pdk/**/*",
-    "build/python/klayout_pex_protobuf/*",
-    "build/*.json",
-    "klayout_pex_protobuf/__init__.py"
+    "klayout_pex_protobuf/__init__.py",
+    # NOTE: explicitly mention generated protobuf files,
+    #       otherwise the .gitignore entries would suppress them
+    { path = "klayout_pex_protobuf/ihp_sg13g2_tech.pb.json", format = ["sdist", "wheel"] },
+    { path = "klayout_pex_protobuf/sky130A_tech.pb.json", format = ["sdist", "wheel"] },
+    { path = "klayout_pex_protobuf/tech_pb2.py", format = ["sdist", "wheel"] },
+    { path = "klayout_pex_protobuf/process_parasitics_pb2.py", format = ["sdist", "wheel"] },
+    { path = "klayout_pex_protobuf/process_stack_pb2.py", format = ["sdist", "wheel"] }
 ]
 
 [tool.poetry.urls]
@@ -70,7 +69,7 @@ pylint-json2html = "^0.5.0"
 licenseheaders = "^0.8.8"
 
 [build-system]
-requires = ["poetry-core"]
+requires = ["poetry-core>=1.0.0", "setuptools>=42"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.pytest.ini_options]

tests/klayout/netlist_expander_test.py

@@ -50,7 +50,7 @@ def klayout_testdata_dir(self) -> str:
     @property
     def tech_info_json_path(self) -> str:
         return os.path.realpath(os.path.join(__file__, '..', '..', '..',
-                                             'build', 'sky130A_tech.pb.json'))
+                                             'klayout_pex_protobuf', 'sky130A_tech.pb.json'))
 
     def test_netlist_expansion(self):
         exp = NetlistExpander()
@@ -82,4 +82,4 @@ def test_netlist_expansion(self):
         debug(f"Wrote expanded netlist to: {out_path}")
 
         allure.attach.file(csv_path, attachment_type=allure.attachment_type.CSV)
-        allure.attach.file(out_path, attachment_type=allure.attachment_type.TEXT)
+        allure.attach.file(out_path, attachment_type=allure.attachment_type.TEXT)