From 7f856abc28b975c20558295f59f4a1ae24e7671b Mon Sep 17 00:00:00 2001 From: Vladimir Milosevic Date: Mon, 12 Aug 2024 11:06:36 +0000 Subject: [PATCH 01/14] Build docker image Remove old pybuda CI files Added Dockerfile Added docker image build workflow --- .github/Dockerfile | 75 +++++++++++++++++++ .github/actions/install-deps/action.yml | 31 -------- .../actions/install-deps/dependencies.json | 12 --- .github/workflows/build-artifacts.yml | 24 ------ .github/workflows/build-image.yml | 41 ++++++++++ .github/workflows/post-commit-workflow.yml | 13 ---- .github/workflows/pull-request-workflow.yml | 13 ---- .gitlab/issue_templates/API.md | 17 ----- .gitlab/issue_templates/Default.md | 21 ------ .gitlab/issue_templates/Feature.md | 15 ---- 10 files changed, 116 insertions(+), 146 deletions(-) create mode 100644 .github/Dockerfile delete mode 100644 .github/actions/install-deps/action.yml delete mode 100644 .github/actions/install-deps/dependencies.json delete mode 100644 .github/workflows/build-artifacts.yml create mode 100644 .github/workflows/build-image.yml delete mode 100644 .github/workflows/post-commit-workflow.yml delete mode 100644 .github/workflows/pull-request-workflow.yml delete mode 100644 .gitlab/issue_templates/API.md delete mode 100644 .gitlab/issue_templates/Default.md delete mode 100644 .gitlab/issue_templates/Feature.md diff --git a/.github/Dockerfile b/.github/Dockerfile new file mode 100644 index 000000000..ce998dfbb --- /dev/null +++ b/.github/Dockerfile @@ -0,0 +1,75 @@ +FROM ubuntu:22.04 +SHELL ["/bin/bash", "-c"] + +# Set environment variables +ENV DEBIAN_FRONTEND=noninteractive +ENV TTMLIR_TOOLCHAIN_DIR=/opt/ttmlir-toolchain +ENV TTFORGE_TOOLCHAIN_DIR=/opt/ttforge-toolchain +ENV PROJECT_NAME=tt-forge + +ENV GITHUB_TOKEN=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 + +# Install dependencies +RUN apt-get update && apt-get install -y \ + software-properties-common \ + build-essential \ + python3-dev \ + python3-venv \ + python3-pip \ + git \ + git-lfs \ + libhwloc-dev \ + pandoc \ + libtbb-dev \ + libcapstone-dev \ + pkg-config \ + linux-tools-generic \ + ninja-build \ + wget \ + libgtest-dev \ + cmake \ + ccache \ + doxygen \ + libgtest-dev \ + libgmock-dev \ + graphviz \ + patchelf \ + libyaml-cpp-dev \ + libboost-all-dev + +# Install clang 17 +RUN wget https://apt.llvm.org/llvm.sh && \ + chmod u+x llvm.sh && \ + ./llvm.sh 17 && \ + apt install -y libc++-17-dev libc++abi-17-dev && \ + ln -s /usr/bin/clang-17 /usr/bin/clang && \ + ln -s /usr/bin/clang++-17 /usr/bin/clang++ + +# Install python packages +RUN pip install cmake \ + pytest + +# Create a directory for the build and toolchain +ARG BUILD_DIR=/home/build +RUN mkdir -p $BUILD_DIR && \ + mkdir -p $TTMLIR_TOOLCHAIN_DIR && \ + mkdir -p $TTFORGE_TOOLCHAIN_DIR + +# Clone the project and update submodules +RUN git clone https://$GITHUB_TOKEN@github.com/tenstorrent/tt-forge.git $BUILD_DIR/$PROJECT_NAME && \ + cd $BUILD_DIR/$PROJECT_NAME && \ + git submodule update --init --recursive -f + +# Build the toolchain +WORKDIR $BUILD_DIR/$PROJECT_NAME +RUN source env/activate && \ + cmake -B env/build env && \ + cmake --build env/build + +# Build project to test the container +RUN source env/activate && \ + cmake -G Ninja -B build . && \ + cmake --build build + +# # Clean up the build directory +RUN rm -rf $BUILD_DIR/$PROJECT_NAME diff --git a/.github/actions/install-deps/action.yml b/.github/actions/install-deps/action.yml deleted file mode 100644 index 1f13767d9..000000000 --- a/.github/actions/install-deps/action.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: "Install dependencies" -description: "Installs dependencies on GitHub Actions runners" - -inputs: - os: - description: 'Runner OS' - required: true - -runs: - using: "composite" - steps: - - name: Verify ubuntu only - shell: bash - run: | - if ! echo ${{ inputs.os }} | grep -q "ubuntu"; then - echo "${{ inputs.os }} does not seem to be ubuntu" - fi - - name: Assert requested os exists in dependencies - shell: bash - run: | - if ! jq -e ".\"${{ inputs.os }}\" != null" $GITHUB_ACTION_PATH/dependencies.json; then - echo "${{ inputs.os }} does not exist as a supported os in $GITHUB_ACTION_PATH/dependencies.json" - fi - - name: Retrieve and install pkg deps based on OS - id: retrieve-pkg-deps - shell: bash - run: | - DEPENDENCIES=$(jq -r --arg os "${{ inputs.os }}" '.[$os] | .[]' $GITHUB_ACTION_PATH/dependencies.json) - echo $DEPENDENCIES - sudo apt update - sudo apt install $DEPENDENCIES diff --git a/.github/actions/install-deps/dependencies.json b/.github/actions/install-deps/dependencies.json deleted file mode 100644 index 2faab1678..000000000 --- a/.github/actions/install-deps/dependencies.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "ubuntu-22.04": [ - "software-properties-common", - "build-essential", - "python3.10-venv", - "libyaml-cpp-dev", - "libboost-all-dev", - "libsndfile1", - "libhwloc-dev", - "libzmq3-dev" - ] -} diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml deleted file mode 100644 index 5e52e7e8f..000000000 --- a/.github/workflows/build-artifacts.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Build artifacts - -on: - workflow_dispatch: - workflow_call: - -env: - PYTHON_VERSION: "python3.10" - -jobs: - build-artifacts: - strategy: - matrix: - arch: ["grayskull"] - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/install-deps - with: - os: ubuntu-22.04 - - name: Update submodule - run: git submodule update --init --recursive - - name: Build for ${{ matrix.arch }} - run: source env_for_silicon.sh \ No newline at end of file diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml new file mode 100644 index 000000000..9480484e0 --- /dev/null +++ b/.github/workflows/build-image.yml @@ -0,0 +1,41 @@ +name: Build and Publish Docker Image + +on: + workflow_dispatch: + workflow_call: + pull_request: + branches: [ "main" ] + push: + branches: [ "main" ] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + sparse-checkout: | + .github/Dockerfile + sparse-checkout-cone-mode: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: .github + file: .github/Dockerfile + push: true + tags: | + ghcr.io/${{ github.repository }}/tt-forge-ubuntu-22-04:${{ github.sha }} + ghcr.io/${{ github.repository }}/tt-forge-ubuntu-22-04:latest diff --git a/.github/workflows/post-commit-workflow.yml b/.github/workflows/post-commit-workflow.yml deleted file mode 100644 index ceb7d58da..000000000 --- a/.github/workflows/post-commit-workflow.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Post commit workflow - -on: - workflow_dispatch: - workflow_call: - push: - branches: - - main - -jobs: - build-artifacts: - uses: ./.github/workflows/build-artifacts.yml - secrets: inherit \ No newline at end of file diff --git a/.github/workflows/pull-request-workflow.yml b/.github/workflows/pull-request-workflow.yml deleted file mode 100644 index c5fbe7958..000000000 --- a/.github/workflows/pull-request-workflow.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Pull request workflow - -on: - workflow_dispatch: - workflow_call: - pull_request: - branches: - - main - -jobs: - build-artifacts: - uses: ./.github/workflows/build-artifacts.yml - secrets: inherit \ No newline at end of file diff --git a/.gitlab/issue_templates/API.md b/.gitlab/issue_templates/API.md deleted file mode 100644 index 6c5dcb482..000000000 --- a/.gitlab/issue_templates/API.md +++ /dev/null @@ -1,17 +0,0 @@ -## API description - -*(Brief usage guide and motivations)* - -## Adoption Impact - -*(Breakdown the impact for adopting the new API and impact on existing FE codebase)* - -## Deprecation -*(Delete this section for new API, or changes with no API interface impact)* - -Procedure: -- FE integrates BE, old interface is still used -- FE implements new API after BE uplift merged to main -- Re-assign this issue on BE to remove the API - -*(List the APIs being replaced and implementation suggetsions)* diff --git a/.gitlab/issue_templates/Default.md b/.gitlab/issue_templates/Default.md deleted file mode 100644 index d4405aec4..000000000 --- a/.gitlab/issue_templates/Default.md +++ /dev/null @@ -1,21 +0,0 @@ -## Description - -*(Summarize the bug encountered concisely)* - -## Reproduce - -*(How one can reproduce the issue - this is very important)* - -```bash -repro command -``` - -## Observed behavior - -*(What actually happens)* - -## Expected behavior - -*(What you should see instead)* - -/label ~state::Screen diff --git a/.gitlab/issue_templates/Feature.md b/.gitlab/issue_templates/Feature.md deleted file mode 100644 index 6e2fcf2d7..000000000 --- a/.gitlab/issue_templates/Feature.md +++ /dev/null @@ -1,15 +0,0 @@ -## Overview - -*(Summarize the feature)* - -## Motivation - -*(Outline what this feature intends to achieve)* - -## Implementation / Code Impact - -*(Outline how the feature might be implemented + impacted code components)* - -## Test Plan - -*(List of test cases to implement / models that cover this feature)* From 6b1dc5d8a9e2cca2ac49162f1b6e990d872e8ff6 Mon Sep 17 00:00:00 2001 From: Vladimir Milosevic Date: Tue, 13 Aug 2024 07:54:56 +0000 Subject: [PATCH 02/14] Adding build workflow --- .github/workflows/build-image.yml | 6 +-- .github/workflows/docker-build.yml | 67 ++++++++++++++++++++++++++++++ .github/workflows/on-pr.yml | 11 +++++ .github/workflows/on-push.yml | 11 +++++ 4 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/docker-build.yml create mode 100644 .github/workflows/on-pr.yml create mode 100644 .github/workflows/on-push.yml diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 9480484e0..5709eeab7 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -3,14 +3,10 @@ name: Build and Publish Docker Image on: workflow_dispatch: workflow_call: - pull_request: - branches: [ "main" ] - push: - branches: [ "main" ] jobs: build: - runs-on: ubuntu-latest + runs-on: self-hosted # Github runners dont have enough storage space to build this image steps: - name: Checkout repository diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 000000000..65ef59862 --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,67 @@ +name: Build in Docker + +on: + workflow_dispatch: + workflow_call: + +jobs: + + build-and-test: + + strategy: + fail-fast: false + matrix: + image: ["ubuntu-22-04"] + build: [ + {runs-on: ubuntu-latest}, + {runs-on: self-hosted}, + ] + + runs-on: ${{ matrix.build.runs-on }} + + container: + image: ghcr.io/${{ github.repository }}/tt-forge-${{ matrix.image }}:latest + options: --user root + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 # Fetch all history and tags + + - name: Git safe dir + run: git config --global --add safe.directory ${{ steps.strings.outputs.work-dir }} + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + create-symlink: true + key: ${{ matrix.build.runs-on }}-runtime-${{ matrix.build.enable_runtime }}-${{ env.SDK_VERSION }} + + - name: Build + shell: bash + run: | + source env/activate + cmake -G Ninja -B build . + cmake --build build + + - name: Run Test + shell: bash + run: | + source env/activate + pytest -svv pybuda/test/mlir/mnist/test_inference.py + pytest -svv pybuda/test/mlir/test_ops.py + continue-on-error: true + + - name: Upload Test Report + uses: actions/upload-artifact@v4 + with: + name: test-reports-${{ matrix.build.runs-on }} + path: build/test/report.xml + + - name: Show Test Report + uses: mikepenz/action-junit-report@v4 + if: success() || failure() + with: + report_paths: build/test/report.xml + check_name: MLIR Tests diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml new file mode 100644 index 000000000..e179fb633 --- /dev/null +++ b/.github/workflows/on-pr.yml @@ -0,0 +1,11 @@ +name: On PR + +on: + workflow_dispatch: + pull_request: + branches: [ "main" ] + +jobs: + docker-build: + uses: ./.github/workflows/docker-build.yml + secrets: inherit diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml new file mode 100644 index 000000000..1bb669f77 --- /dev/null +++ b/.github/workflows/on-push.yml @@ -0,0 +1,11 @@ +name: On push + +on: + workflow_dispatch: + push: + branches: [ "main" ] + +jobs: + docker-build: + uses: ./.github/workflows/docker-build.yml + secrets: inherit From d51abbaf22064978a6f1100c2f2316040bfd7861 Mon Sep 17 00:00:00 2001 From: Vladimir Milosevic Date: Tue, 13 Aug 2024 08:12:54 +0000 Subject: [PATCH 03/14] Add output workdir --- .github/workflows/docker-build.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 65ef59862..bfb08d3af 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -29,6 +29,13 @@ jobs: submodules: recursive fetch-depth: 0 # Fetch all history and tags + - name: Set reusable strings + id: strings + shell: bash + run: | + echo "work-dir=$(pwd)" >> "$GITHUB_OUTPUT" + echo "build-output-dir=$(pwd)/build" >> "$GITHUB_OUTPUT" + - name: Git safe dir run: git config --global --add safe.directory ${{ steps.strings.outputs.work-dir }} From b049f161cd029c6f76b83a02c693681f58f6965a Mon Sep 17 00:00:00 2001 From: Vladimir Milosevic Date: Tue, 13 Aug 2024 10:08:34 +0000 Subject: [PATCH 04/14] Build image on self hosted --- .github/Dockerfile | 4 +--- .github/workflows/build-image.yml | 4 ++++ .github/workflows/on-pr.yml | 4 ++-- .github/workflows/on-push.yml | 4 ++-- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/Dockerfile b/.github/Dockerfile index ce998dfbb..d156248e1 100644 --- a/.github/Dockerfile +++ b/.github/Dockerfile @@ -7,8 +7,6 @@ ENV TTMLIR_TOOLCHAIN_DIR=/opt/ttmlir-toolchain ENV TTFORGE_TOOLCHAIN_DIR=/opt/ttforge-toolchain ENV PROJECT_NAME=tt-forge -ENV GITHUB_TOKEN=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 - # Install dependencies RUN apt-get update && apt-get install -y \ software-properties-common \ @@ -56,7 +54,7 @@ RUN mkdir -p $BUILD_DIR && \ mkdir -p $TTFORGE_TOOLCHAIN_DIR # Clone the project and update submodules -RUN git clone https://$GITHUB_TOKEN@github.com/tenstorrent/tt-forge.git $BUILD_DIR/$PROJECT_NAME && \ +RUN git clone https://github.com/tenstorrent/tt-forge.git $BUILD_DIR/$PROJECT_NAME && \ cd $BUILD_DIR/$PROJECT_NAME && \ git submodule update --init --recursive -f diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 5709eeab7..f1b317c8e 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -3,6 +3,10 @@ name: Build and Publish Docker Image on: workflow_dispatch: workflow_call: + pull_request: + branches: [ "main" ] + push: + branches: [ "main" ] jobs: build: diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index e179fb633..758fa8f22 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -2,8 +2,8 @@ name: On PR on: workflow_dispatch: - pull_request: - branches: [ "main" ] + # pull_request: + # branches: [ "main" ] jobs: docker-build: diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 1bb669f77..b72d0bec7 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -2,8 +2,8 @@ name: On push on: workflow_dispatch: - push: - branches: [ "main" ] + # push: + # branches: [ "main" ] jobs: docker-build: From 49c5084c69a5d1235446b0f3e47f4c6fe5511c26 Mon Sep 17 00:00:00 2001 From: Vladimir Milosevic Date: Tue, 13 Aug 2024 10:15:05 +0000 Subject: [PATCH 05/14] Print user --- .github/workflows/build-image.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index f1b317c8e..e85e21847 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -11,8 +11,15 @@ on: jobs: build: runs-on: self-hosted # Github runners dont have enough storage space to build this image - + steps: + + - run: | + echo Pwd: $(pwd) + echo User: $(whoami) + ls -la /home/ubuntu/actions-runner/_work/tt-forge/tt-forge + shell: bash + - name: Checkout repository uses: actions/checkout@v4 with: From 8f41eeca2817ef7dec025f294efe9dec3f588791 Mon Sep 17 00:00:00 2001 From: Vladimir Milosevic Date: Tue, 13 Aug 2024 11:16:58 +0000 Subject: [PATCH 06/14] Run docker build --- .github/workflows/docker-build.yml | 5 ++--- .github/workflows/on-pr.yml | 4 ++-- .github/workflows/on-push.yml | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index bfb08d3af..cc15ace07 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -11,16 +11,15 @@ jobs: strategy: fail-fast: false matrix: - image: ["ubuntu-22-04"] build: [ - {runs-on: ubuntu-latest}, + # {runs-on: ubuntu-latest}, {runs-on: self-hosted}, ] runs-on: ${{ matrix.build.runs-on }} container: - image: ghcr.io/${{ github.repository }}/tt-forge-${{ matrix.image }}:latest + image: ghcr.io/${{ github.repository }}/tt-forge-ubuntu-22-04:latest options: --user root steps: diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index 758fa8f22..e179fb633 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -2,8 +2,8 @@ name: On PR on: workflow_dispatch: - # pull_request: - # branches: [ "main" ] + pull_request: + branches: [ "main" ] jobs: docker-build: diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index b72d0bec7..1bb669f77 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -2,8 +2,8 @@ name: On push on: workflow_dispatch: - # push: - # branches: [ "main" ] + push: + branches: [ "main" ] jobs: docker-build: From 2d72d4fd8a9866da6ad52c3159987bc877045662 Mon Sep 17 00:00:00 2001 From: Vladimir Milosevic Date: Wed, 14 Aug 2024 07:14:06 +0000 Subject: [PATCH 07/14] Mount device and volumes to docker --- .github/Dockerfile | 4 ++-- .github/workflows/build-image.yml | 4 ---- .github/workflows/docker-build.yml | 17 +++++++++++------ 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/Dockerfile b/.github/Dockerfile index d156248e1..03a16d3d3 100644 --- a/.github/Dockerfile +++ b/.github/Dockerfile @@ -24,7 +24,6 @@ RUN apt-get update && apt-get install -y \ linux-tools-generic \ ninja-build \ wget \ - libgtest-dev \ cmake \ ccache \ doxygen \ @@ -54,7 +53,8 @@ RUN mkdir -p $BUILD_DIR && \ mkdir -p $TTFORGE_TOOLCHAIN_DIR # Clone the project and update submodules -RUN git clone https://github.com/tenstorrent/tt-forge.git $BUILD_DIR/$PROJECT_NAME && \ +# Pass in PAT to clone private repositories +RUN git clone https://$GITHUB_TOKEN@github.com/tenstorrent/tt-forge.git $BUILD_DIR/$PROJECT_NAME && \ cd $BUILD_DIR/$PROJECT_NAME && \ git submodule update --init --recursive -f diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index e85e21847..0f60a7306 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -3,10 +3,6 @@ name: Build and Publish Docker Image on: workflow_dispatch: workflow_call: - pull_request: - branches: [ "main" ] - push: - branches: [ "main" ] jobs: build: diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index cc15ace07..f5c235c09 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -11,17 +11,22 @@ jobs: strategy: fail-fast: false matrix: - build: [ - # {runs-on: ubuntu-latest}, - {runs-on: self-hosted}, - ] + build: + - runs-on: self-hosted runs-on: ${{ matrix.build.runs-on }} container: image: ghcr.io/${{ github.repository }}/tt-forge-ubuntu-22-04:latest - options: --user root - + options: | + --user root + --device /dev/tenstorrent/0 + volumes: + - /dev/hugepages:/dev/hugepages + - /dev/hugepages-1G:/dev/hugepages-1G + - /etc/udev/rules.d:/etc/udev/rules.d + - /lib/modules:/lib/modules + - /opt/tt_metal_infra/provisioning/provisioning_env:/opt/tt_metal_infra/provisioning/provisioning_env steps: - uses: actions/checkout@v4 with: From b8c85389de1f25f6a10d47af57e548f7f13c8c6a Mon Sep 17 00:00:00 2001 From: Vladimir Milosevic Date: Wed, 14 Aug 2024 07:22:49 +0000 Subject: [PATCH 08/14] Fix docker options --- .github/workflows/docker-build.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index f5c235c09..82c75a4a6 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -18,9 +18,7 @@ jobs: container: image: ghcr.io/${{ github.repository }}/tt-forge-ubuntu-22-04:latest - options: | - --user root - --device /dev/tenstorrent/0 + options: --user root --device /dev/tenstorrent/0 volumes: - /dev/hugepages:/dev/hugepages - /dev/hugepages-1G:/dev/hugepages-1G From 5388672acca8d354aa331a32e1ea009148d4c1f5 Mon Sep 17 00:00:00 2001 From: Vladimir Milosevic Date: Wed, 14 Aug 2024 09:49:24 +0000 Subject: [PATCH 09/14] Move ttmlir env build to forge env build --- .github/Dockerfile | 7 +++++-- env/CMakeLists.txt | 9 +++++++++ third_party/build_mlir.sh | 3 --- third_party/build_mlir_env.sh | 9 +++++++++ 4 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 third_party/build_mlir_env.sh diff --git a/.github/Dockerfile b/.github/Dockerfile index 03a16d3d3..917073430 100644 --- a/.github/Dockerfile +++ b/.github/Dockerfile @@ -6,6 +6,7 @@ ENV DEBIAN_FRONTEND=noninteractive ENV TTMLIR_TOOLCHAIN_DIR=/opt/ttmlir-toolchain ENV TTFORGE_TOOLCHAIN_DIR=/opt/ttforge-toolchain ENV PROJECT_NAME=tt-forge +ARG GITHUB_TOKEN # Install dependencies RUN apt-get update && apt-get install -y \ @@ -69,5 +70,7 @@ RUN source env/activate && \ cmake -G Ninja -B build . && \ cmake --build build -# # Clean up the build directory -RUN rm -rf $BUILD_DIR/$PROJECT_NAME +# Clean up the build directory +RUN cd .. && \ + rm -rf $BUILD_DIR/$PROJECT_NAME && \ + unset GITHUB_TOKEN \ No newline at end of file diff --git a/env/CMakeLists.txt b/env/CMakeLists.txt index ebeb8fce2..b93219cd5 100644 --- a/env/CMakeLists.txt +++ b/env/CMakeLists.txt @@ -36,3 +36,12 @@ add_custom_target(python-venv CURRENT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} TTFORGE_VENV_DIR=${TTFORGE_VENV_DIR} bash ${CMAKE_CURRENT_SOURCE_DIR}/create_venv.sh) + +add_custom_target(build_tt_mlir_env ALL + COMMAND ${CMAKE_COMMAND} -E env + TTMLIR_TOOLCHAIN_DIR=${TTMLIR_TOOLCHAIN_DIR} # Export TTMLIR_TOOLCHAIN_DIR to use + TTMLIR_VENV_DIR=${TTMLIR_VENV_DIR} # Export TTMLIR_VENV_DIR to use + bash ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/build_mlir_env.sh + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/tt-mlir + USES_TERMINAL +) \ No newline at end of file diff --git a/third_party/build_mlir.sh b/third_party/build_mlir.sh index 8078e095f..2f4c9a648 100644 --- a/third_party/build_mlir.sh +++ b/third_party/build_mlir.sh @@ -5,9 +5,6 @@ source env/activate -cmake -B env/build env -cmake --build env/build - build_type=${BUILD_TYPE:-Release} c_compiler=${C_COMPILER:-clang} cxx_compiler=${CXX_COMPILER:-clang++} diff --git a/third_party/build_mlir_env.sh b/third_party/build_mlir_env.sh new file mode 100644 index 000000000..0adc0d5ed --- /dev/null +++ b/third_party/build_mlir_env.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# SPDX-FileCopyrightText: (c) 2024 Tenstorrent AI ULC +# +# SPDX-License-Identifier: Apache-2.0 + +source env/activate + +cmake -B env/build env +cmake --build env/build From 37139c574f6927852a4c7244810a4b2d991e9293 Mon Sep 17 00:00:00 2001 From: Vladimir Milosevic Date: Wed, 14 Aug 2024 11:59:43 +0000 Subject: [PATCH 10/14] Only run specified test --- .github/workflows/docker-build.yml | 3 +-- pytest.ini | 10 ++++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 82c75a4a6..42d15a17e 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -58,8 +58,7 @@ jobs: shell: bash run: | source env/activate - pytest -svv pybuda/test/mlir/mnist/test_inference.py - pytest -svv pybuda/test/mlir/test_ops.py + pytest continue-on-error: true - name: Upload Test Report diff --git a/pytest.ini b/pytest.ini index e8d0284e0..de6cbe25b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,14 @@ # TF & some other libraries report a bunch of deprecation warnings [pytest] + +# Ignore specific tests +addopts = -svv --junit-xml=reports/report.xml + +# Where pytest should look for tests +testpaths = + pybuda/test/mlir/test_ops.py + pybuda/test/test_api.py + pybuda/test/test_inference.py + filterwarnings = ignore::DeprecationWarning From ec0106d91299e2d94561e843a7c276733df1be73 Mon Sep 17 00:00:00 2001 From: Vladimir Milosevic Date: Wed, 14 Aug 2024 12:09:19 +0000 Subject: [PATCH 11/14] Run basic tests --- pytest.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index de6cbe25b..a6924e5b5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,7 +8,6 @@ addopts = -svv --junit-xml=reports/report.xml testpaths = pybuda/test/mlir/test_ops.py pybuda/test/test_api.py - pybuda/test/test_inference.py filterwarnings = ignore::DeprecationWarning From 800a8baae870b219b5805bb3a1027b1182e24804 Mon Sep 17 00:00:00 2001 From: Vladimir Milosevic Date: Wed, 14 Aug 2024 12:19:39 +0000 Subject: [PATCH 12/14] Skip failing test --- .github/workflows/docker-build.yml | 4 ++-- .gitignore | 3 +++ pybuda/test/mlir/test_ops.py | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 42d15a17e..c4a05b58f 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -65,11 +65,11 @@ jobs: uses: actions/upload-artifact@v4 with: name: test-reports-${{ matrix.build.runs-on }} - path: build/test/report.xml + path: reports/report.xml - name: Show Test Report uses: mikepenz/action-junit-report@v4 if: success() || failure() with: - report_paths: build/test/report.xml + report_paths: reports/report.xml check_name: MLIR Tests diff --git a/.gitignore b/.gitignore index 4dff4f040..82131f71a 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,6 @@ build_deps/ # ClangD compile_commands.json \n\n# Exclude LFS files to keep the public repo small + +# Test reports +reports/report.xml diff --git a/pybuda/test/mlir/test_ops.py b/pybuda/test/mlir/test_ops.py index bcb145fda..d3c2ddc00 100644 --- a/pybuda/test/mlir/test_ops.py +++ b/pybuda/test/mlir/test_ops.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 import os +import pytest import torch from torch import nn @@ -89,7 +90,7 @@ def forward(self, a): co_out = [co.to("cpu") for co in co_out] assert [torch.allclose(fo, co) for fo, co in zip(fw_out, co_out)] - +@pytest.mark.skip(reason="This is not ready yet") def test_linear(): class Linear(nn.Module): def __init__(self): From 95b3ea777b6d35aa2443e2f7fbdf20f9d3fed676 Mon Sep 17 00:00:00 2001 From: Vladimir Milosevic Date: Wed, 14 Aug 2024 13:48:33 +0000 Subject: [PATCH 13/14] Renamed test report --- .github/workflows/docker-build.yml | 2 +- env/CMakeLists.txt | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index c4a05b58f..365fbe546 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -72,4 +72,4 @@ jobs: if: success() || failure() with: report_paths: reports/report.xml - check_name: MLIR Tests + check_name: TT-Forge Tests diff --git a/env/CMakeLists.txt b/env/CMakeLists.txt index b93219cd5..ae1916656 100644 --- a/env/CMakeLists.txt +++ b/env/CMakeLists.txt @@ -37,11 +37,12 @@ add_custom_target(python-venv TTFORGE_VENV_DIR=${TTFORGE_VENV_DIR} bash ${CMAKE_CURRENT_SOURCE_DIR}/create_venv.sh) +set(TT_MLIR_ROOT_DIR ${CMAKE_SOURCE_DIR}/third_party/tt-mlir) add_custom_target(build_tt_mlir_env ALL COMMAND ${CMAKE_COMMAND} -E env TTMLIR_TOOLCHAIN_DIR=${TTMLIR_TOOLCHAIN_DIR} # Export TTMLIR_TOOLCHAIN_DIR to use TTMLIR_VENV_DIR=${TTMLIR_VENV_DIR} # Export TTMLIR_VENV_DIR to use - bash ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/build_mlir_env.sh - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/tt-mlir + bash ${CMAKE_SOURCE_DIR}/third_party/build_mlir_env.sh + WORKING_DIRECTORY ${TT_MLIR_ROOT_DIR} USES_TERMINAL ) \ No newline at end of file From 1c051ab4b3cef27f472c0de380a1368127348448 Mon Sep 17 00:00:00 2001 From: Vladimir Milosevic Date: Wed, 14 Aug 2024 13:52:56 +0000 Subject: [PATCH 14/14] Revert cmake change --- env/CMakeLists.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/env/CMakeLists.txt b/env/CMakeLists.txt index ae1916656..b93219cd5 100644 --- a/env/CMakeLists.txt +++ b/env/CMakeLists.txt @@ -37,12 +37,11 @@ add_custom_target(python-venv TTFORGE_VENV_DIR=${TTFORGE_VENV_DIR} bash ${CMAKE_CURRENT_SOURCE_DIR}/create_venv.sh) -set(TT_MLIR_ROOT_DIR ${CMAKE_SOURCE_DIR}/third_party/tt-mlir) add_custom_target(build_tt_mlir_env ALL COMMAND ${CMAKE_COMMAND} -E env TTMLIR_TOOLCHAIN_DIR=${TTMLIR_TOOLCHAIN_DIR} # Export TTMLIR_TOOLCHAIN_DIR to use TTMLIR_VENV_DIR=${TTMLIR_VENV_DIR} # Export TTMLIR_VENV_DIR to use - bash ${CMAKE_SOURCE_DIR}/third_party/build_mlir_env.sh - WORKING_DIRECTORY ${TT_MLIR_ROOT_DIR} + bash ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/build_mlir_env.sh + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/tt-mlir USES_TERMINAL ) \ No newline at end of file