From bb3bff1cb4d1a12f7ae0b39f1230185022211389 Mon Sep 17 00:00:00 2001 From: Aleks Knezevic Date: Tue, 17 Sep 2024 11:50:59 +0000 Subject: [PATCH 01/11] build env, wip --- .github/Dockerfile.ci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/Dockerfile.ci b/.github/Dockerfile.ci index 801a85d..ff395cb 100644 --- a/.github/Dockerfile.ci +++ b/.github/Dockerfile.ci @@ -16,7 +16,7 @@ RUN mkdir -p $BUILD_DIR && \ mkdir -p $TTMLIR_TOOLCHAIN_DIR # Clone tt-mlir -RUN git clone git@github.com:tenstorrent/tt-mlir.git $BUILD_DIR/$PROJECT_NAME && \ +RUN git clone https://github.com/tenstorrent/tt-mlir.git $BUILD_DIR/$PROJECT_NAME && \ cd $BUILD_DIR/$PROJECT_NAME && \ git checkout $GIT_SHA From cb05921f4cac76cd72a6c3d11d8ff6992b1a8fce Mon Sep 17 00:00:00 2001 From: Aleks Knezevic Date: Tue, 17 Sep 2024 11:54:47 +0000 Subject: [PATCH 02/11] WIP --- .github/workflows/build-image.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index ee33830..d194093 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -56,18 +56,6 @@ jobs: tags: | ${{ env.CI_IMAGE_NAME}}:${{ github.sha }} - - name: Build and export IRD Docker image - uses: docker/build-push-action@v6 - with: - context: .github - file: .github/Dockerfile.ird - push: true - build-args: | - GIT_SHA=${{ github.sha }} - FROM_IMAGE=ci - tags: | - ${{ env.IRD_IMAGE_NAME}}:${{ github.sha }} - # Tag images as latest - name: Build and push base Docker image From a26338f6fe833d7e9ef102c191dc4e6835a04296 Mon Sep 17 00:00:00 2001 From: Aleks Knezevic Date: Tue, 17 Sep 2024 13:28:19 +0000 Subject: [PATCH 03/11] Somewhat hacky way to build toolchain --- .github/Dockerfile.ci | 6 +- third_party/CMakeLists.txt | 112 ++++++++++++++++++++++--------------- 2 files changed, 70 insertions(+), 48 deletions(-) diff --git a/.github/Dockerfile.ci b/.github/Dockerfile.ci index ff395cb..ad4ecc7 100644 --- a/.github/Dockerfile.ci +++ b/.github/Dockerfile.ci @@ -16,13 +16,15 @@ RUN mkdir -p $BUILD_DIR && \ mkdir -p $TTMLIR_TOOLCHAIN_DIR # Clone tt-mlir -RUN git clone https://github.com/tenstorrent/tt-mlir.git $BUILD_DIR/$PROJECT_NAME && \ +RUN git clone https://github.com/tenstorrent/$PROJECT_NAME.git $BUILD_DIR/$PROJECT_NAME && \ cd $BUILD_DIR/$PROJECT_NAME && \ git checkout $GIT_SHA # Build the toolchain WORKDIR $BUILD_DIR/$PROJECT_NAME -RUN source env/activate && \ +RUN cmake -B toolchain -DTOOLCHAIN=ON third_party/ && \ + cd third_party/tt-mlir/src/tt-mlir/ && \ + source env/activate && \ cmake -B env/build env && \ cmake --build env/build diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt index 8df6da1..4c856db 100644 --- a/third_party/CMakeLists.txt +++ b/third_party/CMakeLists.txt @@ -1,50 +1,70 @@ -include(ExternalProject) -add_subdirectory(pjrt_c_api) +set(TT_MLIR_VERSION "f955a08e8836e41a3333ca20a3bbe193ce91b2a0") +set(LOGURU_VERSION "4adaa185883e3c04da25913579c451d3c32cfac1") -set(TTMLIR_LIB_DIR ${PROJECT_SOURCE_DIR}/third_party/tt-mlir/src/tt-mlir-build/lib/SharedLib) -ExternalProject_Add( - tt-mlir - PREFIX ${TTPJRT_SOURCE_DIR}/third_party/tt-mlir - CMAKE_GENERATOR Ninja - CMAKE_ARGS - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_C_COMPILER=clang-17 - -DCMAKE_CXX_COMPILER=clang++-17 - -DTT_RUNTIME_ENABLE_TTNN=ON - -DTTMLIR_ENABLE_STABLEHLO=ON - -DTTMLIR_ENABLE_RUNTIME=ON - GIT_REPOSITORY git@github.com:tenstorrent/tt-mlir.git - GIT_TAG f955a08e8836e41a3333ca20a3bbe193ce91b2a0 - GIT_PROGRESS ON - INSTALL_COMMAND "" -) +if (TOOLCHAIN STREQUAL "ON") + cmake_minimum_required(VERSION 3.20) + project(ttmlir-toolchain LANGUAGES CXX C) + execute_process( + COMMAND git clone --recursive git@github.com:tenstorrent/tt-mlir.git ${PROJECT_SOURCE_DIR}/tt-mlir/src/tt-mlir + RESULT_VARIABLE result1 + OUTPUT_VARIABLE output1 + ERROR_VARIABLE error_output1 + ) + if (NOT ${result1} EQUAL 0) + message(FATAL_ERROR "Failed to clone tt-mlir: ${error_output1}") + endif() + execute_process( + COMMAND /bin/bash -c "cd ${PROJECT_SOURCE_DIR}/tt-mlir/src/tt-mlir && git checkout ${TT_MLIR_VERSION}" + ) +else() + include(ExternalProject) + add_subdirectory(pjrt_c_api) + set(TTMLIR_LIB_DIR ${PROJECT_SOURCE_DIR}/third_party/tt-mlir/src/tt-mlir-build/lib/SharedLib) + ExternalProject_Add( + tt-mlir + PREFIX ${TTPJRT_SOURCE_DIR}/third_party/tt-mlir + CMAKE_GENERATOR Ninja + CMAKE_ARGS + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_C_COMPILER=clang-17 + -DCMAKE_CXX_COMPILER=clang++-17 + -DTT_RUNTIME_ENABLE_TTNN=ON + -DTTMLIR_ENABLE_STABLEHLO=ON + -DTTMLIR_ENABLE_RUNTIME=ON + GIT_REPOSITORY git@github.com:tenstorrent/tt-mlir.git + GIT_TAG ${TT_MLIR_VERSION} + GIT_PROGRESS ON + INSTALL_COMMAND "" + ) -set_target_properties(tt-mlir PROPERTIES EXCLUDE_FROM_ALL TRUE) + set_target_properties(tt-mlir PROPERTIES EXCLUDE_FROM_ALL TRUE) -# Link to all TTMLIR libraries in ${TTMLIR_INSTALL_DIR}/install/lib -set(TTMLIR_LIB_DIR ${TTMLIR_LIB_DIR} PARENT_SCOPE) -message(STATUS "Linking to TTMLIR libraries in ${TTMLIR_LIB_DIR}") -file(GLOB TTMLIR_LIBRARIES "${TTMLIR_LIB_DIR}/*.so") -foreach(TTMLIR_LIBRARY ${TTMLIR_LIBRARIES}) - get_filename_component(lib_name ${TTMLIR_LIBRARY} NAME_WE) - string(REPLACE "lib" "" lib_name ${lib_name}) # Remove the "lib" prefix if it exists - add_library(${lib_name} SHARED IMPORTED GLOBAL) - set_target_properties(${lib_name} PROPERTIES EXCLUDE_FROM_ALL TRUE IMPORTED_LOCATION ${TTMLIR_LIBRARY}) - add_dependencies(${lib_name} tt-mlir) -endforeach() + # Link to all TTMLIR libraries in ${TTMLIR_INSTALL_DIR}/install/lib + set(TTMLIR_LIB_DIR ${TTMLIR_LIB_DIR} PARENT_SCOPE) + message(STATUS "Linking to TTMLIR libraries in ${TTMLIR_LIB_DIR}") + file(GLOB TTMLIR_LIBRARIES "${TTMLIR_LIB_DIR}/*.so") + foreach(TTMLIR_LIBRARY ${TTMLIR_LIBRARIES}) + get_filename_component(lib_name ${TTMLIR_LIBRARY} NAME_WE) + string(REPLACE "lib" "" lib_name ${lib_name}) # Remove the "lib" prefix if it exists + add_library(${lib_name} SHARED IMPORTED GLOBAL) + set_target_properties(${lib_name} PROPERTIES EXCLUDE_FROM_ALL TRUE IMPORTED_LOCATION ${TTMLIR_LIBRARY}) + add_dependencies(${lib_name} tt-mlir) + endforeach() -set(FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wno-undef -Wno-unsafe-buffer-usage -Wno-disabled-macro-expansion") -ExternalProject_Add( - loguru - PREFIX ${TTPJRT_SOURCE_DIR}/third_party/loguru - CMAKE_GENERATOR Ninja - CMAKE_ARGS - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_C_COMPILER=clang-17 - -DCMAKE_CXX_COMPILER=clang++-17 - -DCMAKE_CXX_FLAGS=${FLAGS} - -DCMAKE_INSTALL_PREFIX=${TTPJRT_SOURCE_DIR}/third_party/loguru/src/loguru-install - GIT_REPOSITORY git@github.com:emilk/loguru.git - GIT_TAG 4adaa185883e3c04da25913579c451d3c32cfac1 - GIT_PROGRESS ON -) + set(FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wno-undef -Wno-unsafe-buffer-usage -Wno-disabled-macro-expansion") + ExternalProject_Add( + loguru + PREFIX ${TTPJRT_SOURCE_DIR}/third_party/loguru + CMAKE_GENERATOR Ninja + CMAKE_ARGS + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_C_COMPILER=clang-17 + -DCMAKE_CXX_COMPILER=clang++-17 + -DCMAKE_CXX_FLAGS=${FLAGS} + -DCMAKE_INSTALL_PREFIX=${TTPJRT_SOURCE_DIR}/third_party/loguru/src/loguru-install + GIT_REPOSITORY git@github.com:emilk/loguru.git + GIT_TAG ${LOGURU_VERSION} + GIT_PROGRESS ON + ) + +endif() \ No newline at end of file From c97400e4dd55e3abac40835f95b3c7cda5b2a350 Mon Sep 17 00:00:00 2001 From: Aleks Knezevic Date: Tue, 17 Sep 2024 14:34:38 +0000 Subject: [PATCH 04/11] Try ssh --- .github/Dockerfile.ci | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/Dockerfile.ci b/.github/Dockerfile.ci index ad4ecc7..4828a45 100644 --- a/.github/Dockerfile.ci +++ b/.github/Dockerfile.ci @@ -15,8 +15,8 @@ RUN echo "Building $PROJECT_NAME at $GIT_SHA" RUN mkdir -p $BUILD_DIR && \ mkdir -p $TTMLIR_TOOLCHAIN_DIR -# Clone tt-mlir -RUN git clone https://github.com/tenstorrent/$PROJECT_NAME.git $BUILD_DIR/$PROJECT_NAME && \ +# Clone the project and update submodules +RUN git clone git@github.com:tenstorrent/$PROJECT_NAME.git $BUILD_DIR/$PROJECT_NAME && \ cd $BUILD_DIR/$PROJECT_NAME && \ git checkout $GIT_SHA From 8dfb35406bd5b03c529e28e74ea9ce56d2fd27f4 Mon Sep 17 00:00:00 2001 From: Aleks Knezevic Date: Tue, 17 Sep 2024 14:54:25 +0000 Subject: [PATCH 05/11] WIP --- .github/Dockerfile.ci | 6 ++---- .github/workflows/build-image.yml | 5 ++++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/Dockerfile.ci b/.github/Dockerfile.ci index 4828a45..4449880 100644 --- a/.github/Dockerfile.ci +++ b/.github/Dockerfile.ci @@ -15,10 +15,8 @@ RUN echo "Building $PROJECT_NAME at $GIT_SHA" RUN mkdir -p $BUILD_DIR && \ mkdir -p $TTMLIR_TOOLCHAIN_DIR -# Clone the project and update submodules -RUN git clone git@github.com:tenstorrent/$PROJECT_NAME.git $BUILD_DIR/$PROJECT_NAME && \ - cd $BUILD_DIR/$PROJECT_NAME && \ - git checkout $GIT_SHA +# Copy the project from host, cloned in build-image.yml +COPY . $BUILD_DIR/$PROJECT_NAME # Build the toolchain WORKDIR $BUILD_DIR/$PROJECT_NAME diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index d194093..e0fc420 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -20,7 +20,10 @@ jobs: run: sudo chmod 777 -R $GITHUB_WORKSPACE - name: Checkout repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + submodules: recursive + lfs: true - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From 37d628df9394cbc1e6cb019f90a5f0e6971db704 Mon Sep 17 00:00:00 2001 From: Aleks Knezevic Date: Tue, 17 Sep 2024 14:57:07 +0000 Subject: [PATCH 06/11] WIP --- .github/workflows/build-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index e0fc420..7966cc9 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -20,7 +20,7 @@ jobs: run: sudo chmod 777 -R $GITHUB_WORKSPACE - name: Checkout repository - - uses: actions/checkout@v4 + uses: actions/checkout@v4 with: submodules: recursive lfs: true From a43721bae456997ed3abaa9b3256bbf997a10c36 Mon Sep 17 00:00:00 2001 From: Aleks Knezevic Date: Tue, 17 Sep 2024 15:10:43 +0000 Subject: [PATCH 07/11] WIP --- .github/workflows/build-image.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 7966cc9..584498c 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -40,7 +40,6 @@ jobs: - name: Build and export base Docker image uses: docker/build-push-action@v6 with: - context: .github file: .github/Dockerfile.base push: true build-args: | @@ -51,7 +50,6 @@ jobs: - name: Build and export CI Docker image uses: docker/build-push-action@v6 with: - context: .github file: .github/Dockerfile.ci push: true build-args: | @@ -64,7 +62,6 @@ jobs: - name: Build and push base Docker image uses: docker/build-push-action@v6 with: - context: .github file: .github/Dockerfile.base push: true build-args: | @@ -75,7 +72,6 @@ jobs: - name: Build and push CI Docker image uses: docker/build-push-action@v6 with: - context: .github file: .github/Dockerfile.ci push: true build-args: | From bdde3dd45f9dd9393b20f449eae1e6d397576fe9 Mon Sep 17 00:00:00 2001 From: Aleks Knezevic Date: Tue, 17 Sep 2024 15:16:35 +0000 Subject: [PATCH 08/11] WIP --- third_party/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt index 4c856db..dde70f6 100644 --- a/third_party/CMakeLists.txt +++ b/third_party/CMakeLists.txt @@ -5,7 +5,7 @@ if (TOOLCHAIN STREQUAL "ON") cmake_minimum_required(VERSION 3.20) project(ttmlir-toolchain LANGUAGES CXX C) execute_process( - COMMAND git clone --recursive git@github.com:tenstorrent/tt-mlir.git ${PROJECT_SOURCE_DIR}/tt-mlir/src/tt-mlir + COMMAND git clone --recursive https://github.com/tenstorrent/tt-mlir.git ${PROJECT_SOURCE_DIR}/tt-mlir/src/tt-mlir RESULT_VARIABLE result1 OUTPUT_VARIABLE output1 ERROR_VARIABLE error_output1 From 8f5b24bf348178d7b16e42f7b5b03ad36fc1d68a Mon Sep 17 00:00:00 2001 From: Aleks Knezevic Date: Tue, 17 Sep 2024 16:49:23 +0000 Subject: [PATCH 09/11] WIP --- .github/workflows/build-and-test.yml | 144 +++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 .github/workflows/build-and-test.yml diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml new file mode 100644 index 0000000..02b86cc --- /dev/null +++ b/.github/workflows/build-and-test.yml @@ -0,0 +1,144 @@ +name: Build and Test + +on: + workflow_dispatch: + workflow_call: + +jobs: + build-ttxla: + + timeout-minutes: 120 + strategy: + fail-fast: false + + name: Build tt-xla + runs-on: ubuntu-latest + + container: + image: ghcr.io/${{ github.repository }}/tt-mlir-ci-ubuntu-22-04:latest #TODO update this to the correct image + options: --user root + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set reusable strings + id: strings + shell: bash + run: | + echo "work-dir=$(pwd)" >> "$GITHUB_OUTPUT" + echo "build-output-dir=$(pwd)/build" >> "$GITHUB_OUTPUT" + echo "install-output-dir=$(pwd)/install" >> "$GITHUB_OUTPUT" + + - name: Git safe dir + run: git config --global --add safe.directory ${{ steps.strings.outputs.work-dir }} + + + # Build project + + - name: Configure CMake + shell: bash + run: | + source venv/activate + cmake -G Ninja \ + -B ${{ steps.strings.outputs.build-output-dir }} + -S ${{ steps.strings.outputs.work-dir }} + + - name: Build + shell: bash + run: | + source venv/activate + cmake --build ${{ steps.strings.outputs.build-output-dir }} + + + + # This is needed to preserve file permissions + # https://github.com/actions/upload-artifact?tab=readme-ov-file#permission-loss + - name: 'Tar install directory' + shell: bash + working-directory: ${{ steps.strings.outputs.install-output-dir }} + run: tar cvf artifact.tar . + + - name: Upload install folder to archive + uses: actions/upload-artifact@v4 + with: + name: install-artifacts + path: ${{ steps.strings.outputs.install-output-dir }}/artifact.tar + + - name: Get the latest tag + shell: bash + run: | + latest_tag=$(git describe --tags --abbrev=0) + latest_tag=${latest_tag#v} + echo "latest_tag=$latest_tag" >> $GITHUB_ENV + commit_count=$(git rev-list ${{ env.latest_tag }}..HEAD --count) + echo "commit_count=$commit_count" >> $GITHUB_ENV + version="${{ env.latest_tag }}.${{ env.commit_count }}" + echo "version=$version" >> $GITHUB_ENV + echo $version + + + # Run tests on TT hardware + + run-tests: + + timeout-minutes: 30 + needs: build-ttxla + strategy: + fail-fast: false + matrix: + build: [ + {runs-on: n150, name: "run"}, + {runs-on: n300, name: "run"}, + ] + + runs-on: + - in-service + - ${{ matrix.build.runs-on }} + + container: + image: ghcr.io/${{ github.repository }}/tt-mlir-ci-ubuntu-22-04:latest + options: --user root --device /dev/tenstorrent/0 + volumes: + - /dev/hugepages:/dev/hugepages + - /dev/hugepages-1G:/dev/hugepages-1G + - /etc/udev/rules.d:/etc/udev/rules.d + - /lib/modules:/lib/modules + - /opt/tt_metal_infra/provisioning/provisioning_env:/opt/tt_metal_infra/provisioning/provisioning_env + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set reusable strings + id: strings + shell: bash + run: | + echo "work-dir=$(pwd)" >> "$GITHUB_OUTPUT" + echo "build-output-dir=$(pwd)/build" >> "$GITHUB_OUTPUT" + echo "install-output-dir=$(pwd)/install" >> "$GITHUB_OUTPUT" + + - name: Git safe dir + run: git config --global --add safe.directory ${{ steps.strings.outputs.work-dir }} + + - name: Use build artifacts + uses: actions/download-artifact@v4 + with: + name: install-artifacts + path: ${{ steps.strings.outputs.install-output-dir }} + + # This is needed to preserve file permissions + # https://github.com/actions/upload-artifact?tab=readme-ov-file#permission-loss + - name: 'Untar install directory' + shell: bash + working-directory: ${{ steps.strings.outputs.install-output-dir }} + run: tar xvf artifact.tar + + + - name: Run tests + shell: bash + run: | + source venv/activate + pytest -v tests/ \ No newline at end of file From ff1faed111a11e671f0556901f5481a75a6add43 Mon Sep 17 00:00:00 2001 From: Aleks Knezevic Date: Tue, 17 Sep 2024 17:22:02 +0000 Subject: [PATCH 10/11] WIP --- .github/workflows/build-and-test.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 02b86cc..5593ee9 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -19,9 +19,11 @@ jobs: options: --user root steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + lfs: true - name: Set reusable strings id: strings From 655ca354d8d9ea8c47b7e6da69cd27fbcfb5dd2e Mon Sep 17 00:00:00 2001 From: Aleks Knezevic Date: Tue, 17 Sep 2024 17:22:34 +0000 Subject: [PATCH 11/11] WIP --- .github/workflows/build-and-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 5593ee9..a8b4032 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -21,9 +21,9 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 - with: - submodules: recursive - lfs: true + with: + submodules: recursive + lfs: true - name: Set reusable strings id: strings