feat: add CIs

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: 'bug: [DESCRIPTION]'
+labels: 'type: bug'
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/discussion-thread.md

@@ -0,0 +1,14 @@
+---
+name: Discussion thread
+about: Start an open ended discussion
+title: 'Discussion: [TOPIC HERE]'
+labels: ''
+assignees: ''
+
+---
+
+**Motivation**
+
+**Discussion**
+
+**Resources**

.github/ISSUE_TEMPLATE/epic-request.md

@@ -0,0 +1,20 @@
+---
+name: Epic request
+about: Suggest an idea for this project
+title: 'epic: [DESCRIPTION]'
+labels: 'type: epic'
+assignees: ''
+
+---
+
+**Problem**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Success Criteria**
+A clear and concise description of what you want to happen.
+
+**Sub Issues**
+- 
+
+**Additional context**
+Add any other context or screenshots about the epic request here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,17 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: 'feat: [DESCRIPTION]'
+labels: 'type: feature request'
+assignees: ''
+
+---
+
+**Problem**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Success Criteria**
+A clear and concise description of what you want to happen.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/pull_request_template.md

@@ -0,0 +1,14 @@
+## Describe Your Changes
+
+-
+
+## Fixes Issues
+
+- Closes #
+- Closes #
+
+## Self Checklist
+
+- [ ] Added relevant comments, esp in complex areas
+- [ ] Updated docs (for bug fixes / features)
+- [ ] Created issues for follow-up changes or refactoring needed

.github/release-drafter.yml

@@ -0,0 +1,26 @@
+categories:
+  - title: '🚀 Features'
+    labels:
+      - 'type: enhancement'
+      - 'type: epic'
+      - 'type: feature request'
+  - title: '🐛 Bug Fixes'
+    labels:
+      - 'type: bug'
+  - title: '🧰 Maintenance'
+    labels: 
+      - 'type: chore'
+      - 'type: ci'
+  - title: '📖 Documentaion'
+    labels: 
+      - 'type: documentation'
+change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
+change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks.
+template: |
+  ## Changes
+
+  $CHANGES
+
+  ## Contributor
+
+  $CONTRIBUTORS

.github/workflows/build.yml

@@ -0,0 +1,146 @@
+name: CI Cortex Release
+
+on:
+  push:
+    tags: ["v[0-9]+.[0-9]+.[0-9]+"]
+    paths: ["/**"]
+  workflow_dispatch:
+
+jobs:
+  create-draft-release:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    outputs:
+      upload_url: ${{ steps.create_release.outputs.upload_url }}
+      version: ${{ steps.get_version.outputs.version }}
+    permissions:
+      contents: write
+    steps:
+      - name: Extract tag name without v prefix
+        id: get_version
+        run: |
+          echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV && echo "::set-output name=version::${GITHUB_REF#refs/tags/v}"
+        env:
+          GITHUB_REF: ${{ github.ref }}
+      - name: Create Draft Release
+        id: create_release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ github.ref_name }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          name: "${{ env.VERSION }}"
+          draft: true
+          prerelease: false
+
+  build-and-test:
+    runs-on: ${{ matrix.runs-on }}
+    needs: [create-draft-release]
+    timeout-minutes: 40
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: "linux"
+            name: "amd64"
+            runs-on: "ubuntu-20-04-cuda-12-0"
+            cmake-flags: "-DTOKENIZER_VERSION=${{github.event.pull_request.head.sha}}"
+            build-deps-cmake-flags: ""
+            ccache-dir: ''
+          - os: "mac"
+            name: "amd64"
+            runs-on: "macos-13"
+            cmake-flags: "-DTOKENIZER_VERSION=${{github.event.pull_request.head.sha}}"
+            build-deps-cmake-flags: ""
+            ccache-dir: ''
+          - os: "mac"
+            name: "arm64"
+            runs-on: "macos-latest"
+            cmake-flags: "-DTOKENIZER_VERSION=${{github.event.pull_request.head.sha}}"
+            build-deps-cmake-flags: ""
+            ccache-dir: ''
+          - os: "windows"
+            name: "amd64"
+            runs-on: "windows-cuda-12-0"
+            cmake-flags: "-DTOKENIZER_VERSION=${{github.event.pull_request.head.sha}}"
+            build-deps-cmake-flags: ""
+            ccache-dir: ''          
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - uses: actions/setup-dotnet@v3
+        if: runner.os == 'Windows'
+        with:
+          dotnet-version: "8.0.x"
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20.x"
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Install choco on Windows
+        if: runner.os == 'Windows'
+        run: |
+          choco install make pkgconfiglite ccache awscli 7zip ninja -y
+
+      - name: Get Cer for code signing
+        if: runner.os == 'macOS'
+        run: base64 -d <<< "$CODE_SIGN_P12_BASE64" > /tmp/codesign.p12
+        shell: bash
+        env:
+          CODE_SIGN_P12_BASE64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
+
+      - uses: apple-actions/import-codesign-certs@v2
+        if: runner.os == 'macOS'
+        with:
+          p12-file-base64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
+          p12-password: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
+
+      - name: Build
+        run: |
+          make build CMAKE_EXTRA_FLAGS="${{ matrix.cmake-flags }}" BUILD_DEPS_CMAKE_EXTRA_FLAGS="${{ matrix.build-deps-cmake-flags }}"
+
+      - name: Pre-package
+        run: |
+          make pre-package
+
+      - name: Code Signing macOS
+        if: runner.os == 'macOS'
+        run: |
+          make codesign CODE_SIGN=true DEVELOPER_ID="${{ secrets.DEVELOPER_ID }}"
+
+      - uses: nick-fields/retry@v3
+        with:
+          continue_on_error: true
+          retry_wait_seconds: 10
+          timeout_minutes: 10
+          max_attempts: 3
+          shell: cmd
+          command: |
+            set PATH=%PATH%;%USERPROFILE%\.dotnet\tools
+            make codesign CODE_SIGN=true AZURE_KEY_VAULT_URI="${{ secrets.AZURE_KEY_VAULT_URI }}" AZURE_CLIENT_ID="${{ secrets.AZURE_CLIENT_ID }}" AZURE_TENANT_ID="${{ secrets.AZURE_TENANT_ID }}" AZURE_CLIENT_SECRET="${{ secrets.AZURE_CLIENT_SECRET }}" AZURE_CERT_NAME="${{ secrets.AZURE_CERT_NAME }}"
+        name: Code Signing Windows
+        if: runner.os == 'Windows'
+
+      - name: Package
+        run: |
+          make package
+
+      - uses: actions/[email protected]
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ needs.create-draft-release.outputs.upload_url }}
+          asset_path: ./tokenizer.tar.gz
+          asset_name: tokenizer-${{ needs.create-draft-release.outputs.version }}-${{ matrix.os }}-${{ matrix.name }}.tar.gz
+          asset_content_type: application/gzip
+
+      - name: Remove build build-deps and build folder for windows
+        if: runner.os == 'Windows'
+        run: |
+          Remove-Item -Recurse -Force build

.github/workflows/quality-gate.yml

@@ -0,0 +1,70 @@
+name: CI Quality Gate Tokenizer.cpp
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+  workflow_dispatch:
+
+env:
+
+jobs:
+  build-and-test:
+    runs-on: ${{ matrix.runs-on }}
+    timeout-minutes: 40
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: "linux"
+            name: "amd64"
+            runs-on: "ubuntu-20-04-cuda-12-0"
+            cmake-flags: "-DTOKENIZER_VERSION=${{github.event.pull_request.head.sha}}"
+            build-deps-cmake-flags: ""
+            ccache-dir: ''
+          - os: "mac"
+            name: "amd64"
+            runs-on: "macos-13"
+            cmake-flags: "-DTOKENIZER_VERSION=${{github.event.pull_request.head.sha}}"
+            build-deps-cmake-flags: ""
+            ccache-dir: ''
+          - os: "mac"
+            name: "arm64"
+            runs-on: "macos-latest"
+            cmake-flags: "-DTOKENIZER_VERSION=${{github.event.pull_request.head.sha}}"
+            build-deps-cmake-flags: ""
+            ccache-dir: ''
+          - os: "windows"
+            name: "amd64"
+            runs-on: "windows-cuda-12-0"
+            cmake-flags: "-DTOKENIZER_VERSION=${{github.event.pull_request.head.sha}}"
+            build-deps-cmake-flags: ""
+            ccache-dir: ''          
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Install choco on Windows
+        if: runner.os == 'Windows'
+        run: |
+          choco install make pkgconfiglite ccache awscli 7zip ninja -y
+
+      - name: Build
+        run: |          
+          make build CMAKE_EXTRA_FLAGS="${{ matrix.cmake-flags }}" BUILD_DEPS_CMAKE_EXTRA_FLAGS="${{ matrix.build-deps-cmake-flags }}"
+
+      - name: Pre-package
+        run: |          
+          make pre-package
+
+      - name: Package
+        run: |          
+          make package
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: tokenizer-${{ matrix.os }}-${{ matrix.name }}
+          path: ./tokenizer