Beta to master

.github/workflows/codeql.yml

@@ -0,0 +1,69 @@
+name: "CodeQL code scanning"
+
+on:
+  push:
+    branches: [ "master", "beta", "rc" ]
+  pull_request:
+    branches: [ "master", "beta", "rc" ]
+  schedule:
+    # Scheduled jobs only run on the default branch.
+    - cron: '35 22 * * 3'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    runs-on: windows-latest
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: python
+          build-mode: none
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      shell: bash
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/regenerate_english_userDocs_translation_source.yml

@@ -0,0 +1,103 @@
+name: Rebuild English User Documentation for Translation 
+
+on:
+  push:
+    branches:
+      - beta
+    paths:
+      - 'user_docs/en/*.md'
+
+jobs:
+  rebuild-translation-source:
+    runs-on: windows-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install lxml requests
+
+    - name: update xliff files
+      shell: pwsh
+      run: |
+        # for any English markdown files changed within the commits of this push,
+        # update the corresponding xliff file (if one exists) to reflect the current markdown file,
+        # keeping existing translation IDs in tact. 
+        $ErrorActionPreference = 'Stop'
+        $changedFiles = git diff --name-only ${{github.event.before}}.. -- user_docs/en/*.md
+        foreach ($file in $changedFiles) {
+            Write-Host "$file has changed"
+            $baseName = [System.IO.Path]::GetFileNameWithoutExtension($file)
+            $xliff = "user_docs/en/$baseName.xliff"
+            $tempXliff = "user_docs/en/$baseName.xliff.temp"
+            $markdown = $file
+            if (Test-Path $xliff) {
+                Write-Host "Updating $xliff with changes from $markdown"
+                python user_docs/markdownTranslate.py updateXliff -x $xliff -m $file -o $tempXliff
+                Write-Host "Renaming $tempXliff to $xliff"
+                move-item -Path $tempXliff -Destination $xliff -Force
+            } else {
+                Write-Host "Ignoring $markdown as it does not have a corresponding xliff file"
+            }
+        }
+      if: success()
+
+    - name: Commit and Push changes
+      run: |
+        $ErrorActionPreference = 'Stop'
+        git config --local user.name "GitHub Actions"
+        git config --local user.email "[email protected]"
+        $filesChanged = git diff --name-only -- *.xliff
+        if ($filesChanged) {
+          Write-Host "xliff files were changed. Committing and pushing changes."
+          foreach ($file in $filesChanged) {
+            git add $file
+            Write-Host "Committing $file"
+            git commit -m "Update $file"
+          }
+          Write-Host "Setting up ssh for push"
+          # Ensure .ssh directory exists
+          New-Item -ItemType Directory -Force -Path ~/.ssh
+          # Add github.com to the list of known hosts for ssh
+          ssh-keyscan github.com >> ~/.ssh/known_hosts
+          # set the private key of the Github deploy key as the ssh key 
+          $privateKey = "${{ secrets.XLIFF_DEPLOY_PRIVATE_KEY }}"
+          $privateKey | Out-File -FilePath ~/.ssh/id_rsa -Encoding ascii
+          # Tell git to use ssh for pushing
+          git remote set-url origin [email protected]:${{ github.repository }}
+          Write-Host "Pushing changes"
+          git push origin HEAD
+        } else {
+          Write-Host "No xliff files were changed. Skipping commit and push."
+        }
+      if: success()
+
+    - name: Crowdin upload
+      # This step must only be run after successfully pushing changes to the repository.
+      # Otherwise if the push fails, subsequent runs may cause new translation IDs to be created,
+      # which will cause needless retranslation of existing strings.
+      env:
+        crowdinProjectID: ${{ vars.CROWDIN_PROJECT_ID }}
+        crowdinAuthToken: ${{ secrets.CROWDIN_AUTH_TOKEN }}
+      run: |
+        # Check if we changed userGuide.xliff in this action.
+        # If we did, upload it to Crowdin.
+        $ErrorActionPreference = 'Stop'
+        $changed = git diff --name-only ${{GITHUB.SHA}}.. -- user_docs/en/userGuide.xliff
+        if ($changed) {
+            Write-Host "Uploading userGuide.xliff to Crowdin"
+            # 18 is the file ID for userGuide.xliff in Crowdin.
+            python appVeyor/crowdinSync.py uploadSourceFile 18 user_docs/en/userguide.xliff
+        } else {
+            Write-Host "Not uploading userGuide.xliff to Crowdin as it has not changed"
+        }

sconstruct

@@ -333,6 +333,17 @@ for po in env.Glob(sourceDir.path + "/locale/*/lc_messages/*.po"):
 styles = os.path.join(userDocsDir.path, "styles.css")
 numberedHeadingsStyle = os.path.join(userDocsDir.path, "numberedHeadings.css")
 
+# Create Non-english md files in user_docs from localized xliff and English skel files
+for xliffFile in env.Glob(os.path.join(userDocsDir.path, "*", "*.xliff")):
+	lang = os.path.basename(xliffFile.path).split(".")[0]
+	if lang == "en":
+		continue
+	mdFile = os.path.splitext(xliffFile.path)[0] + ".md"
+	env.Command(
+		mdFile,
+		xliffFile,
+		[f'@{sys.executable} user_docs/markdownTranslate.py generateMarkdown -x "{xliffFile}" -o "{mdFile}"'],
+	)
 # Allow all markdown files to be converted to html in user_docs
 for mdFile in env.Glob(os.path.join(userDocsDir.path, "*", "*.md")):
 	htmlFile = env.md2html(mdFile)