From bdde9115ec59cc0e02507507810132658bbc7267 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Wed, 12 Jun 2024 16:20:16 -0400 Subject: [PATCH 01/17] [CI] Move the API diff outside of the normal build CI. (#20699) The api diff generates a number of artefacts that are not released. The way it works is that it compiles the project twice and compares the reuslts, therefore it has no dependency on the build pipeline or in the tests. Moving the step to its own pipeline simplifies the migration to the 1ES template AND allows use to identify exactly what failed. The two new pipelines are: - ci: https://dev.azure.com/devdiv/DevDiv/_build/results?buildId=9690468&view=results - pr: https://dev.azure.com/devdiv/DevDiv/_build?definitionId=22268 --------- Co-authored-by: Rolf Bjarne Kvinge --- tools/devops/automation/run-ci-api-diff.yml | 27 +++++ tools/devops/automation/run-pr-api-diff.yml | 27 +++++ .../automation/templates/api-diff-stage.yml | 102 ++++++++++++++++++ .../automation/templates/main-stage.yml | 26 +---- .../templates/pipelines/api-diff-pipeline.yml | 86 +++++++++++++++ .../templates/pipelines/build-pipeline.yml | 6 -- 6 files changed, 244 insertions(+), 30 deletions(-) create mode 100644 tools/devops/automation/run-ci-api-diff.yml create mode 100644 tools/devops/automation/run-pr-api-diff.yml create mode 100644 tools/devops/automation/templates/api-diff-stage.yml create mode 100644 tools/devops/automation/templates/pipelines/api-diff-pipeline.yml diff --git a/tools/devops/automation/run-ci-api-diff.yml b/tools/devops/automation/run-ci-api-diff.yml new file mode 100644 index 000000000000..9d5210603a97 --- /dev/null +++ b/tools/devops/automation/run-ci-api-diff.yml @@ -0,0 +1,27 @@ +# Pipeline that will calculate the api diff on a ci commit. + +trigger: + branches: + include: + - '*' + exclude: + - refs/heads/locfiles/* + - refs/heads/dev/* + paths: + exclude: + - .github + - docs + - CODEOWNERS + - ISSUE_TEMPLATE.md + - LICENSE + - NOTICE.txt + - SECURITY.MD + - README.md + - src/README.md + - tools/mtouch/README.md + - msbuild/Xamarin.Localization.MSBuild/README.md + +extends: + template: templates/pipelines/api-diff-pipeline.yml + parameters: + isPR: false diff --git a/tools/devops/automation/run-pr-api-diff.yml b/tools/devops/automation/run-pr-api-diff.yml new file mode 100644 index 000000000000..3b2681cf6396 --- /dev/null +++ b/tools/devops/automation/run-pr-api-diff.yml @@ -0,0 +1,27 @@ +# Pipeline that will calculate the api diff on a pr commit. + +trigger: none + +pr: + autoCancel: true + branches: + include: + - '*' # yes, you do need the quote, * has meaning in yamls + paths: + exclude: + - .github + - docs + - CODEOWNERS + - ISSUE_TEMPLATE.md + - LICENSE + - NOTICE.txt + - SECURITY.MD + - README.md + - src/README.md + - tools/mtouch/README.md + - msbuild/Xamarin.Localization.MSBuild/README.md + +extends: + template: templates/pipelines/api-diff-pipeline.yml + parameters: + isPR: true diff --git a/tools/devops/automation/templates/api-diff-stage.yml b/tools/devops/automation/templates/api-diff-stage.yml new file mode 100644 index 000000000000..421472d40aaa --- /dev/null +++ b/tools/devops/automation/templates/api-diff-stage.yml @@ -0,0 +1,102 @@ +parameters: + +- name: provisionatorChannel + type: string + default: 'latest' + +- name: pool + type: string + default: automatic + values: + - pr + - ci + - automatic + +- name: isPR + type: boolean + +- name: repositoryAlias + type: string + default: self + +- name: commit + type: string + default: HEAD + +- name: xcodeChannel + type: string + default: Stable + +- name: macOSName + type: string + + + +stages: + +- ${{ if parameters.isPR }}: + - stage: clean + displayName: 'Clean up' + dependsOn: [] + jobs: + - job: + displayName: 'Clean comments' + pool: + vmImage: windows-latest + steps: + - template: ./common/clean.yml + +- stage: configure_build + displayName: 'Configure' + jobs: + + - ${{ if eq(parameters.pool, 'automatic') }}: + - job: AgentPoolSelector # https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml + pool: # Consider using an agentless (server) job here, but would need to host selection logic as an Azure function: https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=azure-devops&tabs=schema#server + vmImage: ubuntu-latest + steps: + - checkout: none # https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=azure-devops&tabs=schema#checkout + + # Selects appropriate agent pool based on trigger type (PR or CI); manually triggered builds target the PR pool + - template: azure-devops-pools/agent-pool-selector.yml@yaml-templates + parameters: + agentPoolPR: $(PRBuildPool) + agentPoolPRUrl: $(PRBuildPoolUrl) + agentPoolCI: $(CIBuildPool) + agentPoolCIUrl: $(CIBuildPoolUrl) + + - job: configure + displayName: 'Configure build' + pool: + vmImage: windows-latest + + variables: + isMain: $[eq(variables['Build.SourceBranch'], 'refs/heads/main')] + isScheduled: $[eq(variables['Build.Reason'], 'Schedule')] + BRANCH_NAME: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ] + + steps: + - template: common/configure.yml + parameters: + repositoryAlias: ${{ parameters.repositoryAlias }} + commit: ${{ parameters.commit }} + testsLabels: '--label=skip-all-tests,run-ios-tests,run-ios-simulator-tests,run-tvos-tests,run-watchos-tests,run-mac-tests,run-maccatalyst-tests,run-dotnet-tests,run-system-permission-tests,run-legacy-xamarin-tests' + statusContext: 'VSTS: simulator tests' + uploadArtifacts: true + +- stage: generate_api_diff + displayName: 'API diff' + dependsOn: [ configure_build ] + jobs: + - template: ./build/api-diff-stage.yml + parameters: + xcodeChannel: ${{ parameters.xcodeChannel }} + macOSName: ${{ parameters.macOSName }} + isPR: ${{ parameters.isPR }} + repositoryAlias: ${{ parameters.repositoryAlias }} + commit: ${{ parameters.commit }} + vsdropsPrefix: ${{ variables.vsdropsPrefix }} + keyringPass: $(pass--lab--mac--builder--keychain) + gitHubToken: $(Github.Token) + xqaCertPass: $(xqa--certificates--password) + pool: ${{ parameters.pool }} diff --git a/tools/devops/automation/templates/main-stage.yml b/tools/devops/automation/templates/main-stage.yml index 1ce2dad1abad..9e78186d7fd2 100644 --- a/tools/devops/automation/templates/main-stage.yml +++ b/tools/devops/automation/templates/main-stage.yml @@ -20,10 +20,6 @@ parameters: type: boolean default: true -- name: enableAPIDiff - type: boolean - default: true - - name: forceInsertion type: boolean default: false @@ -306,7 +302,7 @@ stages: ${{ if and(ne(parameters.dependsOn, ''), ne(parameters.dependsOnResult, '')) }}: condition: eq(dependencies.${{ parameters.dependsOn }}.result, '${{ parameters.dependsOnResult }}') jobs: - + - ${{ if eq(parameters.pool, 'automatic') }}: - job: AgentPoolSelector # https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml pool: # Consider using an agentless (server) job here, but would need to host selection logic as an Azure function: https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=azure-devops&tabs=schema#server @@ -438,22 +434,4 @@ stages: azureContainer: ${{ parameters.azureContainer }} packages: # flatten the pkgs for the parameter - ${{ each pkg in parameters.legacyPackageJobs }}: - - ${{ pkg }} - -- ${{ if eq(parameters.enableAPIDiff, true) }}: - - stage: generate_api_diff - displayName: '${{ parameters.stageDisplayNamePrefix }}API diff' - dependsOn: [ configure_build ] - jobs: - - template: ./build/api-diff-stage.yml - parameters: - xcodeChannel: ${{ parameters.xcodeChannel }} - macOSName: ${{ parameters.macOSName }} - isPR: ${{ parameters.isPR }} - repositoryAlias: ${{ parameters.repositoryAlias }} - commit: ${{ parameters.commit }} - vsdropsPrefix: ${{ variables.vsdropsPrefix }} - keyringPass: $(pass--lab--mac--builder--keychain) - gitHubToken: $(Github.Token) - xqaCertPass: $(xqa--certificates--password) - pool: ${{ parameters.pool }} + - ${{ pkg }} \ No newline at end of file diff --git a/tools/devops/automation/templates/pipelines/api-diff-pipeline.yml b/tools/devops/automation/templates/pipelines/api-diff-pipeline.yml new file mode 100644 index 000000000000..01d9ffb83d26 --- /dev/null +++ b/tools/devops/automation/templates/pipelines/api-diff-pipeline.yml @@ -0,0 +1,86 @@ +# template that can be extended by pipelines that will be used to build the project. This +# allows to share te parameters and resources. + +parameters: + +- name: provisionatorChannel + displayName: Provisionator channel to use + type: string + default: 'latest' + +- name: macOSName # comes from the build agent demand named macOS.Name + displayName: Name of the version of macOS to use + type: string + default: 'Sonoma' + +- name: pool + type: string + displayName: Bot pool to use + default: automatic + values: + - pr + - ci + - automatic + +- name: isPR + displayName: Is PR build + type: boolean + default: false + +- name: testConfigurations + displayName: Test configurations to run + type: object + default: [] + +resources: + repositories: + - repository: self + checkoutOptions: + submodules: true + + - repository: yaml-templates + type: github + name: xamarin/yaml-templates + ref: refs/heads/main + endpoint: xamarin + + - repository: sdk-insertions + type: github + name: xamarin/sdk-insertions + ref: refs/heads/main + endpoint: xamarin + + - repository: maccore + type: github + name: xamarin/maccore + ref: refs/heads/main + endpoint: xamarin + + - repository: release-scripts + type: github + name: xamarin/release-scripts + ref: refs/heads/only_codesign + endpoint: xamarin + +variables: +- ${{ if eq(parameters.isPR, false) }}: + - ${{ if contains(variables['Build.DefinitionName'], 'private') }}: + - template: ../vsts-variables.yml + - template: ../variables.yml + - name: MaciosUploadPrefix + value: '' + - name: DisablePipelineConfigDetector + value: true +- ${{ else }}: + - template: ../variables.yml + - name: MaciosUploadPrefix + value: '' + +stages: +- template: ../api-diff-stage.yml + parameters: + xcodeChannel: Stable + macOSName: ${{ parameters.macOSName }} + isPR: ${{ parameters.isPR }} + provisionatorChannel: ${{ parameters.provisionatorChannel }} + pool: ${{ parameters.pool }} diff --git a/tools/devops/automation/templates/pipelines/build-pipeline.yml b/tools/devops/automation/templates/pipelines/build-pipeline.yml index 83e3571a33d0..d5a3616bd27d 100644 --- a/tools/devops/automation/templates/pipelines/build-pipeline.yml +++ b/tools/devops/automation/templates/pipelines/build-pipeline.yml @@ -27,11 +27,6 @@ parameters: type: boolean default: true -- name: enableAPIDiff - displayName: Enable API diff generation - type: boolean - default: true - - name: forceInsertion displayName: Force Insertion type: boolean @@ -214,7 +209,6 @@ stages: provisionatorChannel: ${{ parameters.provisionatorChannel }} pool: ${{ parameters.pool }} runGovernanceTests: ${{ parameters.runGovernanceTests }} - enableAPIDiff: ${{ parameters.enableAPIDiff }} forceInsertion: ${{ parameters.forceInsertion }} skipESRP: ${{ parameters.skipESRP }} pushNugets: ${{ parameters.pushNugets }} From 8a0acaf547fc4f20f10e0e5ec9ccd5f102b3f930 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Thu, 13 Jun 2024 23:29:44 -0400 Subject: [PATCH 02/17] [CI] Remove the need to refer to the insertions repo in the api diff. (#20725) --- .../automation/templates/pipelines/api-diff-pipeline.yml | 6 ------ .../automation/templates/pipelines/build-pipeline.yml | 2 ++ tools/devops/automation/templates/variables.yml | 1 - 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/tools/devops/automation/templates/pipelines/api-diff-pipeline.yml b/tools/devops/automation/templates/pipelines/api-diff-pipeline.yml index 01d9ffb83d26..0b9e55395b11 100644 --- a/tools/devops/automation/templates/pipelines/api-diff-pipeline.yml +++ b/tools/devops/automation/templates/pipelines/api-diff-pipeline.yml @@ -44,12 +44,6 @@ resources: ref: refs/heads/main endpoint: xamarin - - repository: sdk-insertions - type: github - name: xamarin/sdk-insertions - ref: refs/heads/main - endpoint: xamarin - - repository: maccore type: github name: xamarin/maccore diff --git a/tools/devops/automation/templates/pipelines/build-pipeline.yml b/tools/devops/automation/templates/pipelines/build-pipeline.yml index d5a3616bd27d..76da9c279422 100644 --- a/tools/devops/automation/templates/pipelines/build-pipeline.yml +++ b/tools/devops/automation/templates/pipelines/build-pipeline.yml @@ -184,6 +184,7 @@ variables: - ${{ if eq(parameters.isPR, false) }}: - ${{ if contains(variables['Build.DefinitionName'], 'private') }}: - template: ../vsts-variables.yml + - template: templates/common/vs-release-vars.yml@sdk-insertions - template: ../variables.yml - name: MicrobuildConnector value: 'MicroBuild Signing Task (DevDiv)' @@ -193,6 +194,7 @@ variables: value: true - ${{ else }}: - template: ../variables.yml + - template: templates/common/vs-release-vars.yml@sdk-insertions - name: MicrobuildConnector value: '' - name: MaciosUploadPrefix diff --git a/tools/devops/automation/templates/variables.yml b/tools/devops/automation/templates/variables.yml index 31d9394455ac..a9513ef295c1 100644 --- a/tools/devops/automation/templates/variables.yml +++ b/tools/devops/automation/templates/variables.yml @@ -1,7 +1,6 @@ variables: - ${{ if contains(variables['Build.DefinitionName'], 'private') }}: - template: templates/vsts-variables.yml -- template: templates/common/vs-release-vars.yml@sdk-insertions # used for the insertion steps - group: xamops-azdev-secrets - group: Xamarin-Secrets - group: Xamarin Signing From 38ef781bbe74f6c2ca5c5b1684ddb0dd2b1dbde5 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Fri, 14 Jun 2024 09:15:41 -0400 Subject: [PATCH 03/17] [CI] Remove the need to checkout the sdk-insertions repor to run the tests. (#20726) --- .../automation/templates/pipelines/run-tests-pipeline.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml b/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml index e9a1b80691b7..b21ef5af1148 100644 --- a/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml +++ b/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml @@ -158,12 +158,6 @@ resources: ref: refs/heads/main endpoint: xamarin - - repository: sdk-insertions - type: github - name: xamarin/sdk-insertions - ref: refs/heads/main - endpoint: xamarin - - repository: maccore type: github name: xamarin/maccore From f2c5cc5a6be4cab347729866210962c1987f2a3a Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 15:54:29 +0200 Subject: [PATCH 04/17] [main] Update dependencies from dotnet/installer (#20721) This pull request updates the following dependencies ## From https://github.com/dotnet/installer - **Subscription**: 80cb9ffd-f92f-4fc8-9f8b-08dbca46abfb - **Build**: 20240611.1 - **Date Produced**: June 11, 2024 9:45:46 PM UTC - **Commit**: 8882eb09fcade072ffad12ad477b91bdefa5fe12 - **Branch**: refs/heads/release/8.0.1xx - **Updates**: - **Microsoft.Dotnet.Sdk.Internal**: [from 8.0.107-servicing.24306.1 to 8.0.107-servicing.24311.1][6] [6]: https://github.com/dotnet/installer/compare/0fea32701d...8882eb09fc --- eng/Version.Details.xml | 4 ++-- eng/Versions.props | 2 +- global.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index ae52f17ea508..d7a264372fc5 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,8 +1,8 @@ - + https://github.com/dotnet/installer - 0fea32701d3b882cb06b297b86f8443d446f9814 + 8882eb09fcade072ffad12ad477b91bdefa5fe12 https://github.com/dotnet/runtime diff --git a/eng/Versions.props b/eng/Versions.props index 6f7e5d5fa64e..b535e21f718e 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -2,7 +2,7 @@ - 8.0.107-servicing.24306.1 + 8.0.107-servicing.24311.1 8.0.5 8.0.0-rtm.23524.7 8.0.0-beta.24225.1 diff --git a/global.json b/global.json index 09bfe89d4d03..6bd397a3d5f0 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "8.0.107-servicing.24306.1" + "version": "8.0.107-servicing.24311.1" } } From 6c2a7a9eac6168910e59134fc5e452b758675f2c Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Mon, 17 Jun 2024 20:19:06 +0200 Subject: [PATCH 05/17] [msbuild] The DTPlatformName is supposed to be 'macosx' for Mac Catalyst. (#20720) Change how we compute DTPlatformName so that it's 'macosx' for Mac Catalyst. The PlatformUtils.GetTargetPlatform returns SdkPlatform for all platforms except Mac Catalyst, where it returns the same as for macOS (i.e. 'macosx'). It also returns a lowercased value, so we don't need to do that either. This is a partial fix for https://github.com/xamarin/xamarin-macios/issues/20714. --- .../Tasks/CompileAppManifest.cs | 2 +- .../TaskTests/CompileAppManifestTaskTests.cs | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/CompileAppManifest.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/CompileAppManifest.cs index 88fe882b87eb..3500599ef23a 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/CompileAppManifest.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/CompileAppManifest.cs @@ -527,7 +527,7 @@ void SetXcodeValues (PDictionary plist, IAppleSdk currentSDK) SetValueIfNotNull (plist, "DTCompiler", sdkSettings.DTCompiler); SetValueIfNotNull (plist, "DTPlatformBuild", dtSettings.DTPlatformBuild); SetValueIfNotNull (plist, "DTSDKBuild", sdkSettings.DTSDKBuild); - SetValueIfNotNull (plist, "DTPlatformName", SdkPlatform.ToLowerInvariant ()); + SetValueIfNotNull (plist, "DTPlatformName", PlatformUtils.GetTargetPlatform (SdkPlatform, IsWatchApp)); SetValueIfNotNull (plist, "DTPlatformVersion", dtSettings.DTPlatformVersion); SetValueIfNotNull (plist, "DTSDKName", sdkSettings.CanonicalName); SetValueIfNotNull (plist, "DTXcode", AppleSdkSettings.DTXcode); diff --git a/tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/CompileAppManifestTaskTests.cs b/tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/CompileAppManifestTaskTests.cs index f582ac5c4069..bc84da45a258 100644 --- a/tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/CompileAppManifestTaskTests.cs +++ b/tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/CompileAppManifestTaskTests.cs @@ -138,13 +138,13 @@ public void MacCatalystVersionCheckUnmappedError () } [Test] - [TestCase (ApplePlatform.iOS, true)] - [TestCase (ApplePlatform.iOS, false)] - [TestCase (ApplePlatform.MacCatalyst, false)] - [TestCase (ApplePlatform.TVOS, true)] - [TestCase (ApplePlatform.TVOS, false)] - [TestCase (ApplePlatform.MacOSX, false)] - public void XcodeVariables (ApplePlatform platform, bool isSimulator) + [TestCase (ApplePlatform.iOS, true, "iphonesimulator")] + [TestCase (ApplePlatform.iOS, false, "iphoneos")] + [TestCase (ApplePlatform.MacCatalyst, false, "macosx")] + [TestCase (ApplePlatform.TVOS, true, "appletvsimulator")] + [TestCase (ApplePlatform.TVOS, false, "appletvos")] + [TestCase (ApplePlatform.MacOSX, false, "macosx")] + public void XcodeVariables (ApplePlatform platform, bool isSimulator, string expectedDTPlatformName) { var task = CreateTask (platform: platform); task.SdkIsSimulator = isSimulator; @@ -165,6 +165,7 @@ public void XcodeVariables (ApplePlatform platform, bool isSimulator) var value = plist.GetString (variable)?.Value; Assert.That (value, Is.Not.Null.And.Not.Empty, variable); } + Assert.AreEqual (expectedDTPlatformName, plist.GetString ("DTPlatformName")?.Value, "Expected DTPlatformName"); } } } From 1bb1648b6cdfef67da81c73592c3778a700e24ab Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 19 Jun 2024 15:06:42 +0200 Subject: [PATCH 06/17] [main] Update dependencies from dotnet/xharness (#20733) This pull request updates the following dependencies ## From https://github.com/dotnet/xharness - **Subscription**: 601bc5e1-1cae-44b5-cf5f-08db9342aa2f - **Build**: 20240612.3 - **Date Produced**: June 12, 2024 7:26:05 PM UTC - **Commit**: 6ce15319de72ab6d4c3b0f4c40f59300cffc5450 - **Branch**: refs/heads/main - **Updates**: - **Microsoft.DotNet.XHarness.iOS.Shared**: [from 9.0.0-prerelease.24304.1 to 9.0.0-prerelease.24312.3][1] [1]: https://github.com/dotnet/xharness/compare/03a55dc446...6ce15319de --- eng/Version.Details.xml | 4 ++-- eng/Versions.props | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index d7a264372fc5..8a0e589abfc5 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -78,9 +78,9 @@ https://github.com/dotnet/templating - + https://github.com/dotnet/xharness - 03a55dc4462c3851ce4bb939e71410136ea8f5e3 + 6ce15319de72ab6d4c3b0f4c40f59300cffc5450 diff --git a/eng/Versions.props b/eng/Versions.props index b535e21f718e..fa745c7dbc60 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -11,7 +11,7 @@ 8.0.0 7.0.100-alpha.1.21601.1 0.11.4-alpha.23509.2 - 9.0.0-prerelease.24304.1 + 9.0.0-prerelease.24312.3 $(MicrosoftNETWorkloadEmscriptenCurrentManifest80100Version) $(MicrosoftNETWorkloadEmscriptenCurrentManifest80100Version) From 47806611e763f85578bb4303df354199e0087119 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 19 Jun 2024 15:10:39 +0200 Subject: [PATCH 07/17] [siminstaller] Convert to .NET project. (#20739) In addition to being good by itself, by fixing all the resulting nullability warnings, this random problem goes away: Unhandled Exception: System.NullReferenceException: Object reference not set to an instance of an object at xsiminstaller.MainClass.Main (System.String[] args) [0x006ca] in /Users/builder/azdo/_work/4/s/xamarin-macios/tools/siminstaller/Program.cs:228 [ERROR] FATAL UNHANDLED EXCEPTION: System.NullReferenceException: Object reference not set to an instance of an object at xsiminstaller.MainClass.Main (System.String[] args) [0x006ca] in /Users/builder/azdo/_work/4/s/xamarin-macios/tools/siminstaller/Program.cs:228 Also use HttpClient instead of WebClient (which is deprecated). --- system-dependencies.sh | 7 +- tools/siminstaller/Makefile | 20 +- tools/siminstaller/Program.cs | 269 ++++++++++++++----------- tools/siminstaller/siminstaller.csproj | 57 +----- 4 files changed, 174 insertions(+), 179 deletions(-) diff --git a/system-dependencies.sh b/system-dependencies.sh index 0654b386be0d..84852ff489d6 100755 --- a/system-dependencies.sh +++ b/system-dependencies.sh @@ -1017,11 +1017,12 @@ function check_simulators () IFS=', ' read -r -a SIMS <<< "$EXTRA_SIMULATORS" arraylength=${#SIMS[@]} + INSTALL_SIMULATORS= for (( i=1; i/dev/null); then local action=warn if test -z $OPTIONAL_SIMULATORS; then action=fail @@ -1034,7 +1035,7 @@ function check_simulators () $action " Another possibility is that Apple is not shipping any simulators (yet?) for the new version of Xcode (if the previous list shows no simulators)." else if ! test -z $PROVISION_SIMULATORS; then - if ! mono --debug tools/siminstaller/bin/Debug/siminstaller.exe -q --xcode "$XCODE" "${SIMS[@]}"; then + if ! make -C tools/siminstaller install-simulators INSTALL_SIMULATORS="$INSTALL_SIMULATORS"; then $action "Failed to install extra simulators." else ok "Extra simulators installed successfully: '${FAILED_SIMULATORS//$'\n'/', '}'" diff --git a/tools/siminstaller/Makefile b/tools/siminstaller/Makefile index 96ca1620e21b..cb33ba9dfc98 100644 --- a/tools/siminstaller/Makefile +++ b/tools/siminstaller/Makefile @@ -1,12 +1,22 @@ TOP=../.. include $(TOP)/Make.config -all-local:: bin/Debug/siminstaller.exe +EXECUTABLE=bin/Debug/siminstaller.dll + +all-local:: $(EXECUTABLE) install-local:: all-local -bin/Debug/siminstaller.exe: $(wildcard *.cs) $(wildcard *.csproj) Makefile - $(Q_BUILD) $(SYSTEM_MSBUILD) "/bl:$@.binlog" /restore $(MSBUILD_VERBOSITY) $(wildcard *.csproj) +# we build using the system .NET, because we might execute before we've downloaded our own version of .NET (i.e. while provisioning dependencies) +$(EXECUTABLE): $(wildcard *.cs) $(wildcard *.csproj) Makefile + $(Q_BUILD) cd $(HOME) && dotnet build $(abspath $(CURDIR))/*.csproj "/bl:$@.binlog" $(DOTNET_BUILD_VERBOSITY) + $(Q) touch $@ + +print-simulators: $(EXECUTABLE) + $(Q) cd $(HOME) && dotnet exec $(abspath $<) --xcode=$(abspath $(XCODE_DEVELOPER_ROOT)/../..) --print-simulators + +only-check: $(EXECUTABLE) + $(Q) cd $(HOME) && dotnet exec $(abspath $<) --xcode=$(abspath $(XCODE_DEVELOPER_ROOT)/../..) -q $(INSTALL_SIMULATORS) --only-check -print-simulators: bin/Debug/siminstaller.exe - mono $< --print-simulators --xcode=$(abspath $(XCODE_DEVELOPER_ROOT)/../..) +install-simulators: $(EXECUTABLE) + $(Q) cd $(HOME) && dotnet exec $(abspath $<) --xcode=$(abspath $(XCODE_DEVELOPER_ROOT)/../..) -q $(INSTALL_SIMULATORS) diff --git a/tools/siminstaller/Program.cs b/tools/siminstaller/Program.cs index 1c7acb09c43b..0793c83f3975 100644 --- a/tools/siminstaller/Program.cs +++ b/tools/siminstaller/Program.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.ComponentModel; using System.Diagnostics; using System.IO; using System.Diagnostics.CodeAnalysis; @@ -23,24 +24,26 @@ static string TempDirectory { } } - static bool TryExecuteAndCapture (string filename, string arguments, out string stdout) + static bool TryExecuteAndCapture (out string stdout, string filename, params string [] arguments) { - var rv = TryExecuteAndCapture (filename, arguments, out stdout, out var stderr); + var rv = TryExecuteAndCapture (out stdout, out var stderr, filename, arguments); if (!rv) Console.WriteLine (stderr); return rv; } - static bool TryExecuteAndCapture (string filename, string arguments, out string stdout, out string stderr) + static bool TryExecuteAndCapture (out string stdout, out string stderr, string filename, params string [] arguments) { using (var p = new Process ()) { p.StartInfo.FileName = filename; - p.StartInfo.Arguments = arguments; + foreach (var arg in arguments) + p.StartInfo.ArgumentList.Add (arg); p.StartInfo.RedirectStandardOutput = true; p.StartInfo.RedirectStandardError = true; p.StartInfo.UseShellExecute = false; + var quotedArguments = Xamarin.Utils.StringUtils.FormatArguments (arguments); if (verbose > 0) - Console.WriteLine ($"{filename} {arguments}"); + Console.WriteLine ($"{filename} {quotedArguments}"); var output = new StringBuilder (); var error = new StringBuilder (); var outputDone = new ManualResetEvent (false); @@ -68,12 +71,12 @@ static bool TryExecuteAndCapture (string filename, string arguments, out string stdout = output.ToString (); stderr = error.ToString (); if (verbose > 0 && p.ExitCode != 0) - Console.WriteLine ("Failed to execute '{0} {1}'", filename, arguments); + Console.WriteLine ($"Failed to execute '{filename} {quotedArguments}', exit code: {p.ExitCode}\n{stdout}\n{stderr}"); return p.ExitCode == 0; } } - public static int Main (string [] args) + public async static Task Main (string [] args) { var exit_code = 0; string? xcode_app = null; @@ -122,11 +125,11 @@ public static int Main (string [] args) return 1; } - if (!TryExecuteAndCapture ("/usr/libexec/PlistBuddy", $"-c 'Print :DTXcode' '{plist}'", out var xcodeVersion)) + if (!TryExecuteAndCapture (out var xcodeVersion, "/usr/libexec/PlistBuddy", $"-c", "Print :DTXcode", plist)) return 1; xcodeVersion = xcodeVersion.Trim (); - if (!TryExecuteAndCapture ("/usr/libexec/PlistBuddy", $"-c 'Print :DVTPlugInCompatibilityUUID' '{plist}'", out var xcodeUuid)) + if (!TryExecuteAndCapture (out var xcodeUuid, "/usr/libexec/PlistBuddy", "-c", "Print :DVTPlugInCompatibilityUUID", plist)) return 1; xcodeUuid = xcodeUuid.Trim (); @@ -153,15 +156,18 @@ public static int Main (string [] args) "https://devimages-cdn.apple.com/downloads/xcode/simulators/index2.dvtdownloadableindex", }; var anyFailures = false; + var wc = new HttpClient (); foreach (var url in urls) { var uri = new Uri (url); - var wc = new WebClient (); try { if (verbose > 0) Console.WriteLine ($"Downloading '{uri}'"); else if (anyFailures) Console.WriteLine ($"Attempting fallback url '{uri}'"); - wc.DownloadFile (uri, tmpfile); + + var response = await wc.GetAsync (uri); + using var file = new FileStream (tmpfile, FileMode.Create, FileAccess.Write, FileShare.None); + await response.Content.CopyToAsync (file); } catch (Exception ex) { File.Delete (tmpfile); // Make sure there are no downloaded remnants // 403 means 404 @@ -195,110 +201,118 @@ public static int Main (string [] args) } } - if (!TryExecuteAndCapture ("plutil", $"-convert xml1 -o - '{tmpfile}'", out var xml)) + if (!TryExecuteAndCapture (out var xml, "plutil", "-convert", "xml1", "-o", "-", tmpfile)) return 1; var doc = new XmlDocument (); doc.LoadXml (xml); var downloadables = doc.SelectNodes ("//plist/dict/key[text()='downloadables']/following-sibling::array[1]/dict"); - foreach (XmlNode downloadable in downloadables) { - var nameNode = downloadable.SelectSingleNode ("key[text()='name']/following-sibling::string"); - var versionNode = downloadable.SelectSingleNode ("key[text()='version']/following-sibling::string"); - var sourceNode = downloadable.SelectSingleNode ("key[text()='source']/following-sibling::string"); - var identifierNode = downloadable.SelectSingleNode ("key[text()='identifier']/following-sibling::string"); - var fileSizeNode = downloadable.SelectSingleNode ("key[text()='fileSize']/following-sibling::integer|key[text()='fileSize']/following-sibling::real"); - var installPrefixNode = downloadable.SelectSingleNode ("key[text()='userInfo']/following-sibling::dict/key[text()='InstallPrefix']/following-sibling::string"); - - var version = versionNode.InnerText; - var versions = version.Split ('.'); - var versionMajor = versions [0]; - var versionMinor = versions [1]; - var dict = new Dictionary () { - { "DOWNLOADABLE_VERSION_MAJOR", versionMajor }, - { "DOWNLOADABLE_VERSION_MINOR", versionMinor }, - { "DOWNLOADABLE_VERSION", version }, - }; - - var identifier = Replace (identifierNode.InnerText, dict); - - dict.Add ("DOWNLOADABLE_IDENTIFIER", identifier); + if (downloadables is not null) { + foreach (XmlNode downloadable in downloadables) { + var nameNode = downloadable.SelectSingleNode ("key[text()='name']/following-sibling::string"); + var versionNode = downloadable.SelectSingleNode ("key[text()='version']/following-sibling::string"); + var sourceNode = downloadable.SelectSingleNode ("key[text()='source']/following-sibling::string"); + var identifierNode = downloadable.SelectSingleNode ("key[text()='identifier']/following-sibling::string"); + var fileSizeNode = downloadable.SelectSingleNode ("key[text()='fileSize']/following-sibling::integer|key[text()='fileSize']/following-sibling::real"); + var installPrefixNode = downloadable.SelectSingleNode ("key[text()='userInfo']/following-sibling::dict/key[text()='InstallPrefix']/following-sibling::string"); + + var version = versionNode?.InnerText ?? string.Empty; + var versions = version.Split ('.'); + var versionMajor = versions [0]; + var versionMinor = versions [1]; + var dict = new Dictionary () { + { "DOWNLOADABLE_VERSION_MAJOR", versionMajor }, + { "DOWNLOADABLE_VERSION_MINOR", versionMinor }, + { "DOWNLOADABLE_VERSION", version }, + }; + + var identifier = Replace (identifierNode?.InnerText, dict); + + var name = Replace (nameNode?.InnerText, dict) ?? string.Empty; + var source = Replace (sourceNode?.InnerText, dict); + var installPrefix = Replace (installPrefixNode?.InnerText, dict); + + if (string.IsNullOrEmpty (identifier)) { + if (verbose >= 0) + Console.WriteLine ($"No identifier found for {name}"); + continue; + } - var name = Replace (nameNode.InnerText, dict); - var source = Replace (sourceNode.InnerText, dict); - var installPrefix = Replace (installPrefixNode?.InnerText, dict); + dict.Add ("DOWNLOADABLE_IDENTIFIER", identifier); - if (installPrefix is null) { - // This is just guesswork - var simRuntimeName = name.Replace (" Simulator", ".simruntime"); - installPrefix = $"/Library/Developer/CoreSimulator/Profiles/Runtimes/{simRuntimeName}"; - } + if (installPrefix is null) { + // This is just guesswork + var simRuntimeName = name.Replace (" Simulator", ".simruntime"); + installPrefix = $"/Library/Developer/CoreSimulator/Profiles/Runtimes/{simRuntimeName}"; + } - double.TryParse (fileSizeNode?.InnerText, out var parsedFileSize); - var fileSize = (long) parsedFileSize; + double.TryParse (fileSizeNode?.InnerText, out var parsedFileSize); + var fileSize = (long) parsedFileSize; - var installed = false; - var updateAvailable = false; + var installed = false; + var updateAvailable = false; - if (only_check && !install.Contains (identifier)) - continue; + if (only_check && !install.Contains (identifier)) + continue; - if (IsInstalled (identifier, out var installedVersion)) { - if (installedVersion >= Version.Parse (version)) { - installed = true; - } else { - updateAvailable = true; + if (IsInstalled (identifier, out var installedVersion)) { + if (installedVersion >= Version.Parse (version)) { + installed = true; + } else { + updateAvailable = true; + } } - } - var doInstall = false; - if (install.Contains (identifier)) { - if (force) { - doInstall = true; - if (!only_check && verbose >= 0 && installed) - Console.WriteLine ($"The simulator '{identifier}' is already installed, but will be installed again because --force was specified."); - } else if (installed) { - if (!only_check && verbose >= 0) - Console.WriteLine ($"Not installing '{identifier}' because it's already installed and up-to-date."); - } else { - doInstall = true; + var doInstall = false; + if (install.Contains (identifier)) { + if (force) { + doInstall = true; + if (!only_check && verbose >= 0 && installed) + Console.WriteLine ($"The simulator '{identifier}' is already installed, but will be installed again because --force was specified."); + } else if (installed) { + if (!only_check && verbose >= 0) + Console.WriteLine ($"Not installing '{identifier}' because it's already installed and up-to-date."); + } else { + doInstall = true; + } + install.Remove (identifier); } - install.Remove (identifier); - } - if (print_simulators) { - Console.WriteLine (name); - Console.Write ($" Version: {version}"); - if (updateAvailable) - Console.WriteLine ($" (an earlier version is installed: {installedVersion}"); - else if (!installed) - Console.WriteLine ($" (not installed)"); - else - Console.WriteLine ($" (installed)"); - Console.WriteLine ($" Source: {source}"); - Console.WriteLine ($" Identifier: {identifier}"); - Console.WriteLine ($" InstallPrefix: {installPrefix}"); - } + if (print_simulators) { + Console.WriteLine (name); + Console.Write ($" Version: {version}"); + if (updateAvailable) + Console.WriteLine ($" (an earlier version is installed: {installedVersion}"); + else if (!installed) + Console.WriteLine ($" (not installed)"); + else + Console.WriteLine ($" (installed)"); + Console.WriteLine ($" Source: {source}"); + Console.WriteLine ($" Identifier: {identifier}"); + Console.WriteLine ($" InstallPrefix: {installPrefix}"); + } - if (only_check) { - if (doInstall) { - if (updateAvailable) { - Console.WriteLine (verbose > 0 ? $"The simulator '{name}' is installed, but an update is available." : name); - } else { - Console.WriteLine (verbose > 0 ? $"The simulator '{name}' is not installed." : name); + if (only_check) { + if (doInstall) { + if (updateAvailable) { + Console.WriteLine (verbose > 0 ? $"The simulator '{name}' is installed, but an update is available." : name); + } else { + Console.WriteLine (verbose > 0 ? $"The simulator '{name}' is not installed." : name); + } + exit_code = 1; + } else if (verbose > 0) { + Console.WriteLine ($"The simulator '{name}' is installed."); } - exit_code = 1; - } else if (verbose > 0) { - Console.WriteLine ($"The simulator '{name}' is installed."); } - } - if (doInstall && !only_check) { - Console.WriteLine ($"Installing {name}..."); - if (Install (source, fileSize, installPrefix)) { - Console.WriteLine ($"Installed {name} successfully."); - } else { - Console.WriteLine ($"Failed to install {name}."); - return 1; + if (doInstall && !only_check) { + Console.WriteLine ($"Installing {name}..."); + if (await InstallAsync (source, fileSize, installPrefix)) { + Console.WriteLine ($"Installed {name} successfully."); + } else { + Console.WriteLine ($"Failed to install {name}."); + return 1; + } } } } @@ -313,7 +327,7 @@ public static int Main (string [] args) static bool IsInstalled (string identifier, out Version? installedVersion) { - if (TryExecuteAndCapture ($"pkgutil", $"--pkg-info {identifier}", out var pkgInfo, out _)) { + if (TryExecuteAndCapture (out var pkgInfo, out _, $"pkgutil", "--pkg-info", identifier)) { var lines = pkgInfo.Split ('\n'); var version = lines.First ((v) => v.StartsWith ("version: ", StringComparison.Ordinal)).Substring ("version: ".Length); installedVersion = Version.Parse (version); @@ -324,8 +338,13 @@ static bool IsInstalled (string identifier, out Version? installedVersion) return false; } - static bool Install (string source, long fileSize, string installPrefix) + static async Task InstallAsync (string? source, long fileSize, string installPrefix) { + if (string.IsNullOrEmpty (source)) { + Console.WriteLine ($"Failed to install, no source provided."); + return false; + } + var download_dir = TempDirectory; var filename = Path.GetFileName (source); var download_path = Path.Combine (download_dir, filename); @@ -340,34 +359,49 @@ static bool Install (string source, long fileSize, string installPrefix) } if (download) { var downloadDone = new ManualResetEvent (false); - var wc = new WebClient (); + var wc = new HttpClient (); long lastProgress = 0; var watch = Stopwatch.StartNew (); - wc.DownloadProgressChanged += (sender, progress_args) => { + var downloadProgressChanged = (DownloadProgressChangedEventArgs progress_args) => { var progress = progress_args.BytesReceived * 100 / fileSize; if (progress > lastProgress) { lastProgress = progress; var duration = watch.Elapsed.TotalSeconds; var speed = progress_args.BytesReceived / duration; - var timeLeft = TimeSpan.FromSeconds ((progress_args.TotalBytesToReceive - progress_args.BytesReceived) / speed); + var timeLeft = TimeSpan.FromSeconds ((long) ((progress_args.TotalBytesToReceive - progress_args.BytesReceived) / speed)); Console.WriteLine ($"Downloaded {progress_args.BytesReceived:N0}/{fileSize:N0} bytes = {progress}% in {duration:N1}s ({speed / 1024.0 / 1024.0:N1} MB/s; approximately {timeLeft} left)"); } }; - wc.DownloadFileCompleted += (sender, download_args) => { + var downloadFileCompleted = (AsyncCompletedEventArgs download_args) => { Console.WriteLine ($"Download completed in {watch.Elapsed.TotalSeconds}s"); if (download_args.Error is not null) { Console.WriteLine ($" with error: {download_args.Error}"); } downloadDone.Set (); }; - wc.DownloadFileAsync (new Uri (source), download_path); + + using var file = new FileStream (download_path, FileMode.Create, FileAccess.Write, FileShare.None); + var response = await wc.GetAsync (new Uri (source), HttpCompletionOption.ResponseHeadersRead); + var contentLength = response.Content.Headers.ContentLength; + + using var responseStream = await response.Content.ReadAsStreamAsync (); + + var buffer = new byte [40960]; + long totalBytesRead = 0; + int bytesRead; + while ((bytesRead = await responseStream.ReadAsync (buffer, 0, buffer.Length)) > 0) { + file.Write (buffer, 0, bytesRead); + totalBytesRead += bytesRead; + downloadProgressChanged (new DownloadProgressChangedEventArgs () { BytesReceived = totalBytesRead, TotalBytesToReceive = contentLength ?? -1 }); + } + downloadFileCompleted (new AsyncCompletedEventArgs (null, false, null)); downloadDone.WaitOne (); } var mount_point = Path.Combine (download_dir, filename + "-mount"); Directory.CreateDirectory (mount_point); try { Console.WriteLine ($"Mounting '{download_path}' into '{mount_point}'..."); - if (!TryExecuteAndCapture ("hdiutil", $"attach '{download_path}' -mountpoint '{mount_point}' -quiet -nobrowse", out _)) { + if (!TryExecuteAndCapture (out _, "hdiutil", "attach", download_path, "-mountpoint", mount_point, "-quiet", "-nobrowse")) { Console.WriteLine ("Mount failure!"); return false; } @@ -389,7 +423,7 @@ static bool Install (string source, long fileSize, string installPrefix) if (Directory.Exists (expanded_path)) Directory.Delete (expanded_path, true); Console.WriteLine ($"Expanding '{packages [0]}' into '{expanded_path}'..."); - if (!TryExecuteAndCapture ("pkgutil", $"--expand '{packages [0]}' '{expanded_path}'", out _)) { + if (!TryExecuteAndCapture (out _, "pkgutil", "--expand", packages [0], expanded_path)) { Console.WriteLine ($"Failed to expand {packages [0]}"); return false; } @@ -401,21 +435,21 @@ static bool Install (string source, long fileSize, string installPrefix) // Add the install-location attribute to the pkg-info node var attr = packageInfoDoc.CreateAttribute ("install-location"); attr.Value = installPrefix; - packageInfoDoc.SelectSingleNode ("/pkg-info").Attributes.Append (attr); + packageInfoDoc.SelectSingleNode ("/pkg-info")!.Attributes!.Append (attr); packageInfoDoc.Save (packageInfoPath); - var fixed_path = Path.Combine (Path.GetDirectoryName (download_path), Path.GetFileNameWithoutExtension (download_path) + "-fixed.pkg"); + var fixed_path = Path.Combine (Path.GetDirectoryName (download_path)!, Path.GetFileNameWithoutExtension (download_path) + "-fixed.pkg"); if (File.Exists (fixed_path)) File.Delete (fixed_path); try { Console.WriteLine ($"Creating fixed package '{fixed_path}' from '{expanded_path}'..."); - if (!TryExecuteAndCapture ("pkgutil", $"--flatten '{expanded_path}' '{fixed_path}'", out _)) { + if (!TryExecuteAndCapture (out _, "pkgutil", "--flatten", expanded_path, fixed_path)) { Console.WriteLine ("Failed to create fixed package."); return false; } Console.WriteLine ($"Installing '{fixed_path}'..."); - if (!TryExecuteAndCapture ("sudo", $"installer -pkg '{fixed_path}' -target / -verbose -dumplog", out _)) { + if (!TryExecuteAndCapture (out _, "sudo", "installer", "-pkg", fixed_path, "-target", "/", "-verbose", "-dumplog")) { Console.WriteLine ("Failed to install package."); return false; } @@ -427,7 +461,7 @@ static bool Install (string source, long fileSize, string installPrefix) Directory.Delete (expanded_path, true); } } finally { - if (!TryExecuteAndCapture ("hdiutil", $"detach '{mount_point}' -quiet -force", out _)) + if (!TryExecuteAndCapture (out _, "hdiutil", "detach", mount_point, "-quiet", "-force")) Console.WriteLine ($"Failed to detach {mount_point}"); } } finally { @@ -456,16 +490,7 @@ static bool Install (string source, long fileSize, string installPrefix) } } -#if !NET // the below attributes are no longer needed once we switch to .NET -namespace System.Diagnostics.CodeAnalysis { - [AttributeUsage (AttributeTargets.Parameter | AttributeTargets.Property | AttributeTargets.ReturnValue, AllowMultiple = true, Inherited = false)] - internal sealed class NotNullIfNotNullAttribute : Attribute { - public string ParameterName { get; } - - public NotNullIfNotNullAttribute (string parameterName) - { - ParameterName = parameterName; - } - } +class DownloadProgressChangedEventArgs { + public long TotalBytesToReceive; + public long BytesReceived; } -#endif // !NET diff --git a/tools/siminstaller/siminstaller.csproj b/tools/siminstaller/siminstaller.csproj index 850b5d9b8c4a..05c5a87b8c79 100644 --- a/tools/siminstaller/siminstaller.csproj +++ b/tools/siminstaller/siminstaller.csproj @@ -1,56 +1,15 @@ - - + - Debug - x86 - {DA71C0E6-7A4F-430D-B7D2-D134F4E4DD98} Exe - siminstaller - siminstaller - v4.6.1 - default + net$(BundledNETCoreAppTargetFrameworkVersion) + enable enable + latest + false - - true - full - false - bin\Debug - DEBUG; - prompt - 4 - true - x86 - - - true - bin\Release - prompt - 4 - true - x86 - - - Project - --xcode /Applications/Xcode.app --print-simulators - false - false - - - Project - --xcode /Applications/Xcode.app --install com.apple.pkg.iPhoneSimulatorSDK8_1 - false - false - - - - - + + - - - - - \ No newline at end of file + From 10f3b5ae32bb0d8f1c252fda78e6eba30d1e3ef6 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 19 Jun 2024 15:10:54 +0200 Subject: [PATCH 08/17] [tests] Convert the testgenerator.csproj to a .NET project. (#20740) --- tests/test-libraries/Makefile | 9 ++--- tests/test-libraries/testgenerator.cs | 22 ++++++------ tests/test-libraries/testgenerator.csproj | 42 ++++++----------------- 3 files changed, 26 insertions(+), 47 deletions(-) diff --git a/tests/test-libraries/Makefile b/tests/test-libraries/Makefile index 6a3794a654f4..77566bd562fb 100644 --- a/tests/test-libraries/Makefile +++ b/tests/test-libraries/Makefile @@ -31,11 +31,12 @@ GENERATED_FILES_PATTERN = \ RegistrarTest.generated%cs \ TrampolineTest.generated%cs \ -testgenerator.exe: testgenerator.cs Makefile - $(Q) $(SYSTEM_CSC) -out:$@ $< +TESTGENERATOR=bin/Debug/testgenerator.dll +$(TESTGENERATOR): testgenerator.cs Makefile + $(Q) unset MSBUILD_EXE_PATH && $(DOTNET) build testgenerator.csproj $(DOTNET_BUILD_VERBOSITY) -$(GENERATED_FILES_PATTERN): testgenerator.exe - $(Q) mono --debug $< +$(GENERATED_FILES_PATTERN): $(TESTGENERATOR) + $(Q) $(DOTNET) exec $< libtest-object.m libtest-ar.m: $(Q) ln -fhs libtest.m $@ diff --git a/tests/test-libraries/testgenerator.cs b/tests/test-libraries/testgenerator.cs index 27de6c58eb44..6aa578232032 100644 --- a/tests/test-libraries/testgenerator.cs +++ b/tests/test-libraries/testgenerator.cs @@ -50,16 +50,16 @@ enum Architecture { static Architecture [] strets = structs_and_stret.Select ((v) => v.IndexOf (':') >= 0 ? (Architecture) int.Parse (v.Substring (v.IndexOf (':') + 1)) : Architecture.None).ToArray (); class BindAsData { - public string Managed; - public string Native; - public string ManagedCondition; - public string ManagedNewExpression; - public string Map; - public string MapFrom; - public Version MinXcodeVersion; - public Version MinMacOSVersion; - public string ToNSNumberCastExpression; - public string FromNSNumberCastExpression; + public string? Managed; + public string? Native; + public string? ManagedCondition; + public string? ManagedNewExpression; + public string? Map; + public string? MapFrom; + public Version? MinXcodeVersion; + public Version? MinMacOSVersion; + public string? ToNSNumberCastExpression; + public string? FromNSNumberCastExpression; } static BindAsData [] bindas_nsnumber = new [] { @@ -1651,7 +1651,7 @@ static string GenerateNewExpression (string s, int multiplier = 1) static void Main () { while (Path.GetFileName (Environment.CurrentDirectory) != "test-libraries") - Environment.CurrentDirectory = Path.GetDirectoryName (Environment.CurrentDirectory); + Environment.CurrentDirectory = Path.GetDirectoryName (Environment.CurrentDirectory)!; /* native code */ WriteLibTestStructH (); diff --git a/tests/test-libraries/testgenerator.csproj b/tests/test-libraries/testgenerator.csproj index 53147eb14267..3a6e60b0eb38 100644 --- a/tests/test-libraries/testgenerator.csproj +++ b/tests/test-libraries/testgenerator.csproj @@ -1,39 +1,17 @@ - - + + - Debug - AnyCPU - 8.0.30703 - 2.0 - {CD430449-8E59-4ECD-ADD9-ACF79E9E660B} Exe - testgenerator - testgenerator - v4.5 + net$(BundledNETCoreAppTargetFrameworkVersion) + enable + enable latest + false + false - - true - full - false - bin\Debug - DEBUG; - prompt - 4 - true - - - true - bin\Release - prompt - 4 - true - - - - + - - \ No newline at end of file + + From fc6ce95b52f6f3e21f5b71b3be4e853106be790f Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 19 Jun 2024 15:11:28 +0200 Subject: [PATCH 09/17] [runtime] Convert the bindings-generator.csproj project to a .NET project. (#20743) --- runtime/Makefile | 9 +++-- runtime/bindings-generator.cs | 61 ++++++++++++++----------------- runtime/bindings-generator.csproj | 48 +++++------------------- 3 files changed, 42 insertions(+), 76 deletions(-) diff --git a/runtime/Makefile b/runtime/Makefile index 7485398e2999..deb985193750 100644 --- a/runtime/Makefile +++ b/runtime/Makefile @@ -60,11 +60,12 @@ delegates.%: delegates.%.t4 delegates.t4 Delegates.generated.cs: Delegates.cs.t4 delegates.t4 $(Q_GEN) $(TT) $< -o $@ -bindings-generator.exe: bindings-generator.cs - $(Q) $(SYSTEM_CSC) $< -out:$@ -debug:full -features:strict +BINDINGS_GENERATOR=bin/Debug/bindings-generator.dll +$(BINDINGS_GENERATOR): bindings-generator.cs bindings-generator.csproj + $(Q) $(DOTNET) build bindings-generator.csproj $(DOTNET_BUILD_VERBOSITY) -bindings-generated.m: bindings-generator.exe - $(Q_GEN) $(SYSTEM_MONO) --debug $< $@ +bindings-generated.m: $(BINDINGS_GENERATOR) + $(Q_GEN) $(DOTNET) exec $< $@ # our makefiles don't support building the same source file multiple times with different defines, # so just symlink extension-main.m to another file and compile that instead. diff --git a/runtime/bindings-generator.cs b/runtime/bindings-generator.cs index cc32d75da496..cd369da74973 100644 --- a/runtime/bindings-generator.cs +++ b/runtime/bindings-generator.cs @@ -91,16 +91,7 @@ static IEnumerable GetFunctionData (bool isDotNet = false) { var data = new List (); - Types.NativeHandleType = isDotNet ? Types.NativeHandle : Types.IntPtr; - Types.Vector2d = isDotNet ? Types.NVector2d : Types.OpenTK_Vector2d; - Types.Vector4d = isDotNet ? Types.NVector4d : Types.OpenTK_Vector4d; - Types.Vector2i = isDotNet ? Types.NVector2i : Types.OpenTK_Vector2i; - Types.Vector3i = isDotNet ? Types.NVector3i : Types.OpenTK_Vector3i; - Types.Vector4i = isDotNet ? Types.NVector4i : Types.OpenTK_Vector4i; - Types.Matrix3f = isDotNet ? Types.RMatrix3f : Types.OpenTK_Matrix3f; - Types.Matrix4f = isDotNet ? Types.Numerics_Matrix4f : Types.OpenTK_Matrix4f; - Types.QuatD = isDotNet ? Types.NQuaterniond : Types.OpenTK_QuatD; - Types.MDLVoxelIndexExtent = isDotNet ? Types.MDLVoxelIndexExtent_DotNet : Types.MDLVoxelIndexExtent2; + Types.IsDotNet = isDotNet; data.Add ( new FunctionData { @@ -2992,7 +2983,7 @@ static string GetFuncName (FunctionData func, string overload) return funcName.ToString (); } - static void WriteParametersMarshal (StringWriter writer, ParameterData [] ps) + static void WriteParametersMarshal (StringWriter writer, ParameterData []? ps) { if (ps is null) return; @@ -3007,7 +2998,7 @@ static void WriteParametersMarshal (StringWriter writer, ParameterData [] ps) } } - static void WriteParametersInvoke (StringWriter writer, ParameterData [] ps) + static void WriteParametersInvoke (StringWriter writer, ParameterData []? ps) { if (ps is null) return; @@ -3025,7 +3016,7 @@ static void WriteParametersInvoke (StringWriter writer, ParameterData [] ps) } } - static void WriteParametersNativeDeclaration (StringWriter writer, ParameterData [] parameters, bool isTypedef, FunctionData func) + static void WriteParametersNativeDeclaration (StringWriter writer, ParameterData []? parameters, bool isTypedef, FunctionData func) { if (parameters is null) return; @@ -3122,7 +3113,7 @@ static void Write_objc_msgSend (StringWriter writer, FunctionData func) // body writer.WriteLine ("{"); if (tmpReturnValue) { - writer.WriteLine ("\t{0} rv;", func.ReturnType.NativeType); + writer.WriteLine ("\t{0} rv;", func.ReturnType!.NativeType); } // marshal managed parameters to native format @@ -3185,7 +3176,7 @@ static void Write_objc_msgSendSuper (StringWriter writer, FunctionData func) // body writer.WriteLine ("{"); if (tmpReturnValue) { - writer.WriteLine ("\t{0} rv;", func.ReturnType.NativeType); + writer.WriteLine ("\t{0} rv;", func.ReturnType!.NativeType); } // marshal managed parameters to native format @@ -3354,6 +3345,7 @@ static void Write_objc_msgSendSuper_stret (StringWriter writer, FunctionData fun } public static class Types { + public static bool IsDotNet; public static TypeData Vector2 = new TypeData { ManagedType = "Vector2", NativeType = "vector_float2", @@ -3385,7 +3377,7 @@ public static class Types { NativeWrapperType = "struct Vector4f", RequireMarshal = true, }; - public static TypeData Vector2i; + public static TypeData Vector2i { get => IsDotNet ? Types.NVector2i : Types.OpenTK_Vector2i; } public static TypeData OpenTK_Vector2i = new TypeData { ManagedType = "Vector2i", NativeType = "vector_int2", @@ -3400,7 +3392,7 @@ public static class Types { RequireMarshal = true, IsX86Stret = true, }; - public static TypeData Vector3i; + public static TypeData Vector3i { get => IsDotNet ? Types.NVector3i : Types.OpenTK_Vector3i; } public static TypeData OpenTK_Vector3i = new TypeData { ManagedType = "Vector3i", NativeType = "vector_int3", @@ -3413,7 +3405,7 @@ public static class Types { NativeWrapperType = "struct Vector3i", RequireMarshal = true, }; - public static TypeData Vector4i; + public static TypeData Vector4i { get => IsDotNet ? Types.NVector4i : Types.OpenTK_Vector4i; } public static TypeData OpenTK_Vector4i = new TypeData { ManagedType = "Vector4i", NativeType = "vector_int4", @@ -3426,7 +3418,7 @@ public static class Types { NativeWrapperType = "struct Vector4i", RequireMarshal = true, }; - public static TypeData Vector2d; + public static TypeData Vector2d { get => IsDotNet ? NVector2d : OpenTK_Vector2d; } public static TypeData OpenTK_Vector2d = new TypeData { ManagedType = "Vector2d", NativeType = "vector_double2", @@ -3451,7 +3443,7 @@ public static class Types { NativeWrapperType = "struct Vector4d", // Yes, Vector4d, since NVector3d has 4 doubles. RequireMarshal = true, }; - public static TypeData Vector4d; + public static TypeData Vector4d { get => IsDotNet ? Types.NVector4d : Types.OpenTK_Vector4d; } public static TypeData OpenTK_Vector4d = new TypeData { ManagedType = "Vector4d", NativeType = "vector_double4", @@ -3482,7 +3474,7 @@ public static class Types { IsX86Stret = true, IsX64Stret = false, }; - public static TypeData Matrix3f; + public static TypeData Matrix3f { get => IsDotNet ? Types.RMatrix3f : Types.OpenTK_Matrix3f; } public static TypeData OpenTK_Matrix3f = new TypeData { ManagedType = "Matrix3", NativeType = "matrix_float3x3", @@ -3510,7 +3502,7 @@ public static class Types { IsX86Stret = true, IsX64Stret = true, }; - public static TypeData Matrix4f; + public static TypeData Matrix4f { get => IsDotNet ? Types.Numerics_Matrix4f : Types.OpenTK_Matrix4f; } public static TypeData OpenTK_Matrix4f = new TypeData { ManagedType = "Matrix4", NativeType = "matrix_float4x4", @@ -3570,7 +3562,7 @@ public static class Types { NativeWrapperType = "void *", RequireMarshal = false, }; - public static TypeData NativeHandleType; + public static TypeData NativeHandleType { get => IsDotNet ? NativeHandle : IntPtr; } public static TypeData NativeHandle = new TypeData { ManagedType = "NativeHandle", NativeType = "void *", @@ -3708,7 +3700,7 @@ public static class Types { RequireMarshal = true, }; - public static TypeData QuatD; + public static TypeData QuatD { get => IsDotNet ? Types.NQuaterniond : Types.OpenTK_QuatD; } public static TypeData OpenTK_QuatD = new TypeData { ManagedType = "Quaterniond", NativeType = "simd_quatd", @@ -3734,7 +3726,7 @@ public static class Types { IsX64Stret = true, }; - public static TypeData MDLVoxelIndexExtent; + public static TypeData MDLVoxelIndexExtent { get => IsDotNet ? Types.MDLVoxelIndexExtent_DotNet : Types.MDLVoxelIndexExtent2; } public static TypeData MDLVoxelIndexExtent_DotNet = new TypeData { ManagedType = "MDLVoxelIndexExtent", @@ -3777,9 +3769,9 @@ enum Variants { } class TypeData { - public string ManagedType; - public string NativeWrapperType; - public string NativeType; + public string ManagedType = string.Empty; + public string NativeWrapperType = string.Empty; + public string NativeType = string.Empty; public bool RequireMarshal; public bool IsX86Stret; #pragma warning disable 649 @@ -3790,8 +3782,8 @@ class TypeData { #pragma warning restore 649 public bool IsAnyStret { get { return IsX86Stret || IsX64Stret || IsARMStret; } } public bool IsNativeType; - public TypeData Bit32Type; - public TypeData Bit64Type; + public TypeData? Bit32Type; + public TypeData? Bit64Type; public TypeData AsSpecificNativeType (bool as32bit) { @@ -3802,21 +3794,22 @@ public TypeData AsSpecificNativeType (bool as32bit) } class ParameterData { - public TypeData TypeData; + TypeData? typeData; + public TypeData TypeData { get => typeData!; set => typeData = value; } public bool IsRef; } class FunctionData { - public string Comment; + public string? Comment; public string Prefix = string.Empty; // Variants is a [Flags] enum, specifying which of the objc_msgSend variants // should be generated. You'll usually use "All", which will generate all 4, // or "NonStret", which will just generate objc_msgSend and objc_msgSendSuper. public Variants Variants; // The return type of the function. Use null for void. - public TypeData ReturnType; + public TypeData? ReturnType; // The parameters. Use null for void. - public ParameterData [] Parameters; + public ParameterData []? Parameters; public bool MarshalExceptions; public bool HasNativeType { diff --git a/runtime/bindings-generator.csproj b/runtime/bindings-generator.csproj index d93eb041ebe3..4348bb7b6941 100644 --- a/runtime/bindings-generator.csproj +++ b/runtime/bindings-generator.csproj @@ -1,45 +1,17 @@ - - + + - Debug - x86 - {6B616492-49F0-43EF-8390-F9D1B4CCC632} Exe - zzg - bindings-generator - v4.5 - 8.0.30703 - 2.0 + net$(BundledNETCoreAppTargetFrameworkVersion) + enable + enable + latest + false + false - - true - full - false - bin\Debug - DEBUG; - prompt - 4 - x86 - false - - - full - true - bin\Release - prompt - 4 - true - x86 - - - Project - ${ProjectDir}/bindings-generated.m - - - - + - + From 9255b005929805aeb365e73a2d9699dab6fc92bd Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Wed, 19 Jun 2024 10:53:20 -0400 Subject: [PATCH 10/17] [CI] On-board the build pipeline to 1ES. (#20735) Few comments: 1. Extends cannot be used on templates. So we have to do a single extend and have duplicated code. 2. There are some common templates that we are working around using the use1ES parameter. 3. We are reusing the configure steps on other pipelines. That step should only be done in the build, that change is too big for this PR. 4. The governance template is not longer needed since the 1ES template provides it. --------- Co-authored-by: Rolf Bjarne Kvinge --- tools/devops/automation/build-pipeline.yml | 274 ++++++++++++++++-- .../devops/automation/build-pull-request.yml | 270 +++++++++++++++-- .../automation/templates/api-diff-stage.yml | 1 + .../build/api-diff-build-and-detect.yml | 1 + .../templates/build/build-mac-tests-stage.yml | 3 +- .../templates/build/build-mac-tests.yml | 6 +- .../automation/templates/build/build-pkgs.yml | 36 +-- .../templates/build/build-stage.yml | 3 +- .../automation/templates/build/build.yml | 5 + .../automation/templates/common/configure.yml | 24 +- .../templates/common/mac-agent-logs.yml | 38 ++- .../automation/templates/common/teardown.yml | 5 + .../templates/governance/apiscan.yml | 6 +- .../templates/governance/general.yml | 108 ------- .../automation/templates/governance/stage.yml | 15 +- .../automation/templates/main-stage.yml | 19 +- .../templates/pipelines/build-pipeline.yml | 224 -------------- .../templates/release/publish-nugets.yml | 22 +- .../templates/release/vs-insertion-prep.yml | 34 ++- .../templates/sign-and-notarized/funnel.yml | 43 +-- .../sign-and-notarized/prepare-pkg-stage.yml | 5 +- .../templates/sign-and-notarized/setup.yml | 3 + .../sign-and-notarized/sign-and-notarized.yml | 12 +- .../sign-and-notarized/upload-azure.yml | 24 +- 24 files changed, 698 insertions(+), 483 deletions(-) delete mode 100644 tools/devops/automation/templates/governance/general.yml delete mode 100644 tools/devops/automation/templates/pipelines/build-pipeline.yml diff --git a/tools/devops/automation/build-pipeline.yml b/tools/devops/automation/build-pipeline.yml index 648a2ec8d377..6b6e756fb533 100644 --- a/tools/devops/automation/build-pipeline.yml +++ b/tools/devops/automation/build-pipeline.yml @@ -4,33 +4,267 @@ # YAML build pipeline based on the Jenkins multi-stage (main branch) build workflow # https://jenkins.internalx.com/view/Xamarin.MaciOS/job/macios/job/main/ # https://jenkins.internalx.com/view/Xamarin.MaciOS/job/macios/configure +parameters: + +- name: provisionatorChannel + displayName: Provisionator channel to use + type: string + default: 'latest' + +- name: macOSName # comes from the build agent demand named macOS.Name + displayName: Name of the version of macOS to use + type: string + default: 'Sonoma' + +- name: pool + type: string + displayName: Bot pool to use + default: automatic + values: + - pr + - ci + - automatic + +- name: runGovernanceTests + displayName: Run Governance Checks + type: boolean + default: true + +- name: forceInsertion + displayName: Force Insertion + type: boolean + default: false + +- name: skipESRP + displayName: Skip ESRP + type: boolean + default: false # only to be used when testing the CI and we do not need a signed pkg + +- name: pushNugets + type: boolean + default: true + +- name: pushNugetsToMaestro + type: boolean + default: true + +- name: testConfigurations + displayName: Test configurations to run + type: object + default: [] + +- name: deviceTestsConfigurations + displayName: Device test configurations to run + type: object + default: [ + { + testPrefix: 'iOS64', + stageName: 'ios64b_device', + displayName: 'iOS64 Device Tests', + testPool: 'VSEng-Xamarin-Mac-Devices', + testsLabels: '--label=run-ios-tests,run-non-monotouch-tests,run-monotouch-tests,run-mscorlib-tests', + statusContext: 'VSTS: device tests iOS', + makeTarget: 'vsts-device-tests', + extraBotDemands: [ + 'ios', + ] + }, + { + testPrefix: 'tvos', + stageName: 'tvos_device', + displayName: 'tvOS Device Tests', + testPool: 'VSEng-Xamarin-Mac-Devices', + testsLabels: '--label=run-tvos-tests,run-non-monotouch-tests,run-monotouch-tests,run-mscorlib-tests', + statusContext: 'VSTS: device tests tvOS', + makeTarget: 'vsts-device-tests', + extraBotDemands: [ + 'tvos', + ] + }] + +- name: macTestsConfigurations + displayName: macOS test configurations to run + type: object + default: [ + { + stageName: 'mac_11_m1', + displayName: 'M1 - Mac Big Sur (11)', + macPool: 'VSEng-VSMac-Xamarin-Shared', + useImage: false, + statusContext: 'M1 - Mac Big Sur (11)', + demands: [ + "Agent.OS -equals Darwin", + "macOS.Name -equals BigSur", + "macOS.Architecture -equals arm64", + "Agent.HasDevices -equals False", + "Agent.IsPaired -equals False" + ] + }, + { + stageName: 'mac_12_m1', + displayName: 'M1 - Mac Ventura (12)', + macPool: 'VSEng-VSMac-Xamarin-Shared', + useImage: false, + statusContext: 'M1 - Mac Monterey (12)', + demands: [ + "Agent.OS -equals Darwin", + "macOS.Name -equals Monterey", + "macOS.Architecture -equals arm64", + "Agent.HasDevices -equals False", + "Agent.IsPaired -equals False" + ] + }, + { + stageName: 'mac_13_m1', + displayName: 'M1 - Mac Ventura (13)', + macPool: 'VSEng-VSMac-Xamarin-Shared', + useImage: false, + statusContext: 'M1 - Mac Ventura (13)', + demands: [ + "Agent.OS -equals Darwin", + "macOS.Name -equals Ventura", + "macOS.Architecture -equals arm64", + "Agent.HasDevices -equals False", + "Agent.IsPaired -equals False" + ] + }, + { + stageName: 'mac_14_x64', + displayName: 'X64 - Mac Sonoma (14)', + macPool: 'VSEng-Xamarin-RedmondMacBuildPool-iOS-Untrusted', + useImage: false, + statusContext: 'X64 - Mac Sonoma (14)', + demands: [ + "Agent.OS -equals Darwin", + "macOS.Name -equals Sonoma", + "macOS.Architecture -equals x64", + "Agent.HasDevices -equals False", + "Agent.IsPaired -equals False" + ] + }] + +resources: + repositories: + - repository: self + checkoutOptions: + submodules: true + + - repository: yaml-templates + type: github + name: xamarin/yaml-templates + ref: refs/heads/main + endpoint: xamarin + + - repository: sdk-insertions + type: github + name: xamarin/sdk-insertions + ref: refs/heads/main + endpoint: xamarin + + - repository: maccore + type: github + name: xamarin/maccore + ref: refs/heads/main + endpoint: xamarin + + - repository: release-scripts + type: github + name: xamarin/release-scripts + ref: refs/heads/only_codesign + endpoint: xamarin + + - repository: CustomPipelineTemplates + type: git + name: 1ESPipelineTemplates/MicroBuildTemplate + +variables: +- ${{ if contains(variables['Build.DefinitionName'], 'private') }}: + - template: templates/vsts-variables.yml +- template: templates/common/vs-release-vars.yml@sdk-insertions +- template: templates/variables.yml +- name: MicrobuildConnector + value: 'MicroBuild Signing Task (DevDiv)' +- name: MaciosUploadPrefix + value: '' +- name: DisablePipelineConfigDetector + value: true trigger: branches: include: - - '*' + - '*' exclude: - - refs/heads/locfiles/* - - refs/heads/dev/* + - refs/heads/locfiles/* + - refs/heads/dev/* paths: exclude: - - .github - - docs - - CODEOWNERS - - ISSUE_TEMPLATE.md - - LICENSE - - NOTICE.txt - - SECURITY.MD - - README.md - - src/README.md - - tools/mtouch/README.md - - msbuild/Xamarin.Localization.MSBuild/README.md + - .github + - docs + - CODEOWNERS + - ISSUE_TEMPLATE.md + - LICENSE + - NOTICE.txt + - SECURITY.MD + - README.md + - src/README.md + - tools/mtouch/README.md + - msbuild/Xamarin.Localization.MSBuild/README.md extends: - template: templates/pipelines/build-pipeline.yml + template: azure-pipelines/MicroBuild.1ES.Official.yml@CustomPipelineTemplates parameters: - isPR: false - signingSetupSteps: - - template: ./templates/sign-and-notarized/setup.yml - parameters: - isPR: false + pool: # default pool to be used for validation jobs + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + os: windows + sdl: + baseline: + baselineFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\baselines.gdnbaselines' + suppression: + suppressionFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\suppress.gdnsuppress' + sourceAnalysisPool: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + os: windows + tsa: + configFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\tsa_config.gdntsa' + sbom: + enabled: false # we run our on sbom generation + credscan: + suppressionsFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\CredScanSuppressions.json' + outputFormat: sarif + debugMode: false + batchSize: 16 + policheck: + exclusionsFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\PoliCheckExclusions.xml' + sourceRepositoriesToScan: + runInSingleJob: true # run both maccore and macios in the same job + include: + - repository: maccore + exclude: + - repository: yaml-templates + - repository: sdk-insertions + - repository: release-scripts + stages: + - template: templates/main-stage.yml + parameters: + xcodeChannel: Stable + macOSName: ${{ parameters.macOSName }} + isPR: false + provisionatorChannel: ${{ parameters.provisionatorChannel }} + pool: ${{ parameters.pool }} + runGovernanceTests: ${{ parameters.runGovernanceTests }} + forceInsertion: ${{ parameters.forceInsertion }} + skipESRP: ${{ parameters.skipESRP }} + pushNugets: ${{ parameters.pushNugets }} + pushNugetsToMaestro: ${{ parameters.pushNugetsToMaestro }} + ${{ if ne(length(parameters.testConfigurations), 0)}}: + testConfigurations: ${{ parameters.testConfigurations }} + deviceTestsConfigurations: ${{ parameters.deviceTestsConfigurations }} + macTestsConfigurations: ${{ parameters.macTestsConfigurations }} + azureStorage: ${{ variables['azureStorage'] }} + azureContainer: ${{ variables['azureContainer'] }} + signingSetupSteps: + - template: ./templates/sign-and-notarized/setup.yml + parameters: + isPR: false diff --git a/tools/devops/automation/build-pull-request.yml b/tools/devops/automation/build-pull-request.yml index 82a87a70f0d1..0e3fae13c80a 100644 --- a/tools/devops/automation/build-pull-request.yml +++ b/tools/devops/automation/build-pull-request.yml @@ -5,32 +5,266 @@ # https://jenkins.internalx.com/view/Xamarin.MaciOS/job/macios/job/main/ # https://jenkins.internalx.com/view/Xamarin.MaciOS/job/macios/configure +parameters: + +- name: provisionatorChannel + displayName: Provisionator channel to use + type: string + default: 'latest' + +- name: macOSName # comes from the build agent demand named macOS.Name + displayName: Name of the version of macOS to use + type: string + default: 'Sonoma' + +- name: pool + type: string + displayName: Bot pool to use + default: automatic + values: + - pr + - ci + - automatic + +- name: runGovernanceTests + displayName: Run Governance Checks + type: boolean + default: true + +- name: forceInsertion + displayName: Force Insertion + type: boolean + default: false + +- name: skipESRP + displayName: Skip ESRP + type: boolean + default: false # only to be used when testing the CI and we do not need a signed pkg + +- name: pushNugets + type: boolean + default: true + +- name: pushNugetsToMaestro + type: boolean + default: true + +- name: testConfigurations + displayName: Test configurations to run + type: object + default: [] + +- name: deviceTestsConfigurations + displayName: Device test configurations to run + type: object + default: [ + { + testPrefix: 'iOS64', + stageName: 'ios64b_device', + displayName: 'iOS64 Device Tests', + testPool: 'VSEng-Xamarin-Mac-Devices', + testsLabels: '--label=run-ios-tests,run-non-monotouch-tests,run-monotouch-tests,run-mscorlib-tests', + statusContext: 'VSTS: device tests iOS', + makeTarget: 'vsts-device-tests', + extraBotDemands: [ + 'ios', + ] + }, + { + testPrefix: 'tvos', + stageName: 'tvos_device', + displayName: 'tvOS Device Tests', + testPool: 'VSEng-Xamarin-Mac-Devices', + testsLabels: '--label=run-tvos-tests,run-non-monotouch-tests,run-monotouch-tests,run-mscorlib-tests', + statusContext: 'VSTS: device tests tvOS', + makeTarget: 'vsts-device-tests', + extraBotDemands: [ + 'tvos', + ] + }] + +- name: macTestsConfigurations + displayName: macOS test configurations to run + type: object + default: [ + { + stageName: 'mac_11_m1', + displayName: 'M1 - Mac Big Sur (11)', + macPool: 'VSEng-VSMac-Xamarin-Shared', + useImage: false, + statusContext: 'M1 - Mac Big Sur (11)', + demands: [ + "Agent.OS -equals Darwin", + "macOS.Name -equals BigSur", + "macOS.Architecture -equals arm64", + "Agent.HasDevices -equals False", + "Agent.IsPaired -equals False" + ] + }, + { + stageName: 'mac_12_m1', + displayName: 'M1 - Mac Ventura (12)', + macPool: 'VSEng-VSMac-Xamarin-Shared', + useImage: false, + statusContext: 'M1 - Mac Monterey (12)', + demands: [ + "Agent.OS -equals Darwin", + "macOS.Name -equals Monterey", + "macOS.Architecture -equals arm64", + "Agent.HasDevices -equals False", + "Agent.IsPaired -equals False" + ] + }, + { + stageName: 'mac_13_m1', + displayName: 'M1 - Mac Ventura (13)', + macPool: 'VSEng-VSMac-Xamarin-Shared', + useImage: false, + statusContext: 'M1 - Mac Ventura (13)', + demands: [ + "Agent.OS -equals Darwin", + "macOS.Name -equals Ventura", + "macOS.Architecture -equals arm64", + "Agent.HasDevices -equals False", + "Agent.IsPaired -equals False" + ] + }, + { + stageName: 'mac_14_x64', + displayName: 'X64 - Mac Sonoma (14)', + macPool: 'VSEng-Xamarin-RedmondMacBuildPool-iOS-Untrusted', + useImage: false, + statusContext: 'X64 - Mac Sonoma (14)', + demands: [ + "Agent.OS -equals Darwin", + "macOS.Name -equals Sonoma", + "macOS.Architecture -equals x64", + "Agent.HasDevices -equals False", + "Agent.IsPaired -equals False" + ] + }] + +resources: + repositories: + - repository: self + checkoutOptions: + submodules: true + + - repository: yaml-templates + type: github + name: xamarin/yaml-templates + ref: refs/heads/main + endpoint: xamarin + + - repository: sdk-insertions + type: github + name: xamarin/sdk-insertions + ref: refs/heads/main + endpoint: xamarin + + - repository: maccore + type: github + name: xamarin/maccore + ref: refs/heads/main + endpoint: xamarin + + - repository: release-scripts + type: github + name: xamarin/release-scripts + ref: refs/heads/only_codesign + endpoint: xamarin + + - repository: CustomPipelineTemplates + type: git + name: 1ESPipelineTemplates/MicroBuildTemplate + +variables: +- template: templates/variables.yml +- template: templates/common/vs-release-vars.yml@sdk-insertions +- name: MicrobuildConnector + value: '' +- name: MaciosUploadPrefix + value: '' +- name: Packaging.EnableSBOMSigning + value: false + + trigger: none pr: autoCancel: true branches: include: - - '*' # yes, you do need the quote, * has meaning in yamls + - '*' # yes, you do need the quote, * has meaning in yamls paths: exclude: - - .github - - docs - - CODEOWNERS - - ISSUE_TEMPLATE.md - - LICENSE - - NOTICE.txt - - SECURITY.MD - - README.md - - src/README.md - - tools/mtouch/README.md - - msbuild/Xamarin.Localization.MSBuild/README.md + - .github + - docs + - CODEOWNERS + - ISSUE_TEMPLATE.md + - LICENSE + - NOTICE.txt + - SECURITY.MD + - README.md + - src/README.md + - tools/mtouch/README.md + - msbuild/Xamarin.Localization.MSBuild/README.md extends: - template: templates/pipelines/build-pipeline.yml + template: azure-pipelines/MicroBuild.1ES.Official.yml@CustomPipelineTemplates parameters: - isPR: true - signingSetupSteps: - - template: ./templates/sign-and-notarized/setup.yml - parameters: - isPR: true + pool: # default pool to be used forvalidation jobs + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + os: windows + sdl: + baseline: + baselineFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\baselines.gdnbaselines' + suppression: + suppressionFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\suppress.gdnsuppress' + sourceAnalysisPool: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + os: windows + tsa: + configFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\tsa_config.gdntsa' + sbom: + enabled: false # we run our on sbom generation + credscan: + suppressionsFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\CredScanSuppressions.json' + outputFormat: sarif + debugMode: false + batchSize: 16 + policheck: + exclusionsFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\PoliCheckExclusions.xml' + sourceRepositoriesToScan: + runInSingleJob: true # run both maccore and macios in the same job + include: + - repository: maccore + exclude: + - repository: yaml-templates + - repository: sdk-insertions + - repository: release-scripts + stages: + - template: templates/main-stage.yml + parameters: + xcodeChannel: Stable + macOSName: ${{ parameters.macOSName }} + isPR: true + provisionatorChannel: ${{ parameters.provisionatorChannel }} + pool: ${{ parameters.pool }} + runGovernanceTests: ${{ parameters.runGovernanceTests }} + forceInsertion: ${{ parameters.forceInsertion }} + skipESRP: ${{ parameters.skipESRP }} + pushNugets: ${{ parameters.pushNugets }} + pushNugetsToMaestro: ${{ parameters.pushNugetsToMaestro }} + ${{ if ne(length(parameters.testConfigurations), 0)}}: + testConfigurations: ${{ parameters.testConfigurations }} + deviceTestsConfigurations: ${{ parameters.deviceTestsConfigurations }} + macTestsConfigurations: ${{ parameters.macTestsConfigurations }} + azureStorage: ${{ variables['azureStorage'] }} + azureContainer: ${{ variables['azureContainer'] }} + signingSetupSteps: + - template: ./templates/sign-and-notarized/setup.yml + parameters: + isPR: true diff --git a/tools/devops/automation/templates/api-diff-stage.yml b/tools/devops/automation/templates/api-diff-stage.yml index 421472d40aaa..f824b30f0118 100644 --- a/tools/devops/automation/templates/api-diff-stage.yml +++ b/tools/devops/automation/templates/api-diff-stage.yml @@ -83,6 +83,7 @@ stages: testsLabels: '--label=skip-all-tests,run-ios-tests,run-ios-simulator-tests,run-tvos-tests,run-watchos-tests,run-mac-tests,run-maccatalyst-tests,run-dotnet-tests,run-system-permission-tests,run-legacy-xamarin-tests' statusContext: 'VSTS: simulator tests' uploadArtifacts: true + use1ES: false - stage: generate_api_diff displayName: 'API diff' diff --git a/tools/devops/automation/templates/build/api-diff-build-and-detect.yml b/tools/devops/automation/templates/build/api-diff-build-and-detect.yml index 090bfe267113..678669c58018 100644 --- a/tools/devops/automation/templates/build/api-diff-build-and-detect.yml +++ b/tools/devops/automation/templates/build/api-diff-build-and-detect.yml @@ -44,6 +44,7 @@ steps: gitHubToken: ${{ parameters.gitHubToken }} xqaCertPass: ${{ parameters.xqaCertPass }} makeParallelism: '4' + use1ES: false # detect changes - bash: $(Build.SourcesDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/compare.sh diff --git a/tools/devops/automation/templates/build/build-mac-tests-stage.yml b/tools/devops/automation/templates/build/build-mac-tests-stage.yml index 6d137356fef0..a87b5dca3598 100644 --- a/tools/devops/automation/templates/build/build-mac-tests-stage.yml +++ b/tools/devops/automation/templates/build/build-mac-tests-stage.yml @@ -58,13 +58,12 @@ jobs: RUN_MAC_TESTS: $[ stageDependencies.configure_build.configure.outputs['decisions.RUN_MAC_TESTS'] ] condition: ne(stageDependencies.configure_build.configure.outputs['decisions.RUN_MAC_TESTS'],'') pool: + os: macOS name: $(AgentPoolComputed) demands: - Agent.OS -equals Darwin - macOS.Name -equals ${{ parameters.macOSName }} - XcodeChannel -equals ${{ parameters.xcodeChannel }} - workspace: - clean: all steps: - template: build-mac-tests.yml diff --git a/tools/devops/automation/templates/build/build-mac-tests.yml b/tools/devops/automation/templates/build/build-mac-tests.yml index b66533f1d93d..75d3b85db689 100644 --- a/tools/devops/automation/templates/build/build-mac-tests.yml +++ b/tools/devops/automation/templates/build/build-mac-tests.yml @@ -83,10 +83,10 @@ steps: condition: succeededOrFailed() # Publish all the binlogs we collected in the previous step - - task: PublishPipelineArtifact@1 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Artifact: All binlogs' inputs: - targetPath: $(Build.ArtifactStagingDirectory)/mactests-binlogs - artifactName: '${{ parameters.uploadPrefix }}mactests-binlogs-$(Build.BuildId)-$(System.StageAttempt)-$(System.JobAttempt)' + path: $(Build.ArtifactStagingDirectory)/mactests-binlogs + artifact: '${{ parameters.uploadPrefix }}mactests-binlogs-$(Build.BuildId)-$(System.StageAttempt)-$(System.JobAttempt)' continueOnError: true condition: succeededOrFailed() diff --git a/tools/devops/automation/templates/build/build-pkgs.yml b/tools/devops/automation/templates/build/build-pkgs.yml index ec09973df899..38b79217aeff 100644 --- a/tools/devops/automation/templates/build/build-pkgs.yml +++ b/tools/devops/automation/templates/build/build-pkgs.yml @@ -71,11 +71,11 @@ steps: name: workload_file displayName: 'Generate "WorkloadRollback.json"' - - task: PublishPipelineArtifact@1 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish WorkloadRollback.json' inputs: - targetPath: $(Build.SourcesDirectory)/WorkloadRollback.json - artifactName: '${{ parameters.uploadPrefix }}WorkloadRollback' + path: $(Build.SourcesDirectory)/WorkloadRollback.json + artifact: '${{ parameters.uploadPrefix }}WorkloadRollback' continueOnError: true - bash: | @@ -99,19 +99,19 @@ steps: name: pkg_versions_file displayName: 'Generate PkgsVersions.json' - - task: PublishPipelineArtifact@1 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish PkgsVersions.json' inputs: - targetPath: $(Build.SourcesDirectory)/PkgsVersions.json - artifactName: '${{ parameters.uploadPrefix }}PkgsVersions' + path: $(Build.SourcesDirectory)/PkgsVersions.json + artifact: '${{ parameters.uploadPrefix }}PkgsVersions' continueOnError: true # upload each of the pkgs into the pipeline artifacts - - task: PublishPipelineArtifact@1 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Build Artifacts' inputs: - targetPath: $(Build.SourcesDirectory)/package - artifactName: '${{ parameters.uploadPrefix }}not-signed-package' + path: $(Build.SourcesDirectory)/package + artifact: '${{ parameters.uploadPrefix }}not-signed-package' continueOnError: true - bash: | @@ -124,18 +124,18 @@ steps: continueOnError: true # not a terrible blocking issue timeoutInMinutes: 60 - - task: PublishPipelineArtifact@1 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish test libraries dependencies' inputs: - targetPath: $(Build.SourcesDirectory)/xamarin-macios/tests/package-test-libraries.zip - artifactName: '${{ parameters.uploadPrefix }}package-test-libraries' + path: $(Build.SourcesDirectory)/xamarin-macios/tests/package-test-libraries.zip + artifact: '${{ parameters.uploadPrefix }}package-test-libraries' continueOnError: true - - task: PublishPipelineArtifact@1 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Build.props' inputs: - targetPath: $(Build.SourcesDirectory)/xamarin-macios/Build.props - artifactName: '${{ parameters.uploadPrefix }}Build.props' + path: $(Build.SourcesDirectory)/xamarin-macios/Build.props + artifact: '${{ parameters.uploadPrefix }}Build.props' continueOnError: true - ${{ if eq(parameters.uploadBinlogs, true) }}: @@ -149,10 +149,10 @@ steps: condition: succeededOrFailed() # Publish all the binlogs we collected in the previous step - - task: PublishPipelineArtifact@1 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Artifact: All binlogs' inputs: - targetPath: $(Build.ArtifactStagingDirectory)/all-binlogs - artifactName: '${{ parameters.uploadPrefix }}all-binlogs-$(Build.BuildId)-$(System.StageAttempt)-$(System.JobAttempt)' + path: $(Build.ArtifactStagingDirectory)/all-binlogs + artifact: '${{ parameters.uploadPrefix }}all-binlogs-$(Build.BuildId)-$(System.StageAttempt)-$(System.JobAttempt)' continueOnError: true condition: succeededOrFailed() diff --git a/tools/devops/automation/templates/build/build-stage.yml b/tools/devops/automation/templates/build/build-stage.yml index 2f569d5df47d..70b3441b97bd 100644 --- a/tools/devops/automation/templates/build/build-stage.yml +++ b/tools/devops/automation/templates/build/build-stage.yml @@ -81,13 +81,12 @@ jobs: XHARNESS_LABELS: $[ stageDependencies.configure_build.configure.outputs['labels.xharness_labels'] ] RUN_MAC_TESTS: $[ stageDependencies.configure_build.configure.outputs['decisions.RUN_MAC_TESTS'] ] pool: + os: macOS name: $(AgentPoolComputed) demands: - Agent.OS -equals Darwin - macOS.Name -equals ${{ parameters.macOSName }} - XcodeChannel -equals ${{ parameters.xcodeChannel }} - workspace: - clean: all steps: - template: build-pkgs.yml diff --git a/tools/devops/automation/templates/build/build.yml b/tools/devops/automation/templates/build/build.yml index ec20f9b431a7..83c741c406d7 100644 --- a/tools/devops/automation/templates/build/build.yml +++ b/tools/devops/automation/templates/build/build.yml @@ -39,6 +39,10 @@ parameters: type: number default: 3 +- name: use1ES + type: boolean + default: true + steps: - template: ../common/checkout.yml @@ -207,6 +211,7 @@ steps: - template: ../common/teardown.yml parameters: keyringPass: ${{ parameters.keyringPass }} + use1ES: ${{ parameters.use1ES }} # if we failed, write a comment and set the pipeline to failure. In this case, we do not want to hide the fact that we failed but we also want # to write a comment. diff --git a/tools/devops/automation/templates/common/configure.yml b/tools/devops/automation/templates/common/configure.yml index 5280d27585b1..87a29fa40ba7 100644 --- a/tools/devops/automation/templates/common/configure.yml +++ b/tools/devops/automation/templates/common/configure.yml @@ -7,6 +7,10 @@ parameters: type: boolean default: false +- name: use1ES + type: boolean + default: false + - name: repositoryAlias type: string default: self @@ -98,9 +102,17 @@ steps: # upload config to be consumed later - ${{ if eq(parameters.uploadArtifacts, true) }}: - - task: PublishPipelineArtifact@1 - displayName: 'Publish Artifact: configuration.json' - inputs: - targetPath: '$(Build.ArtifactStagingDirectory)/configuration.json' - artifactName: '${{ parameters.uploadPrefix }}build-configuration' - continueOnError: true + - ${{ if eq(parameters.use1ES, true) }}: + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Artifact: configuration.json' + inputs: + path: '$(Build.ArtifactStagingDirectory)/configuration.json' + artifact: '${{ parameters.uploadPrefix }}build-configuration' + continueOnError: true + - ${{ else }}: + - task: PublishPipelineArtifact@1 + displayName: 'Publish Artifact: configuration.json' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)/configuration.json' + artifactName: '${{ parameters.uploadPrefix }}build-configuration' + continueOnError: true diff --git a/tools/devops/automation/templates/common/mac-agent-logs.yml b/tools/devops/automation/templates/common/mac-agent-logs.yml index 6a6cfdc942d0..2122bf5bd270 100644 --- a/tools/devops/automation/templates/common/mac-agent-logs.yml +++ b/tools/devops/automation/templates/common/mac-agent-logs.yml @@ -33,6 +33,10 @@ parameters: type: string default: '$(MaciosUploadPrefix)' +- name: use1ES + type: boolean + default: false + steps: - pwsh: >- @@ -49,14 +53,26 @@ steps: # only upload when told so - ${{ if eq(parameters.uploadLogs, true) }}: - - task: PublishPipelineArtifact@1 - displayName: 'Publish Artifact: Agent logs' - condition: always() - inputs: - # add or not the working dir to the upload - ${{ if eq(parameters.workingDirectory, '') }}: - targetPath: ${{ parameters.outputPath }} - ${{ else }}: - targetPath: ${{ parameters.workingDirectory }}/${{ parameters.outputPath }} - artifactName: '${{ parameters.uploadPrefix }}$(Agent.Name)-$(System.JobName)-$(System.JobId)-${{ parameters.outputPath }}' - continueOnError: true + - ${{ if eq(parameters.use1ES, true) }}: + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Artifact: Agent logs' + condition: always() + inputs: + ${{ if eq(parameters.workingDirectory, '') }}: + path: ${{ parameters.outputPath }} + ${{ else }}: + path: ${{ parameters.workingDirectory }}/${{ parameters.outputPath }} + artifact: '${{ parameters.uploadPrefix }}$(Agent.Name)-$(System.JobName)-$(System.JobId)-${{ parameters.outputPath }}' + continueOnError: true + - ${{ else }}: + - task: PublishPipelineArtifact@1 + displayName: 'Publish Artifact: Agent logs' + condition: always() + inputs: + # add or not the working dir to the upload + ${{ if eq(parameters.workingDirectory, '') }}: + targetPath: ${{ parameters.outputPath }} + ${{ else }}: + targetPath: ${{ parameters.workingDirectory }}/${{ parameters.outputPath }} + artifactName: '${{ parameters.uploadPrefix }}$(Agent.Name)-$(System.JobName)-$(System.JobId)-${{ parameters.outputPath }}' + continueOnError: true diff --git a/tools/devops/automation/templates/common/teardown.yml b/tools/devops/automation/templates/common/teardown.yml index 78684c9eb20f..cd6b954d6111 100644 --- a/tools/devops/automation/templates/common/teardown.yml +++ b/tools/devops/automation/templates/common/teardown.yml @@ -5,6 +5,10 @@ parameters: - name: keyringPass type: string +- name: use1ES + type: boolean + default: false + steps: - pwsh: | @@ -30,3 +34,4 @@ steps: - template: mac-agent-logs.yml parameters: outputPath: agent-logs.log + use1ES: ${{ parameters.use1ES }} diff --git a/tools/devops/automation/templates/governance/apiscan.yml b/tools/devops/automation/templates/governance/apiscan.yml index ccdf2db38020..cb4395005fd1 100644 --- a/tools/devops/automation/templates/governance/apiscan.yml +++ b/tools/devops/automation/templates/governance/apiscan.yml @@ -88,9 +88,9 @@ steps: GdnBreakOutputSuppressionFile: $(Build.ArtifactStagingDirectory)\GuardianSuppressions\falsepositives.gdnsuppress GdnBreakOutputSuppressionSet: falsepositives -- task: PublishBuildArtifacts@1 +- task: 1ES.PublishPipelineArtifact@1 displayName: Publish Artifact for Guardian Supressions inputs: - PathtoPublish: $(Build.ArtifactStagingDirectory)\GuardianSuppressions - ArtifactName: Guardian Suppressions APIScan $(PLATFORM) $(VERSION) + path: $(Build.ArtifactStagingDirectory)\GuardianSuppressions + artifact: Guardian Suppressions APIScan $(PLATFORM) $(VERSION) condition: failed() diff --git a/tools/devops/automation/templates/governance/general.yml b/tools/devops/automation/templates/governance/general.yml deleted file mode 100644 index e03f2a0cc7f5..000000000000 --- a/tools/devops/automation/templates/governance/general.yml +++ /dev/null @@ -1,108 +0,0 @@ -parameters: - -- name: isPR - type: boolean - -- name: repositoryAlias - type: string - default: self - -- name: commit - type: string - default: HEAD - -- name: uploadPrefix - type: string - default: '$(MaciosUploadPrefix)' - -steps: - -- template: ../common/checkout.yml - parameters: - isPR: ${{ parameters.isPR }} - repositoryAlias: ${{ parameters.repositoryAlias }} - commit: ${{ parameters.commit }} - -- powershell: | - Write-Host 'Source dir $(Build.SourcesDirectory)' - Write-Host 'Working dir $System.DefaultWorkingDirectory)' - - Dir $(Build.SourcesDirectory) - Dir $(System.DefaultWorkingDirectory) - displayName: Show directories - -- powershell: | - Get-ChildItem -Recurse -Path $(Build.SourcesDirectory)/maccore - displayName: Show maccore content - -- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Show Environment' - -- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - -- task: CredScan@3 - displayName: "Run CredScan" - inputs: - suppressionsFile: '$(System.DefaultWorkingDirectory)/maccore/tools/devops/CredScanSuppressions.json' - outputFormat: 'sarif' - verboseOutput: true - -- task: PoliCheck@2 - inputs: - inputType: 'Basic' - targetType: 'F' - targetArgument: '$(Build.SourcesDirectory)' - result: '$(Build.SourcesDirectory)/PoliCheck.xml' - optionsUEPATH: '$(System.DefaultWorkingDirectory)/maccore/tools/devops/PoliCheckExclusions.xml' - -- task: SdtReport@2 - displayName: Create Security Analysis Report - inputs: - GdnExportAllTools: false - GdnExportGdnToolCredScan: true - GdnExportGdnToolCredScanSeverity: 'error' - GdnExportGdnToolPoliCheck: true - GdnExportGdnToolPoliCheckSeverity: 'error' - -- task: PublishSecurityAnalysisLogs@3 - displayName: Publish Security Analysis Logs - inputs: - ArtifactName: 'CodeAnalysisLogs - General' - ArtifactType: 'Container' - AllTools: true - ToolLogsNotFoundAction: 'Standard' - -- task: TSAUpload@2 - inputs: - GdnPublishTsaOnboard: true - GdnPublishTsaConfigFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\tsa_config.gdntsa' - GdnPublishTsaExportedResultsPublishable: true - displayName: Create bugs for compliance tools results - condition: succeededOrFailed() - env: - WorkingDirectory: '$(Build.SourcesDirectory)/xamarin-macios' - -- task: PublishPipelineArtifact@1 - displayName: 'Publish Artifact: Policheck Results' - inputs: - targetPath: '$(Build.SourcesDirectory)/PoliCheck.xml' - artifactName: '${{ parameters.uploadPrefix }}Policheck' - continueOnError: true - condition: always() - -- task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2 - displayName: Break on compliance issues - inputs: - GdnBreakAllTools: true - GdnBreakSuppressionFiles: $(Build.SourcesDirectory)\maccore\tools\devops\falsepositives.gdnsuppress - GdnBreakSuppressionSets: falsepositives - GdnBreakOutputSuppressionFile: $(Build.ArtifactStagingDirectory)\GuardianSuppressions\falsepositives.gdnsuppress - GdnBreakOutputSuppressionSet: falsepositives - -- task: PublishBuildArtifacts@1 - displayName: Publish Artifact for Guardian Supressions - inputs: - PathtoPublish: $(Build.ArtifactStagingDirectory)\GuardianSuppressions - ArtifactName: Guardian Suppressions - condition: failed() diff --git a/tools/devops/automation/templates/governance/stage.yml b/tools/devops/automation/templates/governance/stage.yml index cc15170246c1..1b81ae6ca5d5 100644 --- a/tools/devops/automation/templates/governance/stage.yml +++ b/tools/devops/automation/templates/governance/stage.yml @@ -19,7 +19,7 @@ parameters: stages: - stage: governance_checks - displayName: '${{ parameters.stageDisplayNamePrefix }}Governance Checks' + displayName: '${{ parameters.stageDisplayNamePrefix }}API Scan' dependsOn: [ configure_build, build_packages] jobs: @@ -31,7 +31,6 @@ stages: demands: - ImageOverride -equals 1ESPT-Windows2022 - condition: ne(stageDependencies.configure_build.configure.outputs['apiscan_matrix.APISCAN_MATRIX'],'') strategy: matrix: $[ stageDependencies.configure_build.configure.outputs['apiscan_matrix.APISCAN_MATRIX'] ] @@ -41,15 +40,3 @@ stages: isPR: ${{ parameters.isPR }} repositoryAlias: ${{ parameters.repositoryAlias }} commit: ${{ parameters.commit }} - - - job: general_governance - displayName: 'Governance Checks' - pool: - vmImage: windows-latest - - steps: - - template: ./general.yml - parameters: - isPR: ${{ parameters.isPR }} - repositoryAlias: ${{ parameters.repositoryAlias }} - commit: ${{ parameters.commit }} diff --git a/tools/devops/automation/templates/main-stage.yml b/tools/devops/automation/templates/main-stage.yml index 9e78186d7fd2..806fa7396b8e 100644 --- a/tools/devops/automation/templates/main-stage.yml +++ b/tools/devops/automation/templates/main-stage.yml @@ -292,7 +292,10 @@ stages: - job: displayName: 'Clean comments' pool: - vmImage: windows-latest + name: AzurePipelines-EO + demands: + - ImageOverride -equals 1ESPT-Windows2022 + steps: - template: ./common/clean.yml @@ -302,11 +305,12 @@ stages: ${{ if and(ne(parameters.dependsOn, ''), ne(parameters.dependsOnResult, '')) }}: condition: eq(dependencies.${{ parameters.dependsOn }}.result, '${{ parameters.dependsOnResult }}') jobs: - - ${{ if eq(parameters.pool, 'automatic') }}: - job: AgentPoolSelector # https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml - pool: # Consider using an agentless (server) job here, but would need to host selection logic as an Azure function: https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=azure-devops&tabs=schema#server - vmImage: ubuntu-latest + pool: # Consider using an agentless (server) job here, but would need to host selection logic as an Azure function: https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=azure-devops&tabs=schema#server + name: AzurePipelines-EO + demands: + - ImageOverride -equals 1ESPT-Windows2022 steps: - checkout: none # https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=azure-devops&tabs=schema#checkout @@ -321,7 +325,9 @@ stages: - job: configure displayName: 'Configure build' pool: - vmImage: windows-latest + name: AzurePipelines-EO + demands: + - ImageOverride -equals 1ESPT-Windows2022 variables: isMain: $[eq(variables['Build.SourceBranch'], 'refs/heads/main')] @@ -338,6 +344,7 @@ stages: testsLabels: '--label=skip-all-tests,run-ios-tests,run-ios-simulator-tests,run-tvos-tests,run-watchos-tests,run-mac-tests,run-maccatalyst-tests,run-dotnet-tests,run-system-permission-tests,run-legacy-xamarin-tests' statusContext: 'VSTS: simulator tests' uploadArtifacts: true + use1ES: true - stage: build_packages displayName: '${{ parameters.stageDisplayNamePrefix }}Build' @@ -434,4 +441,4 @@ stages: azureContainer: ${{ parameters.azureContainer }} packages: # flatten the pkgs for the parameter - ${{ each pkg in parameters.legacyPackageJobs }}: - - ${{ pkg }} \ No newline at end of file + - ${{ pkg }} diff --git a/tools/devops/automation/templates/pipelines/build-pipeline.yml b/tools/devops/automation/templates/pipelines/build-pipeline.yml deleted file mode 100644 index 76da9c279422..000000000000 --- a/tools/devops/automation/templates/pipelines/build-pipeline.yml +++ /dev/null @@ -1,224 +0,0 @@ -# template that can be extended by pipelines that will be used to build the project. This -# allows to share te parameters and resources. - -parameters: - -- name: provisionatorChannel - displayName: Provisionator channel to use - type: string - default: 'latest' - -- name: macOSName # comes from the build agent demand named macOS.Name - displayName: Name of the version of macOS to use - type: string - default: 'Sonoma' - -- name: pool - type: string - displayName: Bot pool to use - default: automatic - values: - - pr - - ci - - automatic - -- name: runGovernanceTests - displayName: Run Governance Checks - type: boolean - default: true - -- name: forceInsertion - displayName: Force Insertion - type: boolean - default: false - -- name: skipESRP - displayName: Skip ESRP - type: boolean - default: false # only to be used when testing the CI and we do not need a signed pkg - -- name: pushNugets - type: boolean - default: true - -- name: pushNugetsToMaestro - type: boolean - default: true - -- name: isPR - displayName: Is PR build - type: boolean - default: false - -- name: signingSetupSteps - type: stepList - default: [] - -- name: testConfigurations - displayName: Test configurations to run - type: object - default: [] - -- name: deviceTestsConfigurations - displayName: Device test configurations to run - type: object - default: [ - { - testPrefix: 'iOS64', - stageName: 'ios64b_device', - displayName: 'iOS64 Device Tests', - testPool: 'VSEng-Xamarin-Mac-Devices', - testsLabels: '--label=run-ios-tests,run-non-monotouch-tests,run-monotouch-tests,run-mscorlib-tests', - statusContext: 'VSTS: device tests iOS', - makeTarget: 'vsts-device-tests', - extraBotDemands: [ - 'ios', - ] - }, - { - testPrefix: 'tvos', - stageName: 'tvos_device', - displayName: 'tvOS Device Tests', - testPool: 'VSEng-Xamarin-Mac-Devices', - testsLabels: '--label=run-tvos-tests,run-non-monotouch-tests,run-monotouch-tests,run-mscorlib-tests', - statusContext: 'VSTS: device tests tvOS', - makeTarget: 'vsts-device-tests', - extraBotDemands: [ - 'tvos', - ] - }] - -- name: macTestsConfigurations - displayName: macOS test configurations to run - type: object - default: [ - { - stageName: 'mac_11_m1', - displayName: 'M1 - Mac Big Sur (11)', - macPool: 'VSEng-VSMac-Xamarin-Shared', - useImage: false, - statusContext: 'M1 - Mac Big Sur (11)', - demands: [ - "Agent.OS -equals Darwin", - "macOS.Name -equals BigSur", - "macOS.Architecture -equals arm64", - "Agent.HasDevices -equals False", - "Agent.IsPaired -equals False" - ] - }, - { - stageName: 'mac_12_m1', - displayName: 'M1 - Mac Ventura (12)', - macPool: 'VSEng-VSMac-Xamarin-Shared', - useImage: false, - statusContext: 'M1 - Mac Monterey (12)', - demands: [ - "Agent.OS -equals Darwin", - "macOS.Name -equals Monterey", - "macOS.Architecture -equals arm64", - "Agent.HasDevices -equals False", - "Agent.IsPaired -equals False" - ] - }, - { - stageName: 'mac_13_m1', - displayName: 'M1 - Mac Ventura (13)', - macPool: 'VSEng-VSMac-Xamarin-Shared', - useImage: false, - statusContext: 'M1 - Mac Ventura (13)', - demands: [ - "Agent.OS -equals Darwin", - "macOS.Name -equals Ventura", - "macOS.Architecture -equals arm64", - "Agent.HasDevices -equals False", - "Agent.IsPaired -equals False" - ] - }, - { - stageName: 'mac_14_x64', - displayName: 'X64 - Mac Sonoma (14)', - macPool: 'VSEng-Xamarin-RedmondMacBuildPool-iOS-Untrusted', - useImage: false, - statusContext: 'X64 - Mac Sonoma (14)', - demands: [ - "Agent.OS -equals Darwin", - "macOS.Name -equals Sonoma", - "macOS.Architecture -equals x64", - "Agent.HasDevices -equals False", - "Agent.IsPaired -equals False" - ] - }] - -resources: - repositories: - - repository: self - checkoutOptions: - submodules: true - - - repository: yaml-templates - type: github - name: xamarin/yaml-templates - ref: refs/heads/main - endpoint: xamarin - - - repository: sdk-insertions - type: github - name: xamarin/sdk-insertions - ref: refs/heads/main - endpoint: xamarin - - - repository: maccore - type: github - name: xamarin/maccore - ref: refs/heads/main - endpoint: xamarin - - - repository: release-scripts - type: github - name: xamarin/release-scripts - ref: refs/heads/only_codesign - endpoint: xamarin - - -variables: -- ${{ if eq(parameters.isPR, false) }}: - - ${{ if contains(variables['Build.DefinitionName'], 'private') }}: - - template: ../vsts-variables.yml - - template: templates/common/vs-release-vars.yml@sdk-insertions - - template: ../variables.yml - - name: MicrobuildConnector - value: 'MicroBuild Signing Task (DevDiv)' - - name: MaciosUploadPrefix - value: '' - - name: DisablePipelineConfigDetector - value: true -- ${{ else }}: - - template: ../variables.yml - - template: templates/common/vs-release-vars.yml@sdk-insertions - - name: MicrobuildConnector - value: '' - - name: MaciosUploadPrefix - value: '' - - name: Packaging.EnableSBOMSigning - value: false - -stages: -- template: ../main-stage.yml - parameters: - xcodeChannel: Stable - macOSName: ${{ parameters.macOSName }} - isPR: ${{ parameters.isPR }} - provisionatorChannel: ${{ parameters.provisionatorChannel }} - pool: ${{ parameters.pool }} - runGovernanceTests: ${{ parameters.runGovernanceTests }} - forceInsertion: ${{ parameters.forceInsertion }} - skipESRP: ${{ parameters.skipESRP }} - pushNugets: ${{ parameters.pushNugets }} - pushNugetsToMaestro: ${{ parameters.pushNugetsToMaestro }} - ${{ if ne(length(parameters.testConfigurations), 0)}}: - testConfigurations: ${{ parameters.testConfigurations }} - deviceTestsConfigurations: ${{ parameters.deviceTestsConfigurations }} - macTestsConfigurations: ${{ parameters.macTestsConfigurations }} - azureStorage: ${{ variables['azureStorage'] }} - azureContainer: ${{ variables['azureContainer'] }} - signingSetupSteps: ${{ parameters.signingSetupSteps }} diff --git a/tools/devops/automation/templates/release/publish-nugets.yml b/tools/devops/automation/templates/release/publish-nugets.yml index 61c7a9c06756..6eeb6dee1dc9 100644 --- a/tools/devops/automation/templates/release/publish-nugets.yml +++ b/tools/devops/automation/templates/release/publish-nugets.yml @@ -37,12 +37,15 @@ steps: patterns: | *.nupkg -- task: NuGetCommand@2 - displayName: Publish Nugets to dotnet6 +- task: 1ES.PublishNuget@1 + displayName: Publish Nuget package inputs: - command: push - packagesToPush: $(Build.SourcesDirectory)/package/*.nupkg + useDotNetTask: false # The default is false to use the NuGetCommand task. Set to true to use the DotNetCoreCLI task to publish packages. + packagesToPush: '$(Build.SourcesDirectory)/package/*.nupkg' + packageParentPath: '$(Build.SourcesDirectory)/package' nuGetFeedType: external + allowPackageConflicts: true # Optional. NuGetCommand task only. + publishPackageMetadata: true # Optional publishFeedCredentials: variables['nuget-feed'] - task: DownloadPipelineArtifact@2 @@ -52,10 +55,13 @@ steps: patterns: | *.nupkg -- task: NuGetCommand@2 - displayName: Publish Nugets to dotnet6 +- task: 1ES.PublishNuget@1 + displayName: Publish MSI Nugets package inputs: - command: push - packagesToPush: $(Build.SourcesDirectory)/vs-msi-nugets/*.nupkg + useDotNetTask: false + packagesToPush: '$(Build.SourcesDirectory)/vs-msi-nugets/*.nupkg' + packageParentPath: '$(Build.SourcesDirectory)/vs-msi-nugets' nuGetFeedType: external + allowPackageConflicts: true + publishPackageMetadata: true publishFeedCredentials: variables['nuget-feed'] diff --git a/tools/devops/automation/templates/release/vs-insertion-prep.yml b/tools/devops/automation/templates/release/vs-insertion-prep.yml index 7329f10ec978..34ff0254f22a 100644 --- a/tools/devops/automation/templates/release/vs-insertion-prep.yml +++ b/tools/devops/automation/templates/release/vs-insertion-prep.yml @@ -52,6 +52,7 @@ stages: # Check - "xamarin-macios (Prepare Release Sign NuGets)" - template: sign-artifacts/jobs/v2.yml@yaml-templates parameters: + use1ESTemplate: true signedArtifactName: '${{ parameters.uploadPrefix }}nuget-signed' artifactName: '${{ parameters.uploadPrefix }}not-signed-package' signType: Real @@ -61,6 +62,7 @@ stages: # Check - "xamarin-macios (Prepare Release Convert NuGet to MSI)" - template: nuget-msi-convert/job/v3.yml@yaml-templates parameters: + use1ESTemplate: true yamlResourceName: yaml-templates dependsOn: signing artifactName: '${{ parameters.uploadPrefix }}nuget-signed' @@ -109,7 +111,9 @@ stages: - ${{ if eq(parameters.isPR, false) }}: - group: Publish-Build-Assets pool: - vmImage: windows-latest + name: AzurePipelines-EO + demands: + - ImageOverride -equals 1ESPT-Windows2022 steps: - task: DownloadPipelineArtifact@2 inputs: @@ -118,6 +122,17 @@ stages: patterns: | *.nupkg + - task: 1ES.PublishNuget@1 + displayName: Publish Nugets package + inputs: + useDotNetTask: false # The default is false to use the NuGetCommand task. Set to true to use the DotNetCoreCLI task to publish packages. + packagesToPush: '$(Build.SourcesDirectory)/package/*.nupkg' + packageParentPath: '$(Build.SourcesDirectory)/package' + nuGetFeedType: external + allowPackageConflicts: true # Optional. NuGetCommand task only. + publishPackageMetadata: true # Optional + publishFeedCredentials: dnceng-dotnet8 + - task: DownloadPipelineArtifact@2 inputs: artifactName: '${{ parameters.uploadPrefix }}vs-msi-nugets' @@ -125,6 +140,17 @@ stages: patterns: | *.nupkg + - task: 1ES.PublishNuget@1 + displayName: Publish MSI Nugets package + inputs: + useDotNetTask: false + packagesToPush: '$(Build.SourcesDirectory)/${{ parameters.uploadPrefix }}vs-msi-nugets/*.nupkg' + packageParentPath: '$(Build.SourcesDirectory)/${{ parameters.uploadPrefix }}vs-msi-nugets' + nuGetFeedType: external + allowPackageConflicts: true + publishPackageMetadata: true + publishFeedCredentials: dnceng-dotnet8 + - pwsh: | mkdir $(Build.SourcesDirectory)/nugets-blob cp $(Build.SourcesDirectory)/package/* $(Build.SourcesDirectory)/nugets-blob @@ -205,10 +231,10 @@ stages: workingDirectory: $(Build.SourcesDirectory)\.. condition: and(succeeded(), eq('${{ parameters.pushNugetsToMaestro }}', 'true')) - - task: PublishPipelineArtifact@1 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Artifact: maestro-binlogs' inputs: - targetPath: $(Build.ArtifactStagingDirectory)/maestro-binlogs - artifactName: ${{ parameters.uploadPrefix }}maestro-binlogs-$(System.JobAttempt) + path: $(Build.ArtifactStagingDirectory)/maestro-binlogs + artifact: ${{ parameters.uploadPrefix }}maestro-binlogs-$(System.JobAttempt) condition: and(succeededOrFailed(), eq('${{ parameters.pushNugetsToMaestro }}', 'true')) continueOnError: true diff --git a/tools/devops/automation/templates/sign-and-notarized/funnel.yml b/tools/devops/automation/templates/sign-and-notarized/funnel.yml index 3d18351b6cf2..90ab46b39667 100644 --- a/tools/devops/automation/templates/sign-and-notarized/funnel.yml +++ b/tools/devops/automation/templates/sign-and-notarized/funnel.yml @@ -29,14 +29,15 @@ jobs: displayName: 'Collect signed artifacts' condition: and(not(failed()), not(canceled())) # default is succeded(), but that fails if there are any skipped jobs, so change the condition to !failed && !cancelled timeoutInMinutes: 1000 - pool: - vmImage: internal-macos-11 - workspace: - clean: all variables: + AgentPoolComputed: $[ stageDependencies.configure_build.AgentPoolSelector.outputs['setAgentPool.AgentPoolComputed'] ] ${{ each pkg in parameters.packages }}: ${{ pkg.conditionVariable }}: $[ stageDependencies.configure_build.configure.outputs['configure_platforms.${{ pkg.conditionVariable }}'] ] + pool: + os: macOS + name: $(AgentPoolComputed) + steps: # DO NOT USE THE checkout.yml template. The reason is that the template changes the hash which results in a problem with the artifacts scripts @@ -105,11 +106,11 @@ jobs: allowFailedBuilds: true path: $(Build.SourcesDirectory)/package-internal - - task: PublishPipelineArtifact@1 - displayName: 'Publish Build Internal Artifacts' + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Build Artifacts (notarized)' inputs: - targetPath: $(Build.SourcesDirectory)/package-internal - artifactName: '${{ parameters.uploadPrefix }}package-internal' + path: $(Build.SourcesDirectory)/package-internal + artifact: '${{ parameters.uploadPrefix }}package-internal' continueOnError: true # download msbuild.zip and bundle.zip to the 'package' dir, so that they're uploaded into the 'package' artifact, @@ -131,11 +132,11 @@ jobs: ls -la "$BUILD_SOURCESDIRECTORY"/package displayName: Copy msbuild.zip and bundle.zip to the package artifact - - task: PublishPipelineArtifact@1 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Build Artifacts (notarized)' inputs: - targetPath: $(Build.SourcesDirectory)/package - artifactName: '${{ parameters.uploadPrefix }}package' + path: $(Build.SourcesDirectory)/package + artifact: '${{ parameters.uploadPrefix }}package' continueOnError: true # This job uploads the pkgs generated by the build step in the azure blob storage. This has to be done in a different job @@ -144,6 +145,16 @@ jobs: - job: upload_azure_blob displayName: 'Upload packages to Azure & SBOM' timeoutInMinutes: 1000 + templateContext: # disable some of the sdl tasks for this job + sdl: + credscan: # run in the sdl stage + enabled: false + binskim: # job does not compile anything + enabled: false + eslint: # no js or anything similar + enabled: false + bandit: # we have no python + enabled: false dependsOn: - funnel_job condition: and(not(failed()), not(canceled())) # default is succeded(), but that fails if there are any skipped jobs, so change the condition to !failed && !cancelled @@ -157,9 +168,7 @@ jobs: pool: name: azurepipelines-EO demands: - - ImageOverride -equals AzurePipelinesWindows2022compliant - workspace: - clean: all + - ImageOverride -equals 1ESPT-Windows2022 steps: - template: upload-azure.yml parameters: @@ -181,9 +190,9 @@ jobs: TESTS_BOT: $[ stageDependencies.build_packages.build.outputs['build.TESTS_BOT'] ] # we build in a diff bot than the ones used for the comments GIT_HASH: $[ stageDependencies.build_packages.build.outputs['fix_commit.GIT_HASH'] ] pool: - vmImage: windows-latest - workspace: - clean: all + name: AzurePipelines-EO + demands: + - ImageOverride -equals 1ESPT-Windows2022 steps: - template: artifact-github-comment.yml parameters: diff --git a/tools/devops/automation/templates/sign-and-notarized/prepare-pkg-stage.yml b/tools/devops/automation/templates/sign-and-notarized/prepare-pkg-stage.yml index 577992c305f3..54c4bf3a98e5 100644 --- a/tools/devops/automation/templates/sign-and-notarized/prepare-pkg-stage.yml +++ b/tools/devops/automation/templates/sign-and-notarized/prepare-pkg-stage.yml @@ -34,9 +34,8 @@ jobs: condition: ne(stageDependencies.configure_build.configure.outputs['configure_platforms.${{ pkg.conditionVariable }}'],'') timeoutInMinutes: 1000 pool: - vmImage: internal-macos-11 - workspace: - clean: all + name: 'VSEng-Xamarin-RedmondMacBuildPool-iOS-Trusted' # always use the trusted pool for signing. + os: macOS steps: - template: sign-and-notarized.yml diff --git a/tools/devops/automation/templates/sign-and-notarized/setup.yml b/tools/devops/automation/templates/sign-and-notarized/setup.yml index b885f8071a2b..b6c212bf4f7a 100644 --- a/tools/devops/automation/templates/sign-and-notarized/setup.yml +++ b/tools/devops/automation/templates/sign-and-notarized/setup.yml @@ -53,6 +53,9 @@ steps: version: 3.x displayName: 'Install .NET Core SDK 3.x needed for ESRP' + - pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 + displayName: 'Show Environment' + - task: MicroBuildSigningPlugin@4 displayName: 'Install Signing Plugin' inputs: diff --git a/tools/devops/automation/templates/sign-and-notarized/sign-and-notarized.yml b/tools/devops/automation/templates/sign-and-notarized/sign-and-notarized.yml index 88eb8635fcf1..7efb659c46bb 100644 --- a/tools/devops/automation/templates/sign-and-notarized/sign-and-notarized.yml +++ b/tools/devops/automation/templates/sign-and-notarized/sign-and-notarized.yml @@ -108,11 +108,11 @@ steps: condition: ${{ and(eq(parameters.condition, true), eq(variables['System.debug'], 'true')) }} # Upload the esrp temp dir - - task: PublishPipelineArtifact@1 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Artifact: ESRP temp' inputs: - targetPath: '$(Build.ArtifactStagingDirectory)/esrp.zip' - artifactName: '${{ parameters.uploadPrefix }}ESRP-tmp-${{ parameters.packageName }}' + path: '$(Build.ArtifactStagingDirectory)/esrp.zip' + artifact: '${{ parameters.uploadPrefix }}ESRP-tmp-${{ parameters.packageName }}' continueOnError: true condition: ${{ and(eq(parameters.condition, true), eq(variables['System.debug'], 'true')) }} @@ -124,9 +124,9 @@ steps: condition: ${{ parameters.condition }} # always upload no matter what, since if we are not signing we need the artifact in the pipeline -- task: PublishPipelineArtifact@1 +- task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Notarized Classic Artifacts' inputs: - targetPath: $(Build.SourcesDirectory)/package - artifactName: '${{ parameters.uploadPrefix }}classic-${{ parameters.packageName }}-signed' + path: $(Build.SourcesDirectory)/package + artifact: '${{ parameters.uploadPrefix }}classic-${{ parameters.packageName }}-signed' continueOnError: true diff --git a/tools/devops/automation/templates/sign-and-notarized/upload-azure.yml b/tools/devops/automation/templates/sign-and-notarized/upload-azure.yml index 7582f45d505f..94af45c53a12 100644 --- a/tools/devops/automation/templates/sign-and-notarized/upload-azure.yml +++ b/tools/devops/automation/templates/sign-and-notarized/upload-azure.yml @@ -38,6 +38,10 @@ steps: persistCredentials: true path: s/xamarin-macios +- checkout: maccore + clean: true + persistCredentials: true # hugely important, else there are some scripts that check a single file from maccore that will fail + # checkout an extra repo to ensure that we have the same tree structure in the working directory in all pipelines. # if you delete this checkout the unified pipeline will have issues. - checkout: yaml-templates @@ -88,7 +92,7 @@ steps: name: upload inputs: SourcePath: $(Build.SourcesDirectory)/artifacts/${{ parameters.uploadPrefix }}package - azureSubscription: 'Xamarin - RelEng (BosStorageMirror)' + azureSubscription: 'Azure Releng (7b4817ae-218f-464a-bab1-a9df2d99e1e5)' Destination: AzureBlob storage: ${{ parameters.azureStorage }} ContainerName: ${{ parameters.azureContainer }} @@ -98,7 +102,7 @@ steps: displayName: 'Publish manifest to Azure' inputs: SourcePath: $(Build.SourcesDirectory)/artifacts/${{ parameters.uploadPrefix }}package/manifest - azureSubscription: 'Xamarin - RelEng (BosStorageMirror)' + azureSubscription: 'Azure Releng (7b4817ae-218f-464a-bab1-a9df2d99e1e5)' Destination: AzureBlob storage: ${{ parameters.azureStorage }} ContainerName: ${{ parameters.azureContainer }} @@ -108,7 +112,7 @@ steps: displayName: 'Publish manifest to Azure as latest' inputs: SourcePath: $(Build.SourcesDirectory)/artifacts/${{ parameters.uploadPrefix }}package/manifest - azureSubscription: 'Xamarin - RelEng (BosStorageMirror)' + azureSubscription: 'Azure Releng (7b4817ae-218f-464a-bab1-a9df2d99e1e5)' Destination: AzureBlob storage: ${{ parameters.azureStorage }} ContainerName: ${{ parameters.azureContainer }} @@ -118,7 +122,7 @@ steps: displayName: 'Publish manifest to Azure per commit' inputs: SourcePath: $(Build.SourcesDirectory)/artifacts/${{ parameters.uploadPrefix }}package/manifest - azureSubscription: 'Xamarin - RelEng (BosStorageMirror)' + azureSubscription: 'Azure Releng (7b4817ae-218f-464a-bab1-a9df2d99e1e5)' Destination: AzureBlob storage: ${{ parameters.azureStorage }} ContainerName: ${{ parameters.azureContainer }} @@ -162,11 +166,11 @@ steps: # upload the artifacts.json to the build pipeline artifacts so that it can be consumed by other jobs to # get the required urls -- task: PublishPipelineArtifact@1 +- task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Build Artifacts' inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/${{ parameters.uploadPrefix }}package/artifacts.json - artifactName: '${{ parameters.uploadPrefix }}pkg-info' + path: $(Build.SourcesDirectory)/artifacts/${{ parameters.uploadPrefix }}package/artifacts.json + artifact: '${{ parameters.uploadPrefix }}pkg-info' continueOnError: true - pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 @@ -393,9 +397,9 @@ steps: } displayName: 'SBOM: Cleanup' -- task: PublishBuildArtifacts@1 +- task: 1ES.PublishPipelineArtifact@1 displayName: 'SBOM: Publish' inputs: - artifactName: '${{ parameters.uploadPrefix }}sbom' - pathToPublish: $(Build.ArtifactStagingDirectory)/sbom + artifact: '${{ parameters.uploadPrefix }}sbom' + path: $(Build.ArtifactStagingDirectory)/sbom condition: succeeded() From c8555964677c8e2a142f2045c28d913ee63304c0 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Wed, 19 Jun 2024 21:54:12 -0400 Subject: [PATCH 11/17] [CI] Reduce dependecies on the maccore repo. (#20748) Move the config files to the macios repo since they have no private info. --- tools/devops/automation/build-pipeline.yml | 10 +- .../devops/automation/build-pull-request.yml | 10 +- .../common/publis-pipeline-artifact.yml | 41 + .../governance/CredScanSuppressions.json | 901 ++++++++++++++++++ .../devops/governance/PoliCheckExclusions.xml | 7 + .../devops/governance/baselines.gdnbaselines | 127 +++ tools/devops/governance/suppress.gdnsuppress | 117 +++ tools/devops/governance/tsa_config.gdntsa | 19 + 8 files changed, 1222 insertions(+), 10 deletions(-) create mode 100644 tools/devops/automation/templates/common/publis-pipeline-artifact.yml create mode 100644 tools/devops/governance/CredScanSuppressions.json create mode 100644 tools/devops/governance/PoliCheckExclusions.xml create mode 100644 tools/devops/governance/baselines.gdnbaselines create mode 100644 tools/devops/governance/suppress.gdnsuppress create mode 100644 tools/devops/governance/tsa_config.gdntsa diff --git a/tools/devops/automation/build-pipeline.yml b/tools/devops/automation/build-pipeline.yml index 6b6e756fb533..4ae5c34830d8 100644 --- a/tools/devops/automation/build-pipeline.yml +++ b/tools/devops/automation/build-pipeline.yml @@ -219,24 +219,24 @@ extends: os: windows sdl: baseline: - baselineFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\baselines.gdnbaselines' + baselineFile: '$(System.DefaultWorkingDirectory)\\xamarin-macios\\tools\\devops\\governance\\baselines.gdnbaselines' suppression: - suppressionFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\suppress.gdnsuppress' + suppressionFile: '$(System.DefaultWorkingDirectory)\\xamarin-macios\\tools\\devops\\governance\\suppress.gdnsuppress' sourceAnalysisPool: name: AzurePipelines-EO image: 1ESPT-Windows2022 os: windows tsa: - configFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\tsa_config.gdntsa' + configFile: '$(System.DefaultWorkingDirectory)\\xamarin-macios\\tools\\devops\\governance\\tsa_config.gdntsa' sbom: enabled: false # we run our on sbom generation credscan: - suppressionsFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\CredScanSuppressions.json' + suppressionsFile: '$(System.DefaultWorkingDirectory)\\xamarin-macios\\tools\\devops\\governance\\CredScanSuppressions.json' outputFormat: sarif debugMode: false batchSize: 16 policheck: - exclusionsFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\PoliCheckExclusions.xml' + exclusionsFile: '$(System.DefaultWorkingDirectory)\\xamarin-macios\\tools\\devops\\governance\\PoliCheckExclusions.xml' sourceRepositoriesToScan: runInSingleJob: true # run both maccore and macios in the same job include: diff --git a/tools/devops/automation/build-pull-request.yml b/tools/devops/automation/build-pull-request.yml index 0e3fae13c80a..d4a4c14e2f23 100644 --- a/tools/devops/automation/build-pull-request.yml +++ b/tools/devops/automation/build-pull-request.yml @@ -219,24 +219,24 @@ extends: os: windows sdl: baseline: - baselineFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\baselines.gdnbaselines' + baselineFile: '$(System.DefaultWorkingDirectory)\\xamarin-macios\\tools\\devops\\governance\\baselines.gdnbaselines' suppression: - suppressionFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\suppress.gdnsuppress' + suppressionFile: '$(System.DefaultWorkingDirectory)\\xamarin-macios\\tools\\devops\\governance\\suppress.gdnsuppress' sourceAnalysisPool: name: AzurePipelines-EO image: 1ESPT-Windows2022 os: windows tsa: - configFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\tsa_config.gdntsa' + configFile: '$(System.DefaultWorkingDirectory)\\xamarin-macios\\tools\\devops\\governance\\tsa_config.gdntsa' sbom: enabled: false # we run our on sbom generation credscan: - suppressionsFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\CredScanSuppressions.json' + suppressionsFile: '$(System.DefaultWorkingDirectory)\\xamarin-macios\\tools\\devops\\governance\\CredScanSuppressions.json' outputFormat: sarif debugMode: false batchSize: 16 policheck: - exclusionsFile: '$(System.DefaultWorkingDirectory)\\maccore\\tools\\devops\\PoliCheckExclusions.xml' + exclusionsFile: '$(System.DefaultWorkingDirectory)\\xamarin-macios\\tools\\devops\\governance\\PoliCheckExclusions.xml' sourceRepositoriesToScan: runInSingleJob: true # run both maccore and macios in the same job include: diff --git a/tools/devops/automation/templates/common/publis-pipeline-artifact.yml b/tools/devops/automation/templates/common/publis-pipeline-artifact.yml new file mode 100644 index 000000000000..751a45bf76b4 --- /dev/null +++ b/tools/devops/automation/templates/common/publis-pipeline-artifact.yml @@ -0,0 +1,41 @@ +parameters: + +- name: use1ES + type: boolean + default: false + +- name: displayName + type: string + +- name: path + type: string + +- name: artifact + type: string + +- name: continueOnError + type: boolean + default: false + +- name: condition + type: boolean + default: always() + +steps: +- ${{ if eq(parameters.use1ES, true) }}: + - task: 1ES.PublishPipelineArtifact@1 + displayName: ${{ parameters.displayName }} + condition: ${{ parameters.condition }} + inputs: + path: ${{ parameters.path }} + artifact: ${{ parameters.artifact }} + continueOnError: ${{ parameters.continueOnError }} +- ${{ else }}: + - task: PublishPipelineArtifact@1 + displayName: ${{ parameters.displayName }} + condition: ${{ parameters.condition }} + inputs: + targetPath: ${{ parameters.path }} + artifactName: ${{ parameters.artifact }} + continueOnError: ${{ parameters.continueOnError }} + diff --git a/tools/devops/governance/CredScanSuppressions.json b/tools/devops/governance/CredScanSuppressions.json new file mode 100644 index 000000000000..61eef43fede9 --- /dev/null +++ b/tools/devops/governance/CredScanSuppressions.json @@ -0,0 +1,901 @@ +{ + "tool": "Credential Scanner", + "suppressions": [ + { + "placeholder": "12345678", + "_justification": "Dummy secret used in unit tests, it is fake." + }, + { + "file": "src\\cfnetwork.cs", + "_justification": "API definition file that cannot contain passwords at all." + }, + { + "file": "external\\guiunit\\tools\\nant\\bin\\NAnt.Core.xml", + "_justification": "External repo dependency, guiunit." + }, + { + "file": "external\\guiunit\\tools\\nant\\bin\\NAnt.SourceControlTasks.xml", + "_justification": "External repo dependency, guiunit." + }, + { + "file": "external\\guiunit\\tools\\nant\\bin\\NAnt.DotNetTasks.xml", + "_justification": "External repo dependency, guiunit." + }, + { + "file": "external\\guiunit\\tools\\nant\\bin\\NAnt.Core.xml", + "_justification": "External repo dependency, guiunit." + }, + { + "file": "external\\guiunit\\tools\\nant\\bin\\NAnt.Core.xml", + "_justification": "External repo dependency, guiunit." + }, + { + "file": "external\\mono\\data\\net_2_0\\web.config", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\data\\net_4_0\\web.config", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\data\\net_4_5\\web.config", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\data\\net_1_1\\machine.config", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\api-doc-tools\\external\\SharpZipLib\\ICSharpCode.SharpZipLib.Tests\\Zip\\ZipTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\api-doc-tools\\external\\SharpZipLib\\ICSharpCode.SharpZipLib.Tests\\Zip\\ZipTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\api-doc-tools\\external\\SharpZipLib\\ICSharpCode.SharpZipLib.Tests\\Zip\\ZipTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\api-doc-tools\\external\\SharpZipLib\\ICSharpCode.SharpZipLib.Tests\\Zip\\ZipTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\api-doc-tools\\external\\SharpZipLib\\ICSharpCode.SharpZipLib.Tests\\Zip\\ZipTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\api-doc-tools\\external\\SharpZipLib\\ICSharpCode.SharpZipLib.Tests\\Zip\\ZipTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\api-doc-tools\\external\\SharpZipLib\\ICSharpCode.SharpZipLib.Tests\\Zip\\ZipTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\api-doc-tools\\external\\SharpZipLib\\ICSharpCode.SharpZipLib.Tests\\Zip\\ZipTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\api-doc-tools\\external\\SharpZipLib\\ICSharpCode.SharpZipLib.Tests\\Zip\\ZipTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\api-doc-tools\\external\\SharpZipLib\\ICSharpCode.SharpZipLib.Tests\\Zip\\ZipTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\api-doc-tools\\external\\SharpZipLib\\ICSharpCode.SharpZipLib.Tests\\Zip\\ZipTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\api-doc-tools\\external\\SharpZipLib\\ICSharpCode.SharpZipLib.Tests\\Zip\\ZipTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\api-doc-tools\\external\\SharpZipLib\\ICSharpCode.SharpZipLib.Tests\\Zip\\ZipTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\api-doc-tools\\external\\SharpZipLib\\ICSharpCode.SharpZipLib.Tests\\Zip\\ZipTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\api-doc-tools\\external\\SharpZipLib\\ICSharpCode.SharpZipLib.Tests\\Zip\\ZipTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\macios-binaries\\docs\\macOS\\classic\\XamMac.xml", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\aspnetwebstack\\test\\System.Web.Helpers.Test\\WebMailTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\macios-binaries\\docs\\macOS\\classic\\XamMac.CFNetwork.xml", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\aspnetwebstack\\test\\System.Web.Helpers.Test\\CryptoTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\aspnetwebstack\\test\\System.Web.Helpers.Test\\CryptoTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\aspnetwebstack\\test\\System.Web.Helpers.Test\\CryptoTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\aspnetwebstack\\test\\System.Web.Helpers.Test\\CryptoTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\aspnetwebstack\\test\\System.Web.WebPages.Test\\Validation\\ValidatorTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\aspnetwebstack\\test\\System.Web.WebPages.Test\\Validation\\ValidatorTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\aspnetwebstack\\test\\System.Web.WebPages.Test\\Validation\\ValidatorTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\aspnetwebstack\\test\\System.Web.WebPages.Test\\Validation\\ValidatorTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\boringssl\\ssl\\test\\runner\\ecdsa_key.pem", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\boringssl\\ssl\\test\\runner\\channel_id_key.pem", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\boringssl\\ssl\\test\\runner\\key.pem", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\Common\\src\\Interop\\OSX\\System.Security.Cryptography.Native.Apple\\Interop.SecKeyRef.Export.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\Common\\tests\\System\\Net\\Prerequisites\\Deployment\\config.ps1", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\Common\\tests\\System\\Net\\Prerequisites\\Deployment\\setup_certificates.ps1", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\Common\\tests\\System\\Net\\Prerequisites\\Deployment\\setup_certificates.ps1", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\Common\\tests\\System\\Net\\Configuration.Certificates.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Data.SqlClient\\tests\\FunctionalTests\\ExceptionTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Data.SqlClient\\tests\\FunctionalTests\\SqlConnectionBasicTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Data.SqlClient\\tests\\Tools\\TDS\\TDS.Servers\\TDSServerArguments.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Data.SqlClient\\tests\\Tools\\TDS\\TDS.Servers\\TdsServerCertificate.pfx", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Diagnostics.Process\\tests\\ProcessStartInfoTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.DirectoryServices.AccountManagement\\tests\\PrincipalTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.DirectoryServices.AccountManagement\\tests\\UserPrincipalTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Data.Common\\tests\\System\\Data\\Common\\DbConnectionStringBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Data.Common\\tests\\System\\Data\\Common\\DbConnectionStringBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Data.Common\\tests\\System\\Data\\Common\\DbConnectionStringBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Data.Common\\tests\\System\\Data\\Common\\DbConnectionStringBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Data.Common\\tests\\System\\Data\\Common\\DbConnectionStringBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Data.Common\\tests\\System\\Data\\Common\\DbConnectionStringBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Data.Common\\tests\\System\\Data\\Common\\DbConnectionStringBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Data.Common\\tests\\System\\Data\\Common\\DbConnectionStringBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Management\\src\\System\\Management\\ManagementScope.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Management\\src\\System\\Management\\ManagementScope.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Management\\src\\System\\Management\\ManagementScope.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Management\\src\\System\\Management\\ManagementScope.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Net.Http\\tests\\FunctionalTests\\HttpClientHandlerTest.DefaultProxyCredentials.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Net.Http\\tests\\FunctionalTests\\PostScenarioTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Net.Requests\\src\\System\\Net\\FtpControlStream.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Algorithms\\tests\\Rfc2898Tests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Algorithms\\tests\\Rfc2898Tests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Algorithms\\tests\\Rfc2898Tests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Algorithms\\tests\\Rfc2898Tests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Algorithms\\tests\\Rfc2898Tests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Algorithms\\tests\\Rfc2898Tests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Algorithms\\tests\\Rfc2898Tests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Csp\\tests\\PasswordDeriveBytesTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Pkcs\\tests\\Certificates.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Pkcs\\tests\\Certificates.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Pkcs\\tests\\Certificates.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Pkcs\\tests\\Certificates.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Pkcs\\tests\\Certificates.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Pkcs\\tests\\Certificates.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Pkcs\\tests\\Certificates.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Pkcs\\tests\\Certificates.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.X509Certificates\\tests\\ExportTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.X509Certificates\\tests\\ExportTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.X509Certificates\\tests\\ExportTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.X509Certificates\\tests\\ExportTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.X509Certificates\\tests\\ExportTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.X509Certificates\\tests\\PfxTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.X509Certificates\\tests\\TestData.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Xml\\tests\\EncryptedXmlTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.X509Certificates\\tests\\TestData.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Xml\\tests\\TestHelpers.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Xml\\tests\\SignedXmlTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Xml\\tests\\SignedXmlTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Xml\\tests\\SignedXmlTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Xml\\tests\\SignedXmlTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Xml\\tests\\SignedXmlTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Xml\\tests\\SignedXmlTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Xml\\tests\\EncryptedXmlTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\corefx\\src\\System.Security.Cryptography.Xml\\tests\\TestHelpers.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\Newtonsoft.Json\\Src\\Newtonsoft.Json.Tests\\Newtonsoft.Json.Tests.Metro_TemporaryKey.pfx", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\nuget-buildtasks\\src\\Microsoft.NuGet.Build.Tasks.Tests\\Json\\Win10.json", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\rx\\Rx\\NET\\Test\\Rx\\WindowsStoreApp8_NuGet\\WindowsStoreApp8_NuGet_TemporaryKey.pfx", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\external\\rx\\Rx\\NET\\Test\\Rx\\WpfXbapApp45_NuGet\\WpfXbapApp45_NuGet_TemporaryKey.pfx", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Compat.ICSharpCode.SharpZipLib\\SharpZipLib.key", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\corlib\\Test\\System.Security.Cryptography\\RSAOAEPKeyExchangeDeformatterTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\corlib\\Test\\System.Security.Cryptography\\RSACryptoServiceProviderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\corlib\\Test\\System.Security.Cryptography\\RSAPKCS1KeyExchangeDeformatterTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\corlib\\Test\\System.Security.Cryptography\\RSACryptoServiceProviderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\corlib\\Test\\System.Security.Cryptography\\RSACryptoServiceProviderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\corlib\\Test\\System.Security.Cryptography\\RSACryptoServiceProviderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\corlib\\Test\\System.Security.Cryptography\\RSACryptoServiceProviderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\corlib\\Test\\System.Security.Cryptography\\RSACryptoServiceProviderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\corlib\\Test\\System.Security.Cryptography\\RSACryptoServiceProviderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\corlib\\Test\\System.Security.Cryptography\\DSACryptoServiceProviderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\corlib\\Test\\System.Security.Cryptography\\DSACryptoServiceProviderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\corlib\\Test\\System.Security.Cryptography\\RSACryptoServiceProviderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\corlib\\Test\\System.Security.Cryptography\\RSATest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\ICSharpCode.SharpZipLib\\SharpZipLib.key", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.Http\\samples\\auth.xml", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.Security\\Test\\Mono.Security\\StrongNameTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.Security\\Test\\Mono.Security.Cryptography\\PKCS8Test.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.Security\\Test\\Mono.Security.Cryptography\\RSAManagedTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.Security\\Test\\Mono.Security.Cryptography\\RSAManagedTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.Security\\Test\\Mono.Security.Protocol.Ntlm\\ChallengeResponseTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.Security\\Test\\Mono.Security.Cryptography\\RSAManagedTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.Security\\Test\\Mono.Security.Cryptography\\CryptoConvertTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.Security\\Test\\Mono.Security.Cryptography\\RSAManagedTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.Security\\Test\\Mono.Security.Cryptography\\CryptoConvertTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.Security\\Test\\Mono.Security.Cryptography\\PKCS1Test.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.Security\\Test\\Mono.Security.Protocol.Ntlm\\Type3MessageTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.Security\\Test\\Mono.Security.Protocol.Ntlm\\Type3MessageTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.Security\\Test\\tools\\mutual\\server_key.pem", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.Security\\Test\\tools\\server\\mutual.pem", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.Security\\Test\\tools\\npgsql\\npdbtest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.Security\\Test\\Mono.Security.X509\\PKCS12Test.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.Security\\Test\\Mono.Security.Cryptography\\RSAManagedTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.Security\\Test\\Mono.Security.Cryptography\\RSAManagedTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.Security\\Test\\tools\\mutual\\client.p12", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.ServiceModel.IdentitySelectors\\Test\\resources\\rupert.xml", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\Mono.ServiceModel.IdentitySelectors\\Test\\resources\\managed.xml", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\referencesource\\System\\net\\System\\Net\\_FtpControlStream.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System\\Test\\System\\UriBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System\\Test\\System\\UriBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System\\Test\\System.Security.Cryptography.X509Certificates\\X509Certificate2Test.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System\\Test\\System.Security.Cryptography.X509Certificates\\X509Certificate2Test.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System\\Test\\tools\\mutual\\client.p12", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System\\Test\\tools\\mutual\\server_key.pem", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System\\Test\\tools\\server\\mutual.pem", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Configuration\\Test\\standalone\\t33.exe.config", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Configuration\\Test\\standalone\\t34.exe.config", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Configuration\\Test\\System.Configuration\\ConfigurationManagerTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data\\Test\\MySqlTestBed.dll.config", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data\\Test\\ProviderTests\\app-net_1_1.config", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data\\Test\\ProviderTests\\app-net_2_0.config", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data\\Test\\ProviderTests\\app-net_1_1.config", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data\\Test\\ProviderTests\\Common\\ConnectionManager.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data\\Test\\ProviderTests\\app-net_2_0.config", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data\\Test\\ProviderTests\\Common\\ConnectionManager.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data\\Test\\ProviderTests\\System.Data.OleDb.jvm\\OleDbCommand\\OleDbCommand_ExecuteReader.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data\\Test\\ProviderTests\\System.Data.SqlClient\\SqlConnectionTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data\\Test\\System.Data.Common\\DbConnectionStringBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data\\Test\\System.Data.Common\\DbConnectionStringBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data\\Test\\System.Data.Common\\DbConnectionStringBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data\\Test\\System.Data.Common\\DbConnectionStringBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data\\Test\\System.Data.Common\\DbConnectionStringBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data\\Test\\System.Data.Common\\DbConnectionStringBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data\\Test\\System.Data.Common\\DbConnectionStringBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data\\Test\\System.Data.Common\\DbConnectionStringBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data\\Test\\System.Data.Odbc\\OdbcConnectionStringBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data\\Test\\System.Data.Odbc\\OdbcConnectionStringBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data\\Test\\System.Data.Odbc\\OdbcConnectionStringBuilderTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data.Linq\\examples\\DbLinq.Ingres.Example\\Program.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data.Linq\\examples\\DbLinq.Ora.Example\\Program.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data.Linq\\examples\\DbLinq.Ora.Example\\Program.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data.Linq\\examples\\DbLinq.Pgsql.Example\\PgAndrusClient.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data.Linq\\examples\\DbLinq.Pgsql.Example\\ProgramPg.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data.Linq\\src\\DbLinq\\Test\\Providers\\ReadTest_AllTypes_MySql.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data.Linq\\tests\\ConnectionStrings.xml", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data.Linq\\tests\\ConnectionStrings.xml", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data.Linq\\tests\\ConnectionStrings.xml", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data.Linq\\tests\\ConnectionStrings.xml", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data.Linq\\tests\\ConnectionStrings.xml", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data.Linq\\tests\\ConnectionStrings.xml", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data.Linq\\tests\\ConnectionStrings.xml", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data.Linq\\tests\\ConnectionStrings.xml", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data.OracleClient\\Test\\testblob.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data.OracleClient\\Test\\testclob.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data.OracleClient\\Test\\TestOracleClient.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data.OracleClient\\Test\\TestOracleClient.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data.OracleClient\\Test\\TestOracleClient.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Data.OracleClient\\Test\\TestOracleClient.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.IdentityModel\\Test\\Resources\\test.pfx", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.IdentityModel\\Test\\Resources\\test2.pfx", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.IdentityModel\\Test\\System.IdentityModel.Tokens\\BootstrapContextTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.IdentityModel\\Test\\System.IdentityModel.Tokens\\InMemorySymmetricSecurityKeyTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.IdentityModel\\Test\\System.IdentityModel.Tokens\\InMemorySymmetricSecurityKeyTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Security\\Test\\System.Security.Cryptography.Xml\\EncryptedXmlTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Security\\Test\\System.Security.Cryptography.Xml\\EncryptedXmlTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Security\\Test\\System.Security.Cryptography.Xml\\EncryptedXmlTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Security\\Test\\System.Security.Cryptography.Xml\\sample.pfx", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.ServiceModel\\Test\\Resources\\test.pfx", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.ServiceModel\\Test\\Resources\\test2.pfx", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.ServiceModel\\Test\\System.ServiceModel.Security\\WSSecurityTokenSerializerTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.ServiceModel\\Test\\System.ServiceModel.Security\\WSSecurityTokenSerializerTest.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Web\\Test\\standalone\\masterpages\\77278\\Web.Config", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Web\\Test\\standalone\\MD5PasswordAuth_Bug601727\\Web.config", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.XML\\Test\\System.Xml.Serialization\\XmlSerializerTests.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\tools\\security\\makecert.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Net.Http\\corefx\\Configuration.Certificates.cs", + "_justification": "External repo dependency, mono." + }, + { + "file": "external\\mono\\mcs\\class\\System.Net.Http\\TestData\\testservereku.contoso.com.pfx", + "_justification": "External repo dependency, mono." + }, + { + "file": "maccore\\tools\\provisioning-profiles\\certificates-and-profiles\\developer-id-application-luis-aguilera-aug-2024.p12", + "_justification": "Testing certificate." + }, + { + "file": "maccore\\tools\\provisioning-profiles\\certificates-and-profiles\\developer-id-installer-luis-aguilera-aug-2024.p12", + "_justification": "Testing certificate." + }, + { + "file": "maccore\\tools\\provisioning-profiles\\certificates-and-profiles\\la_dev_apple.p12", + "_justification": "Testing certificate." + }, + { + "file": "maccore\\tools\\provisioning-profiles\\certificates-and-profiles\\la_dev_iPhone.p12", + "_justification": "Testing certificate." + }, + { + "file": "maccore\\tools\\provisioning-profiles\\certificates-and-profiles\\la_distr_apple.p12", + "_justification": "Testing certificate." + }, + { + "file": "maccore\\tools\\provisioning-profiles\\certificates-and-profiles\\la_distr_iphone.p12", + "_justification": "Testing certificate." + }, + { + "file": "maccore\\tools\\provisioning-profiles\\certificates-and-profiles\\la_mac_app_dev.p12", + "_justification": "Testing certificate." + }, + { + "file": "maccore\\tools\\provisioning-profiles\\certificates-and-profiles\\la_mac_app_dev.p12", + "_justification": "Testing certificate." + }, + { + "file": "maccore\\tools\\provisioning-profiles\\certificates-and-profiles\\la_mac_app_distr.p12", + "_justification": "Testing certificate." + }, + { + "file": "maccore\\tools\\provisioning-profiles\\certificates-and-profiles\\la_mac_installer_distr.p12", + "_justification": "Testing certificate." + }, + { + "file": "maccore\\tools\\provisioning-profiles\\certificates-and-profiles\\vseng-xamarin-mac-devices-2.p12", + "_justification": "Testing certificate." + }, + { + "file": "maccore\\tools\\provisioning-profiles\\certificates-and-profiles\\vseng-xamarin-mac-devices.p12", + "_justification": "Testing certificate." + } + ] +} diff --git a/tools/devops/governance/PoliCheckExclusions.xml b/tools/devops/governance/PoliCheckExclusions.xml new file mode 100644 index 000000000000..90cc59a86cb2 --- /dev/null +++ b/tools/devops/governance/PoliCheckExclusions.xml @@ -0,0 +1,7 @@ + + + + EXTERNAL|OPENTK + + SAMPLE.JSON|LICENSE.TXT + diff --git a/tools/devops/governance/baselines.gdnbaselines b/tools/devops/governance/baselines.gdnbaselines new file mode 100644 index 000000000000..0ef7f556e090 --- /dev/null +++ b/tools/devops/governance/baselines.gdnbaselines @@ -0,0 +1,127 @@ +{ + "hydrated": false, + "properties": { + "helpUri": "https://eng.ms/docs/microsoft-security/security/azure-security/cloudai-security-fundamentals-engineering/security-integration/guardian-wiki/microsoft-guardian/general/baselines", + "hydrationStatus": "This file does not contain identifying data. It is safe to check into your repo. To hydrate this file with identifying data, run `guardian hydrate --help` and follow the guidance." + }, + "version": "1.0.0", + "baselines": { + "default": { + "name": "default", + "createdDate": "2024-02-26 23:21:09Z", + "lastUpdatedDate": "2024-02-26 23:21:09Z" + } + }, + "results": { + "5bfe7b64543acba3f04cf06473f454f534c681d035e1880706c675bcbf2a113a": { + "signature": "5bfe7b64543acba3f04cf06473f454f534c681d035e1880706c675bcbf2a113a", + "alternativeSignatures": [ + "b3463f284d1d50e828dc74bf242d16b7d75f056e87884a540220b4a64b851e16" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-26 23:21:09Z" + }, + "6460bd0d0ae3c3abd84d1b932d3d50efe43a839ca39ba8fc8c82f2884bec03e0": { + "signature": "6460bd0d0ae3c3abd84d1b932d3d50efe43a839ca39ba8fc8c82f2884bec03e0", + "alternativeSignatures": [ + "c891b899c263256f6945b163b8081181cd69d217bc5f83a88f6e28d80625e8aa" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-26 23:21:09Z" + }, + "2bc38a8affa25c44e59be8f7333ef145adad3f169f8b3b328168ee14b11e7042": { + "signature": "2bc38a8affa25c44e59be8f7333ef145adad3f169f8b3b328168ee14b11e7042", + "alternativeSignatures": [ + "d7adab12bd9358b3f526ae33fa68aa0e6a12d9f4ff3e61d7deff6d76e9730b78" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-26 23:21:09Z" + }, + "56373da5b7a0a5bdf6f2b7c1f0ab602096b2e0ff4260ac26befb8e1761213bbc": { + "signature": "56373da5b7a0a5bdf6f2b7c1f0ab602096b2e0ff4260ac26befb8e1761213bbc", + "alternativeSignatures": [ + "25b435d2a88adfa812a28781f8a868fb2bbac1721861c3f1d0f635432c03e975" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-26 23:21:09Z" + }, + "983f6b09df3eeac83858b1e7b9a020de0422c71ce2895e16acb779b8bebc09c2": { + "signature": "983f6b09df3eeac83858b1e7b9a020de0422c71ce2895e16acb779b8bebc09c2", + "alternativeSignatures": [ + "6f577c0990a9d84fb921c67fb849aa238ea30723aa2cbf002813787a55fd0de6" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-26 23:21:09Z" + }, + "d511eb73fc60ed3083f24a7c317e72790c75ff873133679b03d679030d57a637": { + "signature": "d511eb73fc60ed3083f24a7c317e72790c75ff873133679b03d679030d57a637", + "alternativeSignatures": [ + "b68f84c3d0d226c6781d0a99ee954a3877030fd50b3a60cf2d83fed6853cf0b3" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-26 23:21:09Z" + }, + "7714c68685ca3a216dfa6748ca603d42e6bac9b31546064faa2e28b0b183d927": { + "signature": "7714c68685ca3a216dfa6748ca603d42e6bac9b31546064faa2e28b0b183d927", + "alternativeSignatures": [ + "fc36cfff6406e27041af0f09f30819818ab4c054507d31d58ed645ff033b0c75" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-26 23:21:09Z" + }, + "f5e832c2a0285109108f3c71fd7e1f71bf73628669fea863de3a7318d84c2227": { + "signature": "f5e832c2a0285109108f3c71fd7e1f71bf73628669fea863de3a7318d84c2227", + "alternativeSignatures": [ + "593b4c0b30f2931ec72f92916be95fc00f5bc5b0d974fedfbff09da322c7624b" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-26 23:21:09Z" + }, + "cf36d387e3f106a4da49eecae99a1ffd1dec398cf42d144ed19569d8e9b92fca": { + "signature": "cf36d387e3f106a4da49eecae99a1ffd1dec398cf42d144ed19569d8e9b92fca", + "alternativeSignatures": [ + "f730744e034ada13851e9a7756b4f6b5c96191f72a29b1a5e7680b2e10b6e211" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-26 23:21:09Z" + }, + "b2fd71c6a486b070977a960c2861511d8681f067cb794241f8c80c922ddc9308": { + "signature": "b2fd71c6a486b070977a960c2861511d8681f067cb794241f8c80c922ddc9308", + "alternativeSignatures": [ + "6dbc8a0fad384b58507197a08507be02400ae9da16daa9d9623214538cb7d8aa" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-26 23:21:09Z" + }, + "861cb54eda1c02892adfd49721cdb698d1f7163a63813b7a8ed24f946f778666": { + "signature": "861cb54eda1c02892adfd49721cdb698d1f7163a63813b7a8ed24f946f778666", + "alternativeSignatures": [ + "4473a569ca2bb9b6dbc20a918cedf5d1898fe6dbbe0856271cba0f9076a05670" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-26 23:21:09Z" + } + } +} \ No newline at end of file diff --git a/tools/devops/governance/suppress.gdnsuppress b/tools/devops/governance/suppress.gdnsuppress new file mode 100644 index 000000000000..82b451a43f2a --- /dev/null +++ b/tools/devops/governance/suppress.gdnsuppress @@ -0,0 +1,117 @@ +{ + "hydrated": false, + "properties": { + "helpUri": "https://eng.ms/docs/microsoft-security/security/azure-security/cloudai-security-fundamentals-engineering/security-integration/guardian-wiki/microsoft-guardian/general/suppressions", + "hydrationStatus": "This file does not contain identifying data. It is safe to check into your repo. To hydrate this file with identifying data, run `guardian hydrate --help` and follow the guidance." + }, + "version": "1.0.0", + "suppressionSets": { + "default": { + "name": "default", + "createdDate": "2024-02-28 14:32:01Z", + "lastUpdatedDate": "2024-02-28 14:32:01Z" + } + }, + "results": { + "3c33b5615d37f699ee9a68a88a280d8dbf841aa9aed056e1709947b43079b437": { + "signature": "3c33b5615d37f699ee9a68a88a280d8dbf841aa9aed056e1709947b43079b437", + "alternativeSignatures": [ + "c49bf6d9c5b73b27bf0ebc2114a706304455e974ea0db9154c1c960af876bfe7" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-28 14:32:01Z" + }, + "0695270b7cd2aa1cbb027c79edcdee4701fa38c05649f09104dea99515bb97bd": { + "signature": "0695270b7cd2aa1cbb027c79edcdee4701fa38c05649f09104dea99515bb97bd", + "alternativeSignatures": [ + "b937aac0bb4ee7e132e6957cff49b8d5c3b4c33709fc274896531087088e055c" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-28 14:32:01Z" + }, + "70a198ef3687359ccf92037555a2d590a48b19d22180f038eaf4ab7f993484e4": { + "signature": "70a198ef3687359ccf92037555a2d590a48b19d22180f038eaf4ab7f993484e4", + "alternativeSignatures": [ + "3b49b7293b5d37a3f65177cab5d8788e1d26b4e7fe69791ce584a9c432677601" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-28 14:32:01Z" + }, + "3d7decdd4e599800955a91c698003b508b0845f05fc9fa90d6700ed0fcf6a1ed": { + "signature": "3d7decdd4e599800955a91c698003b508b0845f05fc9fa90d6700ed0fcf6a1ed", + "alternativeSignatures": [ + "42071e59d83712cc795a3560fd52945720794dc7e30196158eb10029e9433cbf" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-28 14:32:01Z" + }, + "9db5ee164b84798c1aeee9b5805f8e84b54fa615ebd9f21233918ecc94c15020": { + "signature": "9db5ee164b84798c1aeee9b5805f8e84b54fa615ebd9f21233918ecc94c15020", + "alternativeSignatures": [ + "5536188ee53aa7c13fa426cea9bcd846378f3b8c4618aa13a7b17a6365465194" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-28 14:32:01Z" + }, + "5b225f9d4de137280a0c92a050fe1f316a72bb12fb6a9414e6057cddf050a310": { + "signature": "5b225f9d4de137280a0c92a050fe1f316a72bb12fb6a9414e6057cddf050a310", + "alternativeSignatures": [ + "2c42b0707363365137aa7c4a103ab70ba66f4459aa18a486c1b6f693a56545fb" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-28 14:32:01Z" + }, + "5d6523d8cc6074453da8205b3bc23cceca00098c768e874f3b96a21497f66533": { + "signature": "5d6523d8cc6074453da8205b3bc23cceca00098c768e874f3b96a21497f66533", + "alternativeSignatures": [ + "fe889e82b885db03e0d319095c82579bb79c37b525683194fd1e29df167a670f" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-28 14:32:01Z" + }, + "10f8341c6e943732912f495f167bcefae77c7fb879ca09ec2e4991b34506a141": { + "signature": "10f8341c6e943732912f495f167bcefae77c7fb879ca09ec2e4991b34506a141", + "alternativeSignatures": [ + "512e56ec02ebea2e8917078cfc582d3dd7a1175e157117777fa0d0c2f61eba7f" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-28 14:32:01Z" + }, + "47cb0eddb2fa84134862cebadad6ab13ad5176def6f6d2185bc76f30365fb9ad": { + "signature": "47cb0eddb2fa84134862cebadad6ab13ad5176def6f6d2185bc76f30365fb9ad", + "alternativeSignatures": [ + "d393e382a8a1c46a4b4c27d1ced12efdb0cbad825f0fd36120b34fd410850b8b" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-28 14:32:01Z" + }, + "81195a0cd1716cb62d704218acbbdab6e3ae5f75bdc31d1dbfc1b074a259debd": { + "signature": "81195a0cd1716cb62d704218acbbdab6e3ae5f75bdc31d1dbfc1b074a259debd", + "alternativeSignatures": [ + "51524c0bcb9c17989249e3100b33c3ecc0c1a53d64d08a19b8a50bce824cd00e" + ], + "memberOf": [ + "default" + ], + "createdDate": "2024-02-28 14:32:01Z" + } + } +} \ No newline at end of file diff --git a/tools/devops/governance/tsa_config.gdntsa b/tools/devops/governance/tsa_config.gdntsa new file mode 100644 index 000000000000..ab4db00975d6 --- /dev/null +++ b/tools/devops/governance/tsa_config.gdntsa @@ -0,0 +1,19 @@ +{ + "codebaseName": "xamarin-macios", + "notificationAliases": [ + "xamarin-macios@microsoft.com" + ], + "codebaseAdmins": [ + "NORTHAMERICA\\mandel", + "NORTHAMERICA\\alexsoto" + ], + "instanceUrl": "https://devdiv.visualstudio.com", + "projectName": "DevDiv", + "areaPath": "DevDiv\\VS Client - Runtime SDKs\\iOS and Mac", + "iterationPath": "DevDiv", + "tools": [ + "ApiScan", + "CredScan", + "PoliCheck" + ] +} From 8720672ff6c2f92d3e71d88584547094728a02a8 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 20 Jun 2024 08:29:01 +0200 Subject: [PATCH 12/17] [main] Update dependencies from dotnet/installer (#20742) This pull request updates the following dependencies ## From https://github.com/dotnet/installer - **Subscription**: 80cb9ffd-f92f-4fc8-9f8b-08dbca46abfb - **Build**: 20240617.4 - **Date Produced**: June 17, 2024 6:20:02 PM UTC - **Commit**: de7be3dce619f5998e041962b4a1062cc44bfd37 - **Branch**: refs/heads/release/8.0.1xx - **Updates**: - **Microsoft.Dotnet.Sdk.Internal**: [from 8.0.107-servicing.24311.1 to 8.0.107-servicing.24317.4][2] [2]: https://github.com/dotnet/installer/compare/8882eb09fc...de7be3dce6 --- eng/Version.Details.xml | 4 ++-- eng/Versions.props | 2 +- global.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8a0e589abfc5..ba7321f6b671 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,8 +1,8 @@ - + https://github.com/dotnet/installer - 8882eb09fcade072ffad12ad477b91bdefa5fe12 + de7be3dce619f5998e041962b4a1062cc44bfd37 https://github.com/dotnet/runtime diff --git a/eng/Versions.props b/eng/Versions.props index fa745c7dbc60..0a51347a5a44 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -2,7 +2,7 @@ - 8.0.107-servicing.24311.1 + 8.0.107-servicing.24317.4 8.0.5 8.0.0-rtm.23524.7 8.0.0-beta.24225.1 diff --git a/global.json b/global.json index 6bd397a3d5f0..6452f1826494 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "8.0.107-servicing.24311.1" + "version": "8.0.107-servicing.24317.4" } } From 29560c5758a45668f622458d98904d7dfc0856a9 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Thu, 20 Jun 2024 18:20:08 -0400 Subject: [PATCH 13/17] [CI] Disable the credscan step on the signing job since 1ES has a bug with it. (#20756) --- tools/devops/automation/templates/release/vs-insertion-prep.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/devops/automation/templates/release/vs-insertion-prep.yml b/tools/devops/automation/templates/release/vs-insertion-prep.yml index 34ff0254f22a..fddb03634d59 100644 --- a/tools/devops/automation/templates/release/vs-insertion-prep.yml +++ b/tools/devops/automation/templates/release/vs-insertion-prep.yml @@ -53,6 +53,7 @@ stages: - template: sign-artifacts/jobs/v2.yml@yaml-templates parameters: use1ESTemplate: true + enabledCredScan: false signedArtifactName: '${{ parameters.uploadPrefix }}nuget-signed' artifactName: '${{ parameters.uploadPrefix }}not-signed-package' signType: Real From e615133676c4e307854ebc0507454361647ac825 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Thu, 20 Jun 2024 18:20:29 -0400 Subject: [PATCH 14/17] [CI] Expose all the CI build variables of the parent (#20749) This change allows cascading pipelines to download the configuration file and use it rather than recalculating the configuration. This is the first step to ensure that cascading pipelines use the same config, a second one is needed to load the configuration in those pipelines. --------- Co-authored-by: Rolf Bjarne Kvinge --- .../devops/automation/scripts/VSTS.Tests.ps1 | 70 +++++++---- tools/devops/automation/scripts/VSTS.psm1 | 113 ++++++++++++++++-- .../automation/templates/common/configure.yml | 5 + 3 files changed, 157 insertions(+), 31 deletions(-) diff --git a/tools/devops/automation/scripts/VSTS.Tests.ps1 b/tools/devops/automation/scripts/VSTS.Tests.ps1 index 8bd1090a66e6..291ebc5ed540 100644 --- a/tools/devops/automation/scripts/VSTS.Tests.ps1 +++ b/tools/devops/automation/scripts/VSTS.Tests.ps1 @@ -211,11 +211,19 @@ Describe 'New-BuildConfiguration' { $buildConfiguration = New-BuildConfiguration -AddTags $false $buildConfiguration | ConvertTo-Json | Should -Be "{ - ""BuildReason"": ""BUILD_REASON"", - ""BuildSourceBranchName"": ""BUILD_SOURCEBRANCHNAME"", - ""BuildSourceBranch"": ""BUILD_SOURCEBRANCH"", - ""BuildId"": ""BUILD_BUILDID"", ""DOTNET_PLATFORMS"": ""iOS tvOS"", + ""PARENT_BUILD_BUILD_BUILDID"": ""BUILD_BUILDID"", + ""PARENT_BUILD_BUILD_BUILDNUMBER"": null, + ""PARENT_BUILD_BUILD_BUILDURI"": null, + ""PARENT_BUILD_BUILD_BINARIESDIRECTORY"": null, + ""PARENT_BUILD_BUILD_DEFINITIONNAME"": null, + ""PARENT_BUILD_BUILD_REASON"": ""BUILD_REASON"", + ""PARENT_BUILD_BUILD_REPOSITORY_ID"": null, + ""PARENT_BUILD_BUILD_REPOSITORY_NAME"": null, + ""PARENT_BUILD_BUILD_REPOSITORY_PROVIDER"": null, + ""PARENT_BUILD_BUILD_REPOSITORY_URI"": null, + ""PARENT_BUILD_BUILD_SOURCEBRANCH"": ""BUILD_SOURCEBRANCH"", + ""PARENT_BUILD_BUILD_SOURCEBRANCHNAME"": ""BUILD_SOURCEBRANCHNAME"", ""INCLUDE_DOTNET_IOS"": null, ""IOS_NUGET_VERSION_NO_METADATA"": null, ""IOS_NUGET_SDK_NAME"": ""iOSNuGetSdkName"", @@ -230,8 +238,7 @@ Describe 'New-BuildConfiguration' { ""tvos-arm64_NUGET_RUNTIME_NAME"": null, ""Commit"": ""BUILD_SOURCEVERSION"", ""Tags"": [ - ""ciBuild"", - ""BUILD_SOURCEBRANCHNAME"" + ""ciBuild"" ] }" } @@ -247,11 +254,19 @@ Describe 'New-BuildConfiguration' { # Write-Host $buildConfiguration $buildConfiguration | Should -Be "{ - ""BuildReason"": ""BUILD_REASON"", - ""BuildSourceBranchName"": ""BUILD_SOURCEBRANCHNAME"", - ""BuildSourceBranch"": ""BUILD_SOURCEBRANCH"", - ""BuildId"": ""BUILD_BUILDID"", ""DOTNET_PLATFORMS"": ""iOS tvOS"", + ""PARENT_BUILD_BUILD_BUILDID"": ""BUILD_BUILDID"", + ""PARENT_BUILD_BUILD_BUILDNUMBER"": null, + ""PARENT_BUILD_BUILD_BUILDURI"": null, + ""PARENT_BUILD_BUILD_BINARIESDIRECTORY"": null, + ""PARENT_BUILD_BUILD_DEFINITIONNAME"": null, + ""PARENT_BUILD_BUILD_REASON"": ""BUILD_REASON"", + ""PARENT_BUILD_BUILD_REPOSITORY_ID"": null, + ""PARENT_BUILD_BUILD_REPOSITORY_NAME"": null, + ""PARENT_BUILD_BUILD_REPOSITORY_PROVIDER"": null, + ""PARENT_BUILD_BUILD_REPOSITORY_URI"": null, + ""PARENT_BUILD_BUILD_SOURCEBRANCH"": ""BUILD_SOURCEBRANCH"", + ""PARENT_BUILD_BUILD_SOURCEBRANCHNAME"": ""BUILD_SOURCEBRANCHNAME"", ""INCLUDE_DOTNET_IOS"": null, ""IOS_NUGET_VERSION_NO_METADATA"": null, ""IOS_NUGET_SDK_NAME"": ""iOSNuGetSdkName"", @@ -266,8 +281,7 @@ Describe 'New-BuildConfiguration' { ""tvos-arm64_NUGET_RUNTIME_NAME"": null, ""Commit"": ""BUILD_SOURCEVERSION"", ""Tags"": [ - ""ciBuild"", - ""BUILD_SOURCEBRANCHNAME"" + ""ciBuild"" ] } " @@ -279,22 +293,34 @@ Describe 'Import-BuildConfiguration' { Context 'import' { It 'gets the right values' { $config = "{ - ""BuildReason"": ""BUILD_REASON"", - ""BuildSourceBranchName"": ""BUILD_SOURCEBRANCHNAME"", - ""BuildSourceBranch"": ""BUILD_SOURCEBRANCH"", - ""BuildId"": ""BUILD_BUILDID"", ""DOTNET_PLATFORMS"": ""iOS tvOS"", + ""PARENT_BUILD_BUILD_BUILDID"": ""BUILD_BUILDID"", + ""PARENT_BUILD_BUILD_BUILDNUMBER"": null, + ""PARENT_BUILD_BUILD_BUILDURI"": null, + ""PARENT_BUILD_BUILD_BINARIESDIRECTORY"": null, + ""PARENT_BUILD_BUILD_DEFINITIONNAME"": null, + ""PARENT_BUILD_BUILD_REASON"": ""BUILD_REASON"", + ""PARENT_BUILD_BUILD_REPOSITORY_ID"": null, + ""PARENT_BUILD_BUILD_REPOSITORY_NAME"": null, + ""PARENT_BUILD_BUILD_REPOSITORY_PROVIDER"": null, + ""PARENT_BUILD_BUILD_REPOSITORY_URI"": null, + ""PARENT_BUILD_BUILD_SOURCEBRANCH"": ""BUILD_SOURCEBRANCH"", + ""PARENT_BUILD_BUILD_SOURCEBRANCHNAME"": ""BUILD_SOURCEBRANCHNAME"", ""INCLUDE_DOTNET_IOS"": null, - ""INCLUDE_DOTNET_TVOS"": ""true"", - ""DOTNET_IOS_RUNTIME_IDENTIFIERS"": ""ios-arm64"", - ""DOTNET_TVOS_RUNTIME_IDENTIFIERS"": ""tvos-arm64"", + ""IOS_NUGET_VERSION_NO_METADATA"": null, ""IOS_NUGET_SDK_NAME"": ""iOSNuGetSdkName"", - ""TVOS_NUGET_REF_NAME"": ""tvOSNuGetRefName"", + ""IOS_NUGET_REF_NAME"": null, + ""DOTNET_IOS_RUNTIME_IDENTIFIERS"": ""ios-arm64"", ""ios-arm64_NUGET_RUNTIME_NAME"": ""iOSNuGetRuntimeName"", + ""INCLUDE_DOTNET_TVOS"": ""true"", + ""TVOS_NUGET_VERSION_NO_METADATA"": null, + ""TVOS_NUGET_SDK_NAME"": null, + ""TVOS_NUGET_REF_NAME"": ""tvOSNuGetRefName"", + ""DOTNET_TVOS_RUNTIME_IDENTIFIERS"": ""tvos-arm64"", + ""tvos-arm64_NUGET_RUNTIME_NAME"": null, ""Commit"": ""BUILD_SOURCEVERSION"", ""Tags"": [ - ""ciBuild"", - ""BUILD_SOURCEBRANCHNAME"" + ""ciBuild"" ] }" diff --git a/tools/devops/automation/scripts/VSTS.psm1 b/tools/devops/automation/scripts/VSTS.psm1 index 16855e6867f7..92e6014c6649 100644 --- a/tools/devops/automation/scripts/VSTS.psm1 +++ b/tools/devops/automation/scripts/VSTS.psm1 @@ -253,6 +253,32 @@ class BuildConfiguration { throw [System.InvalidOperationException]::new("Failed to load configuration file $configFile") } + $defaultBuildVariables = @( + "BUILD_BUILDID", + "BUILD_BUILDNUMBER", + "BUILD_BUILDURI", + "BUILD_BINARIESDIRECTORY", + "BUILD_DEFINITIONNAME", + "BUILD_REASON", + "BUILD_REPOSITORY_ID", + "BUILD_REPOSITORY_NAME", + "BUILD_REPOSITORY_PROVIDER", + "BUILD_REPOSITORY_URI", + "BUILD_SOURCEBRANCH", + "BUILD_SOURCEBRANCHNAME" + ) + + # load the variable name from the parent + foreach ($buildVariable in $defaultBuildVariables) { + $variableName = "PARENT_BUILD_$buildVariable" + $variableValue = $config.$variableName + if ($variableValue) { + Write-Host "##vso[task.setvariable variable=$variableName;isOutput=true]$variableValue" + } else { + Write-Debug "Ignoring variable $variableName" + } + } + $dotnetPlatforms = $config.DOTNET_PLATFORMS.Split(' ', [StringSplitOptions]::RemoveEmptyEntries) Write-Host "##vso[task.setvariable variable=DOTNET_PLATFORMS;isOutput=true]$dotnetPlatforms" foreach ($platform in $dotnetPlatforms) { @@ -287,17 +313,64 @@ class BuildConfiguration { return $config } + [PSCustomObject] Update([string] $configKey, [string] $configValue, [string] $configFile) { + if (-not (Test-Path -Path $configFile -PathType Leaf)) { + throw [System.InvalidOperationException]::new("Configuration file $configFile is missing") + } + + $config = Get-Content $configFile | ConvertFrom-Json + + if (-not $config) { + throw [System.InvalidOperationException]::new("Failed to load configuration file $configFile") + } + + $config | Add-Member -NotePropertyName $configKey -NotePropertyValue $configValue + + $jsonConfiguration = $config | ConvertTo-Json + + Write-Host "Build configuration:" + Write-Host $jsonConfiguration + + if ($configFile) { + Write-Host "Writing configuration to: $configFile" + Set-Content -Path $configFile -Value $jsonConfiguration + } + + return $config + } + [PSCustomObject] Create([bool] $addTags, [string] $configFile) { # we are going to use a custom object to store all the configuration of the build, this later - # will be uploaded as an artifact so that it can be easily shared with the cascade pipelines + # will be uploaded as an artifact so that it can be easily shared with the cascade pipelines, we will + # uses a special prefix for the default variable names so that we do not step on the cascading pipeline + # settings + $configuration = [PSCustomObject]@{ - BuildReason = "$Env:BUILD_REASON" - BuildSourceBranchName = "$Env:BUILD_SOURCEBRANCHNAME" - BuildSourceBranch = "$Env:BUILD_SOURCEBRANCH" - BuildId = "$Env:BUILD_BUILDID" DOTNET_PLATFORMS = "$Env:CONFIGURE_PLATFORMS_DOTNET_PLATFORMS" } + $defaultBuildVariables = @( + "BUILD_BUILDID", + "BUILD_BUILDNUMBER", + "BUILD_BUILDURI", + "BUILD_BINARIESDIRECTORY", + "BUILD_DEFINITIONNAME", + "BUILD_REASON", + "BUILD_REPOSITORY_ID", + "BUILD_REPOSITORY_NAME", + "BUILD_REPOSITORY_PROVIDER", + "BUILD_REPOSITORY_URI", + "BUILD_SOURCEBRANCH", + "BUILD_SOURCEBRANCHNAME" + ) + + # loop over the default build enviroments and add them with a prefix to the configuration objects + foreach ($buildVariable in $defaultBuildVariables) { + $variableName = "PARENT_BUILD_$buildVariable" + $variableValue = [Environment]::GetEnvironmentVariable($buildVariable) + $configuration | Add-Member -NotePropertyName $variableName -NotePropertyValue $variableValue + } + # For each .NET platform we support, add a INCLUDE_DOTNET_ variable specifying whether that platform is enabled or not. $dotnetPlatforms = $configuration.DOTNET_PLATFORMS.Split(' ', [StringSplitOptions]::RemoveEmptyEntries) foreach ($platform in $dotnetPlatforms) { @@ -331,7 +404,7 @@ class BuildConfiguration { # calculate the commit to later share it with the cascade pipelines if ($Env:BUILD_REASON -eq "PullRequest") { - $changeId = $configuration.BuildSourceBranch.Replace("refs/pull/", "").Replace("/merge", "") + $changeId = $configuration.PARENT_BUILD_BUILD_SOURCEBRANCH.Replace("refs/pull/", "").Replace("/merge", "") } else { $changeId = $Env:BUILD_SOURCEVERSION } @@ -345,12 +418,12 @@ class BuildConfiguration { $tags.Add("cronjob") } - if ($configuration.BuildReason -eq "PullRequest" -or (($configuration.BuildReason -eq "Manual") -and ($configuration.BuildSourceBranchName -eq "merge")) ) { + if ($configuration.BuildReason -eq "PullRequest" -or (($configuration.BuildReason -eq "Manual") -and ($configuration.PARENT_BUILD_BUILD_SOURCEBRANCH -eq "merge")) ) { Write-Host "Configuring build from PR." # This is an interesting step, we do know we are dealing with a PR, but we need the PR id to # be able to get the labels, the buildSourceBranch follows the pattern: refs/pull/{ChangeId}/merge # we could use a regexp but then we would have two problems instead of one - $changeId = $configuration.BuildSourceBranch.Replace("refs/pull/", "").Replace("/merge", "") + $changeId = $configuration.PARENT_BUILD_BUILD_SOURCEBRANCH.Replace("refs/pull/", "").Replace("/merge", "") # add a var with the change id, which can be later consumed by some of the old scripts from # jenkins @@ -403,7 +476,7 @@ class BuildConfiguration { $tags.Add("ciBuild") } # set the name of the branch under build - $tags.Add("$($configuration.BuildSourceBranchName)") + $tags.Add("$($configuration.PARENT_BUILD_BUILD_SOURCEBRANCHNAME)") Write-Host "##vso[task.setvariable variable=prBuild;isOutput=true]False" } # Remove empty entries @@ -706,6 +779,27 @@ function Import-BuildConfiguration { return $buildConfiguration.Import($ConfigFile) } +function Edit-BuildConfiguration { + param + ( + + [Parameter(Mandatory)] + [string] + $ConfigKey, + + [Parameter(Mandatory)] + [string] + $ConfigValue, + + [Parameter(Mandatory)] + [string] + $ConfigFile + ) + $buildConfiguration = [BuildConfiguration]::new() + return $buildConfiguration.Update($ConfigKey, $ConfigValue, $ConfigFile) +} + + # export public functions, other functions are private and should not be used ouside the module. Export-ModuleMember -Function Stop-Pipeline Export-ModuleMember -Function Set-PipelineResult @@ -713,4 +807,5 @@ Export-ModuleMember -Function Set-BuildTags Export-ModuleMember -Function New-VSTS Export-ModuleMember -Function New-BuildConfiguration Export-ModuleMember -Function Import-BuildConfiguration +Export-ModuleMember -Function Edit-BuildConfiguration Export-ModuleMember -Function Get-YamlPreview diff --git a/tools/devops/automation/templates/common/configure.yml b/tools/devops/automation/templates/common/configure.yml index 87a29fa40ba7..3d7d23108915 100644 --- a/tools/devops/automation/templates/common/configure.yml +++ b/tools/devops/automation/templates/common/configure.yml @@ -53,6 +53,7 @@ steps: - pwsh: | Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY/xamarin-macios/tools/devops/automation/scripts/MaciosCI.psd1 $jsonPath = Join-Path -Path "$(Build.ArtifactStagingDirectory)" -ChildPath "configuration.json" + Write-Host "##vso[task.setvariable variable=CONFIG_PATH]$jsonPath" New-BuildConfiguration -ConfigFile $jsonPath env: GITHUB_TOKEN: $(GitHub.Token) @@ -81,6 +82,9 @@ steps: Write-Host "$testMatrix" $testMatrix = $testMatrix | ConvertFrom-Json | ConvertTo-Json -Compress Write-Host "##vso[task.setvariable variable=TEST_MATRIX;isOutput=true]$testMatrix" + # update the config file so that we do not recalculate the matrix in other pipelines + Edit-BuildConfiguration -ConfigKey TEST_MATRIX -ConfigValue $testMatrix -ConfigFile $Env:CONFIG_PATH + #CONFIG_PATH name: test_matrix displayName: 'Create tests strategy matrix' env: @@ -97,6 +101,7 @@ steps: $apiScanMatrix = $apiScanMatrix | ConvertFrom-Json | ConvertTo-Json -Compress } Write-Host "##vso[task.setvariable variable=APISCAN_MATRIX;isOutput=true]$apiScanMatrix" + Edit-BuildConfiguration -ConfigKey APISCAN_MATRIX -ConfigValue $apiScanMatrix -ConfigFile $Env:CONFIG_PATH name: apiscan_matrix displayName: 'Create APIScan matrix' From 35690fc99bb94b58275e202d03739855923b708a Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Thu, 20 Jun 2024 18:21:26 -0400 Subject: [PATCH 15/17] [Governance] Adapt the APIScan arguments to the new version of the cmd. (#20757) --- tools/devops/automation/templates/governance/apiscan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/automation/templates/governance/apiscan.yml b/tools/devops/automation/templates/governance/apiscan.yml index cb4395005fd1..9c241139abba 100644 --- a/tools/devops/automation/templates/governance/apiscan.yml +++ b/tools/devops/automation/templates/governance/apiscan.yml @@ -50,7 +50,7 @@ steps: verbosityLevel: standard preserveLogsFolder: true env: - AzureServicesAuthConnectionString: RunAs=App;AppId=$(CLIENT_ID); + AzureServicesAuthConnectionString: RunAs=App;AppId=$(CLIENT_ID) - task: PublishSecurityAnalysisLogs@3 displayName: Publish Security Analysis Logs From fe9f5f15b3d18071a3dc4de7687986af2f18d2b7 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Thu, 20 Jun 2024 18:41:08 -0400 Subject: [PATCH 16/17] [CI] Fix a failing pwsh test. (#20758) --- tools/devops/automation/scripts/VSTS.Tests.ps1 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/devops/automation/scripts/VSTS.Tests.ps1 b/tools/devops/automation/scripts/VSTS.Tests.ps1 index 291ebc5ed540..1f367ac0cdfb 100644 --- a/tools/devops/automation/scripts/VSTS.Tests.ps1 +++ b/tools/devops/automation/scripts/VSTS.Tests.ps1 @@ -238,7 +238,8 @@ Describe 'New-BuildConfiguration' { ""tvos-arm64_NUGET_RUNTIME_NAME"": null, ""Commit"": ""BUILD_SOURCEVERSION"", ""Tags"": [ - ""ciBuild"" + ""ciBuild"", + ""BUILD_SOURCEBRANCHNAME"" ] }" } @@ -281,7 +282,8 @@ Describe 'New-BuildConfiguration' { ""tvos-arm64_NUGET_RUNTIME_NAME"": null, ""Commit"": ""BUILD_SOURCEVERSION"", ""Tags"": [ - ""ciBuild"" + ""ciBuild"", + ""BUILD_SOURCEBRANCHNAME"" ] } " From e15955bb6739dacfbb51743934438439324b30c9 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Fri, 21 Jun 2024 10:41:21 +0200 Subject: [PATCH 17/17] [siminstaller] Fix the binlog location during the build. (#20755) --- tools/siminstaller/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/siminstaller/Makefile b/tools/siminstaller/Makefile index cb33ba9dfc98..957e62c71ff2 100644 --- a/tools/siminstaller/Makefile +++ b/tools/siminstaller/Makefile @@ -9,7 +9,7 @@ install-local:: all-local # we build using the system .NET, because we might execute before we've downloaded our own version of .NET (i.e. while provisioning dependencies) $(EXECUTABLE): $(wildcard *.cs) $(wildcard *.csproj) Makefile - $(Q_BUILD) cd $(HOME) && dotnet build $(abspath $(CURDIR))/*.csproj "/bl:$@.binlog" $(DOTNET_BUILD_VERBOSITY) + $(Q_BUILD) cd $(HOME) && dotnet build $(abspath $(CURDIR))/*.csproj "/bl:$(abspath $@.binlog)" $(DOTNET_BUILD_VERBOSITY) $(Q) touch $@ print-simulators: $(EXECUTABLE)