From a34b790088bb576d6434f997a57c0b1b74aeb97c Mon Sep 17 00:00:00 2001 From: Salome DO Date: Tue, 30 Jan 2024 16:52:26 +0100 Subject: [PATCH] feat: update dev-tools documentation --- docs/dev-tools/DEV_TOOLS.md | 271 +----------------- packages/@o3r/artifactory-tools/README.md | 144 ++++------ packages/@o3r/artifactory-tools/package.json | 2 - .../src/cli/artifact-cleaner.ts | 37 +-- .../src/cli/artifact-retriever.ts | 245 ---------------- .../src/cli/pr-artifact-cleaner.ts | 77 ++--- .../gavc-response/gavc-response.interfaces.ts | 52 ---- .../src/helpers/gavc-response/index.ts | 1 - .../artifactory-tools/src/helpers/index.ts | 1 - .../@o3r/artifactory-tools/src/public_api.ts | 1 - packages/@o3r/azure-tools/README.md | 40 ++- .../@o3r/azure-tools/src/cli/comment-pr.ts | 2 +- packages/@o3r/dev-tools/README.md | 100 ++----- packages/@o3r/dev-tools/package.json | 1 - .../dev-tools/src/cli/artifact-retriever.ts | 251 +++++++++++++++- packages/@o3r/dev-tools/src/cli/comment-pr.ts | 2 +- .../gavc-response/gavc-response.interfaces.ts | 56 +++- yarn.lock | 3 - 18 files changed, 455 insertions(+), 831 deletions(-) delete mode 100644 packages/@o3r/artifactory-tools/src/cli/artifact-retriever.ts delete mode 100644 packages/@o3r/artifactory-tools/src/helpers/gavc-response/gavc-response.interfaces.ts delete mode 100644 packages/@o3r/artifactory-tools/src/helpers/gavc-response/index.ts delete mode 100644 packages/@o3r/artifactory-tools/src/helpers/index.ts delete mode 100644 packages/@o3r/artifactory-tools/src/public_api.ts diff --git a/docs/dev-tools/DEV_TOOLS.md b/docs/dev-tools/DEV_TOOLS.md index f8ba1c5a24..99024eeae5 100644 --- a/docs/dev-tools/DEV_TOOLS.md +++ b/docs/dev-tools/DEV_TOOLS.md @@ -1,272 +1,15 @@ # Otter utils library -Collection of tools to help to main a project with Otter Library +The Otter Framework provides a collection of tools to help your Otter's project maintenance. -## Artifact Retriever +## Artifactory Tools -Get an artifact from the ArtiFactory +There are tools for the supported repository manager [JFrog](https://jfrog.com/artifactory/), including deleting old artifacts and deleting a specified amount of PR build artifacts. -### Usage +Further information on these tools can be found in the `@o3r/artifactory-tools` package [documentation](https://github.com/AmadeusITGroup/otter/blob/main/packages/%40o3r/artifactory-tools/README.md). -```shell -Usage: o3r-artifact-retriever [options] +## Azure Tools -Get an artifact from Artifactory, artifact-group, artifact-name and artifact-version are mandatory +There is also a tool to comment a Pull Request on Azure. -Options: - - --registry Registry URL. It is ignored for Azure Artifacts. - --repository-manager Artifact repository manager. Supported managers are JFrog, Azure Artifacts (default: JFrog) - --organization Azure Artifacts organization - --project Azure Artifacts project - --feed Azure Artifacts feed - -a, --artifact-name Artifact name - -v, --artifact-version Artifact version - -g, --artifact-group Artifact group name - -r, --artifact-repos Artifact repositories (default: ) - -u, --username Artifactory username (default from ARTIFACTORY_USERNAME) - -p, --password Artifactory user password (default from ARTIFACTORY_PASSWORD) - -o, --out Output file name (default: ./built/${name}.jar) - --use-package-version Use the package version as artifact version - -h, --help output usage information -``` - -### JFrog Example - -```shell -artifact-retriever.js --registry "https://jfrog.io/repoName" -u -p -v "1.0.0" -g "io.swagger" -a "typescriptFetch-swagger-codegen" --out /path/to/typescriptFetch-swagger-codegen.jar -``` - -### Azure Artifacts Example - -```shell -artifact-retriever.js --repository-manager "Azure Artifacts" --organization "OrganizationName" --project "Otter" --feed "otter" -u -p -v "1.0.0" -g "io.swagger" -a "typescriptFetch-swagger-codegen" --out /path/to/typescriptFetch-swagger-codegen.jar -``` - -## Dependency Validator - -Validator of dependencies. -The purpose is to check if the update is not breaking application using the current package. - -### Usage - -```shell -Usage: dep-validator [options] - -Execute dependency tests - - -Options: - - -u, --username Git username - -p, --password Git user password - -w, --workspace Path to the folder containing the repository to clone - --workspace-project Path to the project root folder inside the workspace - -b, --build Tasks to run to build the dependency - -t, --test Tasks to run to test the dependency - -P, --project Project root path - --include Packages to include in the process - --exclude Packages to exclude from the process - --no-lerna Disable lerna file parsing - --no-trustedVersionNumber Determine if the version number is trusted (used for version number generated in Pull Request) -``` - -## Changelog Generator - -Update changelog file with the new changes. -The purpose is to reduce the number of Pull Requests conflict caused by the edition of the Changelog file. - -### Usage - -```shell -Usage: changelog-generator [options] - -Update ChangeLog with new changes - - -Options: - - -g, --generate-on Versions change when generating a changelog (comma separated) - -u, --username Git username - -p, --password Git user password - -o, --output Path to the changelog file (default: README.md) - --pattern JIRA task pattern (default: '[A-Z]{2,}-[0-9]+') - --config Path to a configuration file - --only-task-done Included only task done in this changelog - -h, --help output usage information -``` - -## Set Version - -Replace the packages version in a monorepo - -### Usage - -```shell -Usage: o3r-set-version [options] - -Replace the packages version in a monorepos - -Options: - - -p, --placeholder Pattern of the version placeholder (default: 0.0.0) - --include Add files pattern to apply the verison replacement (default: */lerna.json,**/package.json,!**/node_modules/**/{package,lerna}.json) - -h, --help output usage information -``` - -## Artifact cleaner - -Clean old artifacts from artifactory repositories - -### Usage - -```shell -Usage: o3r-artifact-cleaner -b [options] - -Clean old artifacts from artifactory repositories, base 64 encoding of username:password is mandatory - -Options: - - --artifactory-url Artifact URL (Required) - -a, --duration-kept All the artifact which have been created since more time than this value(ms) will be deleted (Default to 10080000ms (i.e. 7 days)) - -r, --repositories Artifact repositories to clean up (coma separated) ex : npm-otter-pr,npm-refx-pr (Default to npm-otter-pr) - -t, --type-filter List of artifact type that should be deleted coma separated (ex: jar,tgz) (Default : tgz) - --dry-run List all files that should be deleted without actually deleting them. (Default to false) - -h, --help Output usage information - -Example : yarn o3r-artifact-cleaner -b thisismybase64tokenwithuserandencryptedpassword -``` - -## PR Artifact cleaner - -Clean old PR artifacts by identifying using the build version that is present in the path. If build version is not present in the path this tool cannot yet be used. - -### Usage - -```shell -Usage: o3r-pr-artifact-cleaner -b [options] - -Clean old artifacts from artifactory repositories, base 64 encoding of username:password is mandatory - -Options: - - -u, --artifactory-url Artifact URL - -d, --duration-kept Only artifacts which are older than this value (in days) will be deleted. (Default to 1 day) - -r, --repository Artifact repository to clean up. (Default to dga-maven-built-adt-nce) - -pr, --pr-versions Number of pr versions that will be kept. (Default to 2 last versions) - --dry-run List all files that should be deleted without actually deleting them. (Default to false) - -p, --path Artifact paths to cleanup use matcher from AQL language. Be careful that the path do not include release artifacts (Default to com/amadeus/retailing/*-PR-*) - -h, --help Output usage information - -Example : yarn o3r-pr-artifact-cleaner -b thisismybase64tokenwithuserandencryptedpassword -``` - -## Peer dependencies updater - -Update a package.json with the given dependencies versions and their respective peer dependencies. -Relies on `npm info` to retrieve package information. - -### Usage - -```shell -Usage: peer-dependencies-updater [other packages] - -Update the given packages version and their peer dependencies range in the provided package.json file (defaulted to local ./package.json) - -Options: - - -p, --package-json Path to the package.json file to update. Default: ./package.json - --verbose Display debug log message - --silent Do not exit with error in case of metadata fetch error - -Example : peer-dependencies-updater "@random/package@~2.21.0" "@o3r/core" -``` - -## Scripts - -This package provide generic helpers to support the build chain of Otter and Ama sdk packages - -## Set Version - -Replace the packages version in a monorepo. -This is used to edit the package.json version of a yarn workspaces packages. - -### Usage - -```shell -Usage: o3r-set-version [options] - -Replace the packages version in a monorepos - -Options: - -p, --placeholder Pattern of the version placeholder (default: "0.0.0") - --include Add files pattern to apply the verison replacement (default: ["*/lerna.json","**/package.json","!**/node_modules/**/{package,lerna}.json"]) - -h, --help display help for command -``` - -## Version Harmonize - -> :warning: **Deprecate**: This script is deprecated and will be removed in Otter v12, it is replaced by the JSON ESLint rule [@o3r/json-dependency-versions-harmonize](https://github.com/AmadeusITGroup/otter/blob/main/docs/linter/eslint-plugin/rules/json-dependency-versions-harmonize.md). - -Replace the dependencies version in a monorepo. -This aligns the dependencies range of each packages of a yarn monorepo to the latest range detected in the monorepo. - -### Usage - -```shell -Usage: version-harmonize [options] - -Replace the dependencies version in a monorepos - -Options: - -m, --monorepo Path to the private package.json of the monorepo (default: "") - -t, --dependencyTypes <...types> List of dependency types to update, comma separated (default: ["optionalDependencies","dependencies","devDependencies","peerDependencies","generatorDependencies"]) - -v, --verbose Display debug logs - -a, --alignPeerDependencies Enforce to align the version of the dependencies with the latest range - -h, --help display help for command -``` - -### Otter options - -The `version-harmonize` command is reading the content of the package.json file to get the following options: - -* ignore: List of dependency name patterns to ignore when updating the dependencies range. -* skipPackage: Determine if the current package should be ignored when updating the dependencies range. - -The configuration can be provided in the package.json file as following: - -```json -{ - "name": "@o3r/my-package", - "otter": { - "versionHarmonize": { - "ignore": [ - "typescript", - "webpack" - ] - } - } -} -``` - -## Generate Package Exports - -Edit the generated package.json file to add the exports of the packages based on defined sub-entries. -The sub-entries should be specified as JSON files (`package.json` per default) in the folder to expose. - -### Usage - -```shell -Usage: generate-package-exports [options] - -Update package.json exports - -Options: - --cwd Path to the root of the project (default: "") - -o, --outDir Path to folder containing the package.json to edit (default: "./dist") - -s, --srcDir Path to source folder containing the source code (default: "./src") - -p, --pattern Pattern of the JSON filenames to read to determine sub entries (default: "package.json") - --export-types <...types> Add additional supported export types (default: ["typings","types","node","module","es2015","es2020","esm2015","esm2020","esm","default","require","import"]) - -v, --verbose Display debug logs - -h, --help display help for command -``` +Further information on this tool can be found in the `@o3r/azure-tools` package [documentation](https://github.com/AmadeusITGroup/otter/blob/main/packages/%40o3r/azure-tools/README.md). diff --git a/packages/@o3r/artifactory-tools/README.md b/packages/@o3r/artifactory-tools/README.md index 19ac0ef0e5..278fe0ff10 100644 --- a/packages/@o3r/artifactory-tools/README.md +++ b/packages/@o3r/artifactory-tools/README.md @@ -5,120 +5,80 @@ ## Description -Various artifactory tools +This module provides various tools for the supported repository manager [JFrog](https://jfrog.com/artifactory/), including deleting old artifacts and deleting a specified amount of PR build artifacts. -## Artifact Retriever +## Artifact cleaner (JFrog) -### Usage - -```shell -Usage: o3r-artifact-retriever [options] - -Gets an artifact from Artifactory: artifact-group, artifact-name and artifact-version are mandatory - -Options: - - --registry Registry URL. It is ignored for Azure Artifacts. - --repository-manager Artifact repository manager. Supported managers are JFrog, Azure Artifacts (default: JFrog) - --organization Azure Artifacts organization - --project Azure Artifacts project - --feed Azure Artifacts feed - -a, --artifact-name Artifact name - -v, --artifact-version Artifact version - -g, --artifact-group Artifact group name - -r, --artifact-repos Artifact repositories (default: ) - -u, --username Artifactory username (default from ARTIFACTORY_USERNAME) - -p, --password Artifactory user password (default from ARTIFACTORY_PASSWORD) - -o, --out Output file name (default: ./built/${name}.jar) - --use-package-version Use the package version as artifact version - -h, --help output usage information -``` +Cleans old artifacts from JFrog artifactory repositories. Based on the provided (or default) milliseconds value, this script will delete artifacts created before the specified date +and artifacts that have not been downloaded since the specified date. The search for the artifacts to delete is limited to the specified repositories. -> **Info**: *password* and *username* options can be provided via Environment variables. +Further information can be found in the JFrog documentation [here](https://jfrog.com/help/r/jfrog-rest-apis/artifacts-not-downloaded-since). -### JFrog Example +### Usage ```shell -artifact-retriever.js --registry "https://jfrog.io/repoName" -u -p -v "1.0.0" -g "io.swagger" -a "typescriptFetch-swagger-codegen" --out /path/to/typescriptFetch-swagger-codegen.jar +o3r-artifact-cleaner [options] ``` -### Azure Artifacts Example +The required options include: +* the JFrog artifactory URL (`--artifactory-url`) +* the authentication (`--basicAuth `) +* the artifact repositories to clean up (`--repositories `) -```shell -artifact-retriever.js --repository-manager "Azure Artifacts" --organization "AmadeusDigitalAirline" --project "Otter" --feed "otter" -u -p -v "1.0.0" -g "io.swagger" -a "typescriptFetch-swagger-codegen" --out /path/to/typescriptFetch-swagger-codegen.jar -``` +### Options -## Dependency Validator +| Option | Alias | Value Type | Default Value | Description | +|----------------------------------------------------------|:-----:|----------------------|---------------------------------|------------------------------------------------------------------------------------------------| +| `--artifactory-url `
**(Required)** | | `string` | | JFrog artifactory URL | +| `--basicAuth `
**(Required)** | `-b` | `string` | | Base64 encoding of username:password (password already encrypted from artifactory UI) | +| `--repositories `
**(Required)** | `-r` | `string` | | Artifact repositories to clean up (comma separated)
ex: `'repo1,repo2'` | +| `--dry-run ` | | `boolean` | `false` | List all files that would be deleted without actually deleting them | +| `--duration-kept ` | `-d` | `number` or `string` | `604800000`
(i.e., 7 days) | All artifacts older than this value (in ms) or not downloaded since this value will be deleted | +| `--type-filter ` | `-t` | `string` | `'tgz,json'` | List of artifact types that should be deleted (comma separated)
ex: `'jar,tgz'` | +| `--help` | `-h` | | | Output usage information | -Validator of dependencies. -The purpose is to check if the update is not breaking the application using the current package. - -### Usage +### Example ```shell -Usage: dep-validator [options] - -Execute dependency tests - - -Options: - - -u, --username Git username - -p, --password Git user password - -w, --workspace Path to the folder containing the repository to clone - --workspace-project Path to the project root folder inside the workspace - -b, --build Tasks to run to build the dependency - -t, --test Tasks to run to test the dependency - -P, --project Project root path - --include Packages to include in the process - --exclude Packages to exclude from the process - --no-lerna Disable lerna file parsing - --no-trustedVersionNumber Determine if the version number is trusted (used for version number generated in Pull Request) +yarn o3r-artifact-cleaner --artifactory-url "https://jfrog.io/repoName" -b thisismybase64tokenwithuserandencryptedpassword ``` -## Artifact cleaner +## PR Artifact cleaner (JFrog) -Cleans old artifacts from artifactory repositories +Cleans old PR artifacts from JFrog by using the build version that is present in the path. If the build version is not present in the path, this tool cannot be used. +Based on the provided (or default) days value, this script will delete artifacts created before the specified date, within the provided repository matching the provided path. + +Further information can be found in the JFrog documentation [here](https://jfrog.com/help/r/jfrog-rest-apis/artifactory-query-language-aql). ### Usage ```shell -Usage: o3r-artifact-cleaner -b [options] - -Cleans old artifacts from artifactory repositories, base 64 encoding of username:password is mandatory - -Options: - - --artifactory-url Artifact URL (Required) - -a, --duration-kept All the artifact which have been created since more time than this value(ms) will be deleted (Default to 10080000ms (i.e. 7 days)) - -r, --repositories Artifact repositories to clean up (coma separated) ex : npm-otter-pr,npm-o3r-pr (Default to npm-otter-pr) - -t, --type-filter List of artifact type that should be deleted coma separated (ex: jar,tgz) (Default : tgz) - --dry-run List all files that should be deleted without actually deleting them. (Default to false) - -h, --help Output usage information - -Example : yarn o3r-artifact-cleaner -b thisismybase64tokenwithuserandencryptedpassword +o3r-pr-artifact-cleaner [options] ``` -## PR Artifact cleaner - -Cleans old PR artifacts by identifying using the the build version that is present in the path. If build version is not present in the path this tool cannot yet be used. - -### Usage +The required options include: +* the JFrog artifactory URL (`--artifactory-url `) +* a method of authentication (`--api-key ` or `--basicAuth `) +* the artifact repository to clean up (`--repository `) +* the artifact paths to clean up (`--path `) + +### Options + +| Option | Alias | Value Type | Default Value | Description | +|----------------------------------------------------------|:-----:|------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------| +| `--artifactory-url `
**(Required)** | `-u` | `string` | | JFrog artifactory URL | +| `--path `
**(Required)** | `-p` | `string` | | Artifact paths to clean up (using matcher from AQL language). Be careful not to include release artifacts in the path.
ex: `sample/path/*-PR-*` | +| `--repository `
**(Required)** | `-r` | `string` | | Artifact repository to clean up | +| `--api-key ` | `-a` | `string` | | Artifactory API key of the user that can be used to log in | +| `--basicAuth ` | `-b` | `string` | | Base64 encoding of username:password (password already encrypted from artifactory UI) | +| `--dry-run ` | | `boolean` | `false` | List all files that would be deleted without actually deleting them | +| `--duration-kept ` | `-d` | `number` | `1` | All artifacts older than this value (in days) will be deleted | +| `--pr-builds ` | `-n` | `number` | `1` | Number of PR build artifacts that will be kept | +| `--verbose` | `-v` | | | Display the executed AQL query | +| `--help` | `-h` | | | Output usage information | + +### Example ```shell -Usage: o3r-pr-artifact-cleaner -b [options] - -Cleans old artifacts from artifactory repositories, base 64 encoding of username:password is mandatory - -Options: - - -u, --artifactory-url Artifact URL - -d, --duration-kept Only artifacts which are older than this value (in days) will be deleted. (Default to 1 day) - -r, --repository Artifact repository to clean up. (Default to dga-maven-built-adt-nce) - -pr, --pr-versions Number of pr versions that will be kept. (Default to 2 last versions) - --dry-run List all files that should be deleted without actually deleting them. (Default to false) - -p, --path Artifact paths to cleanup use matcher from AQL language. Be careful that the path do not include release artifacts (Default to com/amadeus/retailing/*-PR-*) - -h, --help Output usage information - -Example : yarn o3r-pr-artifact-cleaner -b thisismybase64tokenwithuserandencryptedpassword +yarn o3r-pr-artifact-cleaner --artifactory-url "https://jfrog.io/repoName" -b thisismybase64tokenwithuserandencryptedpassword ``` diff --git a/packages/@o3r/artifactory-tools/package.json b/packages/@o3r/artifactory-tools/package.json index 0f176d56a4..1a1d90e23f 100644 --- a/packages/@o3r/artifactory-tools/package.json +++ b/packages/@o3r/artifactory-tools/package.json @@ -17,14 +17,12 @@ }, "bin": { "o3r-artifact-cleaner": "./dist/src/cli/artifact-cleaner.js", - "o3r-artifact-retriever": "./dist/src/cli/artifact-retriever.js", "o3r-pr-artifact-cleaner": "./dist/src/cli/pr-artifact-cleaner.js" }, "dependencies": { "commander": "^11.0.0", "fs-extra": "^11.0.0", "request": "^2.88.0", - "request-promise-native": "^1.0.9", "tslib": "^2.5.3", "winston": "^3.8.2" }, diff --git a/packages/@o3r/artifactory-tools/src/cli/artifact-cleaner.ts b/packages/@o3r/artifactory-tools/src/cli/artifact-cleaner.ts index ec813ac2d6..d5fdf30074 100644 --- a/packages/@o3r/artifactory-tools/src/cli/artifact-cleaner.ts +++ b/packages/@o3r/artifactory-tools/src/cli/artifact-cleaner.ts @@ -4,22 +4,25 @@ import { program } from 'commander'; import {Headers} from 'request'; -import * as request from 'request-promise-native'; import * as winston from 'winston'; program .description('Clean old artifacts from artifactory repositories') .requiredOption('--artifactory-url ', 'Artifactory URL') - .option('-d, --duration-kept ', 'All artifacts which have not been downloaded and are older than this value(ms) will be deleted. Default to 10080000ms (7 days)', '604800000') - .option( + .requiredOption('-b, --basicAuth ', 'Base64 encoding of username:password (password already encrypted from artifactory UI)') + .requiredOption( '-r, --repositories ', - 'Artifact repositories to clean up (coma separated) ex : npm-otter-pr,npm-o3r-pr (Default to npm-otter-pr)', - (repos: string) => repos.split(','), - ['npm-otter-pr'] + 'Artifact repositories to clean up (comma separated) ex: \'repo1,repo2\'', + (repos: string) => repos.split(',') + ) + .option('-d, --duration-kept ', 'All artifacts older than this value (in ms) will be deleted. (Default: 604800000 ms, i.e., 7 days)', (v) => +v, 604800000) + .option( + '-t, --type-filter ', + 'List of artifact types that should be deleted (comma separated) ex: \'jar,tgz\' (Default: [\'tgz\',\'json\'])', + (type: string) => type.split(','), + ['tgz','json'] ) - .option('-t, --type-filter ', 'List of artifact type that should be deleted coma separated (ex: jar,tgz) (Default : tgz)', (type: string) => type.split(','), ['tgz,json']) - .option('--dry-run ', 'List all files that should be deleted without actually deleting them', false) - .option('-b, --basicAuth ', 'Base 64 encoding of username:password (password already encrypted from artifactory UI)') + .option('--dry-run ', 'List all files that would be deleted without actually deleting them', false) .parse(process.argv); const opts = program.opts(); @@ -45,10 +48,6 @@ const matchFilter = (fullUrl: string, types: string[]) => { return false; }; -if (!opts.basicAuth) { - logger.error('Authentication is mandatory, please specify a base 64 encoded user:password'); - process.exit(1); -} let url = opts.artifactoryUrl as string; const options: { headers: Headers } = { headers: { @@ -64,11 +63,15 @@ logger.info(`Url called : ${url}`); void (async () => { logger.info(`Requesting old artifacts using ${url}`); - const responseSearch = await request.get(url, options).promise().catch((e) => { + let responseSearch; + let responseSearchObj: { results: { uri: string }[] }; + try { + responseSearch = await fetch(url, options); + responseSearchObj = await responseSearch.json(); + } catch (e) { logger.warn('No result found ', e); process.exit(0); - }); - const responseSearchObj: { results: { uri: string }[] } = JSON.parse(responseSearch); + } /** uris will contain the list of all artifacts that need to be deleted */ const uris = responseSearchObj.results .map((res) => res.uri) @@ -77,7 +80,7 @@ void (async () => { for (const uri of uris) { logger.info(`Deleting ${uri}...`); if (!opts.dryRun) { - const response = await request.delete(uri, options).promise(); + const response = await fetch(uri, {...options, method: 'DELETE'}); logger.info(response); } } diff --git a/packages/@o3r/artifactory-tools/src/cli/artifact-retriever.ts b/packages/@o3r/artifactory-tools/src/cli/artifact-retriever.ts deleted file mode 100644 index ce766324b0..0000000000 --- a/packages/@o3r/artifactory-tools/src/cli/artifact-retriever.ts +++ /dev/null @@ -1,245 +0,0 @@ -#!/usr/bin/env node -import { program } from 'commander'; -import * as fse from 'fs-extra'; -import * as http from 'node:http'; -import * as https from 'node:https'; -import * as fs from 'node:fs'; -import * as path from 'node:path'; -import { CookieJar, Headers } from 'request'; -import * as request from 'request-promise-native'; -import * as winston from 'winston'; -import { GavcResponse } from '../helpers/gavc-response'; - -const SUPPORTED_REPOSITORY_MANAGERS = ['JFrog', 'Azure Artifacts']; - -program - .description('Get an artifact from an Artifact repository manager') - .requiredOption('--registry ', 'Registry URL. It is ignored for Azure Artifacts.') - .option('--repository-manager ', `Artifact repository manager. Supported managers are ${SUPPORTED_REPOSITORY_MANAGERS.join(', ')}`, 'JFrog') - .option('--organization ', 'Azure Artifacts organization', undefined) - .option('--project ', 'Azure Artifacts project', undefined) - .option('--feed ', 'Azure Artifacts feed', undefined) - .option('-a, --artifact-name ', 'Artifact name', undefined) - .option('-v, --artifact-version ', 'Artifact version', undefined) - .option('-g, --artifact-group-id ', 'Artifact group id', undefined) - .option('-r, --artifact-repos ', 'Artifact repositories', (repos: string) => repos.split(','), []) - .option('-u, --username ', 'Artifactory username', 'mvn-readonly') - .option('-p, --password ', 'Artifactory user password') - .option('--password_env_var ', 'Artifactory user password environment var', undefined) - .option('-o, --out ', 'Output file name (default: ./built/${name}.jar)', undefined) - .option('--silent', 'Prevent exit code 1 if artifact not found (Usage example : dep-checker with post-install scripts', false) - .option('--verbose', 'Display debug log message') - .option('--use-package-version', 'Use the package version as artifact version') - .parse(process.argv); - -const opts = program.opts(); - -const logger = winston.createLogger({ - level: opts.verbose ? 'debug' : 'info', - format: winston.format.simple(), - transports: new winston.transports.Console() -}); - -if (!SUPPORTED_REPOSITORY_MANAGERS.includes(opts.repositoryManager)) { - logger.error(`Unsupported repository manager: ${opts.repositoryManager as string}`); - process.exit(10); -} - -if (opts.repositoryManager === 'Azure Artifacts') { - if (!opts.organization) { - logger.error('No organization specified for Azure Artifacts'); - process.exit(11); - } - if (!opts.project) { - logger.error('No project specified for Azure Artifacts'); - process.exit(12); - } - if (!opts.feed) { - logger.error('No feed specified for Azure Artifacts'); - process.exit(13); - } -} - -if (!opts.artifactVersion && !opts.usePackageVersion) { - logger.error('No version specified'); - process.exit(14); -} - -if (!opts.artifactName) { - logger.error('No artifact name specified'); - process.exit(15); -} - -if (!opts.artifactGroupId) { - logger.error('No group id specified'); - process.exit(16); -} - -let url: string = opts.registry; -const jar = request.jar(); -const options: {headers?: Headers; jar: CookieJar} = {jar}; -const password = opts.password || opts.password_env_var && process.env[opts.password_env_var]; -if (opts.username && password) { - options.headers = { - // eslint-disable-next-line @typescript-eslint/naming-convention - Authorization: 'Basic ' + Buffer.from(`${opts.username as string}:${password as string}`).toString('base64') - }; -} -const name: string = opts.artifactName; -const artifactGroupId: string = opts.artifactGroupId; - -let version: string = opts.usePackageVersion ? require(path.resolve(process.cwd(), 'package.json')).version : opts.artifactVersion; - -const filePath = opts.out || `./built/${name}.jar`; -fse.ensureDirSync(path.resolve(process.cwd(), path.dirname(filePath))); - -/** - * Report error on artifact downloading - * @param e - */ -const reportError = (e: Error) => { - logger.error(e); -}; - - -/** - * Download the specified Artifact on JFrog - */ -async function retrieveArtifactFromJFrog() { - logger.info(`Searching for ${name}@${version}`); - - url += (url.endsWith('/') ? '' : '/') + `api/search/gavc?a=${name}&g=${artifactGroupId}`; - - if (opts.artifactRepos && opts.artifactRepos.length) { - url += `&repos=${(opts.artifactRepos as string[]).join(',')}`; - } - - if (version.startsWith('0.0.0')) { - url += `&v=${version}`; - } - let responseSearch; - try { - responseSearch = await request.get(url, options).promise(); - } catch (err) { - logger.warn('First call to get artifact information failed, retries'); - responseSearch = await request.get(url, options).promise(); - } - // eslint-disable-next-line no-console - console.log(responseSearch); - const responseSearchObj: {results: {uri: string}[]} = JSON.parse(responseSearch); - const uris = responseSearchObj.results - .map((res) => res.uri) - .filter((uri) => uri.endsWith('.jar')) - .sort((a, b) => { - if (a > b) { - return -1; - } else if (a < b) { - return 1; - } - return 0; - }); - - if (uris.length) { - const artifactUrl = uris[0]; - logger.info(`Call to ${artifactUrl}`); - const gavcResponse: string = await request.get(artifactUrl, options).promise(); - logger.debug(`GavcResponse : ${gavcResponse}`); - const parsedGavcResponse: GavcResponse = JSON.parse(gavcResponse); - const downloadUri: string = parsedGavcResponse.downloadUri; - logger.info(`Downloading artifact ${downloadUri}`); - - const file = fs.createWriteStream(path.resolve(process.cwd(), filePath)); - file.on('error', reportError); - - const cookieString = jar.getCookieString(url); - const httpOptions = {headers: options.headers ? {...options.headers, cookie: cookieString} : undefined}; - - if (artifactUrl.startsWith('https')) { - https.get(downloadUri, httpOptions, (response) => response.pipe(file)).on('error', reportError); - } else { - http.get(downloadUri, httpOptions, (response) => response.pipe(file)).on('error', reportError); - } - } else { - if (opts.silent) { - logger.warn('No artifact found, silent mode ON, prevented the failure'); - } else { - logger.error('No artifact found'); - process.exit(2); - } - } -} - -/** - * Get the latest version, if it exists, of the specified artifact - * @param packages Array of packages from the Azure feed - * @param artifactName Name of the artifact to get the latest version from - */ -function getLatestVersion(packages: Record[], artifactName: string) { - const normalizedArtifactName = `${artifactGroupId}:${artifactName}`.toLowerCase(); - for (const pckg of packages) { - if (pckg.normalizedName === artifactName || pckg.normalizedName === normalizedArtifactName) { - const latestVersion = (pckg.versions as {isLatest: boolean; version: string}[]).find(v => v.isLatest); - if (!latestVersion) { - throw new Error(`No latest version found for ${artifactName}`); - } - return latestVersion.version; - } - } - throw new Error(`No package ${artifactName} found`); -} - -/** - * Download the specified Artifact on Azure - */ -async function retrieveArtifactFromAzure() { - try { - if (version.startsWith('0.0.0')) { - // eslint-disable-next-line max-len - const res = await request.get(`https://feeds.dev.azure.com/${opts.organization as string}/${opts.project as string}/_apis/packaging/feeds/${opts.feed as string}/packages?api-version=6.0-preview.1`, options).promise(); - version = getLatestVersion(JSON.parse(res).value, name); - } - logger.info(`Searching for ${name}@${version}`); - - // eslint-disable-next-line max-len - url = `https://pkgs.dev.azure.com/${opts.organization as string}/${opts.project as string}/_apis/packaging/feeds/${opts.feed as string}/maven/${opts.artifactGroupId as string}/${name}/${version}/${name}-${version}.jar/content?api-version=6.0-preview.1`; - - logger.info(`Call to ${url}`); - const downloadUri = await new Promise((resolve, reject) => { - https.get(url, options, (response) => resolve(response.headers.location)).on('error', reject); - }); - if (!downloadUri) { - throw new Error(`Could not get the download url for the artifact ${name}`); - } - logger.info(`Downloading artifact ${downloadUri}`); - - const file = fs.createWriteStream(path.resolve(process.cwd(), filePath)); - file.on('error', reportError); - - if (!downloadUri.startsWith('https')) { - throw new Error('Artifact must be downloaded using https protocol'); - } - https.get(downloadUri, (response) => response.pipe(file)).on('error', reportError); - } catch (e) { - if (opts.silent) { - logger.warn(e); - } else { - logger.error(e); - process.exit(3); - } - } -} -/** - * Download an artifact on the specified artifact repository manager - * @param registry Registry url - */ -function retrieveArtifact() { - switch (opts.repositoryManager) { - case 'JFrog': { - return retrieveArtifactFromJFrog(); - } - case 'Azure Artifacts': { - return retrieveArtifactFromAzure(); - } - } -} -void retrieveArtifact(); diff --git a/packages/@o3r/artifactory-tools/src/cli/pr-artifact-cleaner.ts b/packages/@o3r/artifactory-tools/src/cli/pr-artifact-cleaner.ts index 6e5a133f3a..1e538fae17 100644 --- a/packages/@o3r/artifactory-tools/src/cli/pr-artifact-cleaner.ts +++ b/packages/@o3r/artifactory-tools/src/cli/pr-artifact-cleaner.ts @@ -2,20 +2,19 @@ import { program } from 'commander'; import { Headers, Options } from 'request'; -import * as request from 'request-promise-native'; import * as winston from 'winston'; program .description('Clean pr artifacts from artifactory repositories') .requiredOption('-u, --artifactory-url ', 'Artifactory URL') - .option('-d, --duration-kept ', 'Only artifacts which are older and have not been downloaded during this duration (in days) will be deleted.', (v) => +v, 1) - .option('-pr, --pr-versions ', 'Number of pr versions that will be kept.', (v) => +v, 1) - .option('-r, --repository ', 'Artifact repository to clean up.', 'dga-maven-built-adt-nce') - .option('-p, --path ', 'Artifact paths to cleanup use matcher from AQL language. Be careful that the path do not include release artifacts.', 'com/amadeus/retailing/*-PR-*') - .option('--dry-run', 'List all files that should be deleted without actually deleting them') + .requiredOption('-r, --repository ', 'Artifact repository to clean up.') + .requiredOption('-p, --path ', 'Artifact paths to clean up (using matcher from AQL language). Be careful not to include release artifacts in the path.') + .option('-d, --duration-kept ', 'All artifacts older than this value (in days) will be deleted.', (v) => +v, 1) + .option('-n, --pr-builds ', 'Number of PR build artifacts that will be kept.', (v) => +v, 1) + .option('--dry-run', 'List all files that would be deleted without actually deleting them', false) .option('-a, --api-key ', 'Artifactory API Key of the user that can be used to log in') - .option('-b, --basicAuth ', 'Base 64 encoding of username:password (password already encrypted from artifactory UI)') - .option('-v, --verbose', 'To display the executed AQL query') + .option('-b, --basicAuth ', 'Base64 encoding of username:password (password already encrypted from artifactory UI)') + .option('-v, --verbose', 'Display the executed AQL query') .parse(process.argv); const programOptions = program.opts(); @@ -44,7 +43,7 @@ const logger = winston.createLogger({ }); if (!programOptions.basicAuth && !programOptions.apiKey) { - logger.error('Authentication is mandatory, please specify a base 64 encoded user:password with b parameter or an ApiKey with -a parameter'); + logger.error('Authentication is mandatory, please specify a base64 encoded user:password with the -b parameter or an ApiKey with the -a parameter'); process.exit(1); } if (programOptions.basicAuth && programOptions.apiKey) { @@ -56,7 +55,7 @@ const authHeader: Headers = programOptions.basicAuth ? { Authorization: `Basic $ let url: string = programOptions.artifactoryUrl; url += (url.endsWith('/') ? '' : '/') + 'api/search/aql'; const ageInDays: number = programOptions.durationKept; -const prVersions: number = programOptions.prVersions; +const prBuilds: number = programOptions.prBuilds; const repository: string = programOptions.repository; const path: string = programOptions.path; const options: Options = { @@ -79,59 +78,27 @@ const options: Options = { logger.debug(`AQL search executed : ${options.body}`); logger.info(`Url called : ${url}`); -/** - * @param result - * @param result.repo - * @param result.path - * @param result.name - * @param pathWithoutBuildNumber - */ -function getUniqueId(result: { repo: string; path: string; name: string }, pathWithoutBuildNumber: string) { - let mapId: string; - if (result.path.indexOf('.') !== -1 && result.path.indexOf('-') !== -1) { - const splittedPath = result.path.split('/'); - const partialPathWithBuildNumber = splittedPath.find((currentPath) => currentPath.indexOf('.') !== -1); - if (partialPathWithBuildNumber) { - const splittedPartialPathWithDot = partialPathWithBuildNumber.split('.'); - splittedPartialPathWithDot.splice(splittedPartialPathWithDot.length - 1, 1); - mapId = splittedPartialPathWithDot.join('.'); - } else { - const splittedName = result.name.split('.'); - const splittedNameDash = splittedName[splittedName.length - 2].split('-'); - splittedNameDash.splice(0, 1); - splittedName[splittedName.length - 2] = splittedNameDash.join('-'); - const nameWithoutBuildNumber = splittedName.join('.'); - mapId = pathWithoutBuildNumber + '/' + nameWithoutBuildNumber; - } - } else { - mapId = pathWithoutBuildNumber + result.name; - } - return mapId; -} - // eslint-disable-next-line @typescript-eslint/no-floating-promises (async () => { logger.info(`Requesting old artifacts using ${url}`); let responseSearch: any; + let responseSearchObj: { results: { repo: string; path: string; name: string }[] }; try { - responseSearch = await request.post(url, options).promise(); - } catch { - logger.warn('No result found %o', responseSearch); + responseSearch = await fetch(url, {method: 'POST', headers: authHeader, body: options.body}); + responseSearchObj = await responseSearch.json(); + } catch (e) { + logger.warn('No result found %o', e); process.exit(0); } - const responseSearchObj: { results: { repo: string; path: string; name: string }[] } = JSON.parse(responseSearch); /** uris will contain the list of all artifacts that need to be deleted */ const mapOfKeptItems = new Map(); const mapOfKeptResult = new Map(); const resultToDelete: { repo: string; path: string; name: string }[] = []; const sortedResult = responseSearchObj.results.sort((a, b) => b.name.localeCompare(a.name)); for (const result of sortedResult) { - const splittedPathBySlash = result.path.split('/'); - const folderwithBuildNumber = splittedPathBySlash.filter((value) => value.indexOf('.') !== -1)[0]; - const splittedPath = folderwithBuildNumber.split('.'); - const pathWithoutBuildNumber = splittedPath.slice(0, -1).join('.'); - const currentBuildNumber = +splittedPath[splittedPath.length - 1]; - const mapId: string = getUniqueId(result, pathWithoutBuildNumber); + const splitPath = result.name.split('.'); + const mapId = splitPath.slice(0, -2).join('.'); + const currentBuildNumber = +splitPath[splitPath.length - 2]; const buildNumbers = mapOfKeptItems.get(mapId); if (!buildNumbers) { mapOfKeptItems.set(mapId, [currentBuildNumber]); @@ -144,10 +111,10 @@ function getUniqueId(result: { repo: string; path: string; name: string }, pathW isBuildNumberHigherThanExisting = isBuildNumberHigherThanExisting && (value < currentBuildNumber); }); if (!isBuildNumberAlreadyInMap) { - if (buildNumbers.length >= prVersions && !isBuildNumberHigherThanExisting) { + if (buildNumbers.length >= prBuilds && !isBuildNumberHigherThanExisting) { resultToDelete.push(result); } else { - if (buildNumbers.length >= prVersions && isBuildNumberHigherThanExisting) { + if (buildNumbers.length >= prBuilds && isBuildNumberHigherThanExisting) { const buildNumberToRemove = buildNumbers.shift(); if (buildNumberToRemove) { const resultsToRemove = mapOfKeptResult.get(`${mapId}${buildNumberToRemove}`); @@ -172,11 +139,13 @@ function getUniqueId(result: { repo: string; path: string; name: string }, pathW } logger.debug('Map of build that will be kept: %o', mapOfKeptItems); - const filesToDelete = resultToDelete.map((data) => (programOptions.artifactoryUrl as string) + repository + '/' + data.path + '/' + data.name); + const filesToDelete = resultToDelete.map( + (data) => (programOptions.artifactoryUrl as string) + (programOptions.artifactoryUrl.endsWith('/') ? '' : '/') + repository + '/' + data.path + '/' + data.name + ); for (const uri of filesToDelete) { logger.info(`Deleting ${uri}...`); if (!programOptions.dryRun) { - const response = await request.delete(uri, options).promise(); + const response = await fetch(uri, {method: 'DELETE', headers: authHeader}); logger.info(response); } } diff --git a/packages/@o3r/artifactory-tools/src/helpers/gavc-response/gavc-response.interfaces.ts b/packages/@o3r/artifactory-tools/src/helpers/gavc-response/gavc-response.interfaces.ts deleted file mode 100644 index 7bb9198959..0000000000 --- a/packages/@o3r/artifactory-tools/src/helpers/gavc-response/gavc-response.interfaces.ts +++ /dev/null @@ -1,52 +0,0 @@ -export interface GavcResponse { - /** Ex : mvn-built */ - repo: string; - - /** Ex : /io/swagger/typescriptFetch-swagger-codegen/1.0.0/typescriptFetch-swagger-codegen-1.0.0.jar */ - path: string; - - /** Ex : 2018-07-06T17:14:14.123Z */ - created: string; - - /** Ex : anonymous */ - createdBy: string; - - /** Ex : 2018-07-06T17:14:14.031Z */ - lastModified: string; - - /** Ex : anonymous */ - modifiedBy: string; - - /** Ex : 2018-07-06T17:14:14.031Z */ - lastUpdated: string; - - /** Ex : https://jfrog.io/mvn-built/io/swagger/typescriptFetch-swagger-codegen/1.0.0/typescriptFetch-swagger-codegen-1.0.0.jar */ - downloadUri: string; - - /** Ex : application/java-archive */ - mimeType: string; - - /** Ex : 41800 */ - size: 41800; - - /** - * Ex : - * checksums : { - * sha1 : 4b12dc32f3ef1a10a872297423908b52bc1ce6ea; - * md5 : db17bea6a5f737f0644482055dcf2d36 - * }; - */ - checksums: { sha1: string; md5: string }; - - /** - * Ex : - * originalChecksums : { - * sha1 : b12dc32f3ef1a10a872297423908b52bc1ce6ea; - * md5 : db17bea6a5f737f0644482055dcf2d36 - * }; - */ - originalChecksums: { sha1: string; md5: string }; - - /** Ex : https://jfrog.io/api/storage/mvn-built/io/swagger/typescriptFetch-swagger-codegen/1.0.0/typescriptFetch-swagger-codegen-1.0.0.jar */ - uri: string; -} diff --git a/packages/@o3r/artifactory-tools/src/helpers/gavc-response/index.ts b/packages/@o3r/artifactory-tools/src/helpers/gavc-response/index.ts deleted file mode 100644 index 6bf83fa25e..0000000000 --- a/packages/@o3r/artifactory-tools/src/helpers/gavc-response/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './gavc-response.interfaces'; diff --git a/packages/@o3r/artifactory-tools/src/helpers/index.ts b/packages/@o3r/artifactory-tools/src/helpers/index.ts deleted file mode 100644 index fb75c8397e..0000000000 --- a/packages/@o3r/artifactory-tools/src/helpers/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './gavc-response'; diff --git a/packages/@o3r/artifactory-tools/src/public_api.ts b/packages/@o3r/artifactory-tools/src/public_api.ts deleted file mode 100644 index 82d4b232f5..0000000000 --- a/packages/@o3r/artifactory-tools/src/public_api.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './helpers/index'; diff --git a/packages/@o3r/azure-tools/README.md b/packages/@o3r/azure-tools/README.md index acf5bd1e0a..db8930c201 100644 --- a/packages/@o3r/azure-tools/README.md +++ b/packages/@o3r/azure-tools/README.md @@ -5,30 +5,54 @@ ## Description -Various Azure DevOps tools +This module provides a tool to comment a Pull Request on Azure. ## Comment Azure PR ### Usage ```shell -yarn dlx -p @o3r/azure-tools o3r-comment-pr "[Deployed app]($(url))" -s Closed -I app-link -m Replace -T $(System.AccessToken) +yarn dlx -p @o3r/azure-tools o3r-comment-pr [options] ``` - or - ```shell -npx -p @o3r/azure-tools o3r-comment-pr "[Deployed app]($(url))" -s Closed -I app-link -m Replace -T $(System.AccessToken) +npx -p @o3r/azure-tools o3r-comment-pr [options] ``` -### Options available +The required options include: +* the access token (`--accessToken `) +* the thread identifier (`--threadIdentifier `) + +Additionally, the following environment variables must be provided: +* `SYSTEM_TEAMPROJECT` (`System.TeamProject` in Azure) +* `BUILD_REPOSITORY_NAME` (`Build.Repository.Name` in Azure) +* `SYSTEM_PULLREQUEST_PULLREQUESTID` (must be a number) (`System.PullRequest.PullRequestId` in Azure) +* `SYSTEM_TEAMFOUNDATIONCOLLECTIONURI` (`System.TeamFoundationCollectionUri` in Azure) + +#### Arguments + +| Argument | Description | +|-----------|------------------------------------| +| `comment` | Comment to publish on the Azure PR | + +#### Options + +| Option | Alias | Value Type | Value Options | Default Value | Description | +|---------------------------------------------------|:-----:|------------|--------------------------------------------------------------------------------------------------|---------------|------------------------------------------------------------------------------------------------------------------| +| `--accessToken `
**(Required)** | `-T` | `string` | | | Access token (Required) | +| `--commentStatus ` | `-s` | `string` | `Unknown`
`Active`
`Fixed`
`WontFix`
`Closed`
`ByDesign`
`Pending` | `Closed` | Comment status | +| `--mode ` | `-m` | `string` | `Replace`
`Add`
`Skip` | `Add` | Replaces thread if existing
Adds a comment to the existing thread
Do anything if thread already exists | +| `--threadIdentifier ` | `-I` | `string` | | | Thread identifier | +| `--help` | `-h` | | | | Output usage information | + +### Example ```shell -yarn dlx -p @o3r/azure-tools o3r-comment-pr --help +yarn dlx -p @o3r/azure-tools o3r-comment-pr "[Deployed app]($(url))" -s Closed -I app-link -m Replace -T $(System.AccessToken) ``` or ```shell -npx -p @o3r/azure-tools o3r-comment-pr --help +npx -p @o3r/azure-tools o3r-comment-pr "[Deployed app]($(url))" -s Closed -I app-link -m Replace -T $(System.AccessToken) ``` diff --git a/packages/@o3r/azure-tools/src/cli/comment-pr.ts b/packages/@o3r/azure-tools/src/cli/comment-pr.ts index 9a5d02b77a..23eaeda4f4 100644 --- a/packages/@o3r/azure-tools/src/cli/comment-pr.ts +++ b/packages/@o3r/azure-tools/src/cli/comment-pr.ts @@ -62,7 +62,7 @@ void (async () => { const orgUrl = process.env.SYSTEM_TEAMFOUNDATIONCOLLECTIONURI; const prService = new PullRequestService(opts.accessToken, project, orgUrl, logger); - const threads = await prService.findThreadsByIdentifier(repositoryId, pullRequestId, opts.threadIdentifier); + const threads = opts.threadIdentifier ? await prService.findThreadsByIdentifier(repositoryId, pullRequestId, opts.threadIdentifier) : []; if (opts.mode === 'Replace') { await Promise.all(threads.filter(thread => !!thread.id).map((thread) => prService.deleteThread(repositoryId, pullRequestId, thread.id!))); diff --git a/packages/@o3r/dev-tools/README.md b/packages/@o3r/dev-tools/README.md index d2c6f1b85d..9815f8ed79 100644 --- a/packages/@o3r/dev-tools/README.md +++ b/packages/@o3r/dev-tools/README.md @@ -7,6 +7,14 @@ Various CLI scripts to help your CI/CD and your dependency management +> :warning: **Deprecate**: `@o3r/dev-tools` package is deprecated and will no longer be updated as of Otter v12. +> The following CLI scripts have been moved to other otter packages: +> * `artifact-cleaner`: please use `o3r-artifact-cleaner` from the package `@o3r/artifactory-tools` +> * `pr-artifact-cleaner`: please use `o3r-pr-artifact-cleaner` from the package `@o3r/artifactory-tools` +> * `comment-pr`: please use `o3r-comment-pr` from the package `@o3r/azure-tools` +> * `version-harmonize`: replaced by the JSON ESLint rule [@o3r/json-dependency-versions-harmonize](https://github.com/AmadeusITGroup/otter/tree/main/docs/linter/eslint-plugin/rules/json-dependency-versions-harmonize.md) + + ## How to install This package can be used via `npx` command to executed on of the listed command line interfaces. @@ -17,6 +25,8 @@ npx -p @o3r/dev-tools ... ## Artifact Retriever +> :warning: **Deprecate**: This script is deprecated and will be removed in Otter v12. + Gets an artifact from the ArtiFactory ### Usage @@ -58,61 +68,10 @@ artifact-retriever.js --registry "https://jfrog.io/repoName" -u -p

-p -v "1.0.0" -g "io.swagger" -a "typescriptFetch-swagger-codegen" --out /path/to/typescriptFetch-swagger-codegen.jar ``` -## Dependency Validator - -Validator of dependencies. -The purpose is to check if the update is not breaking the application using the current package. - -### Usage - -```shell -Usage: dep-validator [options] - -Execute dependency tests - - -Options: - - -u, --username Git username - -p, --password Git user password - -w, --workspace Path to the folder containing the repository to clone - --workspace-project Path to the project root folder inside the workspace - -b, --build Tasks to run to build the dependency - -t, --test Tasks to run to test the dependency - -P, --project Project root path - --include Packages to include in the process - --exclude Packages to exclude from the process - --no-lerna Disable lerna file parsing - --no-trustedVersionNumber Determine if the version number is trusted (used for version number generated in Pull Request) -``` - -## Changelog Generator - -Updates changelog file with the new changes. -The purpose is to reduce the number of Pull Requests conflicts caused by the edition of the Changelog file. - -### Usage - -```shell -Usage: changelog-generator [options] - -Update ChangeLog with new changes - - -Options: - - -g, --generate-on Versions change when generating a changelog (comma separated) - -u, --username Git username - -p, --password Git user password - -o, --output Path to the changelog file (default: README.md) - --pattern JIRA task pattern (default: '[A-Z]{2,}-[0-9]+') - --config Path to a configuration file - --only-task-done Included only task done in this changelog - -h, --help output usage information -``` - ## Set Version +> :warning: **Deprecate**: This script is deprecated and will be removed in Otter v12. + Replaces the value of the `version` field of the `package.json` matched by the pattern provided to the `--include` options. ### Usage @@ -131,6 +90,8 @@ Options: ## Artifact cleaner +> :warning: **Deprecate**: This script is deprecated, please use `o3r-artifact-cleaner` from the package `@o3r/artifactory-tools` + Cleans old artifacts from artifactory repositories ### Usage @@ -154,6 +115,8 @@ Example : yarn artifact-cleaner -b thisismybase64tokenwithuserandencryptedpasswo ## PR Artifact cleaner +> :warning: **Deprecate**: This is deprecated, please use `o3r-pr-artifact-cleaner` from the package `@o3r/artifactory-tools` + Cleans old PR artifacts by identifying using the the build version that is present in the path. If build version is not present in the path this tool cannot yet be used. ### Usage @@ -178,6 +141,8 @@ Example : yarn pr-artifact-cleaner -b thisismybase64tokenwithuserandencryptedpas ## Peer dependencies updater +> :warning: **Deprecate**: This script is deprecated and will be removed in Otter v12. + Updates a package.json with the given dependencies' versions and their respective peer dependencies. Relies on `npm info` to retrieve package information. @@ -197,33 +162,6 @@ Options: Example : peer-dependencies-updater "@random/package@~2.21.0" "@o3r/core" ``` -## Scripts - -This package provides generic helpers to support the build chain of Otter and Ama sdk packages - -### Scripts available - -* **yarn-check** : Checks if the current NPM client used is [Yarn](https://yarnpkg.com/en/) (`exit(1)` if not the case). -* **version-harmonize** : Harmonizes the version of the dependencies (in a Monorepos) between the main packages and the children packages. - -## Set version - -Replaces the packages' version in a monorepos. -This is used to edit the package.json version of a yarn workspaces' packages. - -### Usage - -```shell -Usage: set-version [options] - -Replace the packages version in a monorepos - -Options: - -p, --placeholder Pattern of the version placeholder (default: "0.0.0") - --include Add files pattern to apply the verison replacement (default: ["*/lerna.json","**/package.json","!**/node_modules/**/{package,lerna}.json"]) - -h, --help display help for command -``` - ## Version Harmonize > :warning: **Deprecate**: This script is deprecated and will be removed in Otter v12, it is replaced by the JSON ESLint rule [@o3r/json-dependency-versions-harmonize](https://github.com/AmadeusITGroup/otter/tree/main/docs/linter/eslint-plugin/rules/json-dependency-versions-harmonize.md). @@ -271,6 +209,8 @@ The configuration can be provided in the package.json file as follows: ## Generate Package Exports +> :warning: **Deprecate**: This script is deprecated and will be removed in Otter v12. + Edits the generated package.json file to add the exports of the packages based on defined sub-entries. The sub-entries should be specified as JSON files (`package.json` per default) in the folder to expose. diff --git a/packages/@o3r/dev-tools/package.json b/packages/@o3r/dev-tools/package.json index 15351cdec1..bb5e9be4de 100644 --- a/packages/@o3r/dev-tools/package.json +++ b/packages/@o3r/dev-tools/package.json @@ -21,7 +21,6 @@ "bin": { "artifact-cleaner": "./dist/src/cli/artifact-cleaner.js", "artifact-retriever": "./dist/src/cli/artifact-retriever.js", - "change-log-sanitizer": "./dist/src/cli/change-log-sanitizer.js", "comment-pr": "./dist/src/cli/comment-pr.js", "generate-package-exports": "./dist/src/cli/generate-package-exports.js", "git-release-management": "./dist/src/cli/git-release-management.js", diff --git a/packages/@o3r/dev-tools/src/cli/artifact-retriever.ts b/packages/@o3r/dev-tools/src/cli/artifact-retriever.ts index 1f2e9acfa8..1509c6ab7c 100644 --- a/packages/@o3r/dev-tools/src/cli/artifact-retriever.ts +++ b/packages/@o3r/dev-tools/src/cli/artifact-retriever.ts @@ -1,9 +1,250 @@ #!/usr/bin/env node -import { spawnSync } from 'node:child_process'; +import { program } from 'commander'; +import * as fse from 'fs-extra'; +import * as http from 'node:http'; +import * as https from 'node:https'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { CookieJar, Headers } from 'request'; +import * as request from 'request-promise-native'; +import * as winston from 'winston'; +import { GavcResponse } from '../helpers/gavc-response'; -console.warn('This CLI is deprecated, please use `o3r-artifact-retriever` from the package `@o3r/artifactory-tools`'); +const SUPPORTED_REPOSITORY_MANAGERS = ['JFrog', 'Azure Artifacts']; -spawnSync('o3r-artifact-retriever', process.argv.slice(2), { - shell: true, - stdio: 'inherit' +program + .description('[DEPRECATED] Get an artifact from an Artifact repository manager') + .requiredOption('--registry ', 'Registry URL. It is ignored for Azure Artifacts.') + .option('--repository-manager ', `Artifact repository manager. Supported managers are ${SUPPORTED_REPOSITORY_MANAGERS.join(', ')}`, 'JFrog') + .option('--organization ', 'Azure Artifacts organization', undefined) + .option('--project ', 'Azure Artifacts project', undefined) + .option('--feed ', 'Azure Artifacts feed', undefined) + .option('-a, --artifact-name ', 'Artifact name', undefined) + .option('-v, --artifact-version ', 'Artifact version', undefined) + .option('-g, --artifact-group-id ', 'Artifact group id', undefined) + .option('-r, --artifact-repos ', 'Artifact repositories', (repos: string) => repos.split(','), []) + .option('-u, --username ', 'Artifactory username', 'mvn-readonly') + .option('-p, --password ', 'Artifactory user password') + .option('--password_env_var ', 'Artifactory user password environment var', undefined) + .option('-o, --out ', 'Output file name (default: ./built/${name}.jar)', undefined) + .option('--silent', 'Prevent exit code 1 if artifact not found (Usage example : dep-checker with post-install scripts', false) + .option('--verbose', 'Display debug log message') + .option('--use-package-version', 'Use the package version as artifact version') + .parse(process.argv); + +const opts = program.opts(); + +const logger = winston.createLogger({ + level: opts.verbose ? 'debug' : 'info', + format: winston.format.simple(), + transports: new winston.transports.Console() }); + +logger.warn('This script is deprecated, will be removed in Otter v12.'); + +if (!SUPPORTED_REPOSITORY_MANAGERS.includes(opts.repositoryManager)) { + logger.error(`Unsupported repository manager: ${opts.repositoryManager as string}`); + process.exit(10); +} + +if (opts.repositoryManager === 'Azure Artifacts') { + if (!opts.organization) { + logger.error('No organization specified for Azure Artifacts'); + process.exit(11); + } + if (!opts.project) { + logger.error('No project specified for Azure Artifacts'); + process.exit(12); + } + if (!opts.feed) { + logger.error('No feed specified for Azure Artifacts'); + process.exit(13); + } +} + +if (!opts.artifactVersion && !opts.usePackageVersion) { + logger.error('No version specified'); + process.exit(14); +} + +if (!opts.artifactName) { + logger.error('No artifact name specified'); + process.exit(15); +} + +if (!opts.artifactGroupId) { + logger.error('No group id specified'); + process.exit(16); +} + +let url: string = opts.registry; +const jar = request.jar(); +const options: {headers?: Headers; jar: CookieJar} = {jar}; +const password = opts.password || opts.password_env_var && process.env[opts.password_env_var]; +if (opts.username && password) { + options.headers = { + // eslint-disable-next-line @typescript-eslint/naming-convention + Authorization: 'Basic ' + Buffer.from(`${opts.username as string}:${password as string}`).toString('base64') + }; +} +const name: string = opts.artifactName; +const artifactGroupId: string = opts.artifactGroupId; + +let version: string = opts.usePackageVersion ? require(path.resolve(process.cwd(), 'package.json')).version : opts.artifactVersion; + +const filePath = opts.out || `./built/${name}.jar`; +fse.ensureDirSync(path.resolve(process.cwd(), path.dirname(filePath))); + +/** + * Report error on artifact downloading + * + * @param e + */ +const reportError = (e: Error) => { + logger.error(e); +}; + + +/** + * Download the specified Artifact on JFrog + */ +async function retrieveArtifactFromJFrog() { + logger.info(`Searching for ${name}@${version}`); + + url += (url.endsWith('/') ? '' : '/') + `api/search/gavc?a=${name}&g=${artifactGroupId}`; + + if (opts.artifactRepos && opts.artifactRepos.length) { + url += `&repos=${(opts.artifactRepos as string[]).join(',')}`; + } + + if (version.startsWith('0.0.0')) { + url += `&v=${version}`; + } + let responseSearch; + try { + responseSearch = await request.get(url, options).promise(); + } catch (err) { + logger.warn('First call to get artifact information failed, retries'); + responseSearch = await request.get(url, options).promise(); + } + // eslint-disable-next-line no-console + console.log(responseSearch); + const responseSearchObj: {results: {uri: string}[]} = JSON.parse(responseSearch); + const uris = responseSearchObj.results + .map((res) => res.uri) + .filter((uri) => uri.endsWith('.jar')) + .sort((a, b) => { + if (a > b) { + return -1; + } else if (a < b) { + return 1; + } + return 0; + }); + + if (uris.length) { + const artifactUrl = uris[0]; + logger.info(`Call to ${artifactUrl}`); + const gavcResponse: string = await request.get(artifactUrl, options).promise(); + logger.debug(`GavcResponse : ${gavcResponse}`); + const parsedGavcResponse: GavcResponse = JSON.parse(gavcResponse); + const downloadUri: string = parsedGavcResponse.downloadUri; + logger.info(`Downloading artifact ${downloadUri}`); + + const file = fs.createWriteStream(path.resolve(process.cwd(), filePath)); + file.on('error', reportError); + + const cookieString = jar.getCookieString(url); + const httpOptions = {headers: options.headers ? {...options.headers, cookie: cookieString} : undefined}; + + if (artifactUrl.startsWith('https')) { + https.get(downloadUri, httpOptions, (response) => response.pipe(file)).on('error', reportError); + } else { + http.get(downloadUri, httpOptions, (response) => response.pipe(file)).on('error', reportError); + } + } else { + if (opts.silent) { + logger.warn('No artifact found, silent mode ON, prevented the failure'); + } else { + logger.error('No artifact found'); + process.exit(2); + } + } +} + +/** + * Get the latest version, if it exists, of the specified artifact + * + * @param packages Array of packages from the Azure feed + * @param artifactName Name of the artifact to get the latest version from + */ +function getLatestVersion(packages: Record[], artifactName: string) { + const normalizedArtifactName = `${artifactGroupId}:${artifactName}`.toLowerCase(); + for (const pckg of packages) { + if (pckg.normalizedName === artifactName || pckg.normalizedName === normalizedArtifactName) { + const latestVersion = (pckg.versions as {isLatest: boolean; version: string}[]).find(v => v.isLatest); + if (!latestVersion) { + throw new Error(`No latest version found for ${artifactName}`); + } + return latestVersion.version; + } + } + throw new Error(`No package ${artifactName} found`); +} + +/** + * Download the specified Artifact on Azure + */ +async function retrieveArtifactFromAzure() { + try { + if (version.startsWith('0.0.0')) { + // eslint-disable-next-line max-len + const res = await request.get(`https://feeds.dev.azure.com/${opts.organization as string}/${opts.project as string}/_apis/packaging/feeds/${opts.feed as string}/packages?api-version=6.0-preview.1`, options).promise(); + version = getLatestVersion(JSON.parse(res).value, name); + } + logger.info(`Searching for ${name}@${version}`); + + // eslint-disable-next-line max-len + url = `https://pkgs.dev.azure.com/${opts.organization as string}/${opts.project as string}/_apis/packaging/feeds/${opts.feed as string}/maven/${opts.artifactGroupId as string}/${name}/${version}/${name}-${version}.jar/content?api-version=6.0-preview.1`; + + logger.info(`Call to ${url}`); + const downloadUri = await new Promise((resolve, reject) => { + https.get(url, options, (response) => resolve(response.headers.location)).on('error', reject); + }); + if (!downloadUri) { + throw new Error(`Could not get the download url for the artifact ${name}`); + } + logger.info(`Downloading artifact ${downloadUri}`); + + const file = fs.createWriteStream(path.resolve(process.cwd(), filePath)); + file.on('error', reportError); + + if (!downloadUri.startsWith('https')) { + throw new Error('Artifact must be downloaded using https protocol'); + } + https.get(downloadUri, (response) => response.pipe(file)).on('error', reportError); + } catch (e) { + if (opts.silent) { + logger.warn(e); + } else { + logger.error(e); + process.exit(3); + } + } +} +/** + * Download an artifact on the specified artifact repository manager + * + * @param registry Registry url + */ +function retrieveArtifact() { + switch (opts.repositoryManager) { + case 'JFrog': { + return retrieveArtifactFromJFrog(); + } + case 'Azure Artifacts': { + return retrieveArtifactFromAzure(); + } + } +} +void retrieveArtifact(); diff --git a/packages/@o3r/dev-tools/src/cli/comment-pr.ts b/packages/@o3r/dev-tools/src/cli/comment-pr.ts index 4c80af6a32..1c34f229c5 100644 --- a/packages/@o3r/dev-tools/src/cli/comment-pr.ts +++ b/packages/@o3r/dev-tools/src/cli/comment-pr.ts @@ -1,7 +1,7 @@ #!/usr/bin/env node import { spawnSync } from 'node:child_process'; -console.warn('This CLI is deprecated, please use `o3r-comment-pr` from the package `@o3r/artifactory-tools`'); +console.warn('This CLI is deprecated, please use `o3r-comment-pr` from the package `@o3r/azure-tools`'); spawnSync('o3r-comment-pr', process.argv.slice(2), { shell: true, diff --git a/packages/@o3r/dev-tools/src/helpers/gavc-response/gavc-response.interfaces.ts b/packages/@o3r/dev-tools/src/helpers/gavc-response/gavc-response.interfaces.ts index 1f8fc3063a..b11484781d 100644 --- a/packages/@o3r/dev-tools/src/helpers/gavc-response/gavc-response.interfaces.ts +++ b/packages/@o3r/dev-tools/src/helpers/gavc-response/gavc-response.interfaces.ts @@ -1,5 +1,55 @@ -import { GavcResponse as GavcRes } from '@o3r/artifactory-tools'; /** - * @deprecated will be removed in v12. please use `GavcResponse` from `@o3r/artifactory-tools` + * @deprecated will be removed in Otter v12. */ -export interface GavcResponse extends GavcRes {} +export interface GavcResponse { + /** Ex : mvn-built */ + repo: string; + + /** Ex : /io/swagger/typescriptFetch-swagger-codegen/1.0.0/typescriptFetch-swagger-codegen-1.0.0.jar */ + path: string; + + /** Ex : 2018-07-06T17:14:14.123Z */ + created: string; + + /** Ex : anonymous */ + createdBy: string; + + /** Ex : 2018-07-06T17:14:14.031Z */ + lastModified: string; + + /** Ex : anonymous */ + modifiedBy: string; + + /** Ex : 2018-07-06T17:14:14.031Z */ + lastUpdated: string; + + /** Ex : https://jfrog.io/mvn-built/io/swagger/typescriptFetch-swagger-codegen/1.0.0/typescriptFetch-swagger-codegen-1.0.0.jar */ + downloadUri: string; + + /** Ex : application/java-archive */ + mimeType: string; + + /** Ex : 41800 */ + size: 41800; + + /** + * Ex : + * checksums : { + * sha1 : 4b12dc32f3ef1a10a872297423908b52bc1ce6ea; + * md5 : db17bea6a5f737f0644482055dcf2d36 + * }; + */ + checksums: { sha1: string; md5: string }; + + /** + * Ex : + * originalChecksums : { + * sha1 : b12dc32f3ef1a10a872297423908b52bc1ce6ea; + * md5 : db17bea6a5f737f0644482055dcf2d36 + * }; + */ + originalChecksums: { sha1: string; md5: string }; + + /** Ex : https://jfrog.io/api/storage/mvn-built/io/swagger/typescriptFetch-swagger-codegen/1.0.0/typescriptFetch-swagger-codegen-1.0.0.jar */ + uri: string; +} diff --git a/yarn.lock b/yarn.lock index cf273cd8d8..a00934424b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6569,7 +6569,6 @@ __metadata: jest-junit: "npm:~16.0.0" nx: "npm:~17.2.0" request: "npm:^2.88.0" - request-promise-native: "npm:^1.0.9" ts-jest: "npm:~29.1.1" ts-node: "npm:~10.9.1" tslib: "npm:^2.5.3" @@ -6577,7 +6576,6 @@ __metadata: winston: "npm:^3.8.2" bin: o3r-artifact-cleaner: ./dist/src/cli/artifact-cleaner.js - o3r-artifact-retriever: ./dist/src/cli/artifact-retriever.js o3r-pr-artifact-cleaner: ./dist/src/cli/pr-artifact-cleaner.js languageName: unknown linkType: soft @@ -7253,7 +7251,6 @@ __metadata: bin: artifact-cleaner: ./dist/src/cli/artifact-cleaner.js artifact-retriever: ./dist/src/cli/artifact-retriever.js - change-log-sanitizer: ./dist/src/cli/change-log-sanitizer.js comment-pr: ./dist/src/cli/comment-pr.js generate-package-exports: ./dist/src/cli/generate-package-exports.js git-release-management: ./dist/src/cli/git-release-management.js