Add config helper utility to improve modifying Jenkins CASC yaml (#516)

Signed-off-by: Tanner Lewis <[email protected]>
Co-authored-by: Sayali Gaikawad <[email protected]>

.gitignore

@@ -6,7 +6,7 @@ node_modules
 .DS_Store
 # We do not want to track this file as it is generated only if user provides certain params.
 resources/jenkins.yaml
-
+.coverage
 
 # CDK asset staging directory
 .cdk.staging

README.md

@@ -125,10 +125,11 @@ Steps:
     {
         "clientId": "example_id",
         "clientSecret": "example_password",
-        "wellKnownOpenIDConfigurationUrl": "https://www.example.com",
-        "tokenServerUrl": "https://example.com/token",
-        "authorizationServerUrl": "https://example.com/authorize",
-        "userInfoServerUrl": "https://example.com/userinfo"
+        "serverConfiguration": {
+            "wellKnown": {
+                "wellKnownOpenIDConfigurationUrl": "https://example.openid.com/.well-known/openid-configuration"
+            }
+        }
     }
     ```
 1. **GitHub Authentication**: Use GitHub as Authentication mechanism for jenkins. This set up uses [github-oauth](https://plugins.jenkins.io/github-oauth/) plugin.
@@ -288,7 +289,7 @@ Useful links
 
 ![Plantuml diagram, see ./diagrams/opensearch-ci-overview.puml for source](./diagrams/opensearch-ci-overview.svg)
 
-Built using [AWS Cloud Development Kit](https://aws.amazon.com/cdk/) the configuration of the CI systems will be available for replication in your own accounts.  The Jenkins instance will be hardened and publically visible, connected to GitHub to make build notifications easy for everyone to see.
+Built using [AWS Cloud Development Kit](https://aws.amazon.com/cdk/) the configuration of the CI systems will be available for replication in your own accounts.  The Jenkins instance will be hardened and publicly visible, connected to GitHub to make build notifications easy for everyone to see.
 
 ## Contributing
 

configHelper/.coveragerc

@@ -0,0 +1,6 @@
+[run]
+source = configHelper
+
+[report]
+omit =
+   */tests/*

configHelper/Pipfile

@@ -0,0 +1,17 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+boto3 = "*"
+argparse = "*"
+pyyaml = "*"
+
+[dev-packages]
+pytest = "*"
+pytest-mock = "*"
+coverage = "*"
+
+[scripts]
+test = "coverage run -m pytest --log-cli-level=INFO"

configHelper/README.md

@@ -0,0 +1,39 @@
+# CI Config Helper
+A configuration utility for populating elements of a Jenkins Configuration as Code yaml from other sources such as AWS Secrets Manager
+
+## Development
+
+To install pipenv:
+```shell
+pip install pipenv
+```
+
+To use this tool:
+
+```shell
+pipenv install
+pipenv run python3 config_helper.py --jenkins-config-file-path=/initial_jenkins.yaml --auth-secret-arn=<SECRET_ARN> --security-realm-id=oic --aws-region=<REGION_NAME>
+```
+
+### Unit Tests
+
+Unit tests can be run from this current `configHelper/` directory by first installing dependencies then running pytest:
+
+```shell
+pipenv install --dev
+pipenv run test
+```
+
+### Coverage
+
+_Code coverage_ metrics can be generated after a unit-test run. A report can either be printed on the command line:
+
+```shell
+pipenv run coverage report
+```
+
+or generated as HTML:
+
+```shell
+pipenv run coverage html
+```

configHelper/configHelper/config_helper.py

@@ -0,0 +1,81 @@
+import argparse
+import boto3
+import logging
+import json
+import yaml
+
+logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s', level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def merge_dicts(dict1, dict2):
+    """Recursively merges two dictionaries."""
+    for key, value in dict2.items():
+        if key in dict1 and isinstance(dict1[key], dict) and isinstance(value, dict):
+            merge_dicts(dict1[key], value)
+        else:
+            dict1[key] = value
+
+def get_secret_as_dict(secret_arn, aws_region):
+    client = boto3.client('secretsmanager', region_name=aws_region)
+    response = client.get_secret_value(SecretId=secret_arn)
+
+    if 'SecretString' in response:
+        secret_string = response['SecretString']
+    else:
+        raise RuntimeError(f'SecretString not found in response when retrieving secret value: {secret_arn}')
+
+    try:
+        secret_dict = json.loads(secret_string)
+    except json.JSONDecodeError as json_error:
+        raise RuntimeError(f"Failed to parse secret string as JSON: {json_error}")
+    return secret_dict
+
+
+def modify_auth_config(yaml_dict, auth_aws_secret_arn, aws_region, auth_type):
+    security_realm_id = "oic" if auth_type.lower() == "oidc" else auth_type.lower()
+    auth_secret_dict = get_secret_as_dict(auth_aws_secret_arn, aws_region)
+    if "securityRealm" not in yaml_dict.get("jenkins", {}):
+        logging.warning('Unable to find jenkins.securityRealm path in initial Jenkins config yaml, creating securityRealm object')
+        yaml_dict.get("jenkins", {})["securityRealm"] = {}
+    if security_realm_id not in yaml_dict.get("jenkins", {}).get("securityRealm", {}):
+        logging.warning(f'Unable to find jenkins.securityRealm.{security_realm_id} path in initial Jenkins config yaml, creating {security_realm_id} object')
+        yaml_dict.get("jenkins", {}).get("securityRealm", {})[security_realm_id] = {}
+    yaml_auth_dict = yaml_dict.get("jenkins", {}).get("securityRealm", {}).get(security_realm_id, {})
+    merge_dicts(yaml_auth_dict, auth_secret_dict)
+
+
+def main():
+    logging.info("Starting process to load auth config into Jenkins config yaml")
+    parser = argparse.ArgumentParser(description="CI config setup helper.")
+    parser.add_argument("--initial-jenkins-config-file-path", type=str, required=True, help="The file path for the initial jenkins config yaml file to load from and be modified in place, e.g. /initial_jenkins.yaml")
+    parser.add_argument("--auth-aws-secret-arn", type=str, required=True, help="The AWS Secrets Manager Secret ARN to pull auth config from and populate into jenkins config")
+    parser.add_argument("--aws-region", type=str, required=True, help="The AWS region for the boto3 client to use, e.g. us-east-1")
+    parser.add_argument("--auth-type", type=str, required=True, help="The Jenkins auth type to use", choices=['oidc', 'github'])
+    args = parser.parse_args()
+    file_path = args.initial_jenkins_config_file_path
+    logging.info(f"The following args were provided: {args}")
+
+    # Read input file data
+    try:
+        with open(file_path, 'r') as file:
+            yaml_data = yaml.safe_load(file)
+    except FileNotFoundError:
+        raise FileNotFoundError(f"Error: The file '{file_path}' does not exist.")
+    except yaml.YAMLError as e:
+        raise ValueError(f"Error parsing YAML file '{file_path}': {e}")
+
+    # Modify input data as needed
+    modify_auth_config(yaml_data, args.auth_aws_secret_arn, args.aws_region, args.auth_type)
+
+    # Write file with updated data
+    try:
+        with open(file_path, 'w') as file:
+            yaml.dump(yaml_data, file, default_flow_style=False)
+    except Exception as e:
+        raise IOError(f"Error writing to file '{file_path}': {e}")
+    logging.info("Process completed to load auth config into Jenkins config yaml")
+
+
+if __name__ == "__main__":
+    main()