remote S3 support

.gitignore

@@ -4,3 +4,5 @@ node_modules
 .nvmrc
 .vscode
 .npmrc
+.nyc_output
+coverage

.travis.yml

@@ -13,6 +13,7 @@ script:
   - if [ "${TRAVIS_PULL_REQUEST}" != "false" ]; then npm audit; else npm audit || true; fi
   - npm run lint
   - npm run snifftest
+  - npm run test
   - if [[ "${TRAVIS_TAG}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$ ]]; then npm version --no-git-tag-version "${TRAVIS_TAG}"; fi
   - docker build --rm -t "quay.io/razee/clustersubscription:${TRAVIS_COMMIT}" .
   - if [ -n "${TRAVIS_TAG}" ]; then docker tag quay.io/razee/clustersubscription:${TRAVIS_COMMIT} quay.io/razee/clustersubscription:${TRAVIS_TAG}; fi

lib/backendservice/backendservice.js

@@ -1,5 +1,5 @@
 /**
- * Copyright 2022 IBM Corp. All Rights Reserved.
+ * Copyright 2022, 2023 IBM Corp. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,23 +15,33 @@
  */
 
 module.exports = class BackendService {
-  constructor (sub, razeeApi, namespace) {
+  constructor (sub, razeeApi, razeeApiKey, namespace) {
     this._namespace = namespace;
     this._secretName = `clustersubscription-${sub.subscriptionUuid}-secret`;
     this._razeeApi = razeeApi;
-
+    this._razeeApiKey = razeeApiKey;
   }
 
   get namespace() {
     return this._namespace;
   }
-
   get secretName() {
     return this._secretName;
   }
-
   get razeeApi() {
     return this._razeeApi;
   }
-
+  get razeeApiKey() {
+    return this._razeeApiKey;
+  }
+
+  renderRequests() {
+    return null;
+  }
+  renderAuth() {
+    return null;
+  }
+  renderSecretData() {
+    return null;
+  }
 };

lib/backendservice/generic.js

@@ -1,5 +1,5 @@
 /**
- * Copyright 2022 IBM Corp. All Rights Reserved.
+ * Copyright 2022, 2023 IBM Corp. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,17 +17,30 @@
 const Mustache = require('mustache');
 const BackendService = require('./backendservice');
 
+/*
+razee (generic) backend:
+- uses razeeApi and sub.url to build url
+- uses razeeApiKey (via secret ref) as razee-org-key header on request
+
+This is the only backend that uses razeeApi and razeeApiKey constructor args.
+
+The naming is historical, but this backend is not truly 'generic' as it explicitly targets Razee auth.
+This backend should **only** be used with Razee endpoint, as it will leak the orgKey to the URL recipient.
+A truly generic backend should avoid razee-org-key by default.
+*/
+
 module.exports = class Generic extends BackendService {
-  constructor(sub, razeeApi, namespace) {
-    super(sub, razeeApi, namespace);
+  constructor(sub, razeeApi, razeeApiKey, namespace) {
+    super(sub, razeeApi, razeeApiKey, namespace);
+
     this._url = `${this.razeeApi}/${sub.url}`;
   }
 
   get url() {
     return this._url;
   }
 
-  render() {
+  renderRequests() {
     const requests = {
       options: {
         url: '{{{url}}}',
@@ -45,6 +58,10 @@ module.exports = class Generic extends BackendService {
       }
     };
     const requestsTemplate = JSON.stringify(requests);
-    return Mustache.render(requestsTemplate, { url: this.url, secretName: this.secretName, namespace: this.namespace });
+    return JSON.parse( Mustache.render( requestsTemplate, { url: this.url, secretName: this.secretName, namespace: this.namespace } ) );
+  }
+
+  renderSecretData() {
+    return { 'razee-api-org-key': Buffer.from(this.razeeApiKey).toString('base64') };
   }
 };

lib/backendservice/git.js

@@ -1,5 +1,5 @@
 /**
- * Copyright 2022 IBM Corp. All Rights Reserved.
+ * Copyright 2022, 2023 IBM Corp. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,9 +17,15 @@
 const Mustache = require('mustache');
 const BackendService = require('./backendservice');
 
+/*
+git backend:
+- uses sub.remote.params to identify git remote details (repo, filePath, etc)
+- has support for git authentication
+*/
+
 module.exports = class Git extends BackendService {
-  constructor(sub, razeeApi, namespace) {
-    super(sub, razeeApi, namespace);
+  constructor(sub, razeeApi, razeeApiKey, namespace) {
+    super(sub, null, null, namespace);  // razeeApi and razeeApiKey are unused for git backend
     this._provider = sub.remote.remoteType;
     this._secretNameGit = 'razee-git';
     this._auth = false;
@@ -69,7 +75,7 @@ module.exports = class Git extends BackendService {
     return this._auth;
   }
 
-  render() {
+  renderRequests() {
     let rendered;
     if (this.ref) {
       const requests = {
@@ -132,6 +138,6 @@ module.exports = class Git extends BackendService {
       }
     }
 
-    return rendered;
+    return JSON.parse(rendered);
   }
 };

lib/backendservice/s3.js

@@ -0,0 +1,106 @@
+/**
+ * Copyright 2023 IBM Corp. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const BackendService = require('./backendservice');
+
+/*
+S3 backend is similar to razee (generic) backend but
+- gets url from sub.remote.parameters instead of building from razeeApi and sub.url
+- has S3 authentication support from a separate secret rather than using razeeApiKey saved in the clustersubscription-[uuid]-secret
+
+Current limitations:
+- Only supports `hmac` for authType (`iam` not fully implemented)
+- The secret must be in the same namespace as the remoteresource (could take from adittional sub.remote.parameters)
+- The secret keys must be `accessKeyId` / `secretAccessKey` (could take from adittional sub.remote.parameters)
+*/
+
+module.exports = class S3 extends BackendService {
+  constructor(sub, razeeApi, razeeApiKey, namespace) {
+    super(sub, null, null, namespace);  // razeeApi and razeeApiKey are unused for S3 backend
+
+    this._url = null; // no default
+    this._secretNameS3 = 'razee-s3';
+    this._authType = 'hmac'; // hmac | iam
+    sub.remote.parameters.forEach(param => {
+      if (param.key == 'secretName') {
+        this._secretNameS3 = param.value;
+      } else if (param.key == 'authType' && param.value.toLowerCase() === 'iam') {
+        this._authType = 'iam';
+      } else if (param.key == 'url') {
+        this._url = param.value;
+      }
+    });
+  }
+
+  get url() {
+    return this._url;
+  }
+  get secretNameS3() {
+    return this._secretNameS3;
+  }
+  get authType() {
+    return this._authType;
+  }
+
+  renderRequests() {
+    const requests = {
+      options: {
+        url: this.url,
+      }
+    };
+    return requests;
+  }
+
+  renderAuth() {
+    const auth = {};
+    if( this.authType === 'hmac' ) {
+      auth[ this.authType ] = {
+        accessKeyIdRef: {
+          valueFrom: {
+            secretKeyRef: {
+              name: this.secretNameS3,
+              key: 'accessKeyId'
+            }
+          }
+        },
+        secretAccessKeyRef: {
+          valueFrom: {
+            secretKeyRef: {
+              name: this.secretNameS3,
+              key: 'secretAccessKey'
+            }
+          }
+        }
+      };
+    }
+    else if( this.authType === 'iam' ) {
+      auth[ this.authType ] = {
+        url: 'incomplete_implementation', // E.g. 'https://iam.cloud.ibm.com/identity/token'
+        grantType: 'incomplete_implementation', // E.g. 'urn:ibm:params:oauth:grant-type:apikey'
+        apiKeyRef: {
+          valueFrom: {
+            secretKeyRef: {
+              name: this.secretNameS3,
+              key: 'apiKey'
+            }
+          }
+        }
+      };
+      throw new Error( 'iam authentication is not yet supported' ); // Requires support for getting correct url and grantType
+    }
+    return( auth );
+  }
+};

lib/remoteResource.js

@@ -8,34 +8,34 @@ const pLimit = require('p-limit');
 const RR_API_VERSION = 'deploy.razee.io/v1alpha2';
 const NAMESPACE = process.env.NAMESPACE;
 
-const createRemoteResources = async (razeeApi, apiKey, subscriptions, clusterId) => {
+const createRemoteResources = async (razeeApi, razeeApiKey, subscriptions, clusterId) => {
   log.info('create remote resources subscription list', { subscriptions });
   try {
     const limit = pLimit(5);
     return Promise.all(subscriptions.map(async sub => {
       return limit(async () => {
-        const apiKeyBase64 = Buffer.from(apiKey).toString('base64');
         let backendService;
         if (sub.url) {
           backendService = 'generic';
         } else if (sub.remote) {
           if (sub.remote.remoteType == 'github' || sub.remote.remoteType == 'gitlab') {
             backendService = 'git';
           }
+          else if (sub.remote.remoteType == 's3') {
+            backendService = 's3';
+          }
         }
-        const BackendService = require(`./backendservice/${backendService}`);
-        const backend = new BackendService(sub, razeeApi, NAMESPACE);
-        const rendered = backend.render();
-
-        const parsed = JSON.parse(rendered);
-        const remoteResourceName = `clustersubscription-${sub.subscriptionUuid}`;
+        const BackendService = require(`./backendservice/${backendService}`); // If backendService is not set to one of the known types above, this will throw an error
+        const backend = new BackendService(sub, razeeApi, razeeApiKey, NAMESPACE);
+
+        // Prep RemoteResource
         const userName = (sub.kubeOwnerName && typeof sub.kubeOwnerName === 'string') ? sub.kubeOwnerName : 'razeedeploy';
         const remoteResourceJson = {
           'apiVersion': RR_API_VERSION,
           'kind': 'RemoteResource',
           'metadata': {
             'namespace': NAMESPACE,
-            'name': remoteResourceName,
+            'name': `clustersubscription-${sub.subscriptionUuid}`,
             'annotations': {
               'deploy.razee.io/clustersubscription': sub.subscriptionUuid,
               'deploy.razee.io/clusterid': clusterId
@@ -53,7 +53,12 @@ const createRemoteResources = async (razeeApi, apiKey, subscriptions, clusterId)
               []
           }
         };
-        remoteResourceJson.spec.requests.push(parsed);
+        const renderedRequests = backend.renderRequests();
+        const renderedAuth = backend.renderAuth();  // If auth is unsupported, this will throw an error
+        if( renderedRequests ) remoteResourceJson.spec.requests.push(renderedRequests);
+        if( renderedAuth ) remoteResourceJson.spec.auth = renderedAuth;
+
+        // Prep Secret
         const secretJson = {
           'apiVersion': 'v1',
           'kind': 'Secret',
@@ -68,16 +73,16 @@ const createRemoteResources = async (razeeApi, apiKey, subscriptions, clusterId)
               'razee/watch-resource': 'lite'
             }
           },
-          'data': {
-            'razee-api-org-key': apiKeyBase64
-          }
+          'data': {}
         };
+        const secretData = backend.renderSecretData();
+        if( secretData ) secretJson.data = secretData;
 
+        // Save Secret & RemoteResource
         await applyResource(secretJson);
         await applyResource(remoteResourceJson);
       });
     }));
-
   } catch (error) {
     log.error('There was an error creating remote resources', { error });
   }