chore(kayenta/aws): Add support for access/secret keys. (#942)

docs/commands.md

@@ -1142,10 +1142,12 @@ hal config canary aws account add ACCOUNT [parameters]
 
 #### Parameters
 `ACCOUNT`: The name of the canary account to operate on.
+ * `--access-key-id`: The default access key used to communicate with AWS.
  * `--bucket`: The name of a storage bucket that your specified account has access to. If you specify a globally unique bucket name that doesn't exist yet, Kayenta will create that bucket for you.
  * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment.
  * `--no-validate`: (*Default*: `false`) Skip validation.
  * `--root-folder`: The root folder in the chosen bucket to place all of the canary service's persistent data in (*Default*: `kayenta`).
+ * `--secret-access-key`: (*Sensitive data* - user will be prompted on standard input) The secret key used to communicate with AWS.
 
 
 ---
@@ -1176,10 +1178,12 @@ hal config canary aws account edit ACCOUNT [parameters]
 
 #### Parameters
 `ACCOUNT`: The name of the canary account to operate on.
+ * `--access-key-id`: The default access key used to communicate with AWS.
  * `--bucket`: The name of a storage bucket that your specified account has access to. If you specify a globally unique bucket name that doesn't exist yet, Kayenta will create that bucket for you.
  * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment.
  * `--no-validate`: (*Default*: `false`) Skip validation.
  * `--root-folder`: The root folder in the chosen bucket to place all of the canary service's persistent data in (*Default*: `kayenta`).
+ * `--secret-access-key`: (*Sensitive data* - user will be prompted on standard input) The secret key used to communicate with AWS.
 
 
 ---
@@ -6184,6 +6188,7 @@ hal deploy apply [parameters]
 #### Parameters
  * `--auto-run`: This command will generate a script to be run on your behalf. By default, the script will run without intervention - if you want to override this, provide "true" or "false" to this flag.
  * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment.
+ * `--exclude-service-names`: (*Default*: `[]`) When supplied, do not install or update the specified Spinnaker services.
  * `--flush-infrastructure-caches`: (*Default*: `false`) WARNING: This is considered an advanced command, and may break your deployment if used incorrectly.
 
 This flushes infrastructure caches (clouddriver) after the deploy succeeds.
@@ -6193,7 +6198,6 @@ This flushes infrastructure caches (clouddriver) after the deploy succeeds.
 This guarantees that no configuration will be generated for this deployment. This is useful for staging artifacts for later manual configuration.
  * `--prep-only`: (*Default*: `false`) This does just the prep work, and not the actual deployment. Only useful at the moment if you want to just clone the repositories for a localgit setup.
  * `--service-names`: (*Default*: `[]`) When supplied, only install or update the specified Spinnaker services.
- * `--exclude-service-names`: (*Default*: `[]`) When supplied, do not install or update the specified Spinnaker services.
 
 
 ---
@@ -6223,9 +6227,9 @@ hal deploy collect-logs [parameters]
 
 #### Parameters
  * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment.
+ * `--exclude-service-names`: (*Default*: `[]`) When supplied, logs from the specified services will be not collected
  * `--no-validate`: (*Default*: `false`) Skip validation.
  * `--service-names`: (*Default*: `[]`) When supplied, logs from only the specified services will be collected.
- * `--exclude-service-names`: (*Default*: `[]`) When supplied, logs from the specified services will be not collected.
 
 
 ---
@@ -6288,9 +6292,9 @@ hal deploy rollback [parameters]
 
 #### Parameters
  * `--deployment`: If supplied, use this Halyard deployment. This will _not_ create a new deployment.
+ * `--exclude-service-names`: (*Default*: `[]`) When supplied, do not install or update the specified Spinnaker services.
  * `--no-validate`: (*Default*: `false`) Skip validation.
  * `--service-names`: (*Default*: `[]`) When supplied, only install or update the specified Spinnaker services.
- * `--exclude-service-names`: (*Default*: `[]`) When supplied, do not install or update the specified Spinnaker services.
 
 
 ---

halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/canary/aws/CommonCanaryAwsCommandProperties.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2018 Google, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.halyard.cli.command.v1.config.canary.aws;
+
+public class CommonCanaryAwsCommandProperties {
+  public static final String ACCESS_KEY_ID_DESCRIPTION = "The default access key used to communicate with AWS.";
+
+  public static final String SECRET_KEY_DESCRIPTION = "The secret key used to communicate with AWS.";
+}

halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/canary/aws/EditCanaryAwsCommand.java

@@ -26,7 +26,6 @@
 import com.netflix.spinnaker.halyard.config.model.v1.canary.AbstractCanaryServiceIntegration;
 import com.netflix.spinnaker.halyard.config.model.v1.canary.Canary;
 import com.netflix.spinnaker.halyard.config.model.v1.canary.aws.AwsCanaryServiceIntegration;
-import com.netflix.spinnaker.halyard.config.model.v1.canary.google.GoogleCanaryServiceIntegration;
 
 @Parameters(separators = "=")
 public class EditCanaryAwsCommand extends AbstractEditCanaryServiceIntegrationCommand {

halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/canary/aws/account/AwsAddCanaryAccountCommand.java

@@ -21,6 +21,7 @@
 import com.netflix.spinnaker.halyard.cli.command.v1.config.canary.CommonCanaryCommandProperties;
 import com.netflix.spinnaker.halyard.cli.command.v1.config.canary.account.AbstractAddCanaryAccountCommand;
 import com.netflix.spinnaker.halyard.cli.command.v1.config.canary.account.CanaryUtils;
+import com.netflix.spinnaker.halyard.cli.command.v1.config.canary.aws.CommonCanaryAwsCommandProperties;
 import com.netflix.spinnaker.halyard.config.model.v1.canary.AbstractCanaryAccount;
 import com.netflix.spinnaker.halyard.config.model.v1.canary.AbstractCanaryServiceIntegration;
 import com.netflix.spinnaker.halyard.config.model.v1.canary.Canary;
@@ -48,12 +49,27 @@ protected String getServiceIntegration() {
   )
   private String rootFolder;
 
+  @Parameter(
+      names = "--access-key-id",
+      description = CommonCanaryAwsCommandProperties.ACCESS_KEY_ID_DESCRIPTION
+  )
+  private String accessKeyId;
+
+  @Parameter(
+      names = "--secret-access-key",
+      description = CommonCanaryAwsCommandProperties.SECRET_KEY_DESCRIPTION,
+      password = true
+  )
+  private String secretAccessKey;
+
   @Override
   protected AbstractCanaryAccount buildAccount(Canary canary, String accountName) {
     AwsCanaryAccount account = (AwsCanaryAccount)new AwsCanaryAccount().setName(accountName);
 
     account.setBucket(bucket);
     account.setRootFolder(isSet(rootFolder) ? rootFolder : account.getRootFolder());
+    account.setAccessKeyId(accessKeyId);
+    account.setSecretAccessKey(secretAccessKey);
 
     AwsCanaryServiceIntegration awsCanaryServiceIntegration =
         (AwsCanaryServiceIntegration)CanaryUtils.getServiceIntegrationByClass(canary, AwsCanaryServiceIntegration.class);

halyard-cli/src/main/java/com/netflix/spinnaker/halyard/cli/command/v1/config/canary/aws/account/AwsEditCanaryAccountCommand.java

@@ -20,6 +20,7 @@
 import com.beust.jcommander.Parameters;
 import com.netflix.spinnaker.halyard.cli.command.v1.config.canary.CommonCanaryCommandProperties;
 import com.netflix.spinnaker.halyard.cli.command.v1.config.canary.account.AbstractEditCanaryAccountCommand;
+import com.netflix.spinnaker.halyard.cli.command.v1.config.canary.aws.CommonCanaryAwsCommandProperties;
 import com.netflix.spinnaker.halyard.config.model.v1.canary.AbstractCanaryAccount;
 import com.netflix.spinnaker.halyard.config.model.v1.canary.aws.AwsCanaryAccount;
 
@@ -43,10 +44,25 @@ protected String getServiceIntegration() {
   )
   private String rootFolder;
 
+  @Parameter(
+      names = "--access-key-id",
+      description = CommonCanaryAwsCommandProperties.ACCESS_KEY_ID_DESCRIPTION
+  )
+  private String accessKeyId;
+
+  @Parameter(
+      names = "--secret-access-key",
+      description = CommonCanaryAwsCommandProperties.SECRET_KEY_DESCRIPTION,
+      password = true
+  )
+  private String secretAccessKey;
+
   @Override
   protected AbstractCanaryAccount editAccount(AwsCanaryAccount account) {
     account.setBucket(isSet(bucket) ? bucket : account.getBucket());
     account.setRootFolder(isSet(rootFolder) ? rootFolder : account.getRootFolder());
+    account.setAccessKeyId(isSet(accessKeyId) ? accessKeyId : account.getAccessKeyId());
+    account.setSecretAccessKey(isSet(secretAccessKey) ? secretAccessKey : account.getSecretAccessKey());
 
     return account;
   }

halyard-config/src/main/java/com/netflix/spinnaker/halyard/config/model/v1/canary/aws/AwsCanaryAccount.java

@@ -33,5 +33,7 @@
 public class AwsCanaryAccount extends AbstractCanaryAccount implements Cloneable {
   private String bucket;
   private String rootFolder = "kayenta";
+  private String accessKeyId;
+  private String secretAccessKey;
   private Set<AbstractCanaryServiceIntegration.SupportedTypes> supportedTypes = new HashSet<>();
 }

halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/KayentaService.java

@@ -17,13 +17,19 @@
 package com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service;
 
 
+import com.netflix.spinnaker.halyard.config.model.v1.canary.AbstractCanaryServiceIntegration;
+import com.netflix.spinnaker.halyard.config.model.v1.canary.Canary;
+import com.netflix.spinnaker.halyard.config.model.v1.canary.aws.AwsCanaryAccount;
+import com.netflix.spinnaker.halyard.config.model.v1.canary.aws.AwsCanaryServiceIntegration;
 import com.netflix.spinnaker.halyard.config.model.v1.node.DeploymentConfiguration;
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.SpinnakerArtifact;
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.SpinnakerRuntimeSettings;
+import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.profile.AwsCredentialsProfileFactoryBuilder;
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.profile.KayentaProfileFactory;
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.profile.Profile;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
+import org.apache.commons.lang.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import retrofit.http.GET;
@@ -32,6 +38,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 @EqualsAndHashCode(callSuper = true)
 @Data
@@ -41,6 +48,9 @@ abstract public class KayentaService extends SpringService<KayentaService.Kayent
   @Autowired
   KayentaProfileFactory kayentaProfileFactory;
 
+  @Autowired
+  AwsCredentialsProfileFactoryBuilder awsCredentialsProfileFactoryBuilder;
+
   @Override
   public SpinnakerArtifact getArtifact() {
     return SpinnakerArtifact.KAYENTA;
@@ -68,6 +78,40 @@ public List<Profile> getProfiles(DeploymentConfiguration deploymentConfiguration
     return profiles;
   }
 
+  protected Optional<Profile> generateAwsProfile(DeploymentConfiguration deploymentConfiguration, SpinnakerRuntimeSettings endpoints, String spinnakerHome) {
+    String name = "aws/kayenta-credentials" + spinnakerHome.replace("/", "_");
+    Canary canary = deploymentConfiguration.getCanary();
+
+    if (canary.isEnabled()) {
+      AwsCanaryServiceIntegration awsCanaryServiceIntegration =
+          (AwsCanaryServiceIntegration)getServiceIntegrationByClass(canary, AwsCanaryServiceIntegration.class);
+
+      // TODO(lwander/duftler): Seems like this approach leaves us open to potential collision between kayenta aws
+      // accounts, and front50 and clouddriver configuration.
+      if (awsCanaryServiceIntegration.isS3Enabled()) {
+        Optional<AwsCanaryAccount> optionalAwsCanaryAccount =
+            awsCanaryServiceIntegration.getAccounts()
+                .stream()
+                .filter(a -> !StringUtils.isEmpty(a.getAccessKeyId()) && !StringUtils.isEmpty(a.getSecretAccessKey()))
+                .findFirst();
+
+        if (optionalAwsCanaryAccount.isPresent()) {
+          AwsCanaryAccount awsCanaryAccount = optionalAwsCanaryAccount.get();
+          String outputFile = awsCredentialsProfileFactoryBuilder.getOutputFile(spinnakerHome);
+
+          return Optional.of(awsCredentialsProfileFactoryBuilder
+              .setArtifact(SpinnakerArtifact.KAYENTA)
+              .setAccessKeyId(awsCanaryAccount.getAccessKeyId())
+              .setSecretAccessKey(awsCanaryAccount.getSecretAccessKey())
+              .build()
+              .getProfile(name, outputFile, deploymentConfiguration, endpoints));
+        }
+      }
+    }
+
+    return Optional.empty();
+  }
+
   public interface Kayenta {
     @GET("/resolvedEnv")
     Map<String, String> resolvedEnv();
@@ -96,4 +140,13 @@ public static class Settings extends SpringServiceSettings {
 
     public Settings() {}
   }
+
+  private static AbstractCanaryServiceIntegration getServiceIntegrationByClass(Canary canary,
+                                                                              Class<? extends AbstractCanaryServiceIntegration> serviceIntegrationClass) {
+    return canary.getServiceIntegrations()
+        .stream()
+        .filter(s -> serviceIntegrationClass.isAssignableFrom(s.getClass()))
+        .findFirst()
+        .orElseThrow(() -> new IllegalArgumentException("Canary service integration of type " + serviceIntegrationClass.getSimpleName() + " not found."));
+  }
 }

halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/google/GoogleKayentaService.java

@@ -18,6 +18,7 @@
 
 import com.netflix.spinnaker.halyard.config.model.v1.node.DeploymentConfiguration;
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.SpinnakerRuntimeSettings;
+import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.profile.Profile;
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.KayentaService;
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.distributed.SidecarService;
 import lombok.Data;
@@ -47,6 +48,13 @@ public List<SidecarService> getSidecars(SpinnakerRuntimeSettings runtimeSettings
     return result;
   }
 
+  @Override
+  public List<Profile> getProfiles(DeploymentConfiguration deploymentConfiguration, SpinnakerRuntimeSettings endpoints) {
+    List<Profile> profiles = super.getProfiles(deploymentConfiguration, endpoints);
+    generateAwsProfile(deploymentConfiguration, endpoints, getHomeDirectory()).ifPresent(p -> profiles.add(p));
+    return profiles;
+  }
+
   @Override
   public Settings buildServiceSettings(DeploymentConfiguration deploymentConfiguration) {
     Settings settings = new Settings();

halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/kubernetes/v1/KubernetesV1KayentaService.java

@@ -17,6 +17,8 @@
 package com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.distributed.kubernetes.v1;
 
 import com.netflix.spinnaker.halyard.config.model.v1.node.DeploymentConfiguration;
+import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.SpinnakerRuntimeSettings;
+import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.profile.Profile;
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.HasServiceSettings;
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.KayentaService;
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.distributed.DistributedLogCollector;
@@ -27,6 +29,8 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
+import java.util.List;
+
 @EqualsAndHashCode(callSuper = true)
 @Component
 @Data
@@ -40,6 +44,14 @@ public DistributedLogCollector getLogCollector() {
     return getLogCollectorFactory().build(this);
   }
 
+  @Override
+  public List<Profile> getProfiles(DeploymentConfiguration deploymentConfiguration, SpinnakerRuntimeSettings endpoints) {
+    List<Profile> profiles = super.getProfiles(deploymentConfiguration, endpoints);
+    generateAwsProfile(deploymentConfiguration, endpoints, getRootHomeDirectory()).ifPresent(profiles::add);
+    generateAwsProfile(deploymentConfiguration, endpoints, getHomeDirectory()).ifPresent(profiles::add);
+    return profiles;
+  }
+
   @Override
   public Settings buildServiceSettings(DeploymentConfiguration deploymentConfiguration) {
     KubernetesSharedServiceSettings kubernetesSharedServiceSettings = new KubernetesSharedServiceSettings(deploymentConfiguration);

halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/distributed/kubernetes/v2/KubernetesV2KayentaService.java

@@ -17,12 +17,16 @@
 package com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.distributed.kubernetes.v2;
 
 import com.netflix.spinnaker.halyard.config.model.v1.node.DeploymentConfiguration;
+import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.SpinnakerRuntimeSettings;
+import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.profile.Profile;
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.KayentaService;
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.ServiceSettings;
 import lombok.experimental.Delegate;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
+import java.util.List;
+
 @Component
 public class KubernetesV2KayentaService extends KayentaService implements KubernetesV2Service<KayentaService.Kayenta> {
   @Delegate
@@ -34,6 +38,14 @@ public boolean isEnabled(DeploymentConfiguration deploymentConfiguration) {
     return deploymentConfiguration.getCanary().isEnabled();
   }
 
+  @Override
+  public List<Profile> getProfiles(DeploymentConfiguration deploymentConfiguration, SpinnakerRuntimeSettings endpoints) {
+    List<Profile> profiles = super.getProfiles(deploymentConfiguration, endpoints);
+    generateAwsProfile(deploymentConfiguration, endpoints, getRootHomeDirectory()).ifPresent(profiles::add);
+    generateAwsProfile(deploymentConfiguration, endpoints, getHomeDirectory()).ifPresent(profiles::add);
+    return profiles;
+  }
+
   @Override
   public ServiceSettings defaultServiceSettings() {
     return new Settings();

halyard-deploy/src/main/java/com/netflix/spinnaker/halyard/deploy/spinnaker/v1/service/local/debian/LocalDebianKayentaService.java

@@ -18,6 +18,8 @@
 
 import com.netflix.spinnaker.halyard.config.model.v1.node.DeploymentConfiguration;
 import com.netflix.spinnaker.halyard.deploy.services.v1.ArtifactService;
+import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.SpinnakerRuntimeSettings;
+import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.profile.Profile;
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.HasServiceSettings;
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.KayentaService;
 import com.netflix.spinnaker.halyard.deploy.spinnaker.v1.service.LogCollector;
@@ -29,6 +31,8 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
+import java.util.List;
+
 @EqualsAndHashCode(callSuper = true)
 @Data
 @Component
@@ -46,6 +50,13 @@ LogCollector getLocalLogCollector() {
     return localLogCollectorFactory.build(this);
   }
 
+  @Override
+  public List<Profile> getProfiles(DeploymentConfiguration deploymentConfiguration, SpinnakerRuntimeSettings endpoints) {
+    List<Profile> profiles = super.getProfiles(deploymentConfiguration, endpoints);
+    generateAwsProfile(deploymentConfiguration, endpoints, getHomeDirectory()).ifPresent(p -> profiles.add(p));
+    return profiles;
+  }
+
   @Override
   public ServiceSettings buildServiceSettings(DeploymentConfiguration deploymentConfiguration) {
     return new Settings().setArtifactId(getArtifactId(deploymentConfiguration.getName()))