Skip to content

Commit

Permalink
Use custom pulumi app synthesizer
Browse files Browse the repository at this point in the history 
Each CDK stack has a [synthesizer](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib-readme.html#stack-synthesizers)
which determines how the CDK stack should be synthesized and deployed.
By default stacks will use the `DefaultStackSynthesizer` which expects
that the AWS account has been
[bootstrapped](https://docs.aws.amazon.com/cdk/v2/guide/bootstrapping-env.html).
This default bootstrapping will create a bunch of IAM roles, an S3
Bucket for file assets, and an ECR Repository for image assets.

In our case we do not want the user to have to worry about bootstrapping
the account with a CloudFormation template or with the CDK CLI.

This PR creates a new synthesizer (`PulumiSynthesizer`) which will
create the required S3 Bucket and ECR Repository on-demand as needed.

Also, since the synthesizer is also responsible for registering assets
and writing those assets to the asset manfests, our synthesizer is able
to simplify the asset publishing. Instead of registering the assets and
writing the manifest and then post processing that manifest to create
the `BucketObjectV2` resources for each asset in the manifest, we are
able to create those resource as the assets are registered.

closes #108
  • Loading branch information
@corymhall
corymhall committed Oct 11, 2024
1 parent 5bb37e9 commit b65bbd7
Show file tree
Hide file tree
Showing 19 changed files with 1,635 additions and 1,381 deletions.
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
"constructs": "^10.0.111"
},
"dependencies": {
"@aws-cdk/app-staging-synthesizer-alpha": "^2.149.0-alpha.0",
"@types/glob": "^8.1.0",
"archiver": "^7.0.1",
"cdk-assets": "^2.154.8",
Expand Down
68 changes: 8 additions & 60 deletions src/assembly/manifest.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,9 @@ import * as path from 'path';
import { AssemblyManifest, Manifest, ArtifactType, ArtifactMetadataEntryType } from '@aws-cdk/cloud-assembly-schema';
import * as fs from 'fs-extra';
import { CloudFormationTemplate } from '../cfn';
import { ArtifactManifest, AssetManifestProperties, LogicalIdMetadataEntry } from 'aws-cdk-lib/cloud-assembly-schema';
import { AssetManifest, DockerImageManifestEntry, FileManifestEntry } from 'cdk-assets';
import { ArtifactManifest, LogicalIdMetadataEntry } from 'aws-cdk-lib/cloud-assembly-schema';
import { StackManifest } from './stack';
import { ConstructTree, StackAsset, StackMetadata } from './types';
import { warn } from '@pulumi/pulumi/log';
import { ConstructTree, StackMetadata } from './types';

/**
* Reads a Cloud Assembly manifest
Expand Down Expand Up @@ -76,20 +74,18 @@ export class AssemblyManifestReader {

const metadata = this.getMetadata(artifact);

const assets = this.getAssetsForStack(artifactId);
if (!this.tree.children) {
throw new Error('Invalid tree.json found');
}
const stackTree = this.tree.children[artifactId];
const stackManifest = new StackManifest(
this.directory,
artifactId,
templateFile,
const stackManifest = new StackManifest({
id: artifactId,
templatePath: templateFile,
metadata,
stackTree,
tree: stackTree,
template,
assets,
);
dependencies: artifact.dependencies ?? [],
});
this._stackManifests.set(artifactId, stackManifest);
}
}
Expand Down Expand Up @@ -123,52 +119,4 @@ export class AssemblyManifestReader {
public get stackManifests(): StackManifest[] {
return Array.from(this._stackManifests.values());
}

/**
* Return a list of assets for a given stack
*
* @param stackId - The artifactId of the stack to find assets for
* @returns a list of `StackAsset` for the given stack
*/
private getAssetsForStack(stackId: string): StackAsset[] {
const assets: (FileManifestEntry | DockerImageManifestEntry)[] = [];
for (const artifact of Object.values(this.manifest.artifacts ?? {})) {
if (
artifact.type === ArtifactType.ASSET_MANIFEST &&
(artifact.properties as AssetManifestProperties)?.file === `${stackId}.assets.json`
) {
assets.push(...this.assetsFromAssetManifest(artifact));
}
}
return assets;
}

/**
* Get a list of assets from the asset manifest.
*
* @param artifact - An ArtifactManifest to extract individual assets from
* @returns a list of file and docker assets found in the manifest
*/
private assetsFromAssetManifest(artifact: ArtifactManifest): StackAsset[] {
const assets: (FileManifestEntry | DockerImageManifestEntry)[] = [];
const fileName = (artifact.properties as AssetManifestProperties).file;
const assetManifest = AssetManifest.fromFile(path.join(this.directory, fileName));
assetManifest.entries.forEach((entry) => {
if (entry.type === 'file') {
const source = (entry as FileManifestEntry).source;
// This will ignore template assets
if (source.path && source.path.startsWith('asset.')) {
assets.push(entry as FileManifestEntry);
}
} else if (entry.type === 'docker-image') {
const source = (entry as DockerImageManifestEntry).source;
if (source.directory && source.directory.startsWith('asset.')) {
assets.push(entry as DockerImageManifestEntry);
}
} else {
warn(`found unexpected asset type: ${entry.type}`);
}
});
return assets;
}
}
77 changes: 44 additions & 33 deletions src/assembly/stack.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import * as path from 'path';
import { DestinationIdentifier, FileManifestEntry } from 'cdk-assets';
import { CloudFormationParameter, CloudFormationResource, CloudFormationTemplate } from '../cfn';
import { ConstructTree, StackAsset, StackMetadata } from './types';
import { ConstructTree, StackMetadata } from './types';
import { FileAssetPackaging, FileDestination } from 'aws-cdk-lib/cloud-assembly-schema';

/**
Expand Down Expand Up @@ -41,6 +41,38 @@ export class FileAssetManifest {
}
}

export interface StackManifestProps {
/**
* The artifactId of the stack
*/
readonly id: string;

/**
* The path to the CloudFormation template file within the assembly
*/
readonly templatePath: string;

/**
* The StackMetadata for the stack
*/
readonly metadata: StackMetadata;

/**
* The construct tree for the App
*/
readonly tree: ConstructTree;

/**
* The actual CloudFormation template being processed
*/
readonly template: CloudFormationTemplate;

/**
* A list of artifact ids that this stack depends on
*/
readonly dependencies: string[];
}

/**
* StackManifest represents a single Stack that needs to be converted
* It contains all the necessary information for this library to fully convert
Expand Down Expand Up @@ -81,42 +113,21 @@ export class StackManifest {
*
*/
private readonly metadata: StackMetadata;
private readonly assets: StackAsset[];
private readonly directory: string;
constructor(
directory: string,
id: string,
templatePath: string,
metadata: StackMetadata,
tree: ConstructTree,
template: CloudFormationTemplate,
assets: StackAsset[],
) {
this.directory = directory;
this.assets = assets;
this.outputs = template.Outputs;
this.parameters = template.Parameters;
this.metadata = metadata;
this.templatePath = templatePath;
this.id = id;
this.constructTree = tree;
if (!template.Resources) {
public readonly dependencies: string[];
constructor(props: StackManifestProps) {
this.dependencies = props.dependencies;
this.outputs = props.template.Outputs;
this.parameters = props.template.Parameters;
this.metadata = props.metadata;
this.templatePath = props.templatePath;
this.id = props.id;
this.constructTree = props.tree;
if (!props.template.Resources) {
throw new Error('CloudFormation template has no resources!');
}
this.resources = template.Resources;
this.resources = props.template.Resources;
}

public get fileAssets(): FileAssetManifest[] {
return this.assets
.filter((asset) => asset.type === 'file')
.flatMap((asset) => new FileAssetManifest(this.directory, asset));
}

// TODO: implement docker assets
// public get dockerAssets(): DockerAssetManifest[] {
//
// }

/**
* Get the CloudFormation logicalId for the CFN resource at the given Construct path
*
Expand Down
64 changes: 46 additions & 18 deletions src/converters/app-converter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import * as pulumi from '@pulumi/pulumi';
import { AssemblyManifestReader, StackManifest } from '../assembly';
import { ConstructInfo, GraphBuilder } from '../graph';
import { StackComponentResource, lift, Mapping } from '../types';
import { ArtifactConverter, FileAssetManifestConverter } from './artifact-converter';
import { ArtifactConverter } from './artifact-converter';
import { CdkConstruct, ResourceMapping } from '../interop';
import { debug } from '@pulumi/pulumi/log';
import {
Expand Down Expand Up @@ -35,30 +35,50 @@ export class AppConverter {
}

convert() {
const assetStackIds = this.host.dependencies.flatMap((dep) => dep.name);
const stackManifests: StackManifest[] = [];
for (const stackManifest of this.manifestReader.stackManifests) {
// Don't process artifact manifests
if (assetStackIds.includes(stackManifest.id)) continue;
stackManifests.push(stackManifest);

const stackConverter = new StackConverter(this.host, stackManifest);
this.stacks.set(stackManifest.id, stackConverter);
this.convertStackManifest(stackManifest);
}

for (const stack of stackManifests) {
const done: { [artifactId: string]: StackConverter } = {};
this.convertStackManifest(stack, done);
}
}

private convertStackManifest(artifact: StackManifest): void {
const dependencies = new Set<ArtifactConverter>();
for (const file of artifact.fileAssets) {
const converter = new FileAssetManifestConverter(this.host, file);
converter.convert();
dependencies.add(converter);
private convertStackManifest(
artifact: StackManifest,
done: { [artifactId: string]: StackConverter },
): StackConverter | undefined {
if (artifact.id in done) {
return done[artifact.id];
}

// TODO add docker asset converter
// for (const image of artifact.dockerAssets) {
// }
const dependencies = new Set<ArtifactConverter>();
for (const d of artifact.dependencies) {
const converter = this.stacks.get(d);
if (converter) {
const c = this.convertStackManifest(converter.stack, done);
if (c !== undefined) {
debug(`${artifact.id} depends on ${d}`);
dependencies.add(c);
}
}
}

const stackConverter = this.stacks.get(artifact.id);
if (!stackConverter) {
throw new Error(`missing CDK Stack for artifact ${artifact.id}`);
}
stackConverter.convert(dependencies);
done[artifact.id] = stackConverter;
return stackConverter;
}
}

Expand All @@ -70,7 +90,16 @@ export class StackConverter extends ArtifactConverter {
readonly resources = new Map<string, Mapping<pulumi.Resource>>();
readonly constructs = new Map<ConstructInfo, pulumi.Resource>();

constructor(host: StackComponentResource, readonly stack: StackManifest) {
private _stackResource?: CdkConstruct;

public get stackResource(): CdkConstruct {
if (!this._stackResource) {
throw new Error('StackConverter has no stack resource');
}
return this._stackResource;
}

constructor(private readonly host: StackComponentResource, readonly stack: StackManifest) {
super(host);
}

Expand All @@ -84,7 +113,7 @@ export class StackConverter extends ArtifactConverter {

for (const n of dependencyGraphNodes) {
if (n.construct.id === this.stack.id) {
const stackResource = new CdkConstruct(
this._stackResource = new CdkConstruct(
`${this.stackComponent.name}/${n.construct.path}`,
n.construct.id,
{
Expand All @@ -96,7 +125,7 @@ export class StackConverter extends ArtifactConverter {
dependsOn: this.stackDependsOn(dependencies),
},
);
this.constructs.set(n.construct, stackResource);
this.constructs.set(n.construct, this._stackResource);
continue;
}

Expand Down Expand Up @@ -145,12 +174,11 @@ export class StackConverter extends ArtifactConverter {

private stackDependsOn(dependencies: Set<ArtifactConverter>): pulumi.Resource[] {
const dependsOn: pulumi.Resource[] = [];
dependsOn.push(...this.host.dependencies);
for (const d of dependencies) {
if (d instanceof FileAssetManifestConverter) {
this.resources.set(d.id, { resource: d.file, resourceType: d.resourceType });
dependsOn.push(d.file);
if (d instanceof StackConverter) {
dependsOn.push(d.stackResource);
}
// TODO: handle docker images
}
return dependsOn;
}
Expand Down
58 changes: 0 additions & 58 deletions src/converters/artifact-converter.ts
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
import * as aws from '@pulumi/aws';
import * as cx from 'aws-cdk-lib/cx-api';
import { getAccountId, getPartition, getRegion } from '@pulumi/aws-native';
import { FileAssetManifest } from '../assembly';
import { FileAssetPackaging } from 'aws-cdk-lib/cloud-assembly-schema';
import { zipDirectory } from '../zip';
import { StackComponentResource } from '../types';

/**
Expand Down Expand Up @@ -35,57 +31,3 @@ export abstract class ArtifactConverter {
});
}
}

/**
* FileAssetManifestConverter handles converting CDK assets into Pulumi resources
*/
export class FileAssetManifestConverter extends ArtifactConverter {
private _file?: aws.s3.BucketObjectv2;
public _id?: string;
public resourceType: string = 'aws:s3:BucketObjectv2';

constructor(host: StackComponentResource, readonly manifest: FileAssetManifest) {
super(host);
}

public get id(): string {
if (!this._id) {
throw new Error('must call convert before accessing file');
}
return this._id;
}

/**
* @returns the underlying bucket object pulumi resource
*/
public get file(): aws.s3.BucketObjectv2 {
if (!this._file) {
throw new Error('must call convert before accessing file');
}
return this._file;
}

/**
* Converts a CDK file asset into a Pulumi aws.s3.BucketObjectv2 resource
*/
public convert(): void {
const name = this.manifest.id.assetId;
const id = this.manifest.id.destinationId;
this._id = `${this.stackComponent.name}/${name}/${id}`;

const outputPath =
this.manifest.packaging === FileAssetPackaging.FILE
? Promise.resolve(this.manifest.path)
: zipDirectory(this.manifest.path, this.manifest.path + '.zip');

this._file = new aws.s3.BucketObjectv2(
this._id,
{
source: outputPath,
bucket: this.resolvePlaceholders(this.manifest.destination.bucketName),
key: this.resolvePlaceholders(this.manifest.destination.objectKey),
},
{ parent: this.stackComponent },
);
}
}
Loading

0 comments on commit b65bbd7

Please sign in to comment.