Automate deployment of design system to CDN (#320)

* Split up build tasks and create new scripts for NPM/CDN deployments

* Remove unused images

* Initial concourse config

* Correct concourse config

* Correct concourse config

* Update pipeline and add readme

* Create script to copy templates into folder for zipping and applying to the release

* zip and add templates to release

* install some zips

* Remove unused dependency

.gitignore

@@ -8,9 +8,12 @@ package-lock.json
 # npm package folders
 /components/
 /page-templates/
+/templates/
 /css/
 /favicons/
 /fonts/
 /img/
 /scripts/
 /scss/
+
+secrets.yml

ci/README.md

@@ -0,0 +1,15 @@
+#### Login to Concourse
+
+```
+fly -t main login -c CONCOURSE_URL
+```
+
+#### Add design system Pipeline
+
+```
+fly -t main set-pipeline -p design-system -c concourse.yml --load-vars-from secrets.yml
+```
+
+```
+fly -t main unpause-pipeline --pipeline design-system
+```

ci/common.js

@@ -0,0 +1,55 @@
+import * as fs from 'fs';
+import { createFolder, copyFile } from './helpers';
+
+export async function copyComponents(componentsPath, newComponentsPath) {
+  await createFolder(newComponentsPath);
+
+  try {
+    const items = await fs.readdirSync(componentsPath).filter(path => !path.includes('.njk'));
+
+    items.forEach(item => copyComponent(item, componentsPath, newComponentsPath));
+  } catch (error) {
+    throw new Error(error);
+  }
+}
+
+async function copyComponent(componentName, componentsPath, newComponentsPath) {
+  try {
+    const componentPath = `${componentsPath}/${componentName}`;
+    const newComponentPath = `${newComponentsPath}/${componentName}`;
+
+    const items = await fs
+      .readdirSync(componentPath)
+      .filter(path => path.includes('.njk') && path.includes('_') && !path.includes('_test-'));
+
+    if (items.length) {
+      await createFolder(newComponentPath);
+    }
+
+    items.forEach(async path => {
+      await copyFile(`${componentPath}/${path}`, `${newComponentPath}/${path}`);
+    });
+  } catch (error) {
+    throw new Error(error);
+  }
+}
+
+export async function copyTemplates(templatesPath, newTemplatesPath) {
+  await createFolder(newTemplatesPath);
+
+  try {
+    const items = await fs.readdirSync(templatesPath).filter(path => path.includes('.njk') && path.includes('_'));
+
+    items.forEach(item => copyTemplate(item, templatesPath, newTemplatesPath));
+  } catch (error) {
+    throw new Error(error);
+  }
+}
+
+async function copyTemplate(templateFileName, templatesPath, newTemplatesPath) {
+  try {
+    await copyFile(`${templatesPath}/${templateFileName}`, `${newTemplatesPath}/${templateFileName}`);
+  } catch (error) {
+    throw new Error(error);
+  }
+}

ci/concourse.yml

@@ -0,0 +1,118 @@
+resource_types:
+- name: slack-notification
+  type: docker-image
+  source:
+    repository: cfcommunity/slack-notification-resource
+    tag: latest
+
+resources:
+- name: ons-design-system-release
+  type: github-release
+  source:
+    owner: ONSdigital
+    repository: design-system
+    access_token: ((github_access_token))
+    release: true
+    pre_release: true
+
+- name: slack-alert
+  type: slack-notification
+  source:
+    url: ((slack_webhook_url))
+
+jobs:
+- name: Release
+  plan:
+  - get: ons-design-system-release
+    params:
+      include_source_tarball: true
+    trigger: true
+
+  - task: CDN Build
+    config:
+      platform: linux
+
+      image_resource:
+        type: docker-image
+        source:
+          repository: node
+          tag: 10.14.2
+
+      inputs:
+      - name: ons-design-system-release
+
+      outputs:
+      - name: dist
+      - name: templates
+
+      run:
+        path: sh
+        args:
+        - -exc
+        - |
+          apt-get update && apt-get install -y --allow-unauthenticated zip
+          
+          cd ons-design-system-release
+          
+          mkdir design-system          
+          tar -xzf source.tar.gz -C design-system --strip-components=1
+
+          cd design-system
+
+          design_system_release=$(cat ../../ons-design-system-release/version)
+
+          yarn
+          RELEASE_VERSION=$design_system_release yarn cdn-bundle
+
+          mkdir ../../dist/$design_system_release
+          cp -R build/* ../../dist/$design_system_release
+          zip -r ../../templates/templates.zip templates/*
+    on_failure:
+      put: slack-alert
+      params:
+          channel: '#pat-lib-notifications'
+          attachments:
+          - pretext: Design System Build Failed
+            color: danger
+            title: Concourse Build $BUILD_ID
+            title_link: http://concourse.dev.eq.ons.digital/builds/$BUILD_ID
+
+  - task: Release to S3
+    params: 
+      AWS_ACCESS_KEY_ID: ((aws_access_key))
+      AWS_SECRET_ACCESS_KEY: ((aws_secret_key))
+      AWS_DEFAULT_REGION: eu-west-1
+    config:
+      platform: linux
+
+      image_resource:
+        type: docker-image
+        source:
+          repository: mesosphere/aws-cli
+
+      inputs:
+      - name: dist
+
+      run:
+        path: sh        
+        args:
+        - -exc
+        - |
+          aws s3 sync --acl public-read dist s3://((s3_bucket_name))/design-system/
+          
+    on_failure:
+      put: slack-alert
+      params:
+        channel: '#pat-lib-notifications'
+        attachments:
+          - pretext: Design System CDN Release Failed
+            color: danger
+            title: Concourse Build $BUILD_ID
+            title_link: http://concourse.dev.eq.ons.digital/builds/$BUILD_ID
+
+  - put: ons-design-system-release
+    params:
+      name: ons-design-system-release/version
+      tag: ons-design-system-release/version
+      globs:
+        - templates/*.zip

ci/generate-npm-package.js

@@ -0,0 +1,65 @@
+import * as fs from 'fs';
+
+import { removeFolder, copyDir, createFolder, copyFile, asyncForEach } from './helpers';
+import { copyComponents, copyTemplates } from './common';
+
+const cwd = process.cwd();
+const sourcePath = `${cwd}/src`;
+const componentsPath = `${sourcePath}/components`;
+const newComponentsPath = `${cwd}/components`;
+const templatesPath = `${sourcePath}/styles/page-template`;
+const newTemplatesPath = `${cwd}/page-templates`;
+const assetFolders = ['css', 'favicons', 'fonts', 'img', 'scripts'];
+const builtAssetsFolders = assetFolders.map(folder => `${cwd}/build/${folder}`);
+const newSassPath = `${cwd}/scss`;
+
+async function removeExistingFolders() {
+  const folders = [newComponentsPath, newTemplatesPath, ...assetFolders];
+
+  await asyncForEach(folders, removeFolder);
+}
+
+async function copyAssets() {
+  await asyncForEach(assetFolders, async (folder, index) => {
+    await createFolder(folder);
+
+    try {
+      const builtPath = builtAssetsFolders[index];
+      const files = await fs.readdirSync(builtPath).filter(path => !path.includes('patternlib'));
+
+      asyncForEach(files, async file => {
+        if (file.match(/(\.\w+)$/)) {
+          await copyFile(`${builtPath}/${file}`, `${folder}/${file}`);
+        } else {
+          const newFolderPath = `${folder}/${file}`;
+          const nestedBuiltPath = `${builtPath}/${file}`;
+          await createFolder(newFolderPath);
+
+          const nestedFiles = await fs.readdirSync(nestedBuiltPath).filter(path => !path.includes('patternlib'));
+
+          asyncForEach(nestedFiles, async nestedFile => {
+            await copyFile(`${nestedBuiltPath}/${nestedFile}`, `${newFolderPath}/${nestedFile}`);
+          });
+        }
+      });
+    } catch (error) {
+      throw new Error(error);
+    }
+  });
+}
+
+async function copyBaseSass() {
+  await createFolder(newSassPath);
+  await copyDir(`${sourcePath}/scss/helpers`, `${newSassPath}/helpers`);
+  await copyDir(`${sourcePath}/scss/vars`, `${newSassPath}/vars`);
+}
+
+async function run() {
+  await removeExistingFolders();
+  await copyComponents(componentsPath, newComponentsPath);
+  await copyTemplates(templatesPath, newTemplatesPath);
+  await copyAssets();
+  await copyBaseSass();
+}
+
+run();

ci/helpers.js

@@ -0,0 +1,48 @@
+import * as fs from 'fs';
+import ncp from 'ncp';
+import rimraf from 'rimraf';
+
+export async function copyDir(from, to) {
+  return new Promise((resolve, reject) => {
+    ncp(from, to, error => {
+      if (error) {
+        throw new Error(error);
+        reject(error);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+export async function createFolder(folderPath) {
+  try {
+    await fs.mkdirSync(folderPath);
+  } catch (error) {
+    throw new Error(error);
+  }
+}
+
+export async function copyFile(filePath, newComponentPath) {
+  try {
+    await fs.copyFileSync(filePath, newComponentPath);
+  } catch (error) {
+    throw new Error(error);
+  }
+}
+
+export async function asyncForEach(array, callback) {
+  for (let index = 0, arrayLength = array.length; index < arrayLength; index++) {
+    await callback(array[index], index, array);
+  }
+}
+
+export async function removeFolder(folderPath) {
+  try {
+    if (await fs.existsSync(folderPath)) {
+      await rimraf.sync(folderPath);
+    }
+  } catch (error) {
+    throw new Error(error);
+  }
+}

ci/prepare-templates-for-zip.js

@@ -0,0 +1,40 @@
+import prependFile from 'prepend-file';
+import { removeFolder, createFolder, asyncForEach } from './helpers';
+import { copyComponents, copyTemplates } from './common';
+
+const cwd = process.cwd();
+const sourcePath = `${cwd}/src`;
+const destPath = `${cwd}/templates`;
+const componentsPath = `${sourcePath}/components`;
+const newComponentsPath = `${destPath}/components`;
+const templatesPath = `${sourcePath}/styles/page-template`;
+const newTemplatesPath = `${destPath}/layout`;
+const copiedPageTemplatePath = `${newTemplatesPath}/_template.njk`;
+
+async function removeExistingFolders() {
+  const folders = [newComponentsPath, newTemplatesPath];
+
+  await asyncForEach(folders, removeFolder);
+}
+
+async function addCDNVersionToPageTemplate() {
+  const version = process.env.RELEASE_VERSION;
+
+  if (version) {
+    const insert = `{% set release_version = '${version}' %}\n`;
+
+    prependFile.sync(copiedPageTemplatePath, insert);
+  } else {
+    throw new Error('RELEASE_VERSION not specified');
+  }
+}
+
+async function run() {
+  await removeExistingFolders();
+  await createFolder(destPath);
+  await copyComponents(componentsPath, newComponentsPath);
+  await copyTemplates(templatesPath, newTemplatesPath);
+  await addCDNVersionToPageTemplate();
+}
+
+run();

ci/secrets.yml.example

@@ -0,0 +1,5 @@
+aws_access_key: 
+aws_secret_key: 
+s3_bucket_name: 
+slack_webhook_url: 
+github_access_token: