Merge pull request #272 from happo/asset-size

Validate assets before uploading them

src/createStaticPackage.js

@@ -1,8 +1,10 @@
-import crypto from 'crypto';
 import { Writable } from 'stream';
+import crypto from 'crypto';
 
 import Archiver from 'archiver';
 
+import validateArchive from './validateArchive';
+
 // We're setting the creation date to the same for all files so that the zip
 // packages created for the same content ends up having the same fingerprint.
 const FILE_CREATION_DATE = new Date('Fri Feb 08 2019 13:31:55 GMT+0100 (CET)');
@@ -31,7 +33,14 @@ export default function createStaticPackage({ tmpdir, publicFolders }) {
       data.push(...chunk);
       done();
     };
+
+    const entries = [];
+    archive.on('entry', (entry) => {
+      entries.push(entry);
+    });
+
     stream.on('finish', () => {
+      validateArchive(archive.pointer(), entries);
       const buffer = Buffer.from(data);
       const hash = crypto.createHash('md5').update(buffer).digest('hex');
       resolve({ buffer, hash });

src/remoteRunner.js

@@ -12,15 +12,22 @@ import createHash from './createHash';
 import ensureTarget from './ensureTarget';
 import loadCSSFile from './loadCSSFile';
 import makeRequest from './makeRequest';
+import validateArchive from './validateArchive';
 
 function staticDirToZipFile(dir) {
   return new Promise((resolve, reject) => {
     const archive = new Archiver('zip');
     const rnd = crypto.randomBytes(4).toString('hex');
     const pathToZipFile = path.join(os.tmpdir(), `happo-static-${rnd}.zip`);
     const output = fs.createWriteStream(pathToZipFile);
+    const entries = [];
 
-    output.on('finish', () => {
+    archive.on('entry', (entry) => {
+      entries.push(entry);
+    });
+
+    output.on('finish', async () => {
+      validateArchive(archive.pointer(), entries);
       resolve(pathToZipFile);
     });
     archive.pipe(output);
@@ -44,6 +51,7 @@ async function resolvePackageData(staticPackage) {
   }
 
   const file = await staticDirToZipFile(staticPackage.path);
+
   const readStream = fs.createReadStream(file);
   const hash = await new Promise((resolve) => {
     const hashCreator = crypto.createHash('md5');
@@ -79,7 +87,9 @@ async function uploadStaticPackage({
       { apiKey, apiSecret },
     );
     logger.info(
-      `${logTag(project)}Reusing existing assets at ${assetsDataRes.path} (previously uploaded on ${assetsDataRes.uploadedAt})`,
+      `${logTag(project)}Reusing existing assets at ${
+        assetsDataRes.path
+      } (previously uploaded on ${assetsDataRes.uploadedAt})`,
     );
     return assetsDataRes.path;
   } catch (e) {

src/validateArchive.js

@@ -0,0 +1,29 @@
+export default function validateArchive(totalBytes, entries) {
+  const totalMegaBytes = Math.round(totalBytes / 1024 / 1024);
+  if (totalMegaBytes < 30) {
+    return;
+  }
+  const messageBits = [
+    `Package size is ${totalMegaBytes} MB (${totalBytes} bytes), maximum is 60 MB.`,
+    "Here are the largest 20 files in the archive. Consider removing ones that aren't necessary.",
+  ];
+  const fileSizes = entries.map((entry) => ({
+    name: entry.name,
+    size: entry.stats ? entry.stats.size : (entry.size || 0),
+  }));
+  fileSizes
+    .sort((a, b) => b.size - a.size)
+    .slice(0, 20)
+    .forEach((file) => {
+      messageBits.push(
+        `${file.name}: ${Math.round(file.size / 1024 / 1024)} MB (${
+          file.size
+        } bytes)`,
+      );
+    });
+
+  if (totalMegaBytes > 60) {
+    throw new Error(messageBits.join('\n'));
+  }
+  console.warn(messageBits.join('\n'));
+}

test/validateArchive-test.js

@@ -0,0 +1,37 @@
+import validateArchive from '../src/validateArchive';
+
+describe('validateArchive', () => {
+  let totalBytes;
+  let entries;
+  let subject;
+
+  beforeEach(() => {
+    totalBytes = 100;
+    entries = [];
+    subject = () => validateArchive(totalBytes, entries);
+  });
+
+  it('does not throw when totalBytes is lower than 30 MB', () => {
+    expect(subject()).toBe(undefined);
+  });
+
+  describe('when the size is larger than 60 MB', () => {
+    beforeEach(() => {
+      totalBytes = 73 * 1024 * 1024 + 2346;
+      entries = [
+        { name: 'rar.png', stats: { size: 95000000 } }, // inside stats object
+        { name: 'dar.png', stats: { size: 78000000 } },
+        { name: 'foo.png', size: 98000000 }, // outside stats object
+        { name: 'bar.png', stats: { size: 8000000 } },
+        { name: 'scar.png', size: 88000000 },
+        { name: 'car.png' }, // no size
+      ];
+    });
+
+    it('throws an error with a list of sorted files by size', () => {
+      expect(subject).toThrow(
+        /Package size is 73 MB.*maximum is 60 MB.*foo.png: 93 MB.*rar.png: 91 MB.*scar.png: 84 MB/s,
+      );
+    });
+  });
+});