working code

bucketVersionsStats.js

@@ -6,6 +6,7 @@ const { doWhilst } = require('async');
 
 const { Logger } = require('werelogs');
 
+const { safeListObjectVersions } = require('./utils/safeList');
 const parseOlderThan = require('./utils/parseOlderThan');
 
 const log = new Logger('s3utils::bucketVersionsStats');
@@ -166,7 +167,8 @@ const logProgressInterval = setInterval(
 );
 
 function _listObjectVersions(bucket, KeyMarker, VersionIdMarker, cb) {
-    return s3.listObjectVersions({
+    safeListObjectVersions(s3, {
+    // s3.listObjectVersions({
         Bucket: bucket,
         MaxKeys: LISTING_LIMIT,
         Prefix: TARGET_PREFIX,
@@ -191,8 +193,11 @@ function listBucket(bucket, cb) {
                     return done(err);
                 }
                 for (const version of data.Versions) {
-                    if (_OLDER_THAN_TIMESTAMP && new Date(version.LastModified) > _OLDER_THAN_TIMESTAMP) {
-                        continue;
+                    if (_OLDER_THAN_TIMESTAMP) {
+                        const parsed = new Date(version.LastModified)
+                        if (Number.isNaN(parsed.getTime()) || parsed > _OLDER_THAN_TIMESTAMP) {
+                            continue;
+                        }
                     }
                     const statObj = version.IsLatest ? stats.current : stats.noncurrent;
                     statObj.count += 1;

utils/safeList.js

@@ -0,0 +1,201 @@
+const xml2js = require('xml2js');
+
+const recoverableErrors = {
+    TimestampParserError: true,
+};
+
+function safeListObjectVersions(s3, params, cb) {
+    const req = s3.listObjectVersions(params);
+    req.on('complete', resp => {
+        if (resp.error) {
+            if (recoverableErrors[resp.error.code]) {
+                return recoverListing(resp, cb);
+            }
+            return cb(resp.error);
+        }
+        return cb(null, resp.data);
+    });
+    req.send();
+}
+
+function recoverListing(resp, cb) {
+    const parser = new xml2js.Parser();
+    parser.parseString(resp.httpResponse.body, (err, data) => {
+        if (err) {
+            // if there is an error parsing the xml, return the original error
+            return cb(resp.error);
+        }
+
+        const { value, error } = extractKeys(data.ListVersionsResult, listingKeys);
+        if (error) {
+            return cb(error);
+        }
+
+        const { value: versions, error: verErr } = safeExtractArray(
+            'Version',
+            data.ListVersionsResult,
+            versionKeys
+        );
+        if (verErr) {
+            return cb(verErr);
+        }
+
+        for (const version of versions) {
+            console.log('version', JSON.stringify(version))
+        }
+
+        const { value: deleteMarkers, error: delErr } = safeExtractArray(
+            'DeleteMarker',
+            data.ListVersionsResult,
+            deleteMarkerKeys
+        );
+        if (delErr) {
+            return cb(delErr);
+        }
+
+        const { value: commonPrefixes, error: comErr } = safeExtractArray(
+            'CommonPrefix',
+            data.ListVersionsResult,
+            commonPrefixKeys
+        );
+        if (comErr) {
+            return cb(comErr);
+        }
+
+        value.Versions = versions;
+        value.DeleteMarkers = deleteMarkers;
+        value.CommonPrefixes = commonPrefixes;
+        return cb(null, value);
+    });
+}
+
+const NO_VALUE = Symbol('noValue');
+
+function extractKeys(data, keys) {
+    const extracted = {};
+    for (const [key, extractor] of Object.entries(keys)) {
+        const { value, error } = extractor(key, data);
+        if (error) {
+            return { error };
+        }
+        if (value !== NO_VALUE) {
+            console.dir(value, {depth: null, colors: true})
+            extracted[key] = value;
+        }
+    }
+    return { value: extracted };
+}
+
+function safeExtract(key, data) {
+    if (data[key] && data[key].length > 0) {
+        return { value: data[key][0] };
+    }
+    return { value: NO_VALUE };
+}
+
+function safeExtractInt(key, data) {
+    if (data[key] && data[key].length > 0) {
+        const parsed = parseInt(data[key][0], 10);
+        if (!Number.isNaN(parsed)) {
+            return { value: parsed };
+        }
+        return data[key][0];
+    }
+
+    return { value: NO_VALUE };
+}
+
+function safeExtractDate(key, data) {
+    console.log('safeExtractDate', key, `"${JSON.stringify(data[key])}"`)
+    if (data[key] && data[key].length > 0) {
+        const parsed = new Date(data[key][0]);
+        if (Number.isNaN(parsed.getTime())) {
+            console.log('safeExtractDate', key, `"${JSON.stringify(data[key][0])}"`, 'failed')
+            return { value: data[key][0] };
+        }
+        return { value: parsed };
+    }
+    return { value: NO_VALUE };
+}
+
+function safeExtractBool(key, data) {
+    if (data[key] && data[key].length > 0) {
+        return { value: data[key][0] === 'true' };
+    }
+    return { value: NO_VALUE };
+}
+
+function safeExtractArray(key, data, arrKeys) {
+    const arr = [];
+    if (data[key]) {
+        for (const item of data[key]) {
+            const { value, error } = extractKeys(item, arrKeys);
+            if (error) {
+                return { error };
+            }
+            arr.push(value);
+            console.dir(value, {depth: null, colors: true})
+        }
+    }
+    return { value: arr };
+}
+
+const listingKeys = {
+    Name: safeExtract,
+    Prefix: safeExtract,
+    KeyMarker: safeExtract,
+    VersionIdMarker: safeExtract,
+    NextKeyMarker: safeExtract,
+    NextVersionIdMarker: safeExtract,
+    Delimiter: safeExtract,
+    MaxKeys: safeExtractInt,
+    IsTruncated: safeExtractBool,
+    EncodingType: safeExtract,
+    RequestCharged: safeExtract,
+};
+
+const versionKeys = {
+    Key: safeExtract,
+    VersionId: safeExtract,
+    IsLatest: safeExtractBool,
+    LastModified: safeExtractDate,
+    ETag: safeExtract,
+    Size: safeExtractInt,
+    StorageClass: safeExtract,
+    Owner: extractOwner,
+};
+
+const deleteMarkerKeys = {
+    Key: safeExtract,
+    VersionId: safeExtract,
+    IsLatest: safeExtractBool,
+    LastModified: safeExtractDate,
+    ETag: safeExtract,
+    Owner: extractOwner,
+    StorageClass: safeExtract,
+};
+
+const commonPrefixKeys = {
+    Prefix: safeExtract,
+};
+
+const ownerKeys = {
+    ID: safeExtract,
+    DisplayName: safeExtract,
+};
+
+function extractOwner(key, data) {
+    if (!data[key] || data[key].length === 0) {
+        return { value: undefined };
+    }
+
+    const { value, error } = extractKeys(data[key][0], ownerKeys);
+    if (error) {
+        return { error };
+    }
+    return { value };
+}
+
+module.exports = {
+    safeListObjectVersions,
+};