fix: review improvements

.github/workflows/schedule.sgid-index-validation.yml

@@ -90,9 +90,7 @@ jobs:
             ```
 
             #### Errors
-            ```
             ${{ steps.validate.outputs.stderr }}
-            ```
 
       - name: 🚦 Check for errors
         if: steps.validate.outputs.exitCode != 0

src/scripts/package.json

@@ -5,6 +5,7 @@
     "glob": "^10.3.10",
     "google-auth-library": "^9.7.0",
     "google-spreadsheet": "^4.1.1",
+    "json-to-markdown-table": "^1.0.0",
     "knex": "^3.1.0",
     "ky": "^1.2.2",
     "pg": "^8.11.3",

src/scripts/validate-sgid-index.mjs

@@ -1,5 +1,6 @@
 import { GoogleAuth, auth } from 'google-auth-library';
 import { GoogleSpreadsheet } from 'google-spreadsheet';
+import jsonToMarkdown from 'json-to-markdown-table';
 import ky from 'ky';
 import ProgressBar from 'progress';
 import * as tsImport from 'ts-import';
@@ -29,18 +30,22 @@ const worksheet = spreadsheet.sheetsByTitle['SGID Index'];
 
 const errors = [];
 function recordError(message, row) {
-  errors.push(`${row.get('displayName')} (${row.get('id')}) | ${message}`);
+  errors.push({
+    displayName: row.get('displayName'),
+    id: row.get('id'),
+    Error: message,
+  });
 }
 
 function toBoolean(value) {
   return value === 'TRUE';
 }
 
 async function openSGIDTableName(row) {
-  const shouldExist = row.get('openSgid');
-  if (!shouldExist === 'TRUE') {
+  if (!toBoolean(row.get('openSgid'))) {
     return;
   }
+
   const cellValue = row.get('openSgidTableName');
   if (!cellValue) {
     recordError('openSgidTableName is empty', row);
@@ -63,8 +68,6 @@ async function productPage(row) {
     return;
   }
 
-  const currentValidation = toBoolean(row.get('productPageValidation'));
-
   const url = cellValue.startsWith('/') ? `https://gis-utah.netlify.app${cellValue}` : cellValue;
 
   let result = await validateUrl(url);
@@ -76,11 +79,6 @@ async function productPage(row) {
   if (!result.valid) {
     recordError(`productPage: ${result.message}`, row);
   }
-
-  if (currentValidation !== result.valid) {
-    row.set('productPageValidation', result.valid);
-    return true;
-  }
 }
 
 async function idGuid(row) {
@@ -97,6 +95,8 @@ async function itemId(row) {
   const cellValue = row.get('itemId')?.trim();
 
   if (!cellValue) {
+    // TODO: internal datasets _should_ have an itemId
+    // also, check for deprecated and openSgid/arcGisOnline field values
     // itemId is not a required field
     return;
   }
@@ -106,7 +106,9 @@ async function itemId(row) {
     return;
   }
 
-  const hubData = await ky(`https://opendata.arcgis.com/api/v3/datasets/${cellValue}_0`).json();
+  const hubData = await ky(
+    `https://opendata.arcgis.com/api/v3/datasets/${cellValue}_${row.get('serverLayerId')}`,
+  ).json();
   const serviceParts = hubData.data.attributes.url.split('/rest/services/');
 
   const newData = {
@@ -148,7 +150,7 @@ async function downloadMetadataCheck(row) {
     return;
   }
 
-  const checks = [
+  const metadataChecks = [
     // sgid index field, metadata field
     ['itemId', 'itemId'],
     ['hubName', 'name'],
@@ -157,7 +159,7 @@ async function downloadMetadataCheck(row) {
     ['serverLayerId', 'layerId'],
   ];
 
-  for (const [sgidIndexField, metadataField] of checks) {
+  for (const [sgidIndexField, metadataField] of metadataChecks) {
     const sgidIndexValue = row.get(sgidIndexField).toString();
     const metadataValue = metadata[metadataField].toString();
 
@@ -229,8 +231,7 @@ for (const row of rows) {
 
 if (errors.length > 0) {
   errors.sort();
-  console.error('errors found:\n');
-  console.error(errors.join('\n'));
+  console.error(jsonToMarkdown(errors, ['displayName', 'id', 'Error']));
   console.log(`\ntotal errors: ${errors.length}`);
   console.log(`updated ${updatedRowsCount} rows`);
   process.exit(1);