Merge pull request #398 from USEPA/feature/update-nces-fetching

Feature/update nces fetching
USEPA · Mar 21, 2024 · e9f2cdc · e9f2cdc
2 parents 2ca29fd + 8e40914
commit e9f2cdc
Show file tree

Hide file tree

Showing 7 changed files with 128 additions and 114 deletions.
diff --git a/app/server/app/utilities/s3.js → app/server/app/config/s3.js b/app/server/app/utilities/s3.js → app/server/app/config/s3.js
diff --git a/app/server/app/index.js b/app/server/app/index.js
@@ -1,7 +1,9 @@
 require("dotenv").config();
 
 const { resolve } = require("node:path");
+const { readFile } = require("node:fs/promises");
 const express = require("express");
+const axios = require("axios").default || require("axios"); // TODO: https://github.com/axios/axios/issues/5011
 const cors = require("cors");
 const helmet = require("helmet");
 const morgan = require("morgan");
@@ -11,17 +13,12 @@ const passport = require("passport");
 const errorHandler = require("./utilities/errorHandler");
 const log = require("./utilities/logger");
 const samlStrategy = require("./config/samlStrategy");
+const { s3BucketUrl } = require("./config/s3");
 const { protectClientRoutes, checkClientRouteExists } = require("./middleware");
 const routes = require("./routes");
 
-const {
-  NODE_ENV,
-  PORT,
-  CLIENT_URL,
-  SERVER_BASE_PATH,
-  CLOUD_SPACE,
-  JSON_PAYLOAD_LIMIT,
-} = process.env;
+const { NODE_ENV, PORT, CLIENT_URL, SERVER_BASE_PATH, JSON_PAYLOAD_LIMIT } =
+  process.env;
 
 const requiredEnvironmentVariables = [
   "SERVER_URL",
@@ -62,75 +59,114 @@ requiredEnvironmentVariables.forEach((variable) => {
   }
 });
 
-const app = express();
-const port = PORT || 3001;
-
-app.use(helmet({ contentSecurityPolicy: false }));
-app.use(helmet.hsts({ maxAge: 31536000 }));
-
-/** Instruct web browsers to disable caching. */
-app.use((req, res, next) => {
-  res.setHeader("Surrogate-Control", "no-store");
-  res.setHeader("Cache-Control", "no-store, no-cache, must-revalidate, proxy-revalidate"); // prettier-ignore
-  res.setHeader("Pragma", "no-cache");
-  res.setHeader("Expires", "0");
-  next();
-});
-
-app.disable("x-powered-by");
-
 /**
- * Enable CORS and logging with morgan for local development only.
- * NOTE: process.env.NODE_ENV set to "development" below to match value defined
- * in create-react-app when client app is run locally via `npm start`
+ * Fetch NCES JSON data from S3 bucket or read from local file system.
  */
-if (NODE_ENV === "development") {
-  app.use(cors({ origin: CLIENT_URL, credentials: true }));
-  app.use(morgan("dev"));
-}
+function fetchNcesData() {
+  const localFilePath = resolve(__dirname, "./content", "nces.json");
+  const s3FileUrl = `${s3BucketUrl}/content/nces.json`;
 
-app.use(express.json({ limit: JSON_PAYLOAD_LIMIT || "5mb" }));
-app.use(cookieParser());
-app.use(express.urlencoded({ extended: true }));
+  const logMessage =
+    NODE_ENV === "development"
+      ? "Reading NCES.json file from disk."
+      : `Fetching NCES.json from S3 bucket.`;
 
-app.use(passport.initialize());
-passport.use("saml", samlStrategy);
-
-/**
- * If SERVER_BASE_PATH is provided, serve routes and static files from there
- * (e.g. /csb).
- */
-const basePath = `${SERVER_BASE_PATH || ""}/`;
-app.use(basePath, routes);
-
-/**
- * Use regex to add trailing slash on static requests
- * (required when using sub path).
- */
-const pathRegex = new RegExp(`^\\${SERVER_BASE_PATH || ""}$`);
-app.all(pathRegex, (req, res) => res.redirect(`${basePath}`));
-
-/**
- * Serve client app's static built files.
- * NOTE: client app's `build` directory contents copied into server app's
- * `public` directory in CI/CD step.
- */
-app.use(basePath, express.static(resolve(__dirname, "public")));
-
-/** Ensure that requested client route exists (otherwise send 404). */
-app.use(checkClientRouteExists);
+  log({ level: "info", message: logMessage });
 
-/** Ensure user is authenticated on all client-side routes except / and /welcome */
-app.use(protectClientRoutes);
+  return Promise.resolve(
+    /**
+     * local development: read file directly from disk
+     * Cloud.gov: fetch file from the public s3 bucket
+     */
+    NODE_ENV === "development"
+      ? readFile(localFilePath, "utf8").then((string) => JSON.parse(string))
+      : axios.get(s3FileUrl).then((res) => res.data),
+  ).catch((error) => {
+    const errorStatus = error.response?.status || 500;
+    const errorMethod = error.response?.config?.method?.toUpperCase();
+    const errorUrl = error.response?.config?.url;
+
+    const logMessage = `S3 Error: ${errorStatus} ${errorMethod} ${errorUrl}`;
+    log({ level: "error", message: logMessage });
 
-/** Serve client-side routes. */
-app.get("*", (req, res) => {
-  res.sendFile(resolve(__dirname, "public/index.html"));
-});
+    process.exitCode = 1;
+  });
+}
 
-app.use(errorHandler);
+fetchNcesData().then((ncesData) => {
+  const app = express();
+  const port = PORT || 3001;
+
+  /** Store NCES JSON data in the Express app's locals object. */
+  app.locals.ncesData = ncesData;
+
+  app.use(helmet({ contentSecurityPolicy: false }));
+  app.use(helmet.hsts({ maxAge: 31536000 }));
+
+  /** Instruct web browsers to disable caching. */
+  app.use((req, res, next) => {
+    res.setHeader("Surrogate-Control", "no-store");
+    res.setHeader("Cache-Control", "no-store, no-cache, must-revalidate, proxy-revalidate"); // prettier-ignore
+    res.setHeader("Pragma", "no-cache");
+    res.setHeader("Expires", "0");
+    next();
+  });
+
+  app.disable("x-powered-by");
+
+  /**
+   * Enable CORS and logging with morgan for local development only.
+   * NOTE: process.env.NODE_ENV set to "development" below to match value defined
+   * in create-react-app when client app is run locally via `npm start`
+   */
+  if (NODE_ENV === "development") {
+    app.use(cors({ origin: CLIENT_URL, credentials: true }));
+    app.use(morgan("dev"));
+  }
 
-app.listen(port, () => {
-  const logMessage = `Server listening on port ${port}`;
-  log({ level: "info", message: logMessage });
+  app.use(express.json({ limit: JSON_PAYLOAD_LIMIT || "5mb" }));
+  app.use(cookieParser());
+  app.use(express.urlencoded({ extended: true }));
+
+  app.use(passport.initialize());
+  passport.use("saml", samlStrategy);
+
+  /**
+   * If SERVER_BASE_PATH is provided, serve routes and static files from there
+   * (e.g. /csb).
+   */
+  const basePath = `${SERVER_BASE_PATH || ""}/`;
+  app.use(basePath, routes);
+
+  /**
+   * Use regex to add trailing slash on static requests
+   * (required when using sub path).
+   */
+  const pathRegex = new RegExp(`^\\${SERVER_BASE_PATH || ""}$`);
+  app.all(pathRegex, (req, res) => res.redirect(`${basePath}`));
+
+  /**
+   * Serve client app's static built files.
+   * NOTE: client app's `build` directory contents copied into server app's
+   * `public` directory in CI/CD step.
+   */
+  app.use(basePath, express.static(resolve(__dirname, "public")));
+
+  /** Ensure that requested client route exists (otherwise send 404). */
+  app.use(checkClientRouteExists);
+
+  /** Ensure user is authenticated on all client-side routes except / and /welcome */
+  app.use(protectClientRoutes);
+
+  /** Serve client-side routes. */
+  app.get("*", (req, res) => {
+    res.sendFile(resolve(__dirname, "public/index.html"));
+  });
+
+  app.use(errorHandler);
+
+  app.listen(port, () => {
+    const logMessage = `Server listening on port ${port}`;
+    log({ level: "info", message: logMessage });
+  });
 });
diff --git a/app/server/app/routes/content.js b/app/server/app/routes/content.js
@@ -3,7 +3,7 @@ const { readFile } = require("node:fs/promises");
 const express = require("express");
 const axios = require("axios").default || require("axios"); // TODO: https://github.com/axios/axios/issues/5011
 // ---
-const { s3BucketUrl } = require("../utilities/s3");
+const { s3BucketUrl } = require("../config/s3");
 const log = require("../utilities/logger");
 
 const { NODE_ENV } = process.env;
@@ -33,16 +33,21 @@ router.get("/", (req, res) => {
     filenames.map((filename) => {
       const localFilePath = resolve(__dirname, "../content", filename);
       const s3FileUrl = `${s3BucketUrl}/content/${filename}`;
-      const logMessage = `Fetching ${filename} from S3 bucket.`;
+
+      const logMessage =
+        NODE_ENV === "development"
+          ? `Reading ${filename} file from disk.`
+          : `Fetching ${filename} from S3 bucket.`;
+
+      log({ level: "info", message: logMessage });
 
       /**
        * local development: read files directly from disk
        * Cloud.gov: fetch files from the public s3 bucket
        */
       return NODE_ENV === "development"
         ? readFile(localFilePath, "utf8")
-        : (log({ level: "info", message: logMessage, req }),
-          axios.get(s3FileUrl).then((res) => res.data));
+        : axios.get(s3FileUrl).then((res) => res.data);
     }),
   )
     .then((data) => {

diff --git a/app/server/app/routes/formioNCES.js b/app/server/app/routes/formioNCES.js
@@ -1,12 +1,8 @@
-const { resolve } = require("node:path");
-const { readFile } = require("node:fs/promises");
 const express = require("express");
-const axios = require("axios").default || require("axios"); // TODO: https://github.com/axios/axios/issues/5011
 // ---
-const { s3BucketUrl } = require("../utilities/s3");
 const log = require("../utilities/logger");
 
-const { NODE_ENV, FORMIO_NCES_API_KEY } = process.env;
+const { FORMIO_NCES_API_KEY } = process.env;
 
 const router = express.Router();
 
@@ -42,41 +38,16 @@ router.get("/:searchText?", (req, res) => {
     return res.json({});
   }
 
-  const localFilePath = resolve(__dirname, "../content", "nces.json");
-  const s3FileUrl = `${s3BucketUrl}/content/nces.json`;
-  const logMessage = `Fetching NCES.json from S3 bucket.`;
+  const result = req.app.locals.ncesData.find((item) => {
+    return item["NCES ID"] === searchText;
+  });
 
-  Promise.resolve(
-    /**
-     * local development: read file directly from disk
-     * Cloud.gov: fetch file from the public s3 bucket
-     */
-    NODE_ENV === "development"
-      ? readFile(localFilePath, "utf8").then((string) => JSON.parse(string))
-      : (log({ level: "info", message: logMessage, req }),
-        axios.get(s3FileUrl).then((res) => res.data)),
-  )
-    .then((data) => {
-      const result = data.find((item) => item["NCES ID"] === searchText);
+  const logMessage =
+    `NCES data searched with NCES ID '${searchText}' resulting in ` +
+    `${result ? "a match" : "no matches"}.`;
+  log({ level: "info", message: logMessage, req });
 
-      const logMessage =
-        `NCES data searched with NCES ID '${searchText}' resulting in ` +
-        `${result ? "a match" : "no matches"}.`;
-      log({ level: "info", message: logMessage, req });
-
-      return res.json({ ...result });
-    })
-    .catch((error) => {
-      const errorStatus = error.response?.status || 500;
-      const errorMethod = error.response?.config?.method?.toUpperCase();
-      const errorUrl = error.response?.config?.url;
-
-      const logMessage = `S3 Error: ${errorStatus} ${errorMethod} ${errorUrl}`;
-      log({ level: "error", message: logMessage, req });
-
-      const errorMessage = `Error getting NCES.json data from S3 bucket.`;
-      return res.status(errorStatus).json({ message: errorMessage });
-    });
+  return res.json({ ...result });
 });
 
 module.exports = router;
diff --git a/app/server/app/utilities/bap.js b/app/server/app/utilities/bap.js
@@ -291,7 +291,7 @@ function setupConnection(req) {
       const logMessage = `Initializing BAP connection: ${userInfo.url}.`;
       log({ level: "info", message: logMessage, req });
 
-      /** Store bapConnection in global express object using req.app.locals. */
+      /** Store BAP connection in the Express app's locals object. */
       req.app.locals.bapConnection = bapConnection;
     })
     .catch((err) => {

diff --git a/assets/NCES.json b/assets/NCES.json
diff --git a/assets/README.md b/assets/README.md
@@ -0,0 +1 @@
+**IMPORTANT:** Do not remove the [`NCES.json`](./NCES.json) file in this directory, as it’s used in the 2022 FRF (fetched from the Formio form definition).
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		IMPORTANT: Do not remove the [`NCES.json`](./NCES.json) file in this directory, as it’s used in the 2022 FRF (fetched from the Formio form definition).