Merge pull request #220 from Niels-IO/newRemoteImageURLNames

Use Hash for remote Image URLs
Niels-IO · Jun 9, 2024 · bdbed46 · bdbed46
2 parents db79f0c + 224410e
commit bdbed46
Show file tree

Hide file tree

Showing 12 changed files with 1,345 additions and 668 deletions.
diff --git a/example/next.config.js b/example/next.config.js
@@ -13,5 +13,6 @@ module.exports = {
     nextImageExportOptimizer_quality: "75",
     nextImageExportOptimizer_storePicturesInWEBP: "true",
     nextImageExportOptimizer_generateAndUseBlurImages: "true",
+    nextImageExportOptimizer_remoteImageCacheTTL: "0",
   },
 };
diff --git a/example/pages/remote.js b/example/pages/remote.js
@@ -38,6 +38,25 @@ export default function Home() {
             basePath={basePath}
           />
         </div>
+        <div
+          style={{
+            position: "relative",
+            width: "50%",
+            height: "200px",
+            marginBottom: "3rem",
+          }}
+        >
+          <ExportedImage
+            src="https://reactapp.dev/images/nextImageExportOptimizer/christopher-gower-m_HRfLhgABo-unsplash-opt-2048.WEBP?ref=next-image-export-optimizer"
+            fill
+            id="test_image_queryParam"
+            style={{ objectFit: "cover" }}
+            priority
+            alt={"test_image_queryParam"}
+            basePath={basePath}
+            // overrideSrc="/test_image.jpg"
+          />
+        </div>
       </main>
     </div>
   );

diff --git a/example/public/images/next-image-export-optimizer-hashes.json b/example/public/images/next-image-export-optimizer-hashes.json
@@ -10,5 +10,6 @@
     "/chris-zhang-Jq8-3Bmh1pQ-unsplash_small.0fa13b23.jpg": "w8j9FhKoGEyo52uc8zEMt7XCeMUZsGCQjEjnWzkams4=",
     "/chris-zhang-Jq8-3Bmh1pQ-unsplash_static.921260e0.jpg": "F4KuoW3LZSTHxrqqDmnFlIcTPSHwtJTKuB2djCCjEnw=",
     "/chris-zhang-Jq8-3Bmh1pQ-unsplash_static_asset.921260e0.jpg": "TKroa8LFPMSjLnRq67yFY71qpejsNlpVOV7TkeASSGA=",
-    "/reactapp.dev_images_nextImageExportOptimizer_christopher-gower-m_HRfLhgABo-unsplash-opt-2048.WEBP": "YtPJvMpqxVbOf++q4Q3h-aYZjThU0rfwOPjZYOtvefg="
+    "/5206242668571649.WEBP": "NBLWVfuacpA+Xj8Z3PMTT8Q8UuFo9Dn2KHG6uMAGcJw=",
+    "/6725071117443837.WEBP": "w5l7YQrMfohCxKHUQ+Z6ml2LJN36eH4gzJ0y6YWuYBU="
 }
diff --git a/example/remoteOptimizedImages.js b/example/remoteOptimizedImages.js
@@ -1,5 +1,6 @@
 module.exports = [
   "https://reactapp.dev/images/nextImageExportOptimizer/christopher-gower-m_HRfLhgABo-unsplash-opt-2048.WEBP",
+  "https://reactapp.dev/images/nextImageExportOptimizer/christopher-gower-m_HRfLhgABo-unsplash-opt-2048.WEBP?ref=next-image-export-optimizer",
   // 'https://example.com/image1.jpg',
   // 'https://example.com/image2.jpg',
   // 'https://example.com/image3.jpg',

diff --git a/example/src/ExportedImage.tsx b/example/src/ExportedImage.tsx
@@ -92,24 +92,39 @@ const generateImageURL = (
   if (!isRemoteImage && generatedImageURL.charAt(0) !== "/") {
     generatedImageURL = "/" + generatedImageURL;
   }
+
   return generatedImageURL;
 };
 
-function urlToFilename(url: string) {
-  // Remove the protocol from the URL
-  let filename = url.replace(/^(https?|ftp):\/\//, "");
-
-  // Replace special characters with underscores
-  filename = filename.replace(/[/\\:*?"<>|#%]/g, "_");
-
-  // Remove control characters
-  // eslint-disable-next-line no-control-regex
-  filename = filename.replace(/[\x00-\x1F\x7F]/g, "");
+// Credits to https://github.com/bryc/code/blob/master/jshash/experimental/cyrb53.js
+// This is a hash function that is used to generate a hash from the image URL
+const hashAlgorithm = (str: string, seed = 0) => {
+  let h1 = 0xdeadbeef ^ seed,
+    h2 = 0x41c6ce57 ^ seed;
+  for (let i = 0, ch; i < str.length; i++) {
+    ch = str.charCodeAt(i);
+    h1 = Math.imul(h1 ^ ch, 2654435761);
+    h2 = Math.imul(h2 ^ ch, 1597334677);
+  }
+  h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507);
+  h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909);
+  h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507);
+  h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909);
 
-  // Trim any leading or trailing spaces
-  filename = filename.trim();
+  return 4294967296 * (2097151 & h2) + (h1 >>> 0);
+};
 
-  return filename;
+function urlToFilename(url: string) {
+  try {
+    const parsedUrl = new URL(url);
+    const extension = parsedUrl.pathname.split(".").pop();
+    if (extension) {
+      return hashAlgorithm(url).toString().concat(".", extension);
+    }
+  } catch (error) {
+    console.error("Error parsing URL", url, error);
+  }
+  return hashAlgorithm(url).toString();
 }
 
 const imageURLForRemoteImage = ({
@@ -142,12 +157,16 @@ const optimizedLoader = ({
   // if it is a static image, we can use the width of the original image to generate a reduced srcset that returns
   // the same image url for widths that are larger than the original image
   if (isStaticImage && originalImageWidth && width > originalImageWidth) {
-    const deviceSizes = (process.env.__NEXT_IMAGE_OPTS?.deviceSizes || [
-      640, 750, 828, 1080, 1200, 1920, 2048, 3840,
-    ]).map(Number);
-    const imageSizes = (process.env.__NEXT_IMAGE_OPTS?.imageSizes || [
-      16, 32, 48, 64, 96, 128, 256, 384,
-    ]).map(Number);
+    const deviceSizes = (
+      process.env.__NEXT_IMAGE_OPTS?.deviceSizes || [
+        640, 750, 828, 1080, 1200, 1920, 2048, 3840,
+      ]
+    ).map(Number);
+    const imageSizes = (
+      process.env.__NEXT_IMAGE_OPTS?.imageSizes || [
+        16, 32, 48, 64, 96, 128, 256, 384,
+      ]
+    ).map(Number);
     let allSizes: number[] = [...deviceSizes, ...imageSizes];
     allSizes = allSizes.filter((v, i, a) => a.indexOf(v) === i);
     allSizes.sort((a, b) => a - b);
@@ -211,6 +230,7 @@ const ExportedImage = forwardRef<HTMLImageElement | null, ExportedImageProps>(
       blurDataURL,
       style,
       onError,
+      overrideSrc,
       ...rest
     },
     ref
@@ -279,6 +299,7 @@ const ExportedImage = forwardRef<HTMLImageElement | null, ExportedImageProps>(
         {...(loading && { loading })}
         {...(className && { className })}
         {...(onLoad && { onLoad })}
+        {...(overrideSrc && { overrideSrc })}
         // if the blurStyle is not "empty", then we take care of the blur behavior ourselves
         // if the blur is complete, we also set the placeholder to empty as it otherwise shows
         // the background image on transparent images
@@ -291,7 +312,7 @@ const ExportedImage = forwardRef<HTMLImageElement | null, ExportedImageProps>(
         style={{ ...style, ...blurStyle }}
         loader={
           imageError || unoptimized === true
-            ? fallbackLoader
+            ? () => fallbackLoader({ src: overrideSrc || src })
             : (e) => optimizedLoader({ src, width: e.width, basePath })
         }
         blurDataURL={automaticallyCalculatedBlurDataURL}

diff --git a/example/src/legacy/ExportedImage.tsx b/example/src/legacy/ExportedImage.tsx
@@ -81,21 +81,35 @@ const generateImageURL = (
   return generatedImageURL;
 };
 
-function urlToFilename(url: string) {
-  // Remove the protocol from the URL
-  let filename = url.replace(/^(https?|ftp):\/\//, "");
-
-  // Replace special characters with underscores
-  filename = filename.replace(/[/\\:*?"<>|#%]/g, "_");
-
-  // Remove control characters
-  // eslint-disable-next-line no-control-regex
-  filename = filename.replace(/[\x00-\x1F\x7F]/g, "");
+// Credits to https://github.com/bryc/code/blob/master/jshash/experimental/cyrb53.js
+// This is a hash function that is used to generate a hash from the image URL
+const hashAlgorithm = (str: string, seed = 0) => {
+  let h1 = 0xdeadbeef ^ seed,
+    h2 = 0x41c6ce57 ^ seed;
+  for (let i = 0, ch; i < str.length; i++) {
+    ch = str.charCodeAt(i);
+    h1 = Math.imul(h1 ^ ch, 2654435761);
+    h2 = Math.imul(h2 ^ ch, 1597334677);
+  }
+  h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507);
+  h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909);
+  h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507);
+  h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909);
 
-  // Trim any leading or trailing spaces
-  filename = filename.trim();
+  return 4294967296 * (2097151 & h2) + (h1 >>> 0);
+};
 
-  return filename;
+function urlToFilename(url: string) {
+  try {
+    const parsedUrl = new URL(url);
+    const extension = parsedUrl.pathname.split(".").pop();
+    if (extension) {
+      return hashAlgorithm(url).toString().concat(".", extension);
+    }
+  } catch (error) {
+    console.error("Error parsing URL", url, error);
+  }
+  return hashAlgorithm(url).toString();
 }
 
 const imageURLForRemoteImage = ({

diff --git a/example/test/e2e/imageSizeTest.spec.mjs b/example/test/e2e/imageSizeTest.spec.mjs
@@ -159,17 +159,27 @@ const correctSrcTransparentImage = {
 };
 
 const correctSrcRemoteImage = {
-  640: `http://localhost:8080${basePath}/nextImageExportOptimizer/reactapp.dev_images_nextImageExportOptimizer_christopher-gower-m_HRfLhgABo-unsplash-opt-2048-opt-640.WEBP`,
-  750: `http://localhost:8080${basePath}/nextImageExportOptimizer/reactapp.dev_images_nextImageExportOptimizer_christopher-gower-m_HRfLhgABo-unsplash-opt-2048-opt-750.WEBP`,
-  777: `http://localhost:8080${basePath}/nextImageExportOptimizer/reactapp.dev_images_nextImageExportOptimizer_christopher-gower-m_HRfLhgABo-unsplash-opt-2048-opt-777.WEBP`,
-  828: `http://localhost:8080${basePath}/nextImageExportOptimizer/reactapp.dev_images_nextImageExportOptimizer_christopher-gower-m_HRfLhgABo-unsplash-opt-2048-opt-828.WEBP`,
-  1080: `http://localhost:8080${basePath}/nextImageExportOptimizer/reactapp.dev_images_nextImageExportOptimizer_christopher-gower-m_HRfLhgABo-unsplash-opt-2048-opt-1080.WEBP`,
-  1200: `http://localhost:8080${basePath}/nextImageExportOptimizer/reactapp.dev_images_nextImageExportOptimizer_christopher-gower-m_HRfLhgABo-unsplash-opt-2048-opt-1200.WEBP`,
-  1920: `http://localhost:8080${basePath}/nextImageExportOptimizer/reactapp.dev_images_nextImageExportOptimizer_christopher-gower-m_HRfLhgABo-unsplash-opt-2048-opt-1920.WEBP`,
-  2048: `http://localhost:8080${basePath}/nextImageExportOptimizer/reactapp.dev_images_nextImageExportOptimizer_christopher-gower-m_HRfLhgABo-unsplash-opt-2048-opt-2048.WEBP`,
-  3840: `http://localhost:8080${basePath}/nextImageExportOptimizer/reactapp.dev_images_nextImageExportOptimizer_christopher-gower-m_HRfLhgABo-unsplash-opt-2048-opt-3840.WEBP`,
+  640: `http://localhost:8080${basePath}/nextImageExportOptimizer/5206242668571649-opt-640.WEBP`,
+  750: `http://localhost:8080${basePath}/nextImageExportOptimizer/5206242668571649-opt-750.WEBP`,
+  777: `http://localhost:8080${basePath}/nextImageExportOptimizer/5206242668571649-opt-777.WEBP`,
+  828: `http://localhost:8080${basePath}/nextImageExportOptimizer/5206242668571649-opt-828.WEBP`,
+  1080: `http://localhost:8080${basePath}/nextImageExportOptimizer/5206242668571649-opt-1080.WEBP`,
+  1200: `http://localhost:8080${basePath}/nextImageExportOptimizer/5206242668571649-opt-1200.WEBP`,
+  1920: `http://localhost:8080${basePath}/nextImageExportOptimizer/5206242668571649-opt-1920.WEBP`,
+  2048: `http://localhost:8080${basePath}/nextImageExportOptimizer/5206242668571649-opt-2048.WEBP`,
+  3840: `http://localhost:8080${basePath}/nextImageExportOptimizer/5206242668571649-opt-3840.WEBP`,
+};
+const correctSrcRemoteImageWithQueryParams = {
+  640: `http://localhost:8080${basePath}/nextImageExportOptimizer/6725071117443837-opt-640.WEBP`,
+  750: `http://localhost:8080${basePath}/nextImageExportOptimizer/6725071117443837-opt-750.WEBP`,
+  777: `http://localhost:8080${basePath}/nextImageExportOptimizer/6725071117443837-opt-777.WEBP`,
+  828: `http://localhost:8080${basePath}/nextImageExportOptimizer/6725071117443837-opt-828.WEBP`,
+  1080: `http://localhost:8080${basePath}/nextImageExportOptimizer/6725071117443837-opt-1080.WEBP`,
+  1200: `http://localhost:8080${basePath}/nextImageExportOptimizer/6725071117443837-opt-1200.WEBP`,
+  1920: `http://localhost:8080${basePath}/nextImageExportOptimizer/6725071117443837-opt-1920.WEBP`,
+  2048: `http://localhost:8080${basePath}/nextImageExportOptimizer/6725071117443837-opt-2048.WEBP`,
+  3840: `http://localhost:8080${basePath}/nextImageExportOptimizer/6725071117443837-opt-3840.WEBP`,
 };
-
 const correctSrcAnimatedPNGImage = {
   640: `http://localhost:8080${basePath}/nextImageExportOptimizer/animated.c00e0188-opt-128.${
     imagesWebP ? "WEBP" : "PNG"
@@ -696,9 +706,30 @@ for (let index = 0; index < widths.length; index++) {
 
       // check the number of images on the page
       const images = await page.$$("img");
-      expect(images.length).toBe(1);
+      expect(images.length).toBe(2);
       const srcset = generateSrcset(widths, correctSrcRemoteImage);
       expect(image.srcset).toBe(srcset);
+
+      const img_query = await page.locator("#test_image_queryParam");
+      await img_query.click();
+
+      const image_query = await getImageById(page, "test_image_queryParam");
+      expect(image_query.currentSrc).toBe(
+        correctSrcRemoteImageWithQueryParams[width.toString()]
+      );
+      expect(image_query.naturalWidth).toBe(width >= 2048 ? 2048 : width);
+      await expect(img_query).toHaveCSS("position", "absolute");
+      await expect(img_query).not.toHaveCSS(
+        "background-image",
+        `url("/images/nextImageExportOptimizer/transparentImage-opt-10.WEBP")`
+      );
+      await expect(img_query).not.toHaveCSS("background-repeat", "no-repeat");
+
+      const srcset_query = generateSrcset(
+        widths,
+        correctSrcRemoteImageWithQueryParams
+      );
+      expect(image_query.srcset).toBe(srcset_query);
     });
     test("should check the image size for the animated test page", async ({
       page,

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "next-image-export-optimizer",
-  "version": "1.13.0",
+  "version": "1.14.0",
   "description": "Optimizes all static images for Next.js static HTML export functionality",
   "main": "dist/ExportedImage.js",
   "types": "dist/ExportedImage.d.ts",

diff --git a/src/optimizeImages.ts b/src/optimizeImages.ts
@@ -9,6 +9,8 @@ const getHash = require("./utils/getHash");
 import { getRemoteImageURLs } from "./utils/getRemoteImageURLs";
 import { downloadImagesInBatches } from "./utils/downloadImagesInBatches";
 
+const urlToFilename = require("./utils/urlToFilename");
+
 const fs = require("fs");
 const sharp = require("sharp");
 const path = require("path");
@@ -201,6 +203,40 @@ const nextImageExportOptimizer = async function () {
         remoteImageURLs.length > 1 ? "s" : ""
       }...`
     );
+
+  // we clear all images in the remote image folder that are not in the remoteImageURLs array
+  const allFilesInRemoteImageFolder: string[] = fs.readdirSync(
+    folderNameForRemoteImages
+  );
+  const encodedRemoteImageURLs = remoteImageURLs.map((url: string) =>
+    urlToFilename(url)
+  );
+
+  function removeLastUpdated(str: string) {
+    const suffix = ".lastUpdated";
+    if (str.endsWith(suffix)) {
+      return str.slice(0, -suffix.length);
+    }
+    return str;
+  }
+
+  for (const filename of allFilesInRemoteImageFolder) {
+    if (
+      encodedRemoteImageURLs.includes(filename) ||
+      encodedRemoteImageURLs.includes(removeLastUpdated(filename))
+    ) {
+      // the filename is in the remoteImageURLs array or the filename without the .lastUpdated suffix
+      // so we do not delete it
+      continue;
+    }
+
+    fs.unlinkSync(path.join(folderNameForRemoteImages, filename));
+
+    console.log(
+      `Deleted ${filename} from remote image folder as it is not retrieved from remoteOptimizedImages.js.`
+    );
+  }
+
   await downloadImagesInBatches(
     remoteImageURLs,
     remoteImageFilenames,
@@ -294,7 +330,6 @@ const nextImageExportOptimizer = async function () {
   // remove duplicate widths from the array
   widths = widths.filter((item, index) => widths.indexOf(item) === index);
 
-
   const progressBar = defineProgressBar();
   if (allImagesInImageFolder.length > 0) {
     console.log(`Using sizes: ${widths.toString()}`);
@@ -611,4 +646,3 @@ if (require.main === module) {
   nextImageExportOptimizer();
 }
 module.exports = nextImageExportOptimizer;
-
diff --git a/src/utils/getRemoteImageURLs.ts b/src/utils/getRemoteImageURLs.ts
@@ -7,7 +7,7 @@ export async function getRemoteImageURLs(
   nextConfigFolder: string,
   folderPathForRemoteImages: string
 ) {
-  let remoteImageURLs = [];
+  let remoteImageURLs: string[] = [];
   const remoteImagesFilePath = path.join(
     nextConfigFolder,
     "remoteOptimizedImages.js"
@@ -30,5 +30,6 @@ export async function getRemoteImageURLs(
       fullPath: filename,
     };
   });
+
   return { remoteImageFilenames, remoteImageURLs };
 }
diff --git a/src/utils/urlToFilename.ts b/src/utils/urlToFilename.ts
@@ -1,16 +1,30 @@
 module.exports = function urlToFilename(url: string) {
-  // Remove the protocol from the URL
-  let filename = url.replace(/^(https?|ftp):\/\//, "");
-
-  // Replace special characters with underscores
-  filename = filename.replace(/[/\\:*?"<>|#%]/g, "_");
-
-  // Remove control characters
-  // eslint-disable-next-line no-control-regex
-  filename = filename.replace(/[\x00-\x1F\x7F]/g, "");
+  try {
+    const parsedUrl = new URL(url);
+    const extension = parsedUrl.pathname.split(".").pop();
+    if (extension) {
+      return hashAlgorithm(url).toString().concat(".", extension);
+    }
+    return hashAlgorithm(url).toString();
+  } catch (error) {
+    console.error("Error parsing URL", url, error);
+  }
+};
 
-  // Trim any leading or trailing spaces
-  filename = filename.trim();
+// Credits to https://github.com/bryc/code/blob/master/jshash/experimental/cyrb53.js
+// This is a hash function that is used to generate a hash from the image URL
+const hashAlgorithm = (str: string, seed = 0) => {
+  let h1 = 0xdeadbeef ^ seed,
+    h2 = 0x41c6ce57 ^ seed;
+  for (let i = 0, ch; i < str.length; i++) {
+    ch = str.charCodeAt(i);
+    h1 = Math.imul(h1 ^ ch, 2654435761);
+    h2 = Math.imul(h2 ^ ch, 1597334677);
+  }
+  h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507);
+  h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909);
+  h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507);
+  h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909);
 
-  return filename;
+  return 4294967296 * (2097151 & h2) + (h1 >>> 0);
 };