Use typed arrays

src/epiviz.gl/data-processor.js

@@ -17,7 +17,9 @@ class DataProcessor {
 
     console.log("Loading data...");
 
-    new SpecificationProcessor(specification, this.indexData.bind(this));
+    const processesSpecification = new SpecificationProcessor(specification);
+
+    this.indexData(processesSpecification);
   }
 
   /**
@@ -49,30 +51,28 @@ class DataProcessor {
     console.log("Reading data...");
 
     // Process the global data in the specification processor
-    if (specificationHelper.data) {
-      for (let track of specificationHelper.tracks) {
-        if (!track.hasOwnData) {
-          const geometryMapper = new GeometryMapper(specificationHelper, track);
-
-          let currentPoint = track.getNextDataPoint();
-          while (currentPoint) {
-            geometryMapper.modifyGeometry(currentPoint.geometry);
-
-            this.data[
-              this.index.add(
-                currentPoint.geometry.coordinates[0],
-                currentPoint.geometry.coordinates[1],
-                currentPoint.geometry.coordinates[0] +
-                  currentPoint.geometry.dimensions[0],
-                currentPoint.geometry.coordinates[1] +
-                  currentPoint.geometry.dimensions[1]
-              )
-            ] = currentPoint;
-
-            currentPoint = track.getNextDataPoint();
-          }
-          break;
+    for (let track of specificationHelper.tracks) {
+      if (!track.hasOwnData) {
+        const geometryMapper = new GeometryMapper(specificationHelper, track);
+
+        let currentPoint = track.getNextDataPoint();
+        while (currentPoint) {
+          geometryMapper.modifyGeometry(currentPoint.geometry);
+
+          this.data[
+            this.index.add(
+              currentPoint.geometry.coordinates[0],
+              currentPoint.geometry.coordinates[1],
+              currentPoint.geometry.coordinates[0] +
+                currentPoint.geometry.dimensions[0],
+              currentPoint.geometry.coordinates[1] +
+                currentPoint.geometry.dimensions[1]
+            )
+          ] = currentPoint;
+
+          currentPoint = track.getNextDataPoint();
         }
+        break;
       }
     }
 
@@ -114,15 +114,14 @@ class DataProcessor {
    * @returns closest point or undefined
    */
   getClosestPoint(point) {
-    let indices = this.index.neighbors(point[0], point[1], 1, 0)
-    let pointToReturn =
-      this.data[indices];
+    let indices = this.index.neighbors(point[0], point[1], 1, 0);
+    let pointToReturn = this.data[indices];
     let distance = 0;
     let isInside = true;
     if (pointToReturn === undefined) {
-      indices = this.index.neighbors(point[0], point[1], 1, 5)
-      if(indices.length === 0) {
-        indices = this.index.neighbors(point[0], point[1], 1)
+      indices = this.index.neighbors(point[0], point[1], 1, 5);
+      if (indices.length === 0) {
+        indices = this.index.neighbors(point[0], point[1], 1);
       }
       pointToReturn = this.data[indices];
       distance = Math.sqrt(
@@ -146,12 +145,11 @@ class DataProcessor {
     const largerX = Math.max(points[0], points[2]);
     const largerY = Math.max(points[1], points[3]);
 
-    let indices = this.index
-      .search(smallerX, smallerY, largerX, largerY)
-
-    let tpoints =  indices.map((i) => this.data[i]);
+    let indices = this.index.search(smallerX, smallerY, largerX, largerY);
+
+    let tpoints = indices.map((i) => this.data[i]);
 
-    return {indices, "points": tpoints};
+    return { indices, points: tpoints };
   }
 
   /**
@@ -199,12 +197,12 @@ class DataProcessor {
         simplifiedBoundingPolygon
       );
 
-      if (tbool) findices.push(candidatePoints.indices[i])
+      if (tbool) findices.push(candidatePoints.indices[i]);
 
       return tbool;
     });
 
-    return {"indices": findices, "points": fpoints}
+    return { indices: findices, points: fpoints };
   }
 }
 

src/epiviz.gl/manager-worker.js

@@ -0,0 +1,87 @@
+import {
+  DATA_WORKER_NAME,
+  WEBGL_WORKER_NAME,
+  cloneArrayBuffer,
+  fetchArrayBuffer,
+  prepareData,
+  transformSpecification,
+} from "./utilities";
+
+const webglWorker = new Worker(
+  new URL("offscreen-webgl-worker.js", import.meta.url),
+  { type: "module" }
+);
+
+webglWorker.onmessage = (message) => {
+  // This is a hack to get around the fact that the webgl worker
+  // is not able to request animation frames. Instead, we will
+  // request animation frames from the manager worker and then
+  // pass the message along to the webgl worker.
+  if (message.data?.command === "requestAnimationFrame") {
+    requestAnimationFrame(() => {
+      webglWorker.postMessage({ type: "animate" });
+    });
+    return;
+  }
+
+  self.postMessage({
+    worker: WEBGL_WORKER_NAME,
+    data: message.data,
+  });
+};
+
+const dataWorker = new Worker(
+  new URL("data-processor-worker.js", import.meta.url),
+  { type: "module" }
+);
+
+dataWorker.onmessage = (message) => {
+  self.postMessage({
+    worker: DATA_WORKER_NAME,
+    data: message.data,
+  });
+};
+
+self.onmessage = (message) => {
+  const { worker, data, action } = message.data;
+
+  if (action === "setSpecification") {
+    transformSpecification(data.specification).then(
+      ({ specification, buffers }) => {
+        const clonedBuffers = buffers.map(cloneArrayBuffer);
+
+        webglWorker.postMessage(
+          {
+            type: "specification",
+            specification,
+          },
+          clonedBuffers
+        );
+
+        dataWorker.postMessage(
+          {
+            type: "init",
+            specification,
+          },
+          buffers
+        );
+      }
+    );
+    return;
+  }
+
+  switch (worker) {
+    case WEBGL_WORKER_NAME:
+      if (data.type === "init") {
+        webglWorker.postMessage(data, [data.canvas]);
+      } else {
+        webglWorker.postMessage(data);
+      }
+      break;
+    case DATA_WORKER_NAME:
+      dataWorker.postMessage(data);
+      break;
+    default:
+      throw new Error(`Unknown worker: ${worker}`);
+  }
+};

src/epiviz.gl/offscreen-webgl-worker.js

@@ -19,6 +19,9 @@ self.onmessage = (message) => {
         ? new OffscreenWebGLDrawer(message.data)
         : new WebGLDrawer(message.data);
       break;
+    case "animate":
+      self.drawer.animate();
+      break;
     case "viewport":
       self.drawer.receiveViewport(message.data);
       break;
@@ -38,6 +41,6 @@ self.onmessage = (message) => {
       self.drawer.gl.viewport(0, 0, message.data.width, message.data.height);
       break;
     default:
-      console.error(`Received unknown message type: ${message.type}`);
+      console.error(`Received unknown message type: ${message.data.type}`);
   }
 };

src/epiviz.gl/specification-processor.js

@@ -102,34 +102,15 @@ class SpecificationProcessor {
    * @param {Object} specification user defined specification
    * @param {Function} callback function to call after all the data has been loaded
    */
-  constructor(specification, callback) {
+  constructor(specification) {
     this.index = 0;
     this.specification = specification;
-    if (typeof specification.defaultData === "string") {
-      // data is a url to get
-      this.dataPromise = fetch(specification.defaultData)
-        .then((response) => response.text())
-        .then((text) => (this.data = text.split("\n")));
-    } else if (specification.defaultData) {
-      // default data is defined, assumed to be an object
-      this.data = specification.defaultData;
-      this.isInlineData = true;
-    }
-    this.tracks = specification.tracks.map((track) => new Track(this, track));
-
-    const allPromises = this.tracks
-      .map((track) => track.dataPromise)
-      .filter((p) => p); // Removes undefined
-    if (this.dataPromise) {
-      allPromises.push(this.dataPromise);
-    }
+    this.tracks = specification.tracks.map(
+      (track) => new Track(specification, track)
+    );
 
     this.xScale = getScaleForSpecification("x", specification);
     this.yScale = getScaleForSpecification("y", specification);
-
-    // When all tracks have acquired their data, call the callback
-    // TODO: Allow tracks to be processed while waiting for others, need to keep in mind order
-    Promise.all(allPromises).then(() => callback(this));
   }
 
   /**
@@ -154,33 +135,49 @@ class Track {
    */
   constructor(specification, track) {
     this.track = track;
-
-    if (typeof track.data === "string") {
-      // Track has its own data to GET
-      this.dataPromise = fetch(track.data)
-        .then((response) => response.text())
-        .then((text) => {
-          this.data = text.split(/[\n\r]+/);
-          this.processHeadersAndMappers();
-          this.hasOwnData = true;
-        });
-    } else if (track.data) {
-      // Track has its own inline data
-      this.data = track.data;
-      this.isInlineData = true;
-      this.processHeadersAndMappers();
+    if (track.data) {
+      // Track has its own data
+      let data;
+      if (track.data.isInlineData) {
+        // Track has its own inline data
+        data = {};
+        for (let i = 0; i < track.data.keys.length; i++) {
+          const key = track.data.keys[i];
+          data[key] = new Int8Array(track.data.defaultDataBuffers[i]);
+        }
+      } else {
+        // Track has its own data file
+        const decodedData = new TextDecoder("utf-8").decode(
+          new Uint8Array(track.data.defaultDataBuffers[0])
+        );
+        data = decodedData.split("\n");
+      }
+      this.data = data;
       this.hasOwnData = true;
-    } else if (specification.data) {
+      this.isInlineData = track.data.isInlineData;
+      this.processHeadersAndMappers();
+    } else if (specification.defaultData) {
       // Track does not have its own data, but the specification has default data
-      this.data = specification.data;
-      this.isInlineData = specification.isInlineData;
+      let defaultData;
+      if (specification.defaultData.isInlineData) {
+        // Specification has inline data
+        defaultData = {};
+        for (let i = 0; i < specification.defaultData.keys.length; i++) {
+          const key = specification.defaultData.keys[i];
+          defaultData[key] = new Int8Array(
+            specification.defaultData.defaultDataBuffers[i]
+          );
+        }
+      } else {
+        // Specification has data file
+        const decodedData = new TextDecoder("utf-8").decode(
+          new Uint8Array(specification.defaultData.defaultDataBuffers[0])
+        );
+        defaultData = decodedData.split("\n");
+      }
+      this.isInlineData = specification.defaultData.isInlineData;
+      this.data = defaultData;
       this.processHeadersAndMappers();
-    } else if (specification.dataPromise) {
-      // Track does not have its own data, but the specification is GETting default data
-      specification.dataPromise.then(() => {
-        this.data = specification.data;
-        this.processHeadersAndMappers();
-      });
     } else {
       console.error(
         `Could not find data (no defaultData in specification and no data specified for this track) for track ${track}.`
@@ -333,7 +330,7 @@ class Track {
     } else {
       return () => DEFAULT_CHANNELS[channel].value;
     }
-  };
+  }
 }
 
 /**