Face detection

face/README.md

@@ -0,0 +1,6 @@
+# Face Detection
+
+Face detection code originally created in this repository:
+
+https://github.com/whyboris/extract-faces-node
+

face/detect.ts

@@ -0,0 +1,32 @@
+const tf = require('@tensorflow/tfjs-node');
+
+const faceapi = require('face-api.js');
+
+import { FullDetection } from './interfaces';
+
+/**
+ * Load the model only once
+ */
+export async function loadModel() {
+  await faceapi.nets.ssdMobilenetv1.loadFromDisk('./weights');
+  await faceapi.nets.ageGenderNet.loadFromDisk('./weights');
+  // await faceapi.nets.faceLandmark68Net.loadFromDisk('./weights');
+  // await faceapi.nets.tinyFaceDetector.loadFromDisk('./weights'); // NOT VERY GOOD, but fast
+}
+
+/**
+ * Use face-api.js to detect a rectangle around the face
+ * @param imgBuffer
+ */
+export async function findTheFaces(imgBuffer: Buffer): Promise<FullDetection[]> {
+
+  const imgTensor = tf.node.decodeJpeg(imgBuffer);
+
+  // const detections = await faceapi.detectAllFaces(imgElement);
+  // const detections = await faceapi.detectAllFaces(imgElement, new faceapi.TinyFaceDetectorOptions());
+  const detections = await faceapi.detectAllFaces(imgTensor).withAgeAndGender(); // changes output format a bit
+
+  // console.log(detections);
+
+  return detections;
+}

face/interfaces.ts

@@ -0,0 +1,32 @@
+export interface CropBox {
+  top: number;
+  left: number;
+  width: number;
+  height: number;
+}
+
+export interface InputMeta {
+  width: number;
+  height: number;
+  eachSSwidth: number;
+}
+
+// Face API ===============================
+
+export interface FullDetection {
+  detection: FaceDetection;
+  gender: Gender;
+}
+
+export interface FaceDetection {
+  _box: FaceBox;
+}
+
+export interface FaceBox {
+  _x: number;
+  _y: number;
+  _width: number;
+  _height: number;
+}
+
+export type Gender = 'male' | 'female';

face/pipeline.ts

@@ -0,0 +1,50 @@
+import { loadModel, findTheFaces } from './detect';
+import { getImageSizes, getSubImageBuffer, getCroppedImageBuffers, saveFinalOutput } from './sharp';
+import { InputMeta, FullDetection, Gender } from './interfaces';
+
+// VARIABLES for now ===============================================================================
+
+const RELATIVE_IMAGE_PATH = './images/bbt4.jpg';
+const CURRENT_NUMBER_OF_SCREENS = 1;  // the number of chunks the image is split into (20 screenshots for example) // HARDCODED FOR NOW
+const OUTPUT_FILE_NAME = './output/bbt.jpg';
+const GENDER = 'female';
+
+// runEverything(RELATIVE_IMAGE_PATH, CURRENT_NUMBER_OF_SCREENS, OUTPUT_FILE_NAME, GENDER);
+
+// ==== PIPELINE ===================================================================================
+
+/**
+ * Full pipeline process
+ * @param inputFile  - relative path to INPUT image
+ * @param numOfScreens - the number of screenshots in the filmstrip
+ * @param outputFile - relative path to OUTPUT image
+ */
+export async function runEverything(inputFile: string, numOfScreens: number, outputFile: string, gender: Gender) {
+
+  await loadModel();
+
+  const sizes: InputMeta = await getImageSizes(inputFile, numOfScreens);
+
+  console.log(sizes);
+
+  const all_faces = [];
+
+  for (let i = 0; i < sizes.width; i = i + sizes.eachSSwidth) {
+
+    const imgBuffer: Buffer = await getSubImageBuffer(i, sizes.eachSSwidth, sizes, inputFile);
+
+    const detections: FullDetection[] = await findTheFaces(imgBuffer);
+
+    // warning -- getCroppedImageBuffers returns an array (possibly empty)
+    // so use `...` spread operator - it will not add any elements if incoming array is empty
+    all_faces.push(...(await getCroppedImageBuffers(detections, imgBuffer, sizes, gender)));
+
+  }
+
+  if (all_faces.length) {
+    saveFinalOutput(all_faces, outputFile, sizes);
+    console.log('File saved:', outputFile);
+  } else {
+    console.log('no faces found!');
+  }
+}

face/sharp.ts

@@ -0,0 +1,130 @@
+const sharp = require('sharp');
+
+import { getBetterBox } from './support';
+
+import { InputMeta, FullDetection, FaceBox, Gender } from './interfaces';
+
+// ====== METHODS ==================================================================================
+
+/**
+ * Inspect original image, return the width, height, and width of each sub-image
+ * Later Video Hub App will just provide these 3 pieces of data
+ * @param imgPath
+ * @param numOfScreens
+ */
+export async function getImageSizes(imgPath: string, numOfScreens: number): Promise<InputMeta> {
+  const fileMeta = await sharp(imgPath).metadata();
+
+  return {
+    width: fileMeta.width,
+    height: fileMeta.height,
+    eachSSwidth: fileMeta.width / numOfScreens
+  };
+}
+
+
+/**
+ * Extract buffer from sub-image
+ * @param offset number of pixels offset in original image (for when you have many sub-images horizontally)
+ * @param width width of the (sub-) image
+ * @param sizes
+ * @param imgPath
+ */
+export async function getSubImageBuffer(offset: number, width: number, sizes: InputMeta, imgPath: string): Promise<Buffer> {
+  const imgBuffer: Buffer = await sharp(imgPath)
+    .extract({
+      left: offset,
+      top: 0,
+      width: width,
+      height: sizes.height,
+    })
+    .toBuffer();
+
+  return imgBuffer;
+}
+
+
+/**
+ * Return the cropped buffer
+ * @param imgBuffer
+ * @param match
+ * @param sizes
+ */
+export async function getFaceCropBuffer(imgBuffer: Buffer, match: FaceBox, sizes: InputMeta) {
+
+  const newBox = getBetterBox(match, sizes);
+
+  console.log(newBox);
+
+  const croppedImageBuffer = await sharp(imgBuffer)
+    .extract(newBox)
+    .resize(sizes.eachSSwidth / 2, sizes.height)
+    .toBuffer();
+
+  return croppedImageBuffer;
+}
+
+
+/**
+ * Save each face in the current image
+ *   - when there is more than one face found in an image
+ *
+ * @param matches
+ * @param imgBuffer
+ * @param sizes
+ *
+ * @returns array of buffers !!!
+ */
+export async function getCroppedImageBuffers(matches: FullDetection[], imgBuffer: Buffer, sizes: InputMeta, gender: Gender) {
+
+  console.log('found', matches.length, 'faces');
+
+  const all_faces = [];
+
+  for (let i = 0; i < matches.length; i++) {
+    const box: FaceBox = matches[i].detection._box;
+    const sex: Gender = matches[i].gender;
+    if (sex === gender) {
+      const croppedBuffer = await getFaceCropBuffer(imgBuffer, box, sizes);
+      all_faces.push(croppedBuffer);
+    }
+  }
+
+  return all_faces;
+}
+
+
+/**
+ * Iteracte across face-crop-buffers, combine them into a single photo, and save as output
+ * @param allFaceBuffers
+ * @param outputFile
+ * @param sizes
+ */
+export function saveFinalOutput(allFaceBuffers: Buffer[], outputFile: string, sizes: InputMeta) {
+
+  console.log('Total of', allFaceBuffers.length, 'faces found!');
+
+  let tracker = 0;
+
+  const composeParams = [];
+
+  allFaceBuffers.forEach((face) => {
+    composeParams.push({
+      input: allFaceBuffers[tracker],
+      top: 0,
+      left: tracker * sizes.eachSSwidth / 2,
+    });
+    tracker++;
+  });
+
+  sharp({
+    create: {
+      width: tracker * sizes.eachSSwidth / 2,
+      height: sizes.height,
+      channels: 3,
+      background: { r: 0, g: 0, b: 50 }
+    }
+  })
+  .composite(composeParams)
+  .toFile(outputFile);
+}

face/support.ts

@@ -0,0 +1,29 @@
+import { CropBox, InputMeta } from './interfaces';
+
+/**
+ * Take cropping box and expand it to include more of the face !!!
+ * @param box
+ * @param sizes
+ */
+export function getBetterBox(box, sizes: InputMeta): CropBox {
+
+  // set y to 1/2 height heigher or 0
+  const new_y: number = (Math.round(box._y) - Math.round(box._height / 2)) < 0 ? 0 : (Math.round(box._y) - Math.round(box._height / 2));
+
+  // set x to 1/2 to the left or 0
+  const new_x: number = (Math.round(box._x) - Math.round(box._width / 2)) < 0 ? 0 : (Math.round(box._x) - Math.round(box._width / 2));
+
+  // make width 2x wider or until it hits right edge
+  const new_w: number = (new_x + Math.round(box._width * 2)) > sizes.eachSSwidth ? sizes.eachSSwidth - new_x : (Math.round(box._width * 2));
+
+  // make height 2x as tall or until it hits bottom of image
+  const new_h: number = (new_y + Math.round(box._height * 2)) > sizes.height ? sizes.height - new_y : (Math.round(box._height * 2));
+
+  return {
+    top: new_y,
+    left: new_x,
+    width: new_w,
+    height: new_h,
+  };
+
+}

i18n/en.json

@@ -131,6 +131,9 @@
     "showFoldersDescription": "Folders view",
     "showFoldersHint": "Show folders",
     "showFoldersMoreInfo": "Only affects Thumbnails, Text, and Clips views",
+    "showFacesDescription": "Faces view",
+    "showFacesHint": "Show Faces",
+    "showFacesMoreInfo": "Show faces found in the extracted screenshots",
     "showFreqDescription": "Word cloud",
     "showFreqHint": "Word cloud",
     "showFreqMoreInfo": "Show the Word cloud which shows the most frequent words in currently shown files",

main.ts

@@ -512,3 +512,46 @@ ipcMain.on('open-file', (event, pathToVhaFile) => {
 ipcMain.on('clear-recent-documents', (event): void => {
   app.clearRecentDocuments();
 });
+
+
+import { runEverything } from './face/pipeline';
+
+// ===========================================================================================
+// EXTRACT FACES !!!!!!!!! - electron messages
+// -------------------------------------------------------------------------------------------
+
+let hack: number = 0;
+
+ipc.on('extract-face', function (event, currentAngularFinalArray: ImageElement[]) {
+
+  const element: ImageElement = currentAngularFinalArray[hack];
+
+  console.log(element.fileName);
+
+  const inputFile: string = path.join(
+    globals.selectedOutputFolder,
+    'vha-' + globals.hubName,
+    '/filmstrips',
+    element.hash + '.jpg');
+
+  console.log(inputFile);
+
+  const outputFile: string = path.join(
+    globals.selectedOutputFolder,
+    'vha-' + globals.hubName,
+    '/faces',
+    element.hash + '.jpg');
+
+  try {
+    runEverything(inputFile, element.screens, outputFile, 'female');
+  } catch (err) {
+    dialog.showMessageBox(win, {
+      message: systemMessages.noSuchFileFound,
+      detail: err,
+      buttons: ['OK']
+    });
+  }
+
+  hack++;
+});
+