feat: record browser windows (#467)

* wip: getwindows command on desktop-capturer

* wip: getPidLists

* wip: get pids in windows

* wip: add pid to streaming param

* wip: windows getwindowrect

* wip: get windows w,h

* fix: ToJson error

* wip: add pid on screen-capture option

* wip: step record

* feat: capture window process

* feat: on windows capture apply width,height

* refactor: gdc to get screenId, pid as surface param

* fix: surface not playing

* feat: kill previously running safari instance and capture safari

* fix: reviewed code, validate deviceWindowInfo, add isStepMessageEventHandler() function

* fix: apply reviewed code check GetWindowThreadProcessId return

packages/typescript-private/device-server/src/bootstrap/bootstrap.module.ts

@@ -1,4 +1,7 @@
 import { Module } from '@nestjs/common';
+import { BootstrapService } from './bootstrap.service';
 
-@Module({})
+@Module({
+  providers: [BootstrapService],
+})
 export class BootstrapModule {}

packages/typescript-private/device-server/src/internal/channel/android-channel.ts

@@ -1,6 +1,7 @@
 import {
   DefaultScreenCaptureOption,
   DeviceSystemInfo,
+  DeviceWindowInfo,
   ErrorResult,
   FilledRuntimeInfo,
   Platform,
@@ -289,6 +290,10 @@ export class AndroidChannel implements DeviceChannel {
     return Adb.isPortOpen(this.serial, port);
   }
 
+  getWindows(): DeviceWindowInfo[] {
+    return [];
+  }
+
   async subscribeLog(args: string[], handler: LogHandler, printable?: Printable): Promise<Closable> {
     const { stdout, stderr } = await Adb.logcatClear(this.serial, printable);
     if (stdout) {

packages/typescript-private/device-server/src/internal/channel/ios-channel.ts

@@ -1,6 +1,7 @@
 import {
   CodeUtil,
   DeviceSystemInfo,
+  DeviceWindowInfo,
   ErrorResult,
   FilledRuntimeInfo,
   Platform,
@@ -384,6 +385,10 @@ export class IosChannel implements DeviceChannel {
     return res?.isListening ?? false;
   }
 
+  getWindows(): DeviceWindowInfo[] {
+    return [];
+  }
+
   private async findDotAppPath(appPath: string): Promise<string> {
     if (!appPath.endsWith('.ipa')) {
       throw new Error('appPath must be ipa');

packages/typescript-private/device-server/src/internal/channel/macos-channel.ts

@@ -1,6 +1,7 @@
 import {
   DefaultDeviceSystemInfo,
   DeviceSystemInfo,
+  DeviceWindowInfo,
   ErrorResult,
   FilledRuntimeInfo,
   Platform,
@@ -22,6 +23,7 @@ import { DeviceWebDriverHandler } from '../../device-webdriver/device-webdriver.
 import { SeleniumDeviceWebDriverHandler } from '../../device-webdriver/selenium.device-webdriver.handler';
 import { GamiumContext } from '../../gamium/gamium.context';
 import { logger } from '../../logger/logger.instance';
+import { DesktopCapturer } from '../externals/index';
 import { DeviceChannel, DeviceChannelOpenParam, DeviceHealthStatus, DeviceServerService, LogHandler } from '../public/device-channel';
 import { DeviceAgentService } from '../services/device-agent/device-agent-service';
 import { NullDeviceAgentService } from '../services/device-agent/null-device-agent-service';
@@ -158,6 +160,10 @@ export class MacosChannel implements DeviceChannel {
     return !isFree;
   }
 
+  async getWindows(): Promise<DeviceWindowInfo[]> {
+    return await DesktopCapturer.getWindows(logger);
+  }
+
   uninstallApp(appPath: string): void {
     throw new Error('Method not implemented.');
   }

packages/typescript-private/device-server/src/internal/channel/windows-channel.ts

@@ -1,6 +1,7 @@
 import {
   DefaultDeviceSystemInfo,
   DeviceSystemInfo,
+  DeviceWindowInfo,
   ErrorResult,
   FilledRuntimeInfo,
   Platform,
@@ -22,6 +23,7 @@ import { DeviceWebDriverHandler } from '../../device-webdriver/device-webdriver.
 import { SeleniumDeviceWebDriverHandler } from '../../device-webdriver/selenium.device-webdriver.handler';
 import { GamiumContext } from '../../gamium/gamium.context';
 import { logger } from '../../logger/logger.instance';
+import { DesktopCapturer } from '../externals/index';
 import { DeviceChannel, DeviceChannelOpenParam, DeviceHealthStatus, DeviceServerService, LogHandler } from '../public/device-channel';
 import { DeviceAgentService } from '../services/device-agent/device-agent-service';
 import { NullDeviceAgentService } from '../services/device-agent/null-device-agent-service';
@@ -180,6 +182,10 @@ export class WindowsChannel implements DeviceChannel {
     return !isFree;
   }
 
+  async getWindows(): Promise<DeviceWindowInfo[]> {
+    return await DesktopCapturer.getWindows(logger);
+  }
+
   uninstallApp(appPath: string): void {
     logger.warn('WindowsChannel.uninstallApp is not implemented yet');
   }

packages/typescript-private/device-server/src/internal/externals/cli/desktop-capturer.ts

@@ -0,0 +1,52 @@
+import { DeviceWindowInfo } from '@dogu-private/types';
+import { Printable, stringify, transformAndValidate } from '@dogu-tech/common';
+import { ChildProcess, HostPaths } from '@dogu-tech/node';
+import fs from 'fs';
+import { isArray } from 'lodash';
+import { registerBootstrapHandler } from '../../../bootstrap/bootstrap.service';
+
+const binPath = (): string => HostPaths.thirdParty.pathMap().common.desktopCapturer;
+
+export async function getWindows(printable: Printable): Promise<DeviceWindowInfo[]> {
+  tryAccessAndFix();
+  const res = await ChildProcess.exec(`${binPath()} windows --info`, {}, printable);
+  if (0 == res.stdout.length) {
+    return [];
+  }
+  const infos = JSON.parse(res.stdout) as DeviceWindowInfo[];
+  if (!isArray(infos)) {
+    throw new Error(`Invalid result: ${res.stdout}`);
+  }
+  for (const info of infos) {
+    await transformAndValidate(DeviceWindowInfo, info, { printable });
+  }
+
+  return infos;
+}
+
+const tryAccessAndFix = (): void => {
+  const bin = binPath();
+  try {
+    fs.accessSync(bin, fs.constants.X_OK);
+  } catch (error) {
+    makeAccessableSync();
+  }
+};
+
+const makeAccessableSync = (): void => {
+  try {
+    fs.chmodSync(binPath(), 0o777);
+  } catch (error) {
+    const cause = error instanceof Error ? error : new Error(stringify(error));
+    throw new Error(`Failed to chmod desktop-capturer`, { cause });
+  }
+};
+
+registerBootstrapHandler(__filename, async () => {
+  try {
+    await fs.promises.chmod(binPath(), 0o777);
+  } catch (error) {
+    const cause = error instanceof Error ? error : new Error(stringify(error));
+    throw new Error(`Failed to chmod desktop-capturer`, { cause });
+  }
+});

packages/typescript-private/device-server/src/internal/externals/index.ts

@@ -1,6 +1,7 @@
 export * as Adb from './cli/adb/adb';
 export * as AdbUtil from './cli/adb/adb-util';
 export * from './cli/adb/info';
+export * as DesktopCapturer from './cli/desktop-capturer';
 export * as IdeviceDiagnostics from './cli/idevicediagnostics';
 export * as IdeviceSyslog from './cli/idevicesyslog';
 export * as IosDeviceAgent from './cli/ios-device-agent';

packages/typescript-private/device-server/src/internal/public/device-channel.ts

@@ -1,5 +1,6 @@
 import {
   DeviceSystemInfo,
+  DeviceWindowInfo,
   ErrorResult,
   FilledRuntimeInfo,
   Platform,
@@ -81,6 +82,7 @@ export interface DeviceChannel {
   subscribeLog(args: string[], handler: LogHandler, printable?: Printable): PromiseOrValue<Closable>;
   joinWifi(ssid: string, password: string): PromiseOrValue<void>;
   isPortListening(port: number): PromiseOrValue<boolean>;
+  getWindows(): PromiseOrValue<DeviceWindowInfo[]>;
 
   // app
   uninstallApp(appPath: string, printable?: Printable): PromiseOrValue<void>;

packages/typescript-private/device-server/src/ws/device/find-windows.service.ts

@@ -0,0 +1,87 @@
+import { OnWebSocketClose, OnWebSocketMessage, WebSocketGatewayBase, WebSocketRegistryValueAccessor, WebSocketService } from '@dogu-private/nestjs-common';
+import { categoryFromPlatform, platformTypeFromPlatform, Serial } from '@dogu-private/types';
+import { closeWebSocketWithTruncateReason, DuplicatedCallGuarder, errorify, Instance } from '@dogu-tech/common';
+import { DeviceFindWindows } from '@dogu-tech/device-client-common';
+import { getChildProcessIds, getProcessesMapMacos } from '@dogu-tech/node';
+import { IncomingMessage } from 'http';
+import WebSocket from 'ws';
+import { DoguLogger } from '../../logger/logger';
+import { ScanService } from '../../scan/scan.service';
+
+interface Value {
+  serial: Serial;
+  parentPid: number;
+  timer: NodeJS.Timer | null;
+  updateGuard: DuplicatedCallGuarder;
+}
+
+@WebSocketService(DeviceFindWindows)
+export class DeviceFindWindowsService
+  extends WebSocketGatewayBase<Value, typeof DeviceFindWindows.sendMessage, typeof DeviceFindWindows.receiveMessage>
+  implements OnWebSocketMessage<Value, typeof DeviceFindWindows.sendMessage, typeof DeviceFindWindows.receiveMessage>, OnWebSocketClose<Value>
+{
+  constructor(private readonly scanService: ScanService, private readonly logger: DoguLogger) {
+    super(DeviceFindWindows, logger);
+  }
+
+  override onWebSocketOpen(webSocket: WebSocket, incommingMessage: IncomingMessage): Value {
+    return { serial: '', parentPid: 0, timer: null, updateGuard: new DuplicatedCallGuarder() };
+  }
+
+  async onWebSocketMessage(webSocket: WebSocket, message: Instance<typeof DeviceFindWindows.sendMessage>, valueAccessor: WebSocketRegistryValueAccessor<Value>): Promise<void> {
+    const { updateGuard } = valueAccessor.get();
+    const { serial, parentPid, isSafari } = message;
+    const deviceChannel = this.scanService.findChannel(serial);
+    if (deviceChannel === null) {
+      throw new Error(`Device with serial ${serial} not found`);
+    }
+
+    const categoryPlatform = categoryFromPlatform(platformTypeFromPlatform(deviceChannel.platform));
+    if (categoryPlatform !== 'desktop') {
+      throw new Error(`Device with serial ${serial} is not desktop`);
+    }
+
+    const timer = setInterval(() => {
+      updateGuard
+        .guard(async () => {
+          const deviceChannel = this.scanService.findChannel(serial);
+          if (deviceChannel === null) {
+            throw new Error(`Device with serial ${serial} not found`);
+          }
+          const windows = await deviceChannel.getWindows();
+          const childProcess = await getChildProcessIds(parentPid, this.logger);
+          if (isSafari && process.platform === 'darwin') {
+            const procs = await getProcessesMapMacos(this.logger);
+            const safariProc = Array.from(procs).find((proc) => proc[1].commandLine.includes('Safari.app/Contents/MacOS/Safari'));
+            if (safariProc) {
+              childProcess.push(safariProc[0]);
+            }
+          }
+          const targetWindow = windows.find((window) => childProcess.includes(window.pid));
+          if (!targetWindow) {
+            return;
+          }
+          this.send(webSocket, {
+            pid: targetWindow.pid,
+            width: targetWindow.width,
+            height: targetWindow.height,
+          });
+          await Promise.resolve();
+        })
+        .catch((error) => {
+          this.logger.error('Failed to find windows', { error: errorify(error) });
+        });
+    }, 1000);
+    valueAccessor.update({ serial, parentPid, timer, updateGuard });
+    await Promise.resolve();
+  }
+
+  async onWebSocketClose(webSocket: WebSocket, event: WebSocket.CloseEvent, valueAccessor: WebSocketRegistryValueAccessor<Value>): Promise<void> {
+    const { timer } = valueAccessor.get();
+    if (timer) {
+      clearInterval(timer);
+    }
+    closeWebSocketWithTruncateReason(webSocket, 1000, 'Finding windows finished');
+    await Promise.resolve();
+  }
+}

packages/typescript-private/device-server/src/ws/ws.module.ts

@@ -5,6 +5,7 @@ import { ScanModule } from '../scan/scan.module';
 import { DeviceHostDownloadSharedResourceWebSocketService } from './device-host/download-shared-resource';
 import { DeviceHostUploadFileService } from './device-host/upload-file.service';
 import { DeviceConnectionSubscribeService } from './device/connection-subscribe.service';
+import { DeviceFindWindowsService } from './device/find-windows.service';
 import { DeviceForwardService } from './device/forward.service';
 import { DeviceInstallAppService } from './device/install-app.service';
 import { DeviceJoinWifiService } from './device/join-wifi.service';
@@ -38,6 +39,7 @@ import { DeviceWebSocketRelayService } from './device/websocket-relay.service';
     DeviceRunAppiumServerService,
     DeviceHostUploadFileService,
     DeviceHostDownloadSharedResourceWebSocketService,
+    DeviceFindWindowsService,
   ],
 })
 export class WsModule {}

packages/typescript-private/host-agent/src/device-job/device-job.events.ts

@@ -1,5 +1,5 @@
 import { DeviceJobStatusInfo, StepStatusInfo } from '@dogu-private/console-host-agent';
-import { DeviceId, DeviceJobLog, OrganizationId, Platform, RoutineDeviceJobId, Serial } from '@dogu-private/types';
+import { BrowserName, DeviceId, DeviceJobLog, OrganizationId, Platform, RoutineDeviceJobId, Serial } from '@dogu-private/types';
 import { createEventDefinition, IsFilledString } from '@dogu-tech/common';
 import { Type } from 'class-transformer';
 import { IsArray, IsDate, IsEnum, IsIn, IsNumber, IsOptional, IsUUID, ValidateNested } from 'class-validator';
@@ -29,6 +29,10 @@ export class OnDeviceJobStartedEventValue extends OnDeviceJobEventValueBase {
   @IsArray()
   stepStatusInfos!: StepStatusInfo[];
 
+  @IsIn(BrowserName)
+  @IsOptional()
+  browserName?: BrowserName;
+
   @IsFilledString()
   recordDeviceRunnerPath!: string;
 }

packages/typescript-private/host-agent/src/device-job/device-job.module.ts

@@ -6,11 +6,22 @@ import { DeviceJobLogProcessRegistry } from './device-job.device-log-process-reg
 import { DeviceJobHeartbeater } from './device-job.heartbeater';
 import { DeviceJobLogger } from './device-job.logger';
 import { DeviceJobRecordingProcessRegistry } from './device-job.recording-process-registry';
+import { DeviceJobRecordingService } from './device-job.recording-service';
+import { DeviceJobRecordingWindowProcessRegistry } from './device-job.recording-windows-process-registry';
 import { DeviceJobUpdater } from './device-job.updater';
 
 @Module({
   imports: [ConsoleClientModule, ProcessorModule],
-  providers: [DeviceJobHeartbeater, DeviceJobUpdater, DeviceJobRecordingProcessRegistry, DeviceJobContextRegistry, DeviceJobLogProcessRegistry, DeviceJobLogger],
+  providers: [
+    DeviceJobHeartbeater,
+    DeviceJobUpdater,
+    DeviceJobRecordingService,
+    DeviceJobRecordingProcessRegistry,
+    DeviceJobRecordingWindowProcessRegistry,
+    DeviceJobContextRegistry,
+    DeviceJobLogProcessRegistry,
+    DeviceJobLogger,
+  ],
   exports: [DeviceJobContextRegistry, DeviceJobLogger],
 })
 export class DeviceJobModule {}