Go live on agents

.dockerignore

@@ -1 +1,19 @@
+# hidden unused files
 .git
+.vscode
+.gitignore
+.travis.yml
+
+# unused folders
+docs
+test
+
+# unused script and files
+README.md
+alive.bat
+browsers.ini.sample
+centos_install.sh
+ios_install.sh
+ubuntu_install.sh
+windows_install.ps1
+windows_post_reboot.ps1

.gitattributes

@@ -0,0 +1,2 @@
+# Ensure that linux files have lf endings even when working on windows
+*.sh text eol=lf

.github/workflows/requirements.txt

@@ -17,3 +17,5 @@ brotli
 xvfbwrapper
 google-cloud-storage
 google-cloud-pubsub
+pytz
+tzlocal

Dockerfile

@@ -7,74 +7,65 @@
 ###     docker run -it -d TAGNAME /bin/bash         - Runs bash
 ###     docker exec -it <IMAGE ID> /bin/bash        - Connects to bash for terminal execution (Needs to be running first)
 
-### EXAMPLE DOCKER COMMANDS FOR RUNNING SERVER & AGENT
-
-###     docker run -d -p 4000:80 <IMAGE ID || <IMAGE TAG>
-###     docker run -d -p 4001:80 --network="host" -e "SERVER_URL=http://localhost:4000/work/" -e "LOCATION=Test" -e "-v" <IMAGE ID || <IMAGE TAG>
-
 ### INSTALLING METHOD ###
 
 ###     Recommend to install with "docker build <GITHUB-REPO-LINK> -t TAGNAME",
 ###     grabs the latest copy of WPT and build time on average takes 10 minutes. 
 
-FROM ubuntu:22.04
-
-### PREVENTs INTERACTIVE PROMPTS WHILE INSTALLING ###
-ARG DEBIAN_FRONTEND=noninteractive
-
-### COPYING ENTIRE DIR TO LOCAL DOCKER /wptagent
-COPY / /wptagent
-RUN apt-get update
+FROM ubuntu:22.04 as production
 
-# Git Clone Install
-# RUN apt-get install -y git
-# RUN git clone -b dockerfile https://github.com/sammeboy635/wptagent.git
-
-RUN curl -sL https://deb.nodesource.com/setup_16.x | bash -
+### TIMEZONE INSIDE THE CONTAINER ###
+ARG TIMEZONE=UTC
 
 ### UPDATE ###
-RUN apt-get update 
+RUN curl -sL https://deb.nodesource.com/setup_16.x | bash -
+RUN apt update 
 
 ### INSTALL APT-GET LIBS ###
-RUN apt-get install -y \
+# DEBIAN_FRONTEND prevents interactive prompts while installing
+# set default timezone beforehand to avoid user interaction for tzdata package
+RUN ln -fs /usr/share/zoneinfo/$TIMEZONE /etc/localtime && DEBIAN_FRONTEND=noninteractive apt install -y \
     python3 python3-pip python3-ujson \
     imagemagick dbus-x11 traceroute software-properties-common psmisc libnss3-tools iproute2 net-tools openvpn \
     libtiff5-dev libjpeg-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python3-tk \
     python3-dev libavutil-dev libmp3lame-dev libx264-dev yasm autoconf automake build-essential libass-dev libfreetype6-dev libtheora-dev \
     libtool libvorbis-dev pkg-config texi2html libtext-unidecode-perl python3-numpy python3-scipy perl \
     adb ethtool nodejs cmake git-core libsdl2-dev libva-dev libvdpau-dev libxcb1-dev libxcb-shm0-dev libxcb-xfixes0-dev texinfo wget \
-    ttf-mscorefonts-installer fonts-noto fonts-roboto fonts-open-sans ffmpeg npm
+    ttf-mscorefonts-installer fonts-noto fonts-roboto fonts-open-sans ffmpeg npm sudo curl xvfb
 
-### Update the font cache
+### UPDATE FONT CACHE ###
 RUN fc-cache -f -v
 
-### UPGRADING PIP AND INSTALLING REQUIRED PACKAGES ###
-RUN python3 -m pip install --upgrade --user pip && \
-    python3 -m pip install --user -r /wptagent/.github/workflows/requirements.txt 
-
 ### INSTALLING LIGHTHOUSE FROM NPM ###
 RUN npm install -g lighthouse
 
 ### INSTALLING CHROME BROWSER ###
-###     Fails to Find all libs needed to run
-# RUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
-#     dpkg -i google-chrome-stable_current_amd64.deb; exit 0 && \
-#     apt -f install -y && \
-#     apt-get install google-chrome-stable
+RUN curl -o /tmp/google-chrome-stable_current_amd64.deb  https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
+    apt install -y /tmp/google-chrome-stable_current_amd64.deb && rm /tmp/google-chrome-stable_current_amd64.deb
 
-### BETTER INSTALLING CHROME BROWSER METHOD ###
-###     Better Installing method but would like to change this to something less complex.
-RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
-    && echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list
-RUN apt-get update && apt-get -y install google-chrome-stable ; exit 0
-RUN apt-get update --fix-missing -y
-RUN apt-get install -f -y
+### UPGRADING PIP AND INSTALLING REQUIRED PACKAGES ###
+COPY /.github/workflows/requirements.txt /tmp/agent_requirements.txt
+RUN python3 -m pip install --upgrade --user pip && \
+    python3 -m pip install --user -r /tmp/agent_requirements.txt && \
+    rm /tmp/agent_requirements.txt
 
-### CLEAN UP ###
-#       We could add some clean up here but in testing it was negotiable
+### COPYING ENTIRE DIR TO LOCAL DOCKER /wptagent ###
+# see .dockerignore for filterd out folders
+# source copy last so we don't need to rebuild all the other layers 
+COPY / /wptagent
+WORKDIR /wptagent
 
+ENTRYPOINT ["/bin/sh", "/wptagent/docker/linux-headless/entrypoint.sh"]
 
-WORKDIR /wptagent
+### DEBUG CONTAINER ###
+FROM production as debug
+
+### INSTALLING DEBUG DEPENDENCIES ###
+RUN pip install debugpy
+
+### COPY DEBUG AGENT AND MOVE REAL ONE ###
+RUN mv wptagent.py wptagent_starter.py
+COPY wptagent_debug.py wptagent.py
 
-### /bin/bash LOCATION OF COMMAND EXECUTION ###
-CMD ["/bin/bash", "/wptagent/docker/linux-headless/entrypoint.sh"]
+### SETTING PRODUCTION BUILD AS DEFAULT ###
+FROM production

docker/linux-headless/entrypoint.sh

@@ -29,4 +29,4 @@ fi
 
 # exec replaces the shell process by the python process and is required to
 # propagate signals (i.e. SIGTERM)
-exec python3 /wptagent/wptagent.py --server "${SERVER_URL}" --location "${LOCATION}" ${EXTRA_ARGS} --xvfb --dockerized -vvvv
+exec python3 /wptagent/wptagent.py --server "${SERVER_URL}" --location "${LOCATION}" ${EXTRA_ARGS} --xvfb --dockerized -vvvv "@"

docs/docker.md

@@ -1,45 +1,85 @@
-# Linux Headless Agent
+# Docker Linux Headless Agent
 
-To run the agent, simply specify a few environment variables with docker:
+The `Dockerfile` has multi-stage definition:
+* **production**: Default stage, produce a image without debug features;
+* **debug**: When running the produced image the wptagent script will wait for a debugger to.attach
+
+## Build the Image
+
+Arguments can be passed at build time:
+* **TIMEZONE**: to set the timezone inside the container. Default `UTC.`
+
+To build the production container with UTC timezone
+```bash
+docker build --tag wptagent .
+```
 
-* `SERVER_URL`: will be passed as `--server` (note: it must end with '/work/')
-* `LOCATION`: will be passed as `--location`
-* `KEY`: will be passed as `--key`
-* `NAME`: will be passed as `--name` (optional)
-* `SHAPER`: will be passed as `--shaper` (optional)
-* `EXTRA_ARGS`: extra command-line options that will be passed through verbatim (optional)
+changing the timezone at build time
+```bash
+docker build --build-arg TIMEZONE=EST .
+```
+
+To build the debug container
+```bash
+docker build --target debug --tag wptagent-debug .
+```
 
 ## Prerequisites to use traffic shaping in docker
 **Experimental**: Running the agent with traffic shaping is experimental. It might
 have influence on the host system network. Running multiple agents on the
 same host might result in incorrect traffic shaping.
 
 For traffic shaping to work correctly, you need to load the ifb module on the **host**:
-
+```bash
     sudo modprobe ifb numifbs=1
+```
 
 Also, the container needs `NET_ADMIN` capabilities, so run the container with 
 `--cap-add=NET_ADMIN`.
 
-To disable traffic-shaping, pass SHAPER="none".
-
-## Container Disk Space Fills Up Quickly
+To disable traffic-shaping, pass environment variable at docker un `SHAPER="none"`.
 
-If you see disk space within the container filling up rapidly and you notice
-core dump files in the /wptagent folder, try adding `--shm-size=1g` to your Docker run
-command. This can help resolve an issue with shared memory and headless Chrome in Docker.
+## Run the container
+To run the agent, simply specify a few environment variables with docker:
 
-## Example
+- `SERVER_URL`: will be passed as `--server` (note: it must end with '/work/')
+- `LOCATION`: will be passed as `--location`
+- `KEY`: will be passed as `--key`
+- `NAME`: will be passed as `--name` (optional)
+- `SHAPER`: will be passed as `--shaper` (optional)
+- `EXTRA_ARGS`: extra command-line options that will be passed through verbatim (optional)
 
-Build the image first (from project root), load ifb and start it the container:
+Build the image first (from project root), load ifb and start it the container.
 
-    sudo docker build --tag wptagent .
+A typical run :
+```bash
     sudo modprobe ifb numifbs=1
-    sudo docker run -d \
+    docker build --tag wptagent .
+    docker run -d \
       -e SERVER_URL="http://my-wpt-server.org/work/" \
       -e LOCATION="docker-location" \
       -e NAME="Docker Test" \
       --cap-add=NET_ADMIN \
       --init \
       wptagent
+```
+
+Additional parameters can be also passed as additional commands. 
+A typical run in debug mode, note that we need to expose the port as `50000`:
+```bash
+sudo modprobe ifb numifbs=1
+docker run -d \
+    -e SERVER_URL=http://127.0.0.1:80/work/ \
+    -e LOCATION=Test \
+    --init \
+    --cap-add=NET_ADMIN \
+    -p 50000:50000 \
+    wptagent-debug
+    --key 123456789
+```
 
+## Container Disk Space Fills Up Quickly
+
+If you see disk space within the container filling up rapidly and you notice
+core dump files in the /wptagent folder, try adding `--shm-size=1g` to your Docker run
+command. This can help resolve an issue with shared memory and headless Chrome in Docker.

internal/chrome_desktop.py

@@ -69,6 +69,13 @@
     'OfflinePagesPrefetching',
     'OptimizationHints',
     'Translate',
+    # Disable noisy Edge features
+    'msAutofillEdgeCoupons',
+    'msShoppingTrigger',
+    'msEdgeShoppingUI',
+    'msEntityExtraction',
+    'msEntityExtractionProactive',
+    'msWebAssist',
 ]
 
 ENABLE_BLINK_FEATURES = [

internal/devtools_browser.py

@@ -854,7 +854,12 @@ def run_lighthouse_test(self, task):
             else:
                 cpu_throttle = '{:.3f}'.format(self.job['throttle_cpu']) if 'throttle_cpu' in self.job else '1'
                 if self.job['dtShaper']:
-                    command.extend(['--throttling-method', 'devtools', '--throttling.requestLatencyMs', '150', '--throttling.downloadThroughputKbps', '1600', '--throttling.uploadThroughputKbps', '768', '--throttling.cpuSlowdownMultiplier', cpu_throttle])
+                    if self.options.android or ('mobile' in self.job and self.job['mobile']):
+                        # 1.6Mbps down, 750Kbps up, 150ms RTT
+                        command.extend(['--throttling-method', 'devtools', '--throttling.requestLatencyMs', '150', '--throttling.downloadThroughputKbps', '1600', '--throttling.uploadThroughputKbps', '750', '--throttling.cpuSlowdownMultiplier', cpu_throttle])
+                    else:
+                        # 10Mbps, 40ms RTT
+                        command.extend(['--throttling-method', 'devtools', '--throttling.requestLatencyMs', '40', '--throttling.downloadThroughputKbps', '10240', '--throttling.uploadThroughputKbps', '10240', '--throttling.cpuSlowdownMultiplier', cpu_throttle])
                 elif 'throttle_cpu_requested' in self.job and self.job['throttle_cpu_requested'] > 1:
                     command.extend(['--throttling-method', 'devtools', '--throttling.requestLatencyMs', '0', '--throttling.downloadThroughputKbps', '0', '--throttling.uploadThroughputKbps', '0', '--throttling.cpuSlowdownMultiplier', cpu_throttle])
                 else:
@@ -874,7 +879,7 @@ def run_lighthouse_test(self, task):
                 command.extend(['--screenEmulation.disabled'])
             if 'user_agent_string' in self.job:
                 sanitized_user_agent = re.sub(r'[^a-zA-Z0-9_\-.;:/()\[\] ]+', '', self.job['user_agent_string'])
-                command.append('--chrome-flags="--user-agent=\'{0}\'"'.format(sanitized_user_agent))
+                command.extend(['--emulatedUserAgent', "'" + sanitized_user_agent + "'"])
             if len(task['block']):
                 for pattern in task['block']:
                     pattern = "'" + pattern.replace("'", "'\\''") + "'"

internal/firefox.py

@@ -261,20 +261,12 @@ def driver_set_pref(self, key, value):
         """Set a Firefox pref at runtime"""
         if self.driver is not None:
             try:
-                script = 'const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");'
-                script += 'Services.prefs.'
-                if isinstance(value, bool):
-                    script += 'setBoolPref'
-                elif isinstance(value, (str, unicode)):
-                    script += 'setStringPref'
-                else:
-                    script += 'setIntPref'
-                script += '({0}, {1});'.format(json.dumps(key), json.dumps(value))
-                logging.debug(script)
+                script = 'const { Preferences } = ChromeUtils.importESModule("resource://gre/modules/Preferences.sys.mjs");'
+                script += f'Preferences.set({json.dumps(key)}, {json.dumps(value)});'
                 self.driver.set_context(self.driver.CONTEXT_CHROME)
                 self.driver.execute_script(script)
-            except Exception:
-                logging.exception("Error setting pref")
+            except Exception as err:
+                logging.exception("Error setting pref %s => %s: %s", key, value, err)
             finally:
                 self.driver.set_context(self.driver.CONTEXT_CONTENT)
 

internal/rfc5424logging/__init__.py

@@ -0,0 +1,10 @@
+"""
+This library offers an logging in rfc5424 format.
+
+Is based on https://github.com/jobec/rfc5424-logging-handler that is not currently maintained.
+
+A few changes needed to be implemented anyway to tailor the logging to wptagent, hence the hard copy.
+"""
+
+from .handler import Rfc5424SysLogHandler
+from .rfc5424logging_context import logging_context