Merge branch 'master' of github.com:WPO-Foundation/wptagent

Dockerfile

@@ -5,6 +5,7 @@ RUN apt-get update && \
   apt-get install -y \
     wget \
     curl \
+    git \
     python \
     python-pip \
     xvfb \
@@ -25,7 +26,7 @@ RUN apt-get update && \
     iproute2 \
     software-properties-common && \
 # Node setup
-  curl -sL https://deb.nodesource.com/setup_10.x | sudo -E bash - && \
+  curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash - && \
   wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
   wget -qO- https://deb.opera.com/archive.key | apt-key add - && \
   echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list && \
@@ -61,12 +62,12 @@ RUN apt-get update && \
     pillow \
     psutil \
     requests \
-    ujson \
+    git+git://github.com/marshallpierce/[email protected] \
     tornado \
     wsaccel \
     xvfbwrapper \
     brotli \
-    fonttools \
+    'fonttools>=3.44.0,<4.0.0' \
     marionette_driver
 
 COPY wptagent.py /wptagent/wptagent.py

LICENSE

@@ -1,3 +1,4 @@
+Copyright 2019 WebPageTest LLC.
 Copyright 2017, Google Inc.
 
 Licensed under the Apache License, Version 2.0 (the "License");

alive.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
-# Copyright 2018 Google Inc. All rights reserved.
+# Copyright 2019 WebPageTest LLC.
+# Copyright 2018 Google Inc.
 # Use of this source code is governed by the Apache 2.0 license that can be
 # found in the LICENSE file.
 """Watchdog helper"""
@@ -25,33 +26,33 @@ def main():
         with open('/proc/uptime', 'r') as f_in:
             uptime_seconds = int(float(f_in.readline().split()[0]))
             if uptime_seconds < 3600:
-                print 'OK: Freshly booted ({0:d} seconds)'.format(uptime_seconds)
+                print('OK: Freshly booted ({0:d} seconds)'.format(uptime_seconds))
                 exit(0)
     elif platform.system() == "Windows":
         uptime_seconds = int(time.time()) - int(psutil.boot_time())
         if uptime_seconds < 3600:
-            print 'OK: Freshly booted ({0:d} seconds)'.format(uptime_seconds)
+            print('OK: Freshly booted ({0:d} seconds)'.format(uptime_seconds))
             exit(0)
 
     # Check if the watchdog file has been updated in the last hour.
     if options.file and os.path.isfile(options.file):
         elapsed = int(time.time() - os.path.getmtime(options.file))
         if elapsed < 3600:
-            print 'OK: File last modified {0:d} seconds ago'.format(elapsed)
+            print('OK: File last modified {0:d} seconds ago'.format(elapsed))
             exit(0)
 
     # Ping the provided address if requested.
     if options.ping and platform.system() != "Windows":
         response = os.system('ping -c 2 -i 0.2 -n -W 1 {0} > /dev/null 2>&1'.format(options.ping))
         if response == 0:
-            print 'OK: ping succeeded'
+            print('OK: ping succeeded')
             # Update the alive file to avoid pinging all the time
             if options.file:
                 with open(options.file, 'a'):
                     os.utime(options.file, None)
             exit(0)
 
-    print 'FAIL: No checks passed'
+    print('FAIL: No checks passed')
     if options.reboot:
         if platform.system() == 'Windows':
             subprocess.call(['shutdown', '/r', '/f'])

docs/install.md

@@ -49,7 +49,7 @@ wptagent currently supports Windows, Linux and OSX for desktop browsers as well
 ## For lighthouse testing
 * NodeJS
     * Ubuntu/Debian:
-        * ```curl -sL https://deb.nodesource.com/setup_10.x | sudo -E bash -```
+        * ```curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash -```
         * ```sudo apt-get install -y nodejs```
 * The lighthouse npm module
     * ```sudo npm install -g lighthouse```

internal/adb.py

@@ -1,4 +1,5 @@
-# Copyright 2017 Google Inc. All rights reserved.
+# Copyright 2019 WebPageTest LLC.
+# Copyright 2017 Google Inc.
 # Use of this source code is governed by the Apache 2.0 license that can be
 # found in the LICENSE file.
 """ADB command-line interface"""
@@ -7,9 +8,13 @@
 import platform
 import re
 import subprocess
+import sys
 from threading import Timer
 import time
-import monotonic
+if (sys.version_info > (3, 0)):
+    from time import monotonic
+else:
+    from monotonic import monotonic
 
 # cSpell:ignore vpndialogs, sysctl, iptables, ifconfig, dstaddr, clientidbase, nsecs
 
@@ -33,7 +38,7 @@ def __init__(self, options, cache_dir):
         self.simplert_path = None
         self.simplert = None
         self.no_network_count = 0
-        self.last_network_ok = monotonic.monotonic()
+        self.last_network_ok = monotonic()
         self.needs_exit = False
         self.rebooted = False
         self.vpn_forwarder = None
@@ -135,7 +140,10 @@ def start(self):
             self.simplert_path = None
             if self.options.simplert is not None and platform.system() == 'Linux':
                 running = False
-                stdout = subprocess.check_output(['ps', 'ax'])
+                if (sys.version_info > (3, 0)):
+                    stdout = subprocess.check_output(['ps', 'ax'], encoding='UTF-8')
+                else:
+                    stdout = subprocess.check_output(['ps', 'ax'])
                 if stdout.find('simple-rt ') > -1:
                     running = True
                     logging.debug('simple-rt is already running')
@@ -218,7 +226,7 @@ def start_screenrecord(self):
                                           '/data/local/tmp/wpt_video.mp4'])
             self.screenrecord = subprocess.Popen(cmd)
         except Exception:
-            pass
+            logging.exception('Error starting screenrecord')
 
     def stop_screenrecord(self, local_file):
         """Stop a screen record and download the video to local_file"""
@@ -248,7 +256,7 @@ def start_tcpdump(self):
             logging.debug(' '.join(cmd))
             self.tcpdump = subprocess.Popen(cmd)
         except Exception:
-            pass
+            logging.exception('Error starting tcpdump')
 
     def stop_tcpdump(self, local_file):
         """Stop a tcpdump capture and download to local_file"""
@@ -305,7 +313,7 @@ def is_installed(self, package):
 
     def cleanup_device(self):
         """Do some device-level cleanup"""
-        start = monotonic.monotonic()
+        start = monotonic()
         # Simulate pressing the home button to dismiss any UI
         self.shell(['input', 'keyevent', '3'])
         # Clear notifications
@@ -343,7 +351,7 @@ def cleanup_device(self):
         if out.find('com.motorola.ccc.ota/com.motorola.ccc.ota.ui.DownloadActivity') >= 0:
             self.shell(['am', 'force-stop', 'com.motorola.ccc.ota'])
         # reboot the phone and exit the agent if it is running EXTREMELY slowly
-        elapsed = monotonic.monotonic() - start
+        elapsed = monotonic() - start
         if elapsed > 300:
             logging.debug("Cleanup took %0.3f seconds. Rebooting the phone and restarting agent",
                           elapsed)
@@ -506,8 +514,8 @@ def check_simplert(self):
             self.su('setprop sys.usb.config adb')
             self.adb(['wait-for-device'])
             # wait up to 30 seconds for the interface to come up
-            end_time = monotonic.monotonic() + 30
-            while not is_ready and monotonic.monotonic() < end_time:
+            end_time = monotonic() + 30
+            while not is_ready and monotonic() < end_time:
                 time.sleep(1)
                 self.dismiss_vpn_dialog()
                 is_ready = self.is_tun_interface_available()
@@ -582,7 +590,6 @@ def check_gnirehtet(self):
         if self.is_tun_interface_available():
             is_ready = True
         elif self.gnirehtet_exe is not None:
-            interface, dns_server = self.options.gnirehtet.split(',', 1)
             if self.gnirehtet is not None:
                 try:
                     subprocess.call([self.gnirehtet_exe, 'stop'])
@@ -608,8 +615,8 @@ def check_gnirehtet(self):
             self.dismiss_vpn_dialog()
             # Simulate pressing the home button to dismiss any UI
             self.shell(['input', 'keyevent', '3'])
-            end = monotonic.monotonic() + 30
-            while not is_ready and monotonic.monotonic() < end:
+            end = monotonic() + 30
+            while not is_ready and monotonic() < end:
                 if self.is_tun_interface_available():
                     is_ready = True
                 else:
@@ -672,7 +679,7 @@ def is_device_ready(self):
             net_ok = False
             if self.ping(self.ping_address) is not None:
                 self.no_network_count = 0
-                self.last_network_ok = monotonic.monotonic()
+                self.last_network_ok = monotonic()
                 self.rebooted = False
                 net_ok = True
             else:
@@ -714,7 +721,7 @@ def is_device_ready(self):
                 is_ready = False
         if not is_ready:
             needs_kick = False
-            elapsed = monotonic.monotonic() - self.last_network_ok
+            elapsed = monotonic() - self.last_network_ok
             if self.no_network_count > 20:
                 needs_kick = True
             elif self.no_network_count > 1 and elapsed > 1800:

internal/android_browser.py

@@ -1,4 +1,5 @@
-# Copyright 2017 Google Inc. All rights reserved.
+# Copyright 2019 WebPageTest LLC.
+# Copyright 2017 Google Inc.
 # Use of this source code is governed by the Apache 2.0 license that can be
 # found in the LICENSE file.
 """Base class support for android browsers"""
@@ -8,12 +9,19 @@
 import os
 import shutil
 import subprocess
+import sys
 import time
-import monotonic
+if (sys.version_info > (3, 0)):
+    from time import monotonic
+    GZIP_TEXT = 'wt'
+else:
+    from monotonic import monotonic
+    GZIP_TEXT = 'w'
 try:
     import ujson as json
 except BaseException:
     import json
+
 from .base_browser import BaseBrowser
 
 
@@ -65,7 +73,7 @@ def prepare(self, job, task):
                                              self.config['package'] + '.md5')
             last_md5 = None
             if os.path.isfile(last_install_file):
-                with open(last_install_file, 'rb') as f_in:
+                with open(last_install_file, 'r') as f_in:
                     last_md5 = f_in.read()
             if last_md5 is None or last_md5 != self.config['md5']:
                 valid = False
@@ -91,7 +99,7 @@ def prepare(self, job, task):
                         if md5 == self.config['md5']:
                             valid = True
                 except Exception:
-                    pass
+                    logging.exception('Error downloading browser update')
                 if os.path.isfile(tmp_file):
                     if valid:
                         # Uninstall the previous install of the same package if we are installing a custom browser.
@@ -100,7 +108,7 @@ def prepare(self, job, task):
                             self.adb.adb(['uninstall', self.config['package']])
                         logging.debug('Installing browser APK')
                         self.adb.adb(['install', '-rg', tmp_file])
-                        with open(last_install_file, 'wb') as f_out:
+                        with open(last_install_file, 'w') as f_out:
                             f_out.write(md5)
                     else:
                         logging.error('Error downloading browser APK')
@@ -269,7 +277,7 @@ def wait_for_processing(self, task):
                     if self.tcpdump_file is not None:
                         os.remove(self.tcpdump_file)
             except Exception:
-                pass
+                logging.exception('Error processing tcpdump')
 
     def step_complete(self, task):
         """All of the processing for the current test step is complete"""
@@ -281,11 +289,11 @@ def step_complete(self, task):
                 task['page_data']['eventName'] = task['step_name']
             if 'run_start_time' in task:
                 task['page_data']['test_run_time_ms'] = \
-                    int(round((monotonic.monotonic() - task['run_start_time']) * 1000.0))
+                    int(round((monotonic() - task['run_start_time']) * 1000.0))
             path = os.path.join(task['dir'], task['prefix'] + '_page_data.json.gz')
             json_page_data = json.dumps(task['page_data'])
             logging.debug('Page Data: %s', json_page_data)
-            with gzip.open(path, 'wb', 7) as outfile:
+            with gzip.open(path, GZIP_TEXT, 7) as outfile:
                 outfile.write(json_page_data)
 
     def screenshot(self, task):

internal/base_browser.py

@@ -1,4 +1,5 @@
-# Copyright 2017 Google Inc. All rights reserved.
+# Copyright 2019 WebPageTest LLC.
+# Copyright 2017 Google Inc.
 # Use of this source code is governed by the Apache 2.0 license that can be
 # found in the LICENSE file.
 """Base class support for browsers"""