Merge pull request #252 from my8100/bugfix_scrapyd_v150

Bugfix scrapyd v150

.circleci/config.yml

@@ -2,6 +2,7 @@
 version: 2.1
 orbs:
   codecov: codecov/[email protected]
+  allure: ayte/[email protected]
 jobs:
   py39: &test-template
     docker:
@@ -15,7 +16,7 @@ jobs:
         default: false
       use-scrapyd-v143:
         type: boolean
-        default: true
+        default: false
       use-git:
         type: boolean
         default: false
@@ -31,16 +32,46 @@ jobs:
       use-mysql:
         type: boolean
         default: false
+      allure-version:
+        description: Allure version to use
+        type: string
+        default: 2.13.1
+      allure-configuration-path:
+        description: Path to Allure configuration, uses default one if omitted
+        type: string
+        default: /usr/local/share/allure/config/allure.yml
+      allure-target-path:
+        description: Path for report directory
+        type: string
+        default: allure-report
+      allure-results-path:
+        description: Path to directory with test results
+        type: string
+        default: allure-results
+      allure-artifact-path:
+        description: Path that will be used when storing result as artifact
+        type: string
+        default: Report/Allure
     steps:
       - run:
           name: Install telnet
           command: |
             sudo apt-get update && sudo apt-get install telnet
+      - run:
+          name: Install Java 11
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y openjdk-11-jdk
+      - run:
+          name: Set JAVA_HOME
+          command: |
+            echo "export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64" >> $BASH_ENV
+            source $BASH_ENV
       - run:
           name: Setup env
           command: |
             mkdir ~/logs
-            ls -l ~
+            ls -la ~
       - checkout
       - when:
           condition: <<parameters.is-py27>>
@@ -154,11 +185,12 @@ jobs:
           name: Run tests
           command: |
             pip list
-            ls -l
+            ls -la
             . venv/bin/activate
             flake8 . --count --exclude=./venv* --select=E9,F63,F7,F82 --show-source --statistics
             coverage erase
-            coverage run --source=scrapydweb -m pytest tests -s -vv -l --disable-warnings
+            # tests/test_schedule.py
+            coverage run --source=scrapydweb -m pytest -s -vv -l --disable-warnings --alluredir=allure-results tests
       - run:
           name: Generate report
           command: |
@@ -169,13 +201,66 @@ jobs:
             coverage html
             coverage xml
             ls -la
+            ls -la allure-results || echo 'ignore error'
             coveralls
+          # https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957
+          # https://circleci.com/docs/configuration-reference/#the-when-attribute
+          when: always
       - store_artifacts:
           path: htmlcov
       - store_artifacts:
           path: coverage.xml
       - codecov/upload:
           file: coverage.xml
+      # https://circleci.com/developer/orbs/orb/codecov/codecov
+      # - codecov/upload
+      # https://circleci.com/developer/orbs/orb/coveralls/coveralls
+      # - coveralls/upload
+      # https://discuss.circleci.com/t/how-can-we-publish-pytest-results-to-circleci-using-allure-reports/37830/2
+      # https://circleci.com/developer/orbs/orb/ayte/allure
+      # - allure/install
+      # - allure/report
+      # https://circleci.com/docs/configuration-reference/#the-when-step
+      - when:
+          condition:
+            equal: [ 1, 1 ]
+          steps:
+            - run:
+                name: Allure archive download
+                command: >-
+                  curl -L https://github.com/allure-framework/allure2/releases/download/<<
+                  parameters.allure-version >>/allure-commandline-<< parameters.allure-version >>.zip -o
+                  /tmp/allure.zip
+                when: always
+            - run:
+                name: Archive extraction
+                command: unzip /tmp/allure.zip
+                when: always
+            - run:
+                name: Allure installation
+                command: sudo mv allure-<< parameters.allure-version >> /usr/local/share/allure
+                when: always
+            - run:
+                name: Allure binary symlinking
+                command: sudo ln -s /usr/local/share/allure/bin/allure /usr/local/bin/allure
+                when: always
+      - when:
+          condition:
+            equal: [ 1, 1 ]
+          steps:
+            - run:
+                name: >-
+                  Allure report generation (<< parameters.allure-results-path >> -> <<
+                  parameters.allure-target-path >>)
+                command: |
+                  allure generate \
+                    --config << parameters.allure-configuration-path >> \
+                    --report-dir << parameters.allure-target-path >> \
+                    << parameters.allure-results-path >>
+                when: always
+            - store_artifacts:
+                path: << parameters.allure-target-path >>
+                destination: << parameters.allure-artifact-path >>
   py27:
     <<: *test-template
     docker:
@@ -192,7 +277,7 @@ jobs:
     <<: *test-template
     docker:
       - image: cimg/python:3.8
-  py39-scrapyd-latest:
+  py39-scrapyd-v143:
     <<: *test-template
     docker:
       - image: cimg/python:3.9
@@ -274,10 +359,14 @@ jobs:
     <<: *test-template
     docker:
       - image: cimg/python:3.12
-  py312-scrapyd-latest:
+  py312-scrapyd-v143:
     <<: *test-template
     docker:
       - image: cimg/python:3.12
+  py313:
+    <<: *test-template
+    docker:
+      - image: cimg/python:3.13
 workflows:
   test:
     jobs:
@@ -287,8 +376,8 @@ workflows:
       # - py37
       - py38
       - py39
-      - py39-scrapyd-latest:
-          use-scrapyd-v143: false
+      - py39-scrapyd-v143:
+          use-scrapyd-v143: true
       - py310-git-postgresql:
           use-git: true
           use-postgresql: true
@@ -304,5 +393,6 @@ workflows:
           use-mysql: true
       - py311
       - py312
-      - py312-scrapyd-latest:
-          use-scrapyd-v143: false
+      - py312-scrapyd-v143:
+          use-scrapyd-v143: true
+      - py313

.gitignore

@@ -5,6 +5,7 @@ scrapydweb_settings_v*.py
 
 tests/*.html
 tests/data/
+allure-results/
 
 
 scrapydweb/data/*

requirements-tests.txt

@@ -4,6 +4,7 @@ coverage
 pytest
 # pytest-cov
 coveralls
+allure-pytest
 
 scrapy
 scrapyd

requirements.txt

@@ -8,7 +8,7 @@ Flask-SQLAlchemy==2.4.0
 idna==2.7
 itsdangerous==2.0.0
 Jinja2==3.0.0
-logparser==0.8.2
+logparser>=0.8.4
 MarkupSafe==2.0.0
 pexpect==4.7.0
 ptyprocess==0.6.0

scrapydweb/__version__.py

@@ -1,7 +1,7 @@
 # coding: utf-8
 
 __title__ = 'scrapydweb'
-__version__ = '1.5.1'
+__version__ = '1.5.2'
 __author__ = 'my8100'
 __author_email__ = '[email protected]'
 __url__ = 'https://github.com/my8100/scrapydweb'

scrapydweb/templates/scrapydweb/jobs.html

@@ -15,7 +15,7 @@
   </style>
 
   {% if SCRAPYD_SERVERS_AMOUNT == 1 and (pageview == 1 or pageview % CHECK_LATEST_VERSION_FREQ == 0) %}
-  <script type="text/javascript" src="https://my8100.herokuapp.com/check_update?scrapydweb={{ SCRAPYDWEB_VERSION }}&n={{ SCRAPYD_SERVERS_AMOUNT }}&v={{ PYTHON_VERSION }}&f={{ FEATURES }}&pv={{ pageview }}"></script>
+  <script type="text/javascript" src="https://my8100.pythonanywhere.com/check_update?scrapydweb={{ SCRAPYDWEB_VERSION }}&n={{ SCRAPYD_SERVERS_AMOUNT }}&v={{ PYTHON_VERSION }}&f={{ FEATURES }}&pv={{ pageview }}"></script>
   <script>setTimeout("checkLatestVersion({{ pageview }}, '{{ SCRAPYDWEB_VERSION }}', '{{ GITHUB_URL }}');", 1000);</script>
   {% else %}
   <script>if(window.localStorage && localStorage.getItem('github') !== null) {localStorage.removeItem('github');}</script>
@@ -31,7 +31,7 @@
 <h2>
   <a class="link" target="_blank" href="{{ url }}">Get the list of jobs of all projects in database.</a>
   <label title="switch to non-database view">
-    <a class="button normal" href="{{ url_jobs_classic }}">Classic</a>
+    <a class="button normal" href="{{ url_jobs_classic }}">Classic View</a>
   </label>
   <label title="schedule a spider run">
     <a class="button safe narrow" href="{{ url_schedule }}">+</a>

scrapydweb/templates/scrapydweb/jobs_classic.html

@@ -15,7 +15,7 @@
   </style>
 
   {% if SCRAPYD_SERVERS_AMOUNT == 1 and (pageview == 1 or pageview % CHECK_LATEST_VERSION_FREQ == 0) %}
-  <script type="text/javascript" src="https://my8100.herokuapp.com/check_update?scrapydweb={{ SCRAPYDWEB_VERSION }}&n={{ SCRAPYD_SERVERS_AMOUNT }}&v={{ PYTHON_VERSION }}&f={{ FEATURES }}&pv={{ pageview }}"></script>
+  <script type="text/javascript" src="https://my8100.pythonanywhere.com/check_update?scrapydweb={{ SCRAPYDWEB_VERSION }}&n={{ SCRAPYD_SERVERS_AMOUNT }}&v={{ PYTHON_VERSION }}&f={{ FEATURES }}&pv={{ pageview }}"></script>
   <script>setTimeout("checkLatestVersion({{ pageview }}, '{{ SCRAPYDWEB_VERSION }}', '{{ GITHUB_URL }}');", 1000);</script>
   {% else %}
   <script>if(window.localStorage && localStorage.getItem('github') !== null) {localStorage.removeItem('github');}</script>
@@ -31,7 +31,7 @@
 <h2>
   <a class="link" target="_blank" href="{{ url }}">Get the list of jobs of all projects after Scrapyd server started.</a>
   <label title="switch to database view">
-    <a class="button normal" href="{{ url_jobs_database }}">Database</a>
+    <a class="button normal" href="{{ url_jobs_database }}">Database View</a>
   </label>
   <label title="schedule a spider run">
     <a class="button safe narrow" href="{{ url_schedule }}">+</a>
@@ -140,7 +140,7 @@ <h3>Running ({{ running_jobs|length }}) <strong id="logparser_last_update" style
 
 
 <div class="table wrap">
-  <h3>Finished ({{ finished_jobs|length }}) <strong style="color: red;">DESC</strong></h3>
+  <h3>Finished ({{ finished_jobs|length }}) <strong style="color: green;">ORDER BY finish DESC</strong></h3>
   <table>
     <thead>
       <tr>

scrapydweb/templates/scrapydweb/jobs_mobileui.html

@@ -39,7 +39,7 @@
   <script type="text/javascript" src="{{ static_js_stacktable }}"></script>
 
   {% if pageview == 1 or pageview % CHECK_LATEST_VERSION_FREQ == 0 %}
-  <script type="text/javascript" src="https://my8100.herokuapp.com/check_update?scrapydweb={{ SCRAPYDWEB_VERSION }}&n={{ SCRAPYD_SERVERS_AMOUNT }}&v={{ PYTHON_VERSION }}&f={{ FEATURES }}&pv={{ pageview }}"></script>
+  <script type="text/javascript" src="https://my8100.pythonanywhere.com/check_update?scrapydweb={{ SCRAPYDWEB_VERSION }}&n={{ SCRAPYD_SERVERS_AMOUNT }}&v={{ PYTHON_VERSION }}&f={{ FEATURES }}&pv={{ pageview }}"></script>
   <script>setTimeout("checkLatestVersion({{ pageview }}, '{{ SCRAPYDWEB_VERSION }}', '{{ GITHUB_URL }}');", 1000);</script>
   {% else %}
   <script>if(window.localStorage && localStorage.getItem('github') !== null) {localStorage.removeItem('github');}</script>
@@ -135,7 +135,7 @@
 
   <tr>
     <th colspan="{{ colspan }}" class="category-head">Finished ({{ finished_jobs|length }})
-    <strong style="color: red;">DESC</strong>
+    <strong style="color: red;">sorted DESC</strong>
     </th>
   </tr>
 {% for job in finished_jobs %}

scrapydweb/templates/scrapydweb/logs_items.html

@@ -49,6 +49,7 @@ <h2>
     <el-table-column prop="content_type" label="Content type" sortable width="135" align="center"></el-table-column>
     <el-table-column prop="content_encoding" label="Encoding" sortable width="95" align="center"></el-table-column>
     <el-table-column prop="size" label="Size" sortable width="100" align="center" :sort-method="sortSize" :sort-orders=['descending','ascending',null]></el-table-column>
+    <el-table-column prop="last_modified" label="Last modified" sortable width="140" align="center"></el-table-column>
 
     {% if project and spider %}
     <el-table-column prop="stats" label="Stats" sortable width="{% if SCRAPYD_SERVERS_AMOUNT > 1 %}105{% else %}65{% endif %}" align="center">
@@ -87,6 +88,7 @@ <h2>
         content_type: '{{ row["content_type"] }}',
         content_encoding: '{{ row["content_encoding"] }}',
         size: '{{ row["size"] }}',
+        last_modified: '{{ row["last_modified"] }}',
 
         {% if project and spider %}
         url_clusterreports: '{{ row["url_clusterreports"] }}',

scrapydweb/templates/scrapydweb/servers.html

@@ -35,7 +35,7 @@
   }
   </style>
   {% if SCRAPYD_SERVERS_AMOUNT > 1 and (pageview == 1 or pageview % CHECK_LATEST_VERSION_FREQ == 0) %}
-  <script type="text/javascript" src="https://my8100.herokuapp.com/check_update?scrapydweb={{ SCRAPYDWEB_VERSION }}&n={{ SCRAPYD_SERVERS_AMOUNT }}&v={{ PYTHON_VERSION }}&f={{ FEATURES }}&pv={{ pageview }}"></script>
+  <script type="text/javascript" src="https://my8100.pythonanywhere.com/check_update?scrapydweb={{ SCRAPYDWEB_VERSION }}&n={{ SCRAPYD_SERVERS_AMOUNT }}&v={{ PYTHON_VERSION }}&f={{ FEATURES }}&pv={{ pageview }}"></script>
   <script>setTimeout("checkLatestVersion({{ pageview }}, '{{ SCRAPYDWEB_VERSION }}', '{{ GITHUB_URL }}');", 2000);</script>
   {% else %}
   <script>if(window.localStorage && localStorage.getItem('github') !== null) {localStorage.removeItem('github');}</script>

scrapydweb/utils/poll.py

@@ -24,17 +24,18 @@
 logger.addHandler(_handler)
 
 IN_WINDOWS = platform.system() == 'Windows'
+# See also scrapydweb/views/dashboard/jobs.py
 JOB_PATTERN = re.compile(r"""
-                            <tr>
-                                <td>(?P<Project>.*?)</td>
-                                <td>(?P<Spider>.*?)</td>
-                                <td>(?P<Job>.*?)</td>
-                                (?:<td>(?P<PID>.*?)</td>)?
-                                (?:<td>(?P<Start>.*?)</td>)?
-                                (?:<td>(?P<Runtime>.*?)</td>)?
-                                (?:<td>(?P<Finish>.*?)</td>)?
-                                (?:<td>(?P<Log>.*?)</td>)?
-                                (?:<td>(?P<Items>.*?)</td>)?
+                            <tr>\s*
+                                <td>(?P<Project>.*?)</td>\s*
+                                <td>(?P<Spider>.*?)</td>\s*
+                                <td>(?P<Job>.*?)</td>\s*
+                                (?:<td>(?P<PID>.*?)</td>\s*)?
+                                (?:<td>(?P<Start>.*?)</td>\s*)?
+                                (?:<td>(?P<Runtime>.*?)</td>\s*)?
+                                (?:<td>(?P<Finish>.*?)</td>\s*)?
+                                (?:<td>(?P<Log>.*?)</td>\s*)?
+                                (?:<td>(?P<Items>.*?)</td>\s*)?
                                 [\w\W]*?  # Temp support for Scrapyd v1.3.0 (not released)
                             </tr>
                           """, re.X)

scrapydweb/vars.py

@@ -89,11 +89,12 @@
                                     <tr\sclass="(?P<odd_even>odd|even)">\n
                                         \s+<td>(?P<filename>.*?)</td>\n
                                         \s+<td>(?P<size>.*?)</td>\n
+                                        (?:\s+<td>(?P<last_modified>.*?)</td>\n)?
                                         \s+<td>(?P<content_type>.*?)</td>\n
                                         \s+<td>(?P<content_encoding>.*?)</td>\n
                                     </tr>
                                 """, re.X)
-DIRECTORY_KEYS = ['odd_even', 'filename', 'size', 'content_type', 'content_encoding']
+DIRECTORY_KEYS = ['odd_even', 'filename', 'size', 'last_modified', 'content_type', 'content_encoding']
 HREF_NAME_PATTERN = re.compile(r'href="(.+?)">(.+?)<')
 
 # For JobsView

scrapydweb/views/dashboard/jobs.py

@@ -27,6 +27,7 @@
 NOT_DELETED = '0'
 DELETED = '1'
 HREF_PATTERN = re.compile(r"""href=['"](.+?)['"]""")  # Temp support for Scrapyd v1.3.0 (not released)
+# See also scrapydweb/utils/poll.py
 JOB_PATTERN = re.compile(r"""
                             <tr>\s*
                                 <td>(?P<Project>.*?)</td>\s*
@@ -426,10 +427,9 @@ def set_kwargs(self):
             ))
             return
 
+        self.finished_jobs.sort(key=lambda x: (x['finish'], x['start']), reverse=True)  # Finished DESC
         if self.JOBS_FINISHED_JOBS_LIMIT > 0:
-            self.finished_jobs = self.finished_jobs[::-1][:self.JOBS_FINISHED_JOBS_LIMIT]
-        else:
-            self.finished_jobs = self.finished_jobs[::-1]
+            self.finished_jobs = self.finished_jobs[:self.JOBS_FINISHED_JOBS_LIMIT]
         self.kwargs.update(dict(
             colspan=14,
             url_jobs_database=url_for('jobs', node=self.node, style='database'),

setup.py

@@ -42,7 +42,7 @@
         "idna==2.7",  # Jun 11, 2018
         "itsdangerous==2.0.0",  # May 12, 2021
         "Jinja2==3.0.0",  # May 12, 2021
-        "logparser==0.8.2",
+        "logparser>=0.8.4",
         "MarkupSafe==2.0.0",  # May 12, 2021
         "pexpect==4.7.0",  # Apr 7, 2019
         "ptyprocess==0.6.0",  # Jun 22, 2018
@@ -71,5 +71,6 @@
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
     ]
 )