From 19d807359e8833dbde1ccc1352700dfaf7fbf579 Mon Sep 17 00:00:00 2001
From: LxL <my8100@gmail.com>
Date: Sun, 6 Oct 2024 23:02:18 +0800
Subject: [PATCH] Release v1.5.1 to support scrapyd v1.5.0 (#240)

* Release v1.5.1 to support scrapyd v1.5.0

* Test py312-scrapyd-v143 in circleci

* Set use-scrapyd-v143 default to true in circleci
---
 .circleci/config.yml                          |  29 ++++++++++++++++--
 scrapydweb/__version__.py                     |   2 +-
 .../static/{v150 => v151}/css/dropdown.css    |   0
 .../{v150 => v151}/css/dropdown_mobileui.css  |   0
 .../css/icon_upload_icon_right.css            |   0
 .../static/{v150 => v151}/css/multinode.css   |   0
 .../static/{v150 => v151}/css/stacktable.css  |   0
 .../static/{v150 => v151}/css/stats.css       |   0
 .../static/{v150 => v151}/css/style.css       |   0
 .../{v150 => v151}/css/style_mobileui.css     |   0
 scrapydweb/static/{v150 => v151}/css/utf8.css |   0
 .../{v150 => v151}/css/utf8_mobileui.css      |   0
 .../element-ui@2.4.6/lib/index.js             |   0
 .../lib/theme-chalk/fonts/element-icons.woff  | Bin
 .../lib/theme-chalk/index.css                 |   0
 scrapydweb/static/{v150 => v151}/icon/fav.ico | Bin
 scrapydweb/static/{v150 => v151}/icon/fav.png | Bin
 ...n-spiderman-face-mask-round-avatar-512.png | Bin
 .../static/{v150 => v151}/icon/spiderman.png  | Bin
 scrapydweb/static/{v150 => v151}/js/common.js |   0
 .../static/{v150 => v151}/js/echarts.min.js   |   0
 .../{v150 => v151}/js/github_buttons.html     |   0
 .../{v150 => v151}/js/github_buttons.js       |   0
 .../static/{v150 => v151}/js/icons_menu.js    |   0
 .../static/{v150 => v151}/js/jquery.min.js    |   0
 .../static/{v150 => v151}/js/multinode.js     |   0
 .../static/{v150 => v151}/js/stacktable.js    |   0
 scrapydweb/static/{v150 => v151}/js/stats.js  |   0
 .../static/{v150 => v151}/js/vue.min.js       |   0
 scrapydweb/templates/scrapydweb/jobs.html     |   2 +-
 scrapydweb/views/dashboard/jobs.py            |  22 ++++++-------
 tests/test_deploy.py                          |   3 +-
 tests/test_deploy_single_scrapyd.py           |   3 +-
 tests/test_projects.py                        |   4 ++-
 tests/utils.py                                |   2 +-
 35 files changed, 48 insertions(+), 19 deletions(-)
 rename scrapydweb/static/{v150 => v151}/css/dropdown.css (100%)
 rename scrapydweb/static/{v150 => v151}/css/dropdown_mobileui.css (100%)
 rename scrapydweb/static/{v150 => v151}/css/icon_upload_icon_right.css (100%)
 rename scrapydweb/static/{v150 => v151}/css/multinode.css (100%)
 rename scrapydweb/static/{v150 => v151}/css/stacktable.css (100%)
 rename scrapydweb/static/{v150 => v151}/css/stats.css (100%)
 rename scrapydweb/static/{v150 => v151}/css/style.css (100%)
 rename scrapydweb/static/{v150 => v151}/css/style_mobileui.css (100%)
 rename scrapydweb/static/{v150 => v151}/css/utf8.css (100%)
 rename scrapydweb/static/{v150 => v151}/css/utf8_mobileui.css (100%)
 rename scrapydweb/static/{v150 => v151}/element-ui@2.4.6/lib/index.js (100%)
 rename scrapydweb/static/{v150 => v151}/element-ui@2.4.6/lib/theme-chalk/fonts/element-icons.woff (100%)
 rename scrapydweb/static/{v150 => v151}/element-ui@2.4.6/lib/theme-chalk/index.css (100%)
 rename scrapydweb/static/{v150 => v151}/icon/fav.ico (100%)
 rename scrapydweb/static/{v150 => v151}/icon/fav.png (100%)
 rename scrapydweb/static/{v150 => v151}/icon/spider-man-spiderman-face-mask-round-avatar-512.png (100%)
 rename scrapydweb/static/{v150 => v151}/icon/spiderman.png (100%)
 rename scrapydweb/static/{v150 => v151}/js/common.js (100%)
 rename scrapydweb/static/{v150 => v151}/js/echarts.min.js (100%)
 rename scrapydweb/static/{v150 => v151}/js/github_buttons.html (100%)
 rename scrapydweb/static/{v150 => v151}/js/github_buttons.js (100%)
 rename scrapydweb/static/{v150 => v151}/js/icons_menu.js (100%)
 rename scrapydweb/static/{v150 => v151}/js/jquery.min.js (100%)
 rename scrapydweb/static/{v150 => v151}/js/multinode.js (100%)
 rename scrapydweb/static/{v150 => v151}/js/stacktable.js (100%)
 rename scrapydweb/static/{v150 => v151}/js/stats.js (100%)
 rename scrapydweb/static/{v150 => v151}/js/vue.min.js (100%)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 4c51eaa2..28390d8d 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -13,6 +13,9 @@ jobs:
       is-py27:
         type: boolean
         default: false
+      use-scrapyd-v143:
+        type: boolean
+        default: true
       use-git:
         type: boolean
         default: false
@@ -126,6 +129,16 @@ jobs:
                   pip install -U git+https://github.com/scrapy/scrapyd.git
                   pip install -U git+https://github.com/my8100/logparser.git
                   pip list
+      - when:
+          condition: <<parameters.use-scrapyd-v143>>
+          steps:
+            - run:
+                name: scrapyd==1.4.3
+                command: |
+                  . venv/bin/activate
+                  pip list
+                  pip install scrapyd==1.4.3
+                  pip list
       - run:
           name: Launch Scrapyd
           command: |
@@ -179,6 +192,10 @@ jobs:
     <<: *test-template
     docker:
       - image: cimg/python:3.8
+  py39-scrapyd-latest:
+    <<: *test-template
+    docker:
+      - image: cimg/python:3.9
   py310-git-postgresql:
     <<: *test-template
     docker:
@@ -257,15 +274,21 @@ jobs:
     <<: *test-template
     docker:
       - image: cimg/python:3.12
+  py312-scrapyd-latest:
+    <<: *test-template
+    docker:
+      - image: cimg/python:3.12
 workflows:
   test:
     jobs:
       # - py27:
       #    is-py27: true
-      - py36
-      - py37
+      # - py36
+      # - py37
       - py38
       - py39
+      - py39-scrapyd-latest:
+          use-scrapyd-v143: false
       - py310-git-postgresql:
           use-git: true
           use-postgresql: true
@@ -281,3 +304,5 @@ workflows:
           use-mysql: true
       - py311
       - py312
+      - py312-scrapyd-latest:
+          use-scrapyd-v143: false
diff --git a/scrapydweb/__version__.py b/scrapydweb/__version__.py
index 1818645c..0fce3959 100644
--- a/scrapydweb/__version__.py
+++ b/scrapydweb/__version__.py
@@ -1,7 +1,7 @@
 # coding: utf-8
 
 __title__ = 'scrapydweb'
-__version__ = '1.5.0'
+__version__ = '1.5.1'
 __author__ = 'my8100'
 __author_email__ = 'my8100@gmail.com'
 __url__ = 'https://github.com/my8100/scrapydweb'
diff --git a/scrapydweb/static/v150/css/dropdown.css b/scrapydweb/static/v151/css/dropdown.css
similarity index 100%
rename from scrapydweb/static/v150/css/dropdown.css
rename to scrapydweb/static/v151/css/dropdown.css
diff --git a/scrapydweb/static/v150/css/dropdown_mobileui.css b/scrapydweb/static/v151/css/dropdown_mobileui.css
similarity index 100%
rename from scrapydweb/static/v150/css/dropdown_mobileui.css
rename to scrapydweb/static/v151/css/dropdown_mobileui.css
diff --git a/scrapydweb/static/v150/css/icon_upload_icon_right.css b/scrapydweb/static/v151/css/icon_upload_icon_right.css
similarity index 100%
rename from scrapydweb/static/v150/css/icon_upload_icon_right.css
rename to scrapydweb/static/v151/css/icon_upload_icon_right.css
diff --git a/scrapydweb/static/v150/css/multinode.css b/scrapydweb/static/v151/css/multinode.css
similarity index 100%
rename from scrapydweb/static/v150/css/multinode.css
rename to scrapydweb/static/v151/css/multinode.css
diff --git a/scrapydweb/static/v150/css/stacktable.css b/scrapydweb/static/v151/css/stacktable.css
similarity index 100%
rename from scrapydweb/static/v150/css/stacktable.css
rename to scrapydweb/static/v151/css/stacktable.css
diff --git a/scrapydweb/static/v150/css/stats.css b/scrapydweb/static/v151/css/stats.css
similarity index 100%
rename from scrapydweb/static/v150/css/stats.css
rename to scrapydweb/static/v151/css/stats.css
diff --git a/scrapydweb/static/v150/css/style.css b/scrapydweb/static/v151/css/style.css
similarity index 100%
rename from scrapydweb/static/v150/css/style.css
rename to scrapydweb/static/v151/css/style.css
diff --git a/scrapydweb/static/v150/css/style_mobileui.css b/scrapydweb/static/v151/css/style_mobileui.css
similarity index 100%
rename from scrapydweb/static/v150/css/style_mobileui.css
rename to scrapydweb/static/v151/css/style_mobileui.css
diff --git a/scrapydweb/static/v150/css/utf8.css b/scrapydweb/static/v151/css/utf8.css
similarity index 100%
rename from scrapydweb/static/v150/css/utf8.css
rename to scrapydweb/static/v151/css/utf8.css
diff --git a/scrapydweb/static/v150/css/utf8_mobileui.css b/scrapydweb/static/v151/css/utf8_mobileui.css
similarity index 100%
rename from scrapydweb/static/v150/css/utf8_mobileui.css
rename to scrapydweb/static/v151/css/utf8_mobileui.css
diff --git a/scrapydweb/static/v150/element-ui@2.4.6/lib/index.js b/scrapydweb/static/v151/element-ui@2.4.6/lib/index.js
similarity index 100%
rename from scrapydweb/static/v150/element-ui@2.4.6/lib/index.js
rename to scrapydweb/static/v151/element-ui@2.4.6/lib/index.js
diff --git a/scrapydweb/static/v150/element-ui@2.4.6/lib/theme-chalk/fonts/element-icons.woff b/scrapydweb/static/v151/element-ui@2.4.6/lib/theme-chalk/fonts/element-icons.woff
similarity index 100%
rename from scrapydweb/static/v150/element-ui@2.4.6/lib/theme-chalk/fonts/element-icons.woff
rename to scrapydweb/static/v151/element-ui@2.4.6/lib/theme-chalk/fonts/element-icons.woff
diff --git a/scrapydweb/static/v150/element-ui@2.4.6/lib/theme-chalk/index.css b/scrapydweb/static/v151/element-ui@2.4.6/lib/theme-chalk/index.css
similarity index 100%
rename from scrapydweb/static/v150/element-ui@2.4.6/lib/theme-chalk/index.css
rename to scrapydweb/static/v151/element-ui@2.4.6/lib/theme-chalk/index.css
diff --git a/scrapydweb/static/v150/icon/fav.ico b/scrapydweb/static/v151/icon/fav.ico
similarity index 100%
rename from scrapydweb/static/v150/icon/fav.ico
rename to scrapydweb/static/v151/icon/fav.ico
diff --git a/scrapydweb/static/v150/icon/fav.png b/scrapydweb/static/v151/icon/fav.png
similarity index 100%
rename from scrapydweb/static/v150/icon/fav.png
rename to scrapydweb/static/v151/icon/fav.png
diff --git a/scrapydweb/static/v150/icon/spider-man-spiderman-face-mask-round-avatar-512.png b/scrapydweb/static/v151/icon/spider-man-spiderman-face-mask-round-avatar-512.png
similarity index 100%
rename from scrapydweb/static/v150/icon/spider-man-spiderman-face-mask-round-avatar-512.png
rename to scrapydweb/static/v151/icon/spider-man-spiderman-face-mask-round-avatar-512.png
diff --git a/scrapydweb/static/v150/icon/spiderman.png b/scrapydweb/static/v151/icon/spiderman.png
similarity index 100%
rename from scrapydweb/static/v150/icon/spiderman.png
rename to scrapydweb/static/v151/icon/spiderman.png
diff --git a/scrapydweb/static/v150/js/common.js b/scrapydweb/static/v151/js/common.js
similarity index 100%
rename from scrapydweb/static/v150/js/common.js
rename to scrapydweb/static/v151/js/common.js
diff --git a/scrapydweb/static/v150/js/echarts.min.js b/scrapydweb/static/v151/js/echarts.min.js
similarity index 100%
rename from scrapydweb/static/v150/js/echarts.min.js
rename to scrapydweb/static/v151/js/echarts.min.js
diff --git a/scrapydweb/static/v150/js/github_buttons.html b/scrapydweb/static/v151/js/github_buttons.html
similarity index 100%
rename from scrapydweb/static/v150/js/github_buttons.html
rename to scrapydweb/static/v151/js/github_buttons.html
diff --git a/scrapydweb/static/v150/js/github_buttons.js b/scrapydweb/static/v151/js/github_buttons.js
similarity index 100%
rename from scrapydweb/static/v150/js/github_buttons.js
rename to scrapydweb/static/v151/js/github_buttons.js
diff --git a/scrapydweb/static/v150/js/icons_menu.js b/scrapydweb/static/v151/js/icons_menu.js
similarity index 100%
rename from scrapydweb/static/v150/js/icons_menu.js
rename to scrapydweb/static/v151/js/icons_menu.js
diff --git a/scrapydweb/static/v150/js/jquery.min.js b/scrapydweb/static/v151/js/jquery.min.js
similarity index 100%
rename from scrapydweb/static/v150/js/jquery.min.js
rename to scrapydweb/static/v151/js/jquery.min.js
diff --git a/scrapydweb/static/v150/js/multinode.js b/scrapydweb/static/v151/js/multinode.js
similarity index 100%
rename from scrapydweb/static/v150/js/multinode.js
rename to scrapydweb/static/v151/js/multinode.js
diff --git a/scrapydweb/static/v150/js/stacktable.js b/scrapydweb/static/v151/js/stacktable.js
similarity index 100%
rename from scrapydweb/static/v150/js/stacktable.js
rename to scrapydweb/static/v151/js/stacktable.js
diff --git a/scrapydweb/static/v150/js/stats.js b/scrapydweb/static/v151/js/stats.js
similarity index 100%
rename from scrapydweb/static/v150/js/stats.js
rename to scrapydweb/static/v151/js/stats.js
diff --git a/scrapydweb/static/v150/js/vue.min.js b/scrapydweb/static/v151/js/vue.min.js
similarity index 100%
rename from scrapydweb/static/v150/js/vue.min.js
rename to scrapydweb/static/v151/js/vue.min.js
diff --git a/scrapydweb/templates/scrapydweb/jobs.html b/scrapydweb/templates/scrapydweb/jobs.html
index 096f7b49..b3167319 100644
--- a/scrapydweb/templates/scrapydweb/jobs.html
+++ b/scrapydweb/templates/scrapydweb/jobs.html
@@ -88,7 +88,7 @@ <h3 id="logparser_alert" style="color: red; display: none;"></h3>
 
     <el-table-column prop="action" label="Action" sortable align="center" width="{% if SCRAPYD_SERVERS_AMOUNT > 1 %}115{% else %}90{% endif %}" :sort-method="sortByIndex">
       <template slot-scope="scope">
-        <span style="color: red;" :style="{ display: scope.row.display_kill }">Kill {{scope.row.pid}}</span>
+        <span style="color: red;" :style="{ display: scope.row.display_kill }">Kill PID {{scope.row.pid}}</span>
         {% if SCRAPYD_SERVERS_AMOUNT > 1 %}
         <a class="state multinode" :style="{ display: scope.row.display_action }" :href="scope.row.url_multinode">{{ g.multinode|safe }}</a>
         {% endif %}
diff --git a/scrapydweb/views/dashboard/jobs.py b/scrapydweb/views/dashboard/jobs.py
index d20a5dcb..811efbc7 100644
--- a/scrapydweb/views/dashboard/jobs.py
+++ b/scrapydweb/views/dashboard/jobs.py
@@ -28,16 +28,16 @@
 DELETED = '1'
 HREF_PATTERN = re.compile(r"""href=['"](.+?)['"]""")  # Temp support for Scrapyd v1.3.0 (not released)
 JOB_PATTERN = re.compile(r"""
-                            <tr>
-                                <td>(?P<Project>.*?)</td>
-                                <td>(?P<Spider>.*?)</td>
-                                <td>(?P<Job>.*?)</td>
-                                (?:<td>(?P<PID>.*?)</td>)?
-                                (?:<td>(?P<Start>.*?)</td>)?
-                                (?:<td>(?P<Runtime>.*?)</td>)?
-                                (?:<td>(?P<Finish>.*?)</td>)?
-                                (?:<td>(?P<Log>.*?)</td>)?
-                                (?:<td>(?P<Items>.*?)</td>)?
+                            <tr>\s*
+                                <td>(?P<Project>.*?)</td>\s*
+                                <td>(?P<Spider>.*?)</td>\s*
+                                <td>(?P<Job>.*?)</td>\s*
+                                (?:<td>(?P<PID>.*?)</td>\s*)?
+                                (?:<td>(?P<Start>.*?)</td>\s*)?
+                                (?:<td>(?P<Runtime>.*?)</td>\s*)?
+                                (?:<td>(?P<Finish>.*?)</td>\s*)?
+                                (?:<td>(?P<Log>.*?)</td>\s*)?
+                                (?:<td>(?P<Items>.*?)</td>\s*)?
                                 [\w\W]*?  # Temp support for Scrapyd v1.3.0 (not released)
                             </tr>
                           """, re.X)
@@ -96,7 +96,7 @@ def __init__(self):
 
     def dispatch_request(self, **kwargs):
         status_code, self.text = self.make_request(self.url, auth=self.AUTH, as_json=False)
-        if status_code != 200 or not re.search(r'<body><h1>Jobs</h1>', self.text):
+        if status_code != 200 or not re.search(r'<h1>Jobs</h1>', self.text):
             kwargs = dict(
                 node=self.node,
                 url=self.url,
diff --git a/tests/test_deploy.py b/tests/test_deploy.py
index 7e66aeb5..54afc5ac 100644
--- a/tests/test_deploy.py
+++ b/tests/test_deploy.py
@@ -31,6 +31,7 @@ def test_auto_packaging_select_option(app, client):
 
 # {'status': 'error', 'message': 'Traceback
 # ...TypeError:...activate_egg(eggpath)...\'tuple\' object is not an iterator\r\n'}
+# egg is not a ZIP file (if using curl, use egg=@path not egg=path)
 def test_addversion(app, client):
     data = {
         '1': 'on',
@@ -39,7 +40,7 @@ def test_addversion(app, client):
         'version': 'fakeversion',
         'file': (BytesIO(b'my file contents'), "fake.egg")
     }
-    req(app, client, view='deploy.upload', kws=dict(node=2), data=data, ins='activate_egg')
+    req(app, client, view='deploy.upload', kws=dict(node=2), data=data, ins=['Fail to deploy project', 'egg'])
 
 
 # <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
diff --git a/tests/test_deploy_single_scrapyd.py b/tests/test_deploy_single_scrapyd.py
index 507361c4..7dc3b5d3 100644
--- a/tests/test_deploy_single_scrapyd.py
+++ b/tests/test_deploy_single_scrapyd.py
@@ -49,13 +49,14 @@ def test_auto_packaging_select_option(app, client):
 
 # {'status': 'error', 'message': 'Traceback
 # ...TypeError:...activate_egg(eggpath)...\'tuple\' object is not an iterator\r\n'}
+# egg is not a ZIP file (if using curl, use egg=@path not egg=path)
 def test_addversion(app, client):
     data = {
         'project': 'fakeproject',
         'version': 'fakeversion',
         'file': (io.BytesIO(b'my file contents'), "fake.egg")
     }
-    req_single_scrapyd(app, client, view='deploy.upload', kws=dict(node=1), data=data, ins='activate_egg')
+    req_single_scrapyd(app, client, view='deploy.upload', kws=dict(node=1), data=data, ins=['Fail to deploy project', 'egg'])
 
 
 # <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
diff --git a/tests/test_projects.py b/tests/test_projects.py
index a2b2f262..a5c3a442 100644
--- a/tests/test_projects.py
+++ b/tests/test_projects.py
@@ -39,7 +39,8 @@ def test_listspiders_del(app, client):
         d['listspiders_fail'] = dict(
             url=url_for('projects', node=1, opt='listspiders',
                         project=cst.FAKE_PROJECT, version_spider_job=cst.FAKE_VERSION),
-            checks=['listspiders.json', 'No such file or directory']
+            # project 'FAKE_PROJECT' not found
+            checks=['listspiders.json', 'See details below']
         )
 
         d['delversion'] = dict(
@@ -66,5 +67,6 @@ def test_listspiders_del(app, client):
                 upload_file_deploy(app, client, filename='demo.zip', project=cst.PROJECT, redirect_project=cst.PROJECT)
             response = client.get(v['url'])
             text = get_text(response)
+            # print('text', text)
             for c in v['checks']:
                 assert c in text
diff --git a/tests/utils.py b/tests/utils.py
index e2767aad..10b587cf 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -117,7 +117,7 @@ def req(app, client, view='', kws=None, url='', data=None,
             with io.open('%s.html' % save, 'wb') as f:
                 f.write(response.data)
         text = get_text(response)
-        # print(text)
+        print('text', text)
         js = {}
         try:
             # js = response.get_json()