Release v1.5.1 to support scrapyd v1.5.0 (#240)

* Release v1.5.1 to support scrapyd v1.5.0

* Test py312-scrapyd-v143 in circleci

* Set use-scrapyd-v143 default to true in circleci

.circleci/config.yml

@@ -13,6 +13,9 @@ jobs:
       is-py27:
         type: boolean
         default: false
+      use-scrapyd-v143:
+        type: boolean
+        default: true
       use-git:
         type: boolean
         default: false
@@ -126,6 +129,16 @@ jobs:
                   pip install -U git+https://github.com/scrapy/scrapyd.git
                   pip install -U git+https://github.com/my8100/logparser.git
                   pip list
+      - when:
+          condition: <<parameters.use-scrapyd-v143>>
+          steps:
+            - run:
+                name: scrapyd==1.4.3
+                command: |
+                  . venv/bin/activate
+                  pip list
+                  pip install scrapyd==1.4.3
+                  pip list
       - run:
           name: Launch Scrapyd
           command: |
@@ -179,6 +192,10 @@ jobs:
     <<: *test-template
     docker:
       - image: cimg/python:3.8
+  py39-scrapyd-latest:
+    <<: *test-template
+    docker:
+      - image: cimg/python:3.9
   py310-git-postgresql:
     <<: *test-template
     docker:
@@ -257,15 +274,21 @@ jobs:
     <<: *test-template
     docker:
       - image: cimg/python:3.12
+  py312-scrapyd-latest:
+    <<: *test-template
+    docker:
+      - image: cimg/python:3.12
 workflows:
   test:
     jobs:
       # - py27:
       #    is-py27: true
-      - py36
-      - py37
+      # - py36
+      # - py37
       - py38
       - py39
+      - py39-scrapyd-latest:
+          use-scrapyd-v143: false
       - py310-git-postgresql:
           use-git: true
           use-postgresql: true
@@ -281,3 +304,5 @@ workflows:
           use-mysql: true
       - py311
       - py312
+      - py312-scrapyd-latest:
+          use-scrapyd-v143: false

scrapydweb/__version__.py

@@ -1,7 +1,7 @@
 # coding: utf-8
 
 __title__ = 'scrapydweb'
-__version__ = '1.5.0'
+__version__ = '1.5.1'
 __author__ = 'my8100'
 __author_email__ = '[email protected]'
 __url__ = 'https://github.com/my8100/scrapydweb'

scrapydweb/templates/scrapydweb/jobs.html

@@ -88,7 +88,7 @@ <h3 id="logparser_alert" style="color: red; display: none;"></h3>
 
     <el-table-column prop="action" label="Action" sortable align="center" width="{% if SCRAPYD_SERVERS_AMOUNT > 1 %}115{% else %}90{% endif %}" :sort-method="sortByIndex">
       <template slot-scope="scope">
-        <span style="color: red;" :style="{ display: scope.row.display_kill }">Kill {{scope.row.pid}}</span>
+        <span style="color: red;" :style="{ display: scope.row.display_kill }">Kill PID {{scope.row.pid}}</span>
         {% if SCRAPYD_SERVERS_AMOUNT > 1 %}
         <a class="state multinode" :style="{ display: scope.row.display_action }" :href="scope.row.url_multinode">{{ g.multinode|safe }}</a>
         {% endif %}

scrapydweb/views/dashboard/jobs.py

@@ -28,16 +28,16 @@
 DELETED = '1'
 HREF_PATTERN = re.compile(r"""href=['"](.+?)['"]""")  # Temp support for Scrapyd v1.3.0 (not released)
 JOB_PATTERN = re.compile(r"""
-                            <tr>
-                                <td>(?P<Project>.*?)</td>
-                                <td>(?P<Spider>.*?)</td>
-                                <td>(?P<Job>.*?)</td>
-                                (?:<td>(?P<PID>.*?)</td>)?
-                                (?:<td>(?P<Start>.*?)</td>)?
-                                (?:<td>(?P<Runtime>.*?)</td>)?
-                                (?:<td>(?P<Finish>.*?)</td>)?
-                                (?:<td>(?P<Log>.*?)</td>)?
-                                (?:<td>(?P<Items>.*?)</td>)?
+                            <tr>\s*
+                                <td>(?P<Project>.*?)</td>\s*
+                                <td>(?P<Spider>.*?)</td>\s*
+                                <td>(?P<Job>.*?)</td>\s*
+                                (?:<td>(?P<PID>.*?)</td>\s*)?
+                                (?:<td>(?P<Start>.*?)</td>\s*)?
+                                (?:<td>(?P<Runtime>.*?)</td>\s*)?
+                                (?:<td>(?P<Finish>.*?)</td>\s*)?
+                                (?:<td>(?P<Log>.*?)</td>\s*)?
+                                (?:<td>(?P<Items>.*?)</td>\s*)?
                                 [\w\W]*?  # Temp support for Scrapyd v1.3.0 (not released)
                             </tr>
                           """, re.X)
@@ -96,7 +96,7 @@ def __init__(self):
 
     def dispatch_request(self, **kwargs):
         status_code, self.text = self.make_request(self.url, auth=self.AUTH, as_json=False)
-        if status_code != 200 or not re.search(r'<body><h1>Jobs</h1>', self.text):
+        if status_code != 200 or not re.search(r'<h1>Jobs</h1>', self.text):
             kwargs = dict(
                 node=self.node,
                 url=self.url,

tests/test_deploy.py

@@ -31,6 +31,7 @@ def test_auto_packaging_select_option(app, client):
 
 # {'status': 'error', 'message': 'Traceback
 # ...TypeError:...activate_egg(eggpath)...\'tuple\' object is not an iterator\r\n'}
+# egg is not a ZIP file (if using curl, use egg=@path not egg=path)
 def test_addversion(app, client):
     data = {
         '1': 'on',
@@ -39,7 +40,7 @@ def test_addversion(app, client):
         'version': 'fakeversion',
         'file': (BytesIO(b'my file contents'), "fake.egg")
     }
-    req(app, client, view='deploy.upload', kws=dict(node=2), data=data, ins='activate_egg')
+    req(app, client, view='deploy.upload', kws=dict(node=2), data=data, ins=['Fail to deploy project', 'egg'])
 
 
 # <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">

tests/test_deploy_single_scrapyd.py

@@ -49,13 +49,14 @@ def test_auto_packaging_select_option(app, client):
 
 # {'status': 'error', 'message': 'Traceback
 # ...TypeError:...activate_egg(eggpath)...\'tuple\' object is not an iterator\r\n'}
+# egg is not a ZIP file (if using curl, use egg=@path not egg=path)
 def test_addversion(app, client):
     data = {
         'project': 'fakeproject',
         'version': 'fakeversion',
         'file': (io.BytesIO(b'my file contents'), "fake.egg")
     }
-    req_single_scrapyd(app, client, view='deploy.upload', kws=dict(node=1), data=data, ins='activate_egg')
+    req_single_scrapyd(app, client, view='deploy.upload', kws=dict(node=1), data=data, ins=['Fail to deploy project', 'egg'])
 
 
 # <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">

tests/test_projects.py

@@ -39,7 +39,8 @@ def test_listspiders_del(app, client):
         d['listspiders_fail'] = dict(
             url=url_for('projects', node=1, opt='listspiders',
                         project=cst.FAKE_PROJECT, version_spider_job=cst.FAKE_VERSION),
-            checks=['listspiders.json', 'No such file or directory']
+            # project 'FAKE_PROJECT' not found
+            checks=['listspiders.json', 'See details below']
         )
 
         d['delversion'] = dict(
@@ -66,5 +67,6 @@ def test_listspiders_del(app, client):
                 upload_file_deploy(app, client, filename='demo.zip', project=cst.PROJECT, redirect_project=cst.PROJECT)
             response = client.get(v['url'])
             text = get_text(response)
+            # print('text', text)
             for c in v['checks']:
                 assert c in text

tests/utils.py

@@ -117,7 +117,7 @@ def req(app, client, view='', kws=None, url='', data=None,
             with io.open('%s.html' % save, 'wb') as f:
                 f.write(response.data)
         text = get_text(response)
-        # print(text)
+        print('text', text)
         js = {}
         try:
             # js = response.get_json()