Fix dependencies and release v1.4.1 (#216)

* Fix dependencies in v1.4.1

* Update config.yml for circleci

* Add py36 for circleci

* Fix test cases

* Add py310 for circleci

.circleci/config.yml

@@ -121,8 +121,10 @@ jobs:
             . venv/bin/activate
             which python
             python --version
+            pip list
             pip install -r requirements.txt
             pip install -r requirements-tests.txt
+            pip list
 
       - when:
           condition: <<parameters.use-git>>
@@ -131,13 +133,16 @@ jobs:
                 name: Git Scrapy, Scrapyd, and LogParser
                 command: |
                   . venv/bin/activate
+                  pip list
                   pip install -U git+https://github.com/scrapy/scrapy.git
                   pip install -U git+https://github.com/scrapy/scrapyd.git
                   pip install -U git+https://github.com/my8100/logparser.git
+                  pip list
 
       - run:
           name: Launch Scrapyd
           command: |
+            pip list
             cd ~
             printf "[scrapyd]\nusername = admin\npassword = 12345\n" > scrapyd.conf
             cat scrapyd.conf
@@ -149,11 +154,12 @@ jobs:
       - run:
           name: Run tests
           command: |
+            pip list
             ls -l
             . venv/bin/activate
             flake8 . --count --exclude=./venv* --select=E9,F63,F7,F82 --show-source --statistics
             coverage erase
-            coverage run --source=scrapydweb -m pytest tests -s -vv -l --disable-warnings -x
+            coverage run --source=scrapydweb -m pytest tests -s -vv -l --disable-warnings
 
       - run:
           name: Generate report
@@ -177,11 +183,16 @@ jobs:
     <<: *test-template
     docker:
       - image: circleci/python:2.7
-      
-  py37-git-postgresql:
+
+  py36:
     <<: *test-template
     docker:
-      - image: circleci/python:3.7
+      - image: circleci/python:3.6
+
+  py38-git-postgresql:
+    <<: *test-template
+    docker:
+      - image: circleci/python:3.8
       # CircleCI PostgreSQL images available at: https://hub.docker.com/r/circleci/postgres/
       - image: circleci/postgres:9.6-alpine
         environment:
@@ -194,10 +205,10 @@ jobs:
       SCRAPYDWEB_TESTMODE: True
       DATABASE_URL: 'postgresql://circleci:fakepassword@localhost:5432'
 
-  py37-git-mysql:
+  py38-git-mysql:
     <<: *test-template
     docker:
-      - image: circleci/python:3.7
+      - image: circleci/python:3.8
       # https://circleci.com/docs/2.0/postgres-config/#example-mysql-project
       # Plugin caching_sha2_password could not be loaded
       # - image: circleci/mysql:8.0.4
@@ -260,18 +271,26 @@ jobs:
     docker:
       - image: circleci/python:3.9
 
+  py310:
+    <<: *test-template
+    docker:
+      - image: circleci/python:3.10
+
 
 workflows:
   test:
     jobs:
       # - py27:
       #    is-py27: true
 
+      - py36
+
       - py37
-      - py37-git-postgresql:
+
+      - py38-git-postgresql:
           use-git: true
           use-postgresql: true
-      - py37-git-mysql:
+      - py38-git-mysql:
           use-git: true
           use-mysql: true
 
@@ -284,4 +303,5 @@ workflows:
           use-mysql: true
 
       - py39
-
+
+      - py310

requirements.txt

@@ -1,11 +1,22 @@
 pip>=19.1.1
-APScheduler>=3.5.3
-flask>=1.0.2
-flask-compress>=1.4.0
-Flask-SQLAlchemy>=2.3.2
+APScheduler==3.5.3
+click==7.0
+colorama==0.4.0
+Flask==1.0.2
+Flask-Compress==1.4.0
+Flask-SQLAlchemy==2.4.0
+idna==2.7
+itsdangerous==1.1.0
+Jinja2==2.10
 logparser==0.8.2
+MarkupSafe==1.1.1
+pexpect==4.7.0
+ptyprocess==0.6.0
+pytz==2018.9
 requests>=2.21.0
 setuptools>=40.6.3
-six>=1.12.0
-SQLAlchemy>=1.2.15
-w3lib>=1.17.0
+six==1.12.0
+SQLAlchemy==1.3.24
+tzlocal==1.5.1
+w3lib==1.19.0
+Werkzeug==0.14.1

scrapydweb/__version__.py

@@ -1,7 +1,7 @@
 # coding: utf-8
 
 __title__ = 'scrapydweb'
-__version__ = '1.4.0'
+__version__ = '1.4.1'
 __author__ = 'my8100'
 __author_email__ = '[email protected]'
 __url__ = 'https://github.com/my8100/scrapydweb'

setup.py

@@ -31,18 +31,29 @@
     packages=find_packages(exclude=("tests", )),
     include_package_data=True,
     zip_safe=False,
-    python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*",
+    python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <=3.9",
     install_requires=[
-        "APScheduler >= 3.5.3",  # Aug 15, 2018
-        "flask >= 1.0.2",  # May 2, 2018
-        "flask-compress >= 1.4.0",  # Jan 5, 2017
-        "Flask-SQLAlchemy >= 2.3.2",  # Oct 11, 2017
-        "logparser == 0.8.2",
-        "requests >= 2.21.0",  # Dec 10, 2018
-        "setuptools >= 40.6.3",  # Dec 11, 2018
-        "six >= 1.12.0",  # Dec 10, 2018
-        "SQLAlchemy >= 1.2.15",  # Dec 12, 2018
-        "w3lib >= 1.17.0",  # Feb 9, 2017
+        "APScheduler==3.5.3",  # Aug 15, 2018
+        "click==7.0",  # Sep 26, 2018
+        "colorama==0.4.0",  # Oct 10, 2018
+        "Flask==1.0.2",  # May 2, 2018
+        "Flask-Compress==1.4.0",  # Jan 5, 2017
+        "Flask-SQLAlchemy==2.4.0",  # Apr 25, 2019
+        "idna==2.7",  # Jun 11, 2018    
+        "itsdangerous==1.1.0",  # Oct 27, 2018
+        "Jinja2==2.10",  # Nov 9, 2017
+        "logparser==0.8.2",
+        "MarkupSafe==1.1.1",  # Feb 24, 2019
+        "pexpect==4.7.0",  # Apr 7, 2019
+        "ptyprocess==0.6.0",  # Jun 22, 2018
+        "pytz==2018.9",  # Jan 7, 2019
+        "requests>=2.21.0",  # Dec 10, 2018
+        "setuptools>=40.6.3",  # Dec 11, 2018
+        "six==1.12.0",  # Dec 10, 2018
+        "SQLAlchemy==1.3.24",  # Mar 31, 2021
+        "tzlocal==1.5.1",  # Dec 1, 2017
+        "w3lib==1.19.0",  # Jan 25, 2018
+        "Werkzeug==0.14.1",  # Jan 1, 2018
     ],
 
     entry_points={
@@ -55,6 +66,8 @@
         "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
         "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7"
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
     ]
 )

tests/test_schedule_single_scrapyd.py

@@ -92,7 +92,7 @@ def test_run(app, client):
     sleep()
 
     ins = [
-        'JOB: %s' % cst.JOBID,
+        '%s.log' % cst.JOBID,
         'USER_AGENT: Mozilla/5.0 (Windows',
         'ROBOTSTXT_OBEY: False',
         'COOKIES_ENABLED: False',

tests/test_tasks_single_scrapyd.py

@@ -191,7 +191,7 @@ def test_run_with_task(app, client):
     metadata['jobid'] = jobid
 
     ins = [
-        'JOB: %s' % jobid,
+        '%s.log' % jobid,
         'USER_AGENT: Mozilla/5.0 (iPhone',
         'ROBOTSTXT_OBEY: True',
         'COOKIES_ENABLED: False',

tests/utils.py

@@ -131,24 +131,24 @@ def req(app, client, view='', kws=None, url='', data=None,
                     print("ins: %s" % ins)
                 except:  # For compatibility with Win10 Python2
                     print("ins: %s" % repr(ins))
-                assert ins in text
+                assert ins in text, "%s is not found in %s" % (ins, text)
             elif isinstance(ins, list):
                 for i in ins:
                     try:
                         print("ins: %s" % i)
                     except:
                         print("ins: %s" % repr(i))
-                    assert i in text
+                    assert i in text, "%s is not found in %s" % (i, text)
             elif ins:
                 raise TypeError("The argument 'ins' should be either a string or a list")
 
             if isinstance(nos, string_types):
                 print("nos: %s" % nos)
-                assert nos not in text
+                assert nos not in text, "%s is found in %s" % (nos, text)
             elif isinstance(nos, list):
                 for n in nos:
                     print("nos: %s" % n)
-                    assert n not in text
+                    assert n not in text, "%s is found in %s" % (n, text)
             elif nos:
                 raise TypeError("The argument 'nos' should be either a string or a list")