diff --git a/.circleci/config.yml b/.circleci/config.yml index 68f3dc8..00fd68f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -121,8 +121,10 @@ jobs: . venv/bin/activate which python python --version + pip list pip install -r requirements.txt pip install -r requirements-tests.txt + pip list - when: condition: <> @@ -131,13 +133,16 @@ jobs: name: Git Scrapy, Scrapyd, and LogParser command: | . venv/bin/activate + pip list pip install -U git+https://github.com/scrapy/scrapy.git pip install -U git+https://github.com/scrapy/scrapyd.git pip install -U git+https://github.com/my8100/logparser.git + pip list - run: name: Launch Scrapyd command: | + pip list cd ~ printf "[scrapyd]\nusername = admin\npassword = 12345\n" > scrapyd.conf cat scrapyd.conf @@ -149,11 +154,12 @@ jobs: - run: name: Run tests command: | + pip list ls -l . venv/bin/activate flake8 . --count --exclude=./venv* --select=E9,F63,F7,F82 --show-source --statistics coverage erase - coverage run --source=scrapydweb -m pytest tests -s -vv -l --disable-warnings -x + coverage run --source=scrapydweb -m pytest tests -s -vv -l --disable-warnings - run: name: Generate report @@ -177,11 +183,16 @@ jobs: <<: *test-template docker: - image: circleci/python:2.7 - - py37-git-postgresql: + + py36: <<: *test-template docker: - - image: circleci/python:3.7 + - image: circleci/python:3.6 + + py38-git-postgresql: + <<: *test-template + docker: + - image: circleci/python:3.8 # CircleCI PostgreSQL images available at: https://hub.docker.com/r/circleci/postgres/ - image: circleci/postgres:9.6-alpine environment: @@ -194,10 +205,10 @@ jobs: SCRAPYDWEB_TESTMODE: True DATABASE_URL: 'postgresql://circleci:fakepassword@localhost:5432' - py37-git-mysql: + py38-git-mysql: <<: *test-template docker: - - image: circleci/python:3.7 + - image: circleci/python:3.8 # https://circleci.com/docs/2.0/postgres-config/#example-mysql-project # Plugin caching_sha2_password could not be loaded # - image: circleci/mysql:8.0.4 @@ -260,6 +271,11 @@ jobs: docker: - image: circleci/python:3.9 + py310: + <<: *test-template + docker: + - image: circleci/python:3.10 + workflows: test: @@ -267,11 +283,14 @@ workflows: # - py27: # is-py27: true + - py36 + - py37 - - py37-git-postgresql: + + - py38-git-postgresql: use-git: true use-postgresql: true - - py37-git-mysql: + - py38-git-mysql: use-git: true use-mysql: true @@ -284,4 +303,5 @@ workflows: use-mysql: true - py39 - + + - py310 diff --git a/requirements.txt b/requirements.txt index 8b9d417..96002c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,22 @@ pip>=19.1.1 -APScheduler>=3.5.3 -flask>=1.0.2 -flask-compress>=1.4.0 -Flask-SQLAlchemy>=2.3.2 +APScheduler==3.5.3 +click==7.0 +colorama==0.4.0 +Flask==1.0.2 +Flask-Compress==1.4.0 +Flask-SQLAlchemy==2.4.0 +idna==2.7 +itsdangerous==1.1.0 +Jinja2==2.10 logparser==0.8.2 +MarkupSafe==1.1.1 +pexpect==4.7.0 +ptyprocess==0.6.0 +pytz==2018.9 requests>=2.21.0 setuptools>=40.6.3 -six>=1.12.0 -SQLAlchemy>=1.2.15 -w3lib>=1.17.0 +six==1.12.0 +SQLAlchemy==1.3.24 +tzlocal==1.5.1 +w3lib==1.19.0 +Werkzeug==0.14.1 diff --git a/scrapydweb/__version__.py b/scrapydweb/__version__.py index 4b798a5..01aa33b 100644 --- a/scrapydweb/__version__.py +++ b/scrapydweb/__version__.py @@ -1,7 +1,7 @@ # coding: utf-8 __title__ = 'scrapydweb' -__version__ = '1.4.0' +__version__ = '1.4.1' __author__ = 'my8100' __author_email__ = 'my8100@gmail.com' __url__ = 'https://github.com/my8100/scrapydweb' diff --git a/scrapydweb/static/v140/css/dropdown.css b/scrapydweb/static/v141/css/dropdown.css similarity index 100% rename from scrapydweb/static/v140/css/dropdown.css rename to scrapydweb/static/v141/css/dropdown.css diff --git a/scrapydweb/static/v140/css/dropdown_mobileui.css b/scrapydweb/static/v141/css/dropdown_mobileui.css similarity index 100% rename from scrapydweb/static/v140/css/dropdown_mobileui.css rename to scrapydweb/static/v141/css/dropdown_mobileui.css diff --git a/scrapydweb/static/v140/css/icon_upload_icon_right.css b/scrapydweb/static/v141/css/icon_upload_icon_right.css similarity index 100% rename from scrapydweb/static/v140/css/icon_upload_icon_right.css rename to scrapydweb/static/v141/css/icon_upload_icon_right.css diff --git a/scrapydweb/static/v140/css/multinode.css b/scrapydweb/static/v141/css/multinode.css similarity index 100% rename from scrapydweb/static/v140/css/multinode.css rename to scrapydweb/static/v141/css/multinode.css diff --git a/scrapydweb/static/v140/css/stacktable.css b/scrapydweb/static/v141/css/stacktable.css similarity index 100% rename from scrapydweb/static/v140/css/stacktable.css rename to scrapydweb/static/v141/css/stacktable.css diff --git a/scrapydweb/static/v140/css/stats.css b/scrapydweb/static/v141/css/stats.css similarity index 100% rename from scrapydweb/static/v140/css/stats.css rename to scrapydweb/static/v141/css/stats.css diff --git a/scrapydweb/static/v140/css/style.css b/scrapydweb/static/v141/css/style.css similarity index 100% rename from scrapydweb/static/v140/css/style.css rename to scrapydweb/static/v141/css/style.css diff --git a/scrapydweb/static/v140/css/style_mobileui.css b/scrapydweb/static/v141/css/style_mobileui.css similarity index 100% rename from scrapydweb/static/v140/css/style_mobileui.css rename to scrapydweb/static/v141/css/style_mobileui.css diff --git a/scrapydweb/static/v140/css/utf8.css b/scrapydweb/static/v141/css/utf8.css similarity index 100% rename from scrapydweb/static/v140/css/utf8.css rename to scrapydweb/static/v141/css/utf8.css diff --git a/scrapydweb/static/v140/css/utf8_mobileui.css b/scrapydweb/static/v141/css/utf8_mobileui.css similarity index 100% rename from scrapydweb/static/v140/css/utf8_mobileui.css rename to scrapydweb/static/v141/css/utf8_mobileui.css diff --git a/scrapydweb/static/v140/element-ui@2.4.6/lib/index.js b/scrapydweb/static/v141/element-ui@2.4.6/lib/index.js similarity index 100% rename from scrapydweb/static/v140/element-ui@2.4.6/lib/index.js rename to scrapydweb/static/v141/element-ui@2.4.6/lib/index.js diff --git a/scrapydweb/static/v140/element-ui@2.4.6/lib/theme-chalk/fonts/element-icons.woff b/scrapydweb/static/v141/element-ui@2.4.6/lib/theme-chalk/fonts/element-icons.woff similarity index 100% rename from scrapydweb/static/v140/element-ui@2.4.6/lib/theme-chalk/fonts/element-icons.woff rename to scrapydweb/static/v141/element-ui@2.4.6/lib/theme-chalk/fonts/element-icons.woff diff --git a/scrapydweb/static/v140/element-ui@2.4.6/lib/theme-chalk/index.css b/scrapydweb/static/v141/element-ui@2.4.6/lib/theme-chalk/index.css similarity index 100% rename from scrapydweb/static/v140/element-ui@2.4.6/lib/theme-chalk/index.css rename to scrapydweb/static/v141/element-ui@2.4.6/lib/theme-chalk/index.css diff --git a/scrapydweb/static/v140/icon/fav.ico b/scrapydweb/static/v141/icon/fav.ico similarity index 100% rename from scrapydweb/static/v140/icon/fav.ico rename to scrapydweb/static/v141/icon/fav.ico diff --git a/scrapydweb/static/v140/icon/fav.png b/scrapydweb/static/v141/icon/fav.png similarity index 100% rename from scrapydweb/static/v140/icon/fav.png rename to scrapydweb/static/v141/icon/fav.png diff --git a/scrapydweb/static/v140/icon/spider-man-spiderman-face-mask-round-avatar-512.png b/scrapydweb/static/v141/icon/spider-man-spiderman-face-mask-round-avatar-512.png similarity index 100% rename from scrapydweb/static/v140/icon/spider-man-spiderman-face-mask-round-avatar-512.png rename to scrapydweb/static/v141/icon/spider-man-spiderman-face-mask-round-avatar-512.png diff --git a/scrapydweb/static/v140/icon/spiderman.png b/scrapydweb/static/v141/icon/spiderman.png similarity index 100% rename from scrapydweb/static/v140/icon/spiderman.png rename to scrapydweb/static/v141/icon/spiderman.png diff --git a/scrapydweb/static/v140/js/common.js b/scrapydweb/static/v141/js/common.js similarity index 100% rename from scrapydweb/static/v140/js/common.js rename to scrapydweb/static/v141/js/common.js diff --git a/scrapydweb/static/v140/js/echarts.min.js b/scrapydweb/static/v141/js/echarts.min.js similarity index 100% rename from scrapydweb/static/v140/js/echarts.min.js rename to scrapydweb/static/v141/js/echarts.min.js diff --git a/scrapydweb/static/v140/js/github_buttons.html b/scrapydweb/static/v141/js/github_buttons.html similarity index 100% rename from scrapydweb/static/v140/js/github_buttons.html rename to scrapydweb/static/v141/js/github_buttons.html diff --git a/scrapydweb/static/v140/js/github_buttons.js b/scrapydweb/static/v141/js/github_buttons.js similarity index 100% rename from scrapydweb/static/v140/js/github_buttons.js rename to scrapydweb/static/v141/js/github_buttons.js diff --git a/scrapydweb/static/v140/js/icons_menu.js b/scrapydweb/static/v141/js/icons_menu.js similarity index 100% rename from scrapydweb/static/v140/js/icons_menu.js rename to scrapydweb/static/v141/js/icons_menu.js diff --git a/scrapydweb/static/v140/js/jquery.min.js b/scrapydweb/static/v141/js/jquery.min.js similarity index 100% rename from scrapydweb/static/v140/js/jquery.min.js rename to scrapydweb/static/v141/js/jquery.min.js diff --git a/scrapydweb/static/v140/js/multinode.js b/scrapydweb/static/v141/js/multinode.js similarity index 100% rename from scrapydweb/static/v140/js/multinode.js rename to scrapydweb/static/v141/js/multinode.js diff --git a/scrapydweb/static/v140/js/stacktable.js b/scrapydweb/static/v141/js/stacktable.js similarity index 100% rename from scrapydweb/static/v140/js/stacktable.js rename to scrapydweb/static/v141/js/stacktable.js diff --git a/scrapydweb/static/v140/js/stats.js b/scrapydweb/static/v141/js/stats.js similarity index 100% rename from scrapydweb/static/v140/js/stats.js rename to scrapydweb/static/v141/js/stats.js diff --git a/scrapydweb/static/v140/js/vue.min.js b/scrapydweb/static/v141/js/vue.min.js similarity index 100% rename from scrapydweb/static/v140/js/vue.min.js rename to scrapydweb/static/v141/js/vue.min.js diff --git a/setup.py b/setup.py index 7b56ae0..de8d854 100644 --- a/setup.py +++ b/setup.py @@ -31,18 +31,29 @@ packages=find_packages(exclude=("tests", )), include_package_data=True, zip_safe=False, - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*", + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <=3.9", install_requires=[ - "APScheduler >= 3.5.3", # Aug 15, 2018 - "flask >= 1.0.2", # May 2, 2018 - "flask-compress >= 1.4.0", # Jan 5, 2017 - "Flask-SQLAlchemy >= 2.3.2", # Oct 11, 2017 - "logparser == 0.8.2", - "requests >= 2.21.0", # Dec 10, 2018 - "setuptools >= 40.6.3", # Dec 11, 2018 - "six >= 1.12.0", # Dec 10, 2018 - "SQLAlchemy >= 1.2.15", # Dec 12, 2018 - "w3lib >= 1.17.0", # Feb 9, 2017 + "APScheduler==3.5.3", # Aug 15, 2018 + "click==7.0", # Sep 26, 2018 + "colorama==0.4.0", # Oct 10, 2018 + "Flask==1.0.2", # May 2, 2018 + "Flask-Compress==1.4.0", # Jan 5, 2017 + "Flask-SQLAlchemy==2.4.0", # Apr 25, 2019 + "idna==2.7", # Jun 11, 2018 + "itsdangerous==1.1.0", # Oct 27, 2018 + "Jinja2==2.10", # Nov 9, 2017 + "logparser==0.8.2", + "MarkupSafe==1.1.1", # Feb 24, 2019 + "pexpect==4.7.0", # Apr 7, 2019 + "ptyprocess==0.6.0", # Jun 22, 2018 + "pytz==2018.9", # Jan 7, 2019 + "requests>=2.21.0", # Dec 10, 2018 + "setuptools>=40.6.3", # Dec 11, 2018 + "six==1.12.0", # Dec 10, 2018 + "SQLAlchemy==1.3.24", # Mar 31, 2021 + "tzlocal==1.5.1", # Dec 1, 2017 + "w3lib==1.19.0", # Jan 25, 2018 + "Werkzeug==0.14.1", # Jan 1, 2018 ], entry_points={ @@ -55,6 +66,8 @@ "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7" + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", ] ) diff --git a/tests/test_schedule_single_scrapyd.py b/tests/test_schedule_single_scrapyd.py index a79175d..7f0ca1e 100644 --- a/tests/test_schedule_single_scrapyd.py +++ b/tests/test_schedule_single_scrapyd.py @@ -92,7 +92,7 @@ def test_run(app, client): sleep() ins = [ - 'JOB: %s' % cst.JOBID, + '%s.log' % cst.JOBID, 'USER_AGENT: Mozilla/5.0 (Windows', 'ROBOTSTXT_OBEY: False', 'COOKIES_ENABLED: False', diff --git a/tests/test_tasks_single_scrapyd.py b/tests/test_tasks_single_scrapyd.py index 406da14..a9a3092 100644 --- a/tests/test_tasks_single_scrapyd.py +++ b/tests/test_tasks_single_scrapyd.py @@ -191,7 +191,7 @@ def test_run_with_task(app, client): metadata['jobid'] = jobid ins = [ - 'JOB: %s' % jobid, + '%s.log' % jobid, 'USER_AGENT: Mozilla/5.0 (iPhone', 'ROBOTSTXT_OBEY: True', 'COOKIES_ENABLED: False', diff --git a/tests/utils.py b/tests/utils.py index 570d2ba..e2767aa 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -131,24 +131,24 @@ def req(app, client, view='', kws=None, url='', data=None, print("ins: %s" % ins) except: # For compatibility with Win10 Python2 print("ins: %s" % repr(ins)) - assert ins in text + assert ins in text, "%s is not found in %s" % (ins, text) elif isinstance(ins, list): for i in ins: try: print("ins: %s" % i) except: print("ins: %s" % repr(i)) - assert i in text + assert i in text, "%s is not found in %s" % (i, text) elif ins: raise TypeError("The argument 'ins' should be either a string or a list") if isinstance(nos, string_types): print("nos: %s" % nos) - assert nos not in text + assert nos not in text, "%s is found in %s" % (nos, text) elif isinstance(nos, list): for n in nos: print("nos: %s" % n) - assert n not in text + assert n not in text, "%s is found in %s" % (n, text) elif nos: raise TypeError("The argument 'nos' should be either a string or a list")