diff --git a/.circleci/config.yml b/.circleci/config.yml index 28390d8..2858c98 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,6 +2,7 @@ version: 2.1 orbs: codecov: codecov/codecov@1.0.2 + allure: ayte/allure@0.1.3 jobs: py39: &test-template docker: @@ -15,7 +16,7 @@ jobs: default: false use-scrapyd-v143: type: boolean - default: true + default: false use-git: type: boolean default: false @@ -31,16 +32,46 @@ jobs: use-mysql: type: boolean default: false + allure-version: + description: Allure version to use + type: string + default: 2.13.1 + allure-configuration-path: + description: Path to Allure configuration, uses default one if omitted + type: string + default: /usr/local/share/allure/config/allure.yml + allure-target-path: + description: Path for report directory + type: string + default: allure-report + allure-results-path: + description: Path to directory with test results + type: string + default: allure-results + allure-artifact-path: + description: Path that will be used when storing result as artifact + type: string + default: Report/Allure steps: - run: name: Install telnet command: | sudo apt-get update && sudo apt-get install telnet + - run: + name: Install Java 11 + command: | + sudo apt-get update + sudo apt-get install -y openjdk-11-jdk + - run: + name: Set JAVA_HOME + command: | + echo "export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64" >> $BASH_ENV + source $BASH_ENV - run: name: Setup env command: | mkdir ~/logs - ls -l ~ + ls -la ~ - checkout - when: condition: <> @@ -154,11 +185,12 @@ jobs: name: Run tests command: | pip list - ls -l + ls -la . venv/bin/activate flake8 . --count --exclude=./venv* --select=E9,F63,F7,F82 --show-source --statistics coverage erase - coverage run --source=scrapydweb -m pytest tests -s -vv -l --disable-warnings + # tests/test_schedule.py + coverage run --source=scrapydweb -m pytest -s -vv -l --disable-warnings --alluredir=allure-results tests - run: name: Generate report command: | @@ -169,13 +201,66 @@ jobs: coverage html coverage xml ls -la + ls -la allure-results || echo 'ignore error' coveralls + # https://discuss.circleci.com/t/make-custom-command-run-always-with-when-always/38957 + # https://circleci.com/docs/configuration-reference/#the-when-attribute + when: always - store_artifacts: path: htmlcov - store_artifacts: path: coverage.xml - codecov/upload: file: coverage.xml + # https://circleci.com/developer/orbs/orb/codecov/codecov + # - codecov/upload + # https://circleci.com/developer/orbs/orb/coveralls/coveralls + # - coveralls/upload + # https://discuss.circleci.com/t/how-can-we-publish-pytest-results-to-circleci-using-allure-reports/37830/2 + # https://circleci.com/developer/orbs/orb/ayte/allure + # - allure/install + # - allure/report + # https://circleci.com/docs/configuration-reference/#the-when-step + - when: + condition: + equal: [ 1, 1 ] + steps: + - run: + name: Allure archive download + command: >- + curl -L https://github.com/allure-framework/allure2/releases/download/<< + parameters.allure-version >>/allure-commandline-<< parameters.allure-version >>.zip -o + /tmp/allure.zip + when: always + - run: + name: Archive extraction + command: unzip /tmp/allure.zip + when: always + - run: + name: Allure installation + command: sudo mv allure-<< parameters.allure-version >> /usr/local/share/allure + when: always + - run: + name: Allure binary symlinking + command: sudo ln -s /usr/local/share/allure/bin/allure /usr/local/bin/allure + when: always + - when: + condition: + equal: [ 1, 1 ] + steps: + - run: + name: >- + Allure report generation (<< parameters.allure-results-path >> -> << + parameters.allure-target-path >>) + command: | + allure generate \ + --config << parameters.allure-configuration-path >> \ + --report-dir << parameters.allure-target-path >> \ + << parameters.allure-results-path >> + when: always + - store_artifacts: + path: << parameters.allure-target-path >> + destination: << parameters.allure-artifact-path >> py27: <<: *test-template docker: @@ -192,7 +277,7 @@ jobs: <<: *test-template docker: - image: cimg/python:3.8 - py39-scrapyd-latest: + py39-scrapyd-v143: <<: *test-template docker: - image: cimg/python:3.9 @@ -274,10 +359,14 @@ jobs: <<: *test-template docker: - image: cimg/python:3.12 - py312-scrapyd-latest: + py312-scrapyd-v143: <<: *test-template docker: - image: cimg/python:3.12 + py313: + <<: *test-template + docker: + - image: cimg/python:3.13 workflows: test: jobs: @@ -287,8 +376,8 @@ workflows: # - py37 - py38 - py39 - - py39-scrapyd-latest: - use-scrapyd-v143: false + - py39-scrapyd-v143: + use-scrapyd-v143: true - py310-git-postgresql: use-git: true use-postgresql: true @@ -304,5 +393,6 @@ workflows: use-mysql: true - py311 - py312 - - py312-scrapyd-latest: - use-scrapyd-v143: false + - py312-scrapyd-v143: + use-scrapyd-v143: true + - py313 diff --git a/.gitignore b/.gitignore index f9b313b..66ac8f0 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ scrapydweb_settings_v*.py tests/*.html tests/data/ +allure-results/ scrapydweb/data/* diff --git a/requirements-tests.txt b/requirements-tests.txt index 4849842..cf6b5a9 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -4,6 +4,7 @@ coverage pytest # pytest-cov coveralls +allure-pytest scrapy scrapyd diff --git a/requirements.txt b/requirements.txt index dd3f526..5656af3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ Flask-SQLAlchemy==2.4.0 idna==2.7 itsdangerous==2.0.0 Jinja2==3.0.0 -logparser==0.8.2 +logparser>=0.8.4 MarkupSafe==2.0.0 pexpect==4.7.0 ptyprocess==0.6.0 diff --git a/scrapydweb/__version__.py b/scrapydweb/__version__.py index 0fce395..27d54ff 100644 --- a/scrapydweb/__version__.py +++ b/scrapydweb/__version__.py @@ -1,7 +1,7 @@ # coding: utf-8 __title__ = 'scrapydweb' -__version__ = '1.5.1' +__version__ = '1.5.2' __author__ = 'my8100' __author_email__ = 'my8100@gmail.com' __url__ = 'https://github.com/my8100/scrapydweb' diff --git a/scrapydweb/static/v151/css/dropdown.css b/scrapydweb/static/v152/css/dropdown.css similarity index 100% rename from scrapydweb/static/v151/css/dropdown.css rename to scrapydweb/static/v152/css/dropdown.css diff --git a/scrapydweb/static/v151/css/dropdown_mobileui.css b/scrapydweb/static/v152/css/dropdown_mobileui.css similarity index 100% rename from scrapydweb/static/v151/css/dropdown_mobileui.css rename to scrapydweb/static/v152/css/dropdown_mobileui.css diff --git a/scrapydweb/static/v151/css/icon_upload_icon_right.css b/scrapydweb/static/v152/css/icon_upload_icon_right.css similarity index 100% rename from scrapydweb/static/v151/css/icon_upload_icon_right.css rename to scrapydweb/static/v152/css/icon_upload_icon_right.css diff --git a/scrapydweb/static/v151/css/multinode.css b/scrapydweb/static/v152/css/multinode.css similarity index 100% rename from scrapydweb/static/v151/css/multinode.css rename to scrapydweb/static/v152/css/multinode.css diff --git a/scrapydweb/static/v151/css/stacktable.css b/scrapydweb/static/v152/css/stacktable.css similarity index 100% rename from scrapydweb/static/v151/css/stacktable.css rename to scrapydweb/static/v152/css/stacktable.css diff --git a/scrapydweb/static/v151/css/stats.css b/scrapydweb/static/v152/css/stats.css similarity index 100% rename from scrapydweb/static/v151/css/stats.css rename to scrapydweb/static/v152/css/stats.css diff --git a/scrapydweb/static/v151/css/style.css b/scrapydweb/static/v152/css/style.css similarity index 100% rename from scrapydweb/static/v151/css/style.css rename to scrapydweb/static/v152/css/style.css diff --git a/scrapydweb/static/v151/css/style_mobileui.css b/scrapydweb/static/v152/css/style_mobileui.css similarity index 100% rename from scrapydweb/static/v151/css/style_mobileui.css rename to scrapydweb/static/v152/css/style_mobileui.css diff --git a/scrapydweb/static/v151/css/utf8.css b/scrapydweb/static/v152/css/utf8.css similarity index 100% rename from scrapydweb/static/v151/css/utf8.css rename to scrapydweb/static/v152/css/utf8.css diff --git a/scrapydweb/static/v151/css/utf8_mobileui.css b/scrapydweb/static/v152/css/utf8_mobileui.css similarity index 100% rename from scrapydweb/static/v151/css/utf8_mobileui.css rename to scrapydweb/static/v152/css/utf8_mobileui.css diff --git a/scrapydweb/static/v151/element-ui@2.4.6/lib/index.js b/scrapydweb/static/v152/element-ui@2.4.6/lib/index.js similarity index 100% rename from scrapydweb/static/v151/element-ui@2.4.6/lib/index.js rename to scrapydweb/static/v152/element-ui@2.4.6/lib/index.js diff --git a/scrapydweb/static/v151/element-ui@2.4.6/lib/theme-chalk/fonts/element-icons.woff b/scrapydweb/static/v152/element-ui@2.4.6/lib/theme-chalk/fonts/element-icons.woff similarity index 100% rename from scrapydweb/static/v151/element-ui@2.4.6/lib/theme-chalk/fonts/element-icons.woff rename to scrapydweb/static/v152/element-ui@2.4.6/lib/theme-chalk/fonts/element-icons.woff diff --git a/scrapydweb/static/v151/element-ui@2.4.6/lib/theme-chalk/index.css b/scrapydweb/static/v152/element-ui@2.4.6/lib/theme-chalk/index.css similarity index 100% rename from scrapydweb/static/v151/element-ui@2.4.6/lib/theme-chalk/index.css rename to scrapydweb/static/v152/element-ui@2.4.6/lib/theme-chalk/index.css diff --git a/scrapydweb/static/v151/icon/fav.ico b/scrapydweb/static/v152/icon/fav.ico similarity index 100% rename from scrapydweb/static/v151/icon/fav.ico rename to scrapydweb/static/v152/icon/fav.ico diff --git a/scrapydweb/static/v151/icon/fav.png b/scrapydweb/static/v152/icon/fav.png similarity index 100% rename from scrapydweb/static/v151/icon/fav.png rename to scrapydweb/static/v152/icon/fav.png diff --git a/scrapydweb/static/v151/icon/spider-man-spiderman-face-mask-round-avatar-512.png b/scrapydweb/static/v152/icon/spider-man-spiderman-face-mask-round-avatar-512.png similarity index 100% rename from scrapydweb/static/v151/icon/spider-man-spiderman-face-mask-round-avatar-512.png rename to scrapydweb/static/v152/icon/spider-man-spiderman-face-mask-round-avatar-512.png diff --git a/scrapydweb/static/v151/icon/spiderman.png b/scrapydweb/static/v152/icon/spiderman.png similarity index 100% rename from scrapydweb/static/v151/icon/spiderman.png rename to scrapydweb/static/v152/icon/spiderman.png diff --git a/scrapydweb/static/v151/js/common.js b/scrapydweb/static/v152/js/common.js similarity index 100% rename from scrapydweb/static/v151/js/common.js rename to scrapydweb/static/v152/js/common.js diff --git a/scrapydweb/static/v151/js/echarts.min.js b/scrapydweb/static/v152/js/echarts.min.js similarity index 100% rename from scrapydweb/static/v151/js/echarts.min.js rename to scrapydweb/static/v152/js/echarts.min.js diff --git a/scrapydweb/static/v151/js/github_buttons.html b/scrapydweb/static/v152/js/github_buttons.html similarity index 100% rename from scrapydweb/static/v151/js/github_buttons.html rename to scrapydweb/static/v152/js/github_buttons.html diff --git a/scrapydweb/static/v151/js/github_buttons.js b/scrapydweb/static/v152/js/github_buttons.js similarity index 100% rename from scrapydweb/static/v151/js/github_buttons.js rename to scrapydweb/static/v152/js/github_buttons.js diff --git a/scrapydweb/static/v151/js/icons_menu.js b/scrapydweb/static/v152/js/icons_menu.js similarity index 100% rename from scrapydweb/static/v151/js/icons_menu.js rename to scrapydweb/static/v152/js/icons_menu.js diff --git a/scrapydweb/static/v151/js/jquery.min.js b/scrapydweb/static/v152/js/jquery.min.js similarity index 100% rename from scrapydweb/static/v151/js/jquery.min.js rename to scrapydweb/static/v152/js/jquery.min.js diff --git a/scrapydweb/static/v151/js/multinode.js b/scrapydweb/static/v152/js/multinode.js similarity index 100% rename from scrapydweb/static/v151/js/multinode.js rename to scrapydweb/static/v152/js/multinode.js diff --git a/scrapydweb/static/v151/js/stacktable.js b/scrapydweb/static/v152/js/stacktable.js similarity index 100% rename from scrapydweb/static/v151/js/stacktable.js rename to scrapydweb/static/v152/js/stacktable.js diff --git a/scrapydweb/static/v151/js/stats.js b/scrapydweb/static/v152/js/stats.js similarity index 100% rename from scrapydweb/static/v151/js/stats.js rename to scrapydweb/static/v152/js/stats.js diff --git a/scrapydweb/static/v151/js/vue.min.js b/scrapydweb/static/v152/js/vue.min.js similarity index 100% rename from scrapydweb/static/v151/js/vue.min.js rename to scrapydweb/static/v152/js/vue.min.js diff --git a/scrapydweb/templates/scrapydweb/jobs.html b/scrapydweb/templates/scrapydweb/jobs.html index b316731..1c96269 100644 --- a/scrapydweb/templates/scrapydweb/jobs.html +++ b/scrapydweb/templates/scrapydweb/jobs.html @@ -15,7 +15,7 @@ {% if SCRAPYD_SERVERS_AMOUNT == 1 and (pageview == 1 or pageview % CHECK_LATEST_VERSION_FREQ == 0) %} - + {% else %} @@ -31,7 +31,7 @@

Get the list of jobs of all projects in database.