diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..3e30413 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,7 @@ +[run] +branch = True +source = pybluedot +omit = pybluedot/tests/* + +[report] +ignore-errors = True \ No newline at end of file diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..9e1596f --- /dev/null +++ b/.pylintrc @@ -0,0 +1,118 @@ +# The format of this file isn't really documented; just use --generate-rcfile +[MASTER] +# Add to the black list. It should be a base name, not a +# path. You may set this option multiple times. +# +# Note the 'openstack' below is intended to match only +# neutron.openstack.common. If we ever have another 'openstack' +# dirname, then we'll need to expand the ignore features in pylint :/ +ignore=.git,tests,openstack + +[MESSAGES CONTROL] +# NOTE(gus): This is a long list. A number of these are important and +# should be re-enabled once the offending code is fixed (or marked +# with a local disable) +disable= +# "F" Fatal errors that prevent further processing + import-error, +# "I" Informational noise + locally-disabled, +# "E" Error for important programming issues (likely bugs) + access-member-before-definition, + no-member, + no-method-argument, + no-self-argument, +# "W" Warnings for stylistic problems or minor programming issues + abstract-method, + arguments-differ, + attribute-defined-outside-init, + bad-builtin, + bad-indentation, + broad-except, + dangerous-default-value, + deprecated-lambda, + expression-not-assigned, + fixme, + global-statement, + no-init, + non-parent-init-called, + not-callable, + protected-access, + redefined-builtin, + redefined-outer-name, + signature-differs, + star-args, + super-init-not-called, + super-on-old-class, + unpacking-non-sequence, + unused-argument, + unused-import, + unused-variable, +# "C" Coding convention violations + bad-continuation, + invalid-name, + missing-docstring, + superfluous-parens, +# "R" Refactor recommendations + abstract-class-little-used, + abstract-class-not-used, + duplicate-code, + interface-not-implemented, + no-self-use, + too-few-public-methods, + too-many-ancestors, + too-many-arguments, + too-many-branches, + too-many-instance-attributes, + too-many-lines, + too-many-locals, + too-many-public-methods, + too-many-return-statements, + too-many-statements + +[BASIC] +# Variable names can be 1 to 31 characters long, with lowercase and underscores +variable-rgx=[a-z_][a-z0-9_]{0,30}$ + +# Argument names can be 2 to 31 characters long, with lowercase and underscores +argument-rgx=[a-z_][a-z0-9_]{1,30}$ + +# Method names should be at least 3 characters long +# and be lowecased with underscores +method-rgx=([a-z_][a-z0-9_]{2,}|setUp|tearDown)$ + +# Module names matching neutron-* are ok (files in bin/) +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+)|(pybluedot-[a-z0-9_-]+))$ + +# Don't require docstrings on tests. +no-docstring-rgx=((__.*__)|([tT]est.*)|setUp|tearDown)$ + +[FORMAT] +# Maximum number of characters on a single line. +max-line-length=79 + +[VARIABLES] +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +# _ is used by our localization +additional-builtins=_ + +[CLASSES] +# List of interface methods to ignore, separated by a comma. +ignore-iface-methods= + +[IMPORTS] +# Deprecated modules which should not be used, separated by a comma +deprecated-modules= +# In most openstack projects, the use of the stdlib's json package is discouraged. +# However this isn't an openstack project so I'm allowing this +# json + + +[TYPECHECK] +# List of module names for which member attributes should not be checked +ignored-modules=six.moves,_MovedItems + +[REPORTS] +# Tells whether to display a full report or only the messages +reports=no diff --git a/.testr.conf b/.testr.conf new file mode 100644 index 0000000..8b86b3c --- /dev/null +++ b/.testr.conf @@ -0,0 +1,4 @@ +[DEFAULT] +test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 OS_LOG_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./ ${OS_TEST_PATH:-./pybluedot/tests/unit} $LISTOPT $IDOPTION | cat +test_id_option=--load-list $IDFILE +test_list_option=--list \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 94a93b0..c1f6484 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1 +1,29 @@ -random notes about contributing can go here +# How to Contribute + +`pybluedot` uses Travis CI for Continuous Integration. This runs both unit tests as well as linting checks to ensure code quality. + +It's very easy to run these checks for yourself, without even having to push any code or open a PR. All you need to do is install virtualenv and tox: + + pip install virtualenv tox + +Then, you can run these three commands from inside this repository (they are the same commands that Travis will run): + +- `tox -epep8`: This runs PEP8 and other style checks on your code +- `tox -epy27`: This runs unit tests +- `tox -ecover`: This runs coverage testing + +If those three commands run without error, then you're ready to open a Pull Request. If not, please address those issues (of course, feel free to reach out on Slack for any assistance with this). + +# Tips + +## Cryptography Failure on OSX + +If you are developing on OSX and are seeing this error when running Tox: + + fatal error: too many errors emitted, stopping now [-ferror-limit=] + + 20 errors generated. + + error: command 'cc' failed with exit status 1 + +Make sure you're running the latest version of virtualenv and pip, then destroy and re-run Tox, and you should be good. \ No newline at end of file diff --git a/pybluedot/__init__.py b/pybluedot/__init__.py index 6d6e477..e69de29 100644 --- a/pybluedot/__init__.py +++ b/pybluedot/__init__.py @@ -1,4 +0,0 @@ -""" -keeping this __init__.py very light for now. We can expand later if needed. - QHM -""" - diff --git a/pybluedot/tests/__init__.py b/pybluedot/tests/__init__.py index 6d6e477..e69de29 100644 --- a/pybluedot/tests/__init__.py +++ b/pybluedot/tests/__init__.py @@ -1,4 +0,0 @@ -""" -keeping this __init__.py very light for now. We can expand later if needed. - QHM -""" - diff --git a/pybluedot/tests/base.py b/pybluedot/tests/base.py new file mode 100644 index 0000000..0a299e8 --- /dev/null +++ b/pybluedot/tests/base.py @@ -0,0 +1,20 @@ +import unittest + + +class DietTestCase(unittest.TestCase): + """Same great taste, less filling. + + This class provides testing functionality that is common across all tests. + """ + + def setUp(self): + """Perform setup activies for unit tests + """ + + pass + + def tearDown(self): + """Perform teardown activies for unit tests + """ + + pass diff --git a/pybluedot/tests/test_pybluedot_basic.py b/pybluedot/tests/test_pybluedot_basic.py deleted file mode 100644 index c89a6aa..0000000 --- a/pybluedot/tests/test_pybluedot_basic.py +++ /dev/null @@ -1,2 +0,0 @@ -from pybluedot import commands - diff --git a/pybluedot/tests/unit/__init__.py b/pybluedot/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pybluedot/tests/unit/test_sample.py b/pybluedot/tests/unit/test_sample.py new file mode 100644 index 0000000..fc2aad8 --- /dev/null +++ b/pybluedot/tests/unit/test_sample.py @@ -0,0 +1,30 @@ +from pybluedot.tests import base + + +class SampleTestSuite(base.DietTestCase): + """This test suite provides some common things relevant to the + tests in this file. All tests should inherit from this class + """ + + def setUp(self): + """Perform setup activities + """ + + super(SampleTestSuite, self).setUp() + + def tearDown(self): + """Perform teardown activities + """ + + super(SampleTestSuite, self).tearDown() + + +class test_sample_foo(SampleTestSuite): + """This is a sample unit test. Implement the testing here + """ + + def runTest(self): + """Runs the unit test + """ + + self.assertTrue(True) diff --git a/requirements.txt b/requirements.txt index c42592f..f229360 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -requests=2.12.3 +requests diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..574f715 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,24 @@ +[metadata] +name = pybluedot +version = 0.0.1 +summary = pybluedot is a Python library/command-line utility that allows for easy display of NASA data +description-file = + README.md +author = Commitmas 2016 +author-email = commitmas@vbrownbag.com +home-page = https://github.com/commitmas/pybluedot +classifier = + Environment :: MegaAwesome + Intended Audience :: Information Technology + Intended Audience :: System Administrators + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.4 + +[entry_points] +console_scripts = + bdot = pybluedot.cli:main diff --git a/setup.py b/setup.py index f1f5f9a..377f971 100644 --- a/setup.py +++ b/setup.py @@ -1,13 +1,11 @@ -import os -from setuptools import setup -setup( - name = "pybluedot", - version = "0.0.1", - author = "Matt Oswalt", - author_email = "not_sure@email.com", - description = ("A Python library/command-line utility that allows for easy display of NASA data."), - license = "Apache-2.0", - url = "https://github.com/commitmas/pybluedot", - packages = ['pybluedot', 'tests'], - ) +import setuptools + +try: + import multiprocessing +except ImportError: + pass + +setuptools.setup( + setup_requires=['pbr>=1.8'], + pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..81cddc8 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,24 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +hacking<0.11,>=0.10.0 + +astroid==1.3.8 +cliff>=1.14.0 # Apache-2.0 +coverage==3.6 +fixtures>=1.3.1 +mock>=1.2 +python-subunit>=0.0.18 +requests-mock>=0.6.0 # Apache-2.0 +sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 +oslosphinx>=2.5.0 # Apache-2.0 +testrepository>=0.0.18 +testtools>=1.4.0 +testresources>=0.2.4 +testscenarios>=0.4 +WebTest>=2.0 +oslotest>=1.10.0 # Apache-2.0 +os-testr>=0.4.1 +tempest-lib>=0.10.0 +ddt>=0.7.0 +pylint==1.4.4 # GNU GPL v2 \ No newline at end of file diff --git a/tools/abandon_old_reviews.sh b/tools/abandon_old_reviews.sh new file mode 100755 index 0000000..2cb08a7 --- /dev/null +++ b/tools/abandon_old_reviews.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# WARNING! +# Please do not run this script without talking to the Neutron PTL. Auto +# abandoning people's changes is a good thing, but must be done with care. +# +# before you run this modify your .ssh/config to create a +# review.openstack.org entry: +# +# Host review.openstack.org +# User +# Port 29418 +# + +# Note: due to gerrit bug somewhere, this double posts messages. :( + +# first purge the all reviews that are more than 4w old and blocked by a core -2 + +set -o errexit + +function abandon_review { + local gitid=$1 + shift + local msg=$@ + echo "Abandoning $gitid" + # echo ssh review.openstack.org gerrit review $gitid --abandon --message \"$msg\" + ssh review.openstack.org gerrit review $gitid --abandon --message \"$msg\" +} + +PROJECTS="(project:openstack/neutron OR project:openstack/neutron-fwaas OR \ + project:openstack/neutron-lbaas OR project:openstack/neutron-vpnaas OR \ + project:openstack/python-neutronclient OR project:openstack/neutron-specs)" + +blocked_reviews=$(ssh review.openstack.org "gerrit query --current-patch-set --format json $PROJECTS status:open age:4w label:Code-Review<=-2" | jq .currentPatchSet.revision | grep -v null | sed 's/"//g') + +blocked_msg=$(cat < 4 weeks without comment and currently blocked by a +core reviewer with a -2. We are abandoning this for now. + +Feel free to reactivate the review by pressing the restore button and +contacting the reviewer with the -2 on this review to ensure you +address their concerns. + +EOF +) + +# For testing, put in a git rev of something you own and uncomment +# blocked_reviews="b6c4218ae4d75b86c33fa3d37c27bc23b46b6f0f" + +for review in $blocked_reviews; do + # echo ssh review.openstack.org gerrit review $review --abandon --message \"$msg\" + echo "Blocked review $review" + abandon_review $review $blocked_msg +done + +# then purge all the reviews that are > 4w with no changes and Jenkins has -1ed + +failing_reviews=$(ssh review.openstack.org "gerrit query --current-patch-set --format json $PROJECTS status:open age:4w NOT label:Verified>=1,jenkins" | jq .currentPatchSet.revision | grep -v null | sed 's/"//g') + +failing_msg=$(cat < 4 weeks without comment, and failed Jenkins the last +time it was checked. We are abandoning this for now. + +Feel free to reactivate the review by pressing the restore button and +leaving a 'recheck' comment to get fresh test results. + +EOF +) + +for review in $failing_reviews; do + echo "Failing review $review" + abandon_review $review $failing_msg +done diff --git a/tools/check_unit_test_structure.sh b/tools/check_unit_test_structure.sh new file mode 100755 index 0000000..c1acc40 --- /dev/null +++ b/tools/check_unit_test_structure.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +# This script identifies the unit test modules that do not correspond +# directly with a module in the code tree. See TESTING.rst for the +# intended structure. + +neutron_path=$(cd "$(dirname "$0")/.." && pwd) +base_test_path=neutron/tests/unit +test_path=$neutron_path/$base_test_path + +test_files=$(find ${test_path} -iname 'test_*.py') + +ignore_regexes=( + # The following vendor plugins are not required to confrm to the + # structural requirements. + "^plugins/brocade.*$" + "^plugins/ibm.*$" + # The following open source plugin tests are not actually unit + # tests and are ignored pending their relocation to the functional + # test tree. + "^plugins/ml2/drivers/mech_sriov/mech_driver/test_mech_sriov_nic_switch.py$" + "^plugins/ml2/test_security_group.py$" + "^plugins/ml2/test_port_binding.py$" + "^plugins/ml2/test_extension_driver_api.py$" + "^plugins/ml2/test_ext_portsecurity.py$" + "^plugins/ml2/test_agent_scheduler.py$" + "^plugins/ml2/drivers/openvswitch/agent/test_agent_scheduler.py$" + "^plugins/ml2/drivers/openvswitch/agent/test_ovs_tunnel.py$" + "^plugins/openvswitch/test_agent_scheduler.py$" +) + +error_count=0 +ignore_count=0 +total_count=0 +for test_file in ${test_files[@]}; do + relative_path=${test_file#$test_path/} + expected_path=$(dirname $neutron_path/neutron/$relative_path) + test_filename=$(basename "$test_file") + expected_filename=${test_filename#test_} + # Module filename (e.g. foo/bar.py -> foo/test_bar.py) + filename=$expected_path/$expected_filename + # Package dir (e.g. foo/ -> test_foo.py) + package_dir=${filename%.py} + if [ ! -f "$filename" ] && [ ! -d "$package_dir" ]; then + for ignore_regex in ${ignore_regexes[@]}; do + if [[ "$relative_path" =~ $ignore_regex ]]; then + ((ignore_count++)) + continue 2 + fi + done + echo "Unexpected test file: $base_test_path/$relative_path" + ((error_count++)) + fi + ((total_count++)) +done + +if [ "$ignore_count" -ne 0 ]; then + echo "$ignore_count unmatched test modules were ignored" +fi + +if [ "$error_count" -eq 0 ]; then + echo 'Success! All test modules match targets in the code tree.' + exit 0 +else + echo "Failure! $error_count of $total_count test modules do not match targets in the code tree." + exit 1 +fi diff --git a/tools/clean.sh b/tools/clean.sh new file mode 100755 index 0000000..b79f035 --- /dev/null +++ b/tools/clean.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +rm -rf ./*.deb ./*.tar.gz ./*.dsc ./*.changes +rm -rf */*.deb +rm -rf ./plugins/**/build/ ./plugins/**/dist +rm -rf ./plugins/**/lib/neutron_*_plugin.egg-info ./plugins/neutron-* diff --git a/tools/coding-checks.sh b/tools/coding-checks.sh new file mode 100644 index 0000000..4926dee --- /dev/null +++ b/tools/coding-checks.sh @@ -0,0 +1,58 @@ +#!/bin/sh + +set -eu + +usage () { + echo "Usage: $0 [OPTION]..." + echo "Run pybluedot's coding check(s)" + echo "" + echo " -Y, --pylint [] Run pylint check on the entire pybluedot module or just files changed in basecommit (e.g. HEAD~1)" + echo " -h, --help Print this usage message" + echo + exit 0 +} + +process_options () { + i=1 + while [ $i -le $# ]; do + eval opt=\$$i + case $opt in + -h|--help) usage;; + -Y|--pylint) pylint=1;; + *) scriptargs="$scriptargs $opt" + esac + i=$((i+1)) + done +} + +run_pylint () { + local target="${scriptargs:-all}" + + if [ "$target" = "all" ]; then + files="pybluedot" + else + case "$target" in + *HEAD~[0-9]*) files=$(git diff --diff-filter=AM --name-only $target -- "*.py");; + *) echo "$target is an unrecognized basecommit"; exit 1;; + esac + fi + + echo "Running pylint..." + echo "You can speed this up by running it on 'HEAD~[0-9]' (e.g. HEAD~1, this change only)..." + if [ -n "${files}" ]; then + pylint --rcfile=.pylintrc --output-format=colorized ${files} + else + echo "No python changes in this commit, pylint check not required." + exit 0 + fi +} + +scriptargs= +pylint=1 + +process_options $@ + +if [ $pylint -eq 1 ]; then + run_pylint + exit 0 +fi diff --git a/tools/configure_for_func_testing.sh b/tools/configure_for_func_testing.sh new file mode 100755 index 0000000..3a431b0 --- /dev/null +++ b/tools/configure_for_func_testing.sh @@ -0,0 +1,246 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +set -e + + +# Control variable used to determine whether to execute this script +# directly or allow the gate_hook to import. +IS_GATE=${IS_GATE:-False} + + +if [[ "$IS_GATE" != "True" ]] && [[ "$#" -lt 1 ]]; then + >&2 echo "Usage: $0 /path/to/devstack [-i] +Configure a host to run Neutron's functional test suite. + +-i Install Neutron's package dependencies. By default, it is assumed + that devstack has already been used to deploy neutron to the + target host and that package dependencies need not be installed. + +Warning: This script relies on devstack to perform extensive +modification to the underlying host. It is recommended that it be +invoked only on a throw-away VM." + exit 1 +fi + + +# Skip the first argument +OPTIND=2 +while getopts ":i" opt; do + case $opt in + i) + INSTALL_BASE_DEPENDENCIES=True + ;; + esac + +done + +# Default to environment variables to permit the gate_hook to override +# when sourcing. +VENV=${VENV:-dsvm-functional} +DEVSTACK_PATH=${DEVSTACK_PATH:-$1} +PROJECT_NAME=${PROJECT_NAME:-neutron} +REPO_BASE=${GATE_DEST:-$(cd $(dirname "$0")/../.. && pwd)} +# The gate should automatically install dependencies. +INSTALL_BASE_DEPENDENCIES=${INSTALL_BASE_DEPENDENCIES:-$IS_GATE} + + +if [ ! -f "$DEVSTACK_PATH/stack.sh" ]; then + >&2 echo "Unable to find devstack at '$DEVSTACK_PATH'. Please verify that the specified path points to a valid devstack repo." + exit 1 +fi + + +set -x + + +function _init { + # Subsequently-called devstack functions depend on the following variables. + HOST_IP=127.0.0.1 + FILES=$DEVSTACK_PATH/files + TOP_DIR=$DEVSTACK_PATH + + source $DEVSTACK_PATH/stackrc + + # Allow the gate to override values set by stackrc. + DEST=${GATE_DEST:-$DEST} + STACK_USER=${GATE_STACK_USER:-$STACK_USER} +} + + +function _install_base_deps { + echo_summary "Installing base dependencies" + + INSTALL_TESTONLY_PACKAGES=True + PACKAGES=$(get_packages general neutron,q-agt,q-l3) + # Do not install 'python-' prefixed packages other than + # python-dev*. Neutron's functional testing relies on deployment + # to a tox env so there is no point in installing python + # dependencies system-wide. + PACKAGES=$(echo $PACKAGES | perl -pe 's|python-(?!dev)[^ ]*||g') + install_package $PACKAGES +} + + +function _install_rpc_backend { + echo_summary "Installing rabbitmq" + + RABBIT_USERID=${RABBIT_USERID:-stackrabbit} + RABBIT_HOST=${RABBIT_HOST:-$SERVICE_HOST} + RABBIT_PASSWORD=${RABBIT_HOST:-secretrabbit} + + source $DEVSTACK_PATH/lib/rpc_backend + + enable_service rabbit + install_rpc_backend + restart_rpc_backend +} + + +function _install_databases { + echo_summary "Installing databases" + + # Avoid attempting to configure the db if it appears to already + # have run. The setup as currently defined is not idempotent. + if mysql openstack_citest > /dev/null 2>&1 < /dev/null; then + echo_summary "DB config appears to be complete, skipping." + return 0 + fi + + MYSQL_PASSWORD=${MYSQL_PASSWORD:-secretmysql} + DATABASE_PASSWORD=${DATABASE_PASSWORD:-secretdatabase} + + source $DEVSTACK_PATH/lib/database + + enable_service mysql + initialize_database_backends + install_database + configure_database_mysql + + enable_service postgresql + initialize_database_backends + install_database + configure_database_postgresql + + # Set up the 'openstack_citest' user and database in each backend + tmp_dir=$(mktemp -d) + trap "rm -rf $tmp_dir" EXIT + + cat << EOF > $tmp_dir/mysql.sql +CREATE DATABASE openstack_citest; +CREATE USER 'openstack_citest'@'localhost' IDENTIFIED BY 'openstack_citest'; +CREATE USER 'openstack_citest' IDENTIFIED BY 'openstack_citest'; +GRANT ALL PRIVILEGES ON *.* TO 'openstack_citest'@'localhost'; +GRANT ALL PRIVILEGES ON *.* TO 'openstack_citest'; +FLUSH PRIVILEGES; +EOF + /usr/bin/mysql -u root < $tmp_dir/mysql.sql + + cat << EOF > $tmp_dir/postgresql.sql +CREATE USER openstack_citest WITH CREATEDB LOGIN PASSWORD 'openstack_citest'; +CREATE DATABASE openstack_citest WITH OWNER openstack_citest; +EOF + + # User/group postgres needs to be given access to tmp_dir + setfacl -m g:postgres:rwx $tmp_dir + sudo -u postgres /usr/bin/psql --file=$tmp_dir/postgresql.sql +} + + +function _install_agent_deps { + echo_summary "Installing agent dependencies" + + source $DEVSTACK_PATH/lib/neutron-legacy + + ENABLED_SERVICES=q-agt,q-dhcp,q-l3 + install_neutron_agent_packages +} + + +# Set up the rootwrap sudoers for neutron to target the rootwrap +# configuration deployed in the venv. +function _install_rootwrap_sudoers { + echo_summary "Installing rootwrap sudoers file" + + PROJECT_VENV=$REPO_BASE/$PROJECT_NAME/.tox/$VENV + ROOTWRAP_SUDOER_CMD="$PROJECT_VENV/bin/neutron-rootwrap $PROJECT_VENV/etc/neutron/rootwrap.conf *" + ROOTWRAP_DAEMON_SUDOER_CMD="$PROJECT_VENV/bin/neutron-rootwrap-daemon $PROJECT_VENV/etc/neutron/rootwrap.conf" + TEMPFILE=$(mktemp) + cat << EOF > $TEMPFILE +# A bug in oslo.rootwrap [1] prevents commands executed with 'ip netns +# exec' from being automatically qualified with a prefix from +# rootwrap's configured exec_dirs. To work around this problem, add +# the venv bin path to a user-specific secure_path. +# +# While it might seem preferable to set a command-specific +# secure_path, this would only ensure the correct path for 'ip netns +# exec' and the command targeted for execution in the namespace would +# not inherit the path. +# +# 1: https://bugs.launchpad.net/oslo.rootwrap/+bug/1417331 +# +Defaults:$STACK_USER secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PROJECT_VENV/bin" +$STACK_USER ALL=(root) NOPASSWD: $ROOTWRAP_SUDOER_CMD +$STACK_USER ALL=(root) NOPASSWD: $ROOTWRAP_DAEMON_SUDOER_CMD +EOF + chmod 0440 $TEMPFILE + sudo chown root:root $TEMPFILE + # Name the functional testing rootwrap to ensure that it will be + # loaded after the devstack rootwrap (50_stack_sh if present) so + # that the functional testing secure_path (a superset of what + # devstack expects) will not be overwritten. + sudo mv $TEMPFILE /etc/sudoers.d/60-neutron-func-test-rootwrap +} + + +function _install_post_devstack { + echo_summary "Performing post-devstack installation" + + _install_databases + _install_rootwrap_sudoers + + # Installing python-openvswitch from packages is a stop-gap while + # python-openvswitch remains unavailable from pypi. This also + # requires that sitepackages=True be set in tox.ini to allow the + # venv to use the installed package. Once python-openvswitch + # becomes available on pypi, this will no longer be required. + # + # NOTE: the package name 'python-openvswitch' is common across + # supported distros. + install_package python-openvswitch +} + + +function configure_host_for_func_testing { + echo_summary "Configuring host for functional testing" + + if [[ "$INSTALL_BASE_DEPENDENCIES" == "True" ]]; then + # Installing of the following can be achieved via devstack by + # installing neutron, so their installation is conditional to + # minimize the work to do on a devstack-configured host. + _install_base_deps + _install_agent_deps + _install_rpc_backend + fi + _install_post_devstack +} + + +_init + + +if [[ "$IS_GATE" != "True" ]]; then + configure_host_for_func_testing +fi diff --git a/tools/copy_api_tests_from_tempest.sh b/tools/copy_api_tests_from_tempest.sh new file mode 100755 index 0000000..7084451 --- /dev/null +++ b/tools/copy_api_tests_from_tempest.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +# This script is intended to allow repeatable migration of the neutron +# api tests from tempest. The intention is to allow development to +# continue in Tempest while the migration strategy evolves. + +set -e + +if [[ "$#" -ne 1 ]]; then + >&2 echo "Usage: $0 /path/to/tempest +Migrate neutron's api tests from a tempest repo." + exit 1 +fi + +TEMPEST_PATH=${TEMPEST_PATH:-$1} + +if [ ! -f "$TEMPEST_PATH/run_tempest.sh" ]; then + >&2 echo "Unable to find tempest at '$TEMPEST_PATH'. Please verify that the specified path points to a valid tempest repo." + exit 1 +fi + +NEUTRON_PATH=${NEUTRON_PATH:-$(cd "$(dirname "$0")/.." && pwd)} +NEUTRON_TEST_PATH=$NEUTRON_PATH/neutron/tests + +function copy_files { + local tempest_dep_paths=( + 'tempest' + 'tempest/common' + 'tempest/common/generator' + 'tempest/common/utils' + 'tempest/services' + 'tempest/services/identity' + 'tempest/services/identity/v2' + 'tempest/services/identity/v2/json' + 'tempest/services/identity/v3' + 'tempest/services/identity/v3/json' + 'tempest/services/network' + 'tempest/services/network/json' + ) + for tempest_dep_path in ${tempest_dep_paths[@]}; do + local target_path=$NEUTRON_TEST_PATH/$tempest_dep_path + if [[ ! -d "$target_path" ]]; then + mkdir -p "$target_path" + fi + cp $TEMPEST_PATH/$tempest_dep_path/*.py "$target_path" + done + local paths_to_remove=( + "$NEUTRON_TEST_PATH/tempest/clients.py" + ) + for path_to_remove in ${paths_to_remove[@]}; do + if [ -f "$path_to_remove" ]; then + rm "$path_to_remove" + fi + done + + # Tests are now maintained in neutron/tests/api + cp $TEMPEST_PATH/tempest/api/network/*.py $NEUTRON_TEST_PATH/api + cp $TEMPEST_PATH/tempest/api/network/admin/*.py \ + $NEUTRON_TEST_PATH/api/admin +} + +function rewrite_imports { + regexes=( + 's/tempest.common.generator/neutron.tests.tempest.common.generator/' + "s/tempest.api.network/neutron.tests.api/" + 's/tempest.test/neutron.tests.tempest.test/' + 's/from tempest.openstack.common import lockutils/from oslo_concurrency import lockutils/' + 's/from tempest.openstack.common import importutils/from oslo_utils import importutils/' + 's/tempest.openstack.common/neutron.openstack.common/' + 's/from tempest(?!_lib) import clients/from neutron.tests.api import clients/' + 's/from tempest(?!_lib)/from neutron.tests.tempest/' + 's/CONF.lock_path/CONF.oslo_concurrency.lock_path/' + ) + files=$(find "$NEUTRON_TEST_PATH/tempest" "$NEUTRON_TEST_PATH/api" -name '*.py') + for ((i = 0; i < ${#regexes[@]}; i++)); do + perl -p -i -e "${regexes[$i]}" $files + done +} + +copy_files +rewrite_imports diff --git a/tools/deploy_rootwrap.sh b/tools/deploy_rootwrap.sh new file mode 100755 index 0000000..27d9a36 --- /dev/null +++ b/tools/deploy_rootwrap.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -eu + +if [ "$#" -ne 3 ]; then + >&2 echo "Usage: $0 /path/to/neutron /path/to/target/etc /path/to/target/bin +Deploy Neutron's rootwrap configuration. + +Warning: Any existing rootwrap files at the specified etc path will be +removed by this script. + +Optional: set OS_SUDO_TESTING=1 to deploy the filters required by +Neutron's functional testing suite." + exit 1 +fi + +OS_SUDO_TESTING=${OS_SUDO_TESTING:-0} + +neutron_path=$1 +target_etc_path=$2 +target_bin_path=$3 + +src_conf_path=${neutron_path}/etc +src_conf=${src_conf_path}/rootwrap.conf +src_rootwrap_path=${src_conf_path}/neutron/rootwrap.d + +dst_conf_path=${target_etc_path}/neutron +dst_conf=${dst_conf_path}/rootwrap.conf +dst_rootwrap_path=${dst_conf_path}/rootwrap.d + +if [[ -d "$dst_rootwrap_path" ]]; then + rm -rf ${dst_rootwrap_path} +fi +mkdir -p -m 755 ${dst_rootwrap_path} + +cp -p ${src_rootwrap_path}/* ${dst_rootwrap_path}/ +cp -p ${src_conf} ${dst_conf} +sed -i "s:^filters_path=.*$:filters_path=${dst_rootwrap_path}:" ${dst_conf} +sed -i "s:^\(exec_dirs=.*\)$:\1,${target_bin_path}:" ${dst_conf} + +if [[ "$OS_SUDO_TESTING" = "1" ]]; then + sed -i 's/use_syslog=False/use_syslog=True/g' ${dst_conf} + sed -i 's/syslog_log_level=ERROR/syslog_log_level=DEBUG/g' ${dst_conf} + cp -p ${neutron_path}/neutron/tests/contrib/functional-testing.filters \ + ${dst_rootwrap_path}/ +fi diff --git a/tools/install_venv.py b/tools/install_venv.py new file mode 100644 index 0000000..f8fb8fa --- /dev/null +++ b/tools/install_venv.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Copyright 2010 OpenStack Foundation. +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Installation script for Neutron's development virtualenv +""" +from __future__ import print_function + +import os +import sys + +import install_venv_common as install_venv + + +def print_help(): + help = """ + Neutron development environment setup is complete. + + Neutron development uses virtualenv to track and manage Python dependencies + while in development and testing. + + To activate the Neutron virtualenv for the extent of your current shell + session you can run: + + $ source .venv/bin/activate + + Or, if you prefer, you can run commands in the virtualenv on a case by case + basis by running: + + $ tools/with_venv.sh + + Also, make test will automatically use the virtualenv. + """ + print(help) + + +def main(argv): + root = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + venv = os.path.join(root, '.venv') + pip_requires = os.path.join(root, 'requirements.txt') + test_requires = os.path.join(root, 'test-requirements.txt') + py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1]) + project = 'Neutron' + install = install_venv.InstallVenv(root, venv, pip_requires, test_requires, + py_version, project) + options = install.parse_args(argv) + install.check_python_version() + install.check_dependencies() + install.create_virtualenv(no_site_packages=options.no_site_packages) + install.install_dependencies() + print_help() + + +if __name__ == '__main__': + main(sys.argv) diff --git a/tools/install_venv_common.py b/tools/install_venv_common.py new file mode 100644 index 0000000..e279159 --- /dev/null +++ b/tools/install_venv_common.py @@ -0,0 +1,172 @@ +# Copyright 2013 OpenStack Foundation +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides methods needed by installation script for OpenStack development +virtual environments. + +Since this script is used to bootstrap a virtualenv from the system's Python +environment, it should be kept strictly compatible with Python 2.6. + +Synced in from openstack-common +""" + +from __future__ import print_function + +import optparse +import os +import subprocess +import sys + + +class InstallVenv(object): + + def __init__(self, root, venv, requirements, + test_requirements, py_version, + project): + self.root = root + self.venv = venv + self.requirements = requirements + self.test_requirements = test_requirements + self.py_version = py_version + self.project = project + + def die(self, message, *args): + print(message % args, file=sys.stderr) + sys.exit(1) + + def check_python_version(self): + if sys.version_info < (2, 6): + self.die("Need Python Version >= 2.6") + + def run_command_with_code(self, cmd, redirect_output=True, + check_exit_code=True): + """Runs a command in an out-of-process shell. + + Returns the output of that command. Working directory is self.root. + """ + if redirect_output: + stdout = subprocess.PIPE + else: + stdout = None + + proc = subprocess.Popen(cmd, cwd=self.root, stdout=stdout) + output = proc.communicate()[0] + if check_exit_code and proc.returncode != 0: + self.die('Command "%s" failed.\n%s', ' '.join(cmd), output) + return (output, proc.returncode) + + def run_command(self, cmd, redirect_output=True, check_exit_code=True): + return self.run_command_with_code(cmd, redirect_output, + check_exit_code)[0] + + def get_distro(self): + if (os.path.exists('/etc/fedora-release') or + os.path.exists('/etc/redhat-release')): + return Fedora( + self.root, self.venv, self.requirements, + self.test_requirements, self.py_version, self.project) + else: + return Distro( + self.root, self.venv, self.requirements, + self.test_requirements, self.py_version, self.project) + + def check_dependencies(self): + self.get_distro().install_virtualenv() + + def create_virtualenv(self, no_site_packages=True): + """Creates the virtual environment and installs PIP. + + Creates the virtual environment and installs PIP only into the + virtual environment. + """ + if not os.path.isdir(self.venv): + print('Creating venv...', end=' ') + if no_site_packages: + self.run_command(['virtualenv', '-q', '--no-site-packages', + self.venv]) + else: + self.run_command(['virtualenv', '-q', self.venv]) + print('done.') + else: + print("venv already exists...") + pass + + def pip_install(self, *args): + self.run_command(['tools/with_venv.sh', + 'pip', 'install', '--upgrade'] + list(args), + redirect_output=False) + + def install_dependencies(self): + print('Installing dependencies with pip (this can take a while)...') + + # First things first, make sure our venv has the latest pip and + # setuptools and pbr + self.pip_install('pip>=1.4') + self.pip_install('setuptools') + self.pip_install('pbr') + + self.pip_install('-r', self.requirements, '-r', self.test_requirements) + + def parse_args(self, argv): + """Parses command-line arguments.""" + parser = optparse.OptionParser() + parser.add_option('-n', '--no-site-packages', + action='store_true', + help="Do not inherit packages from global Python " + "install.") + return parser.parse_args(argv[1:])[0] + + +class Distro(InstallVenv): + + def check_cmd(self, cmd): + return bool(self.run_command(['which', cmd], + check_exit_code=False).strip()) + + def install_virtualenv(self): + if self.check_cmd('virtualenv'): + return + + if self.check_cmd('easy_install'): + print('Installing virtualenv via easy_install...', end=' ') + if self.run_command(['easy_install', 'virtualenv']): + print('Succeeded') + return + else: + print('Failed') + + self.die('ERROR: virtualenv not found.\n\n%s development' + ' requires virtualenv, please install it using your' + ' favorite package management tool' % self.project) + + +class Fedora(Distro): + """This covers all Fedora-based distributions. + + Includes: Fedora, RHEL, CentOS, Scientific Linux + """ + + def check_pkg(self, pkg): + return self.run_command_with_code(['rpm', '-q', pkg], + check_exit_code=False)[1] == 0 + + def install_virtualenv(self): + if self.check_cmd('virtualenv'): + return + + if not self.check_pkg('python-virtualenv'): + self.die("Please install 'python-virtualenv'.") + + super(Fedora, self).install_virtualenv() diff --git a/tools/misc-sanity-checks.sh b/tools/misc-sanity-checks.sh new file mode 100644 index 0000000..3abf82a --- /dev/null +++ b/tools/misc-sanity-checks.sh @@ -0,0 +1,41 @@ +#! /bin/sh + +# Copyright (C) 2014 VA Linux Systems Japan K.K. +# Copyright (C) 2014 YAMAMOTO Takashi +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +TMPDIR=`mktemp -d /tmp/${0##*/}.XXXXXX` || exit 1 +export TMPDIR +trap "rm -rf $TMPDIR" EXIT + +FAILURES=$TMPDIR/failures + + +check_no_symlinks_allowed () { + # Symlinks break the package build process, so ensure that they + # do not slip in, except hidden symlinks. + if [ $(find . -type l ! -path '*/.*' | wc -l) -ge 1 ]; then + echo "Symlinks are not allowed!" >>$FAILURES + fi +} + +# Add your checks here... +check_no_symlinks_allowed + +# Fail, if there are emitted failures +if [ -f $FAILURES ]; then + cat $FAILURES + exit 1 +fi diff --git a/tools/pecan_server.sh b/tools/pecan_server.sh new file mode 100755 index 0000000..ef36919 --- /dev/null +++ b/tools/pecan_server.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Copyright (c) 2015 Mirantis, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# A script useful to develop changes to the codebase. It launches the pecan +# API server and will reload it whenever the code changes if inotifywait is +# installed. + +inotifywait --help >/dev/null 2>&1 +if [[ $? -ne 1 ]]; then + USE_INOTIFY=0 +else + USE_INOTIFY=1 +fi + +DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../ +source "$DIR/.tox/py27/bin/activate" +COMMAND="python -c 'from neutron.cmd.eventlet import server; server.main_wsgi_pecan()'" + +function cleanup() { + kill $PID + exit 0 +} + +if [[ $USE_INOTIFY -eq 1 ]]; then + trap cleanup INT + while true; do + eval "$COMMAND &" + PID=$! + inotifywait -e modify -r $DIR/neutron/ + kill $PID + done +else + eval $COMMAND +fi diff --git a/tools/split.sh b/tools/split.sh new file mode 100755 index 0000000..995d937 --- /dev/null +++ b/tools/split.sh @@ -0,0 +1,109 @@ +#!/bin/sh +# +# This script has been shamelessly copied and tweaked from original copy: +# +# https://github.com/openstack/oslo-incubator/blob/master/tools/graduate.sh +# +# Use this script to export a Neutron module to a separate git repo. +# +# You can call this script Call script like so: +# +# ./split.sh +# +# The file should be a text file like the one below: +# +# /path/to/file/file1 +# /path/to/file/file2 +# ... +# /path/to/file/fileN +# +# Such a list can be generated with a command like this: +# +# find $path -type f # path is the base dir you want to list files for + +set -e + +if [ $# -lt 2 ]; then + echo "Usage $0 " + exit 1 +fi + +set -x + +file_list_path="$1" +project_name="$2" +files_to_keep=$(cat $file_list_path) + + +# Build the grep pattern for ignoring files that we want to keep +keep_pattern="\($(echo $files_to_keep | sed -e 's/^/\^/' -e 's/ /\\|\^/g')\)" +# Prune all other files in every commit +pruner="git ls-files | grep -v \"$keep_pattern\" | git update-index --force-remove --stdin; git ls-files > /dev/stderr" + +# Find all first commits with listed files and find a subset of them that +# predates all others + +roots="" +for file in $files_to_keep; do + file_root=$(git rev-list --reverse HEAD -- $file | head -n1) + fail=0 + for root in $roots; do + if git merge-base --is-ancestor $root $file_root; then + fail=1 + break + elif !git merge-base --is-ancestor $file_root $root; then + new_roots="$new_roots $root" + fi + done + if [ $fail -ne 1 ]; then + roots="$new_roots $file_root" + fi +done + +# Purge all parents for those commits + +set_roots=" +if [ 1 -eq 0 $(for root in $roots; do echo " -o \"\$GIT_COMMIT\" = '$root' "; done) ]; then + echo ''; +else + cat; +fi" + +# Enhance git_commit_non_empty_tree to skip merges with: +# a) either two equal parents (commit that was about to land got purged as well +# as all commits on mainline); +# b) or with second parent being an ancestor to the first one (just as with a) +# but when there are some commits on mainline). +# In both cases drop second parent and let git_commit_non_empty_tree to decide +# if commit worth doing (most likely not). + +skip_empty=$(cat << \EOF +if [ $# = 5 ] && git merge-base --is-ancestor $5 $3; then + git_commit_non_empty_tree $1 -p $3 +else + git_commit_non_empty_tree "$@" +fi +EOF +) + +# Filter out commits for unrelated files +echo "Pruning commits for unrelated files..." +git filter-branch \ + --index-filter "$pruner" \ + --parent-filter "$set_roots" \ + --commit-filter "$skip_empty" \ + --tag-name-filter cat \ + -- --all + +# Generate the new .gitreview file +echo "Generating new .gitreview file..." +cat > .gitreview <