diff --git a/README.md b/README.md index 0743310ca6..398d7cf8dc 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,46 @@ platform provides services for collecting and storing data from buildings and devices and provides an environment for developing applications which interact with that data. +## Upgrading to VOLTTRON 8.x + +VOLTTRON 8 introduces three changes that require an explict upgrade step when upgrading from a earlier VOLTTRON version + + 1. Dynamic RPC authorization feature - This requires a modification to the auth file. If you have a pre-existing + instance of VOLTTRON running on an older version, the auth file will need to be updated. + 2. Historian agents now store the cache database (backup.sqlite file) in + /agents///.agent-data directory instead of + /agents// directory. In future all core agents will write data only + to the .agent-data subdirectory. This is because vctl install --force backs up and restores + only the contents of this directory. + 3. SQLHistorians (historian version 4.0.0 and above) now use a new database schema where metadata is stored in + topics table instead of separate metadata table. SQLHistorians with version >= 4.0.0 can work with existing + database with older schema however the historian agent code should be upgraded to newer version (>=4.0.0) to run + with VOLTTRON 8 core. + +To upgrade: + + 1. If upgrading historian, make sure historians are not in auto start mode. To remove any historian from auto start + mode use the command 'vctl disable . This is necessary so that the old + sqlhistorian does not automatically start after step 5. + 2. Update volttron source code version to VOLTTRON 8 + 3. activate the volttron environment, and run ```python bootstrap.py --force```. If you have + any additional bootstrap options that you need (rabbitmq, web, drivers, etc.) include these in the above command. + 4. Run ```volttron-upgrade``` to update the auth file and move historian cache files into agent-data directory. + Note that the upgrade script will only move the backup.sqlite file and will not move sqlite historian's db file + if they are within the install directory. If using a SQLite historian, please backup the database file of + sqlite historian before upgrading to the latest historian version. + 5. Start VOLTTRON + 6. Run ```vctl install --force --vip-identity --agent-config ``` to upgrade + to the latest historian version. vctl install --force will backup the cache in .agent-data + folder, installs the latest version of the historian and restore the contents of + .agent-data folder. + +### Upgrading aggregate historians + +VOLTTRON 8 also comes with updated SQL aggregate historian schema. However, there is no automated upgrade path for +aggregate historian. To upgrade an existing aggregate historian please refer to the CHANGELOG.md within +SQLAggregateHistorian source directory + ## Features - [Message Bus](https://volttron.readthedocs.io/en/latest/platform-features/message-bus/index.html) allows agents to subscribe to data sources and publish results and messages. @@ -295,6 +335,7 @@ There are several walkthroughs to explore additional aspects of the platform: - [RabbitMQ setup with Federation and Shovel plugins](https://volttron.readthedocs.io/en/latest/deploying-volttron/multi-platform/multi-platform-rabbitmq-deployment.html) - [Backward compatibility with the RabbitMQ message bus](https://volttron.readthedocs.io/en/latest/deploying-volttron/multi-platform/multi-platform-multi-bus.html) + ## Acquiring Third Party Agent Code Third party agents are available under the volttron-applications repository. In diff --git a/bootstrap.py b/bootstrap.py index e7e94ead13..836f44c2cf 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -133,7 +133,7 @@ def update(operation, verbose=None, upgrade=False, offline=False, optional_requi # We must install wheel first to eliminate a bunch of scary looking # errors at first install. # TODO Look towards fixing the packaging so that it works with 0.31 - pip('install', ['wheel==0.30'], verbose, True, offline=offline) + # option_requirements contains wheel as first entry # Build option_requirements separately to pass install options build_option = '--build-option' if wheeling else '--install-option' diff --git a/docs/source/conf.py b/docs/source/conf.py index ed745b292a..70d3290f4a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -43,9 +43,9 @@ def __getattr__(cls, name): author = 'The VOLTTRON Community' # The short X.Y version -version = '8.1' +version = '8.1.1' # The full version, including alpha/beta/rc tags -release = '8.1' +release = '8.1.1' # -- General configuration --------------------------------------------------- diff --git a/docs/source/volttron-topics/change-log/scalability/scalability-improvements.rst b/docs/source/deploying-volttron/scalability/scalability-improvements.rst similarity index 100% rename from docs/source/volttron-topics/change-log/scalability/scalability-improvements.rst rename to docs/source/deploying-volttron/scalability/scalability-improvements.rst diff --git a/docs/source/volttron-topics/change-log/scalability/scalability.rst b/docs/source/deploying-volttron/scalability/scalability.rst similarity index 100% rename from docs/source/volttron-topics/change-log/scalability/scalability.rst rename to docs/source/deploying-volttron/scalability/scalability.rst diff --git a/docs/source/volttron-topics/change-log/scalability/testing-driver-scalability.rst b/docs/source/deploying-volttron/scalability/testing-driver-scalability.rst similarity index 100% rename from docs/source/volttron-topics/change-log/scalability/testing-driver-scalability.rst rename to docs/source/deploying-volttron/scalability/testing-driver-scalability.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 82490ca042..4b196a82c0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -137,6 +137,7 @@ at our bi-weekly office-hours and on Slack. To be invited to office-hours or sla platform-features/config-store/configuration-store platform-features/security/volttron-security + .. toctree:: :caption: VOLTTRON Core Service Agents :hidden: @@ -146,6 +147,7 @@ at our bi-weekly office-hours and on Slack. To be invited to office-hours or sla volttron-api/services/*/modules + .. toctree:: :caption: VOLTTRON Core Operations Agents :hidden: @@ -155,16 +157,16 @@ at our bi-weekly office-hours and on Slack. To be invited to office-hours or sla volttron-api/ops/*/modules + .. toctree:: :caption: VOLTTRON Topics :hidden: :titlesonly: :maxdepth: 1 + Releases volttron-topics/troubleshooting/index volttron-topics/volttron-applications/index - volttron-topics/change-log/index - Indices and tables diff --git a/docs/source/volttron-topics/VOLTTRON-releases/index.rst b/docs/source/volttron-topics/VOLTTRON-releases/index.rst new file mode 100644 index 0000000000..b762c36d03 --- /dev/null +++ b/docs/source/volttron-topics/VOLTTRON-releases/index.rst @@ -0,0 +1,13 @@ +.. _VOLTTRON-Releases: + +========== +VOLTTRON Releases +========== + +This section includes individual documents describing important changes to platform components. For information on specific release, please refer to the corresponding document. + + +.. toctree:: + + release-history + upgrading-versions diff --git a/docs/source/volttron-topics/VOLTTRON-releases/release-history.rst b/docs/source/volttron-topics/VOLTTRON-releases/release-history.rst new file mode 100644 index 0000000000..4b7b750090 --- /dev/null +++ b/docs/source/volttron-topics/VOLTTRON-releases/release-history.rst @@ -0,0 +1,124 @@ +.. _Release-History: + +=============== +Release History +=============== + +VOLTTRON Release Documentation for version 5.1.0 and above is found on GitHub. +`https://github.com/VOLTTRON/volttron/releases `_ + + +VOLTTRON 8.1.1 Maintenance Release +================================== + +`https://github.com/VOLTTRON/volttron/releases/tag/8.1.1 `_ + + +VOLTTRON 8.1 Release +==================== + +`https://github.com/VOLTTRON/volttron/releases/tag/8.1 `_ + + +VOLTTRON 8.0 Full Release +========================= + +`https://github.com/VOLTTRON/volttron/releases/tag/8.0.0 `_ + + +VOLTTRON 7.0.1 Update +===================== + +`https://github.com/VOLTTRON/volttron/releases/tag/7.0.1 `_ + + +VOLTTRON 8.0 Release Candidate +============================== + +`https://github.com/VOLTTRON/volttron/releases/tag/8.0rc1 `_ + + +VOLTTRON 7.0 Release +==================== + +`https://github.com/VOLTTRON/volttron/releases/tag/7.0 `_ + + +VOLTTRON 7.0 Release Candidate +============================== + +`https://github.com/VOLTTRON/volttron/releases/tag/7.0rc1 `_ + + +VOLTTRON 6.0 Release +==================== + +`https://github.com/VOLTTRON/volttron/releases/tag/6.0 `_ + + +VOLTTRON 6.0 Release Candidate +============================== + +`https://github.com/VOLTTRON/volttron/releases/tag/6.0rc1 `_ + + +VOLTTRON 5.1.0 Release +====================== + +`https://github.com/VOLTTRON/volttron/releases/tag/5.1.0 `_ + + +VOLTTRON 5.0 Release +==================== + +- Tagging service for attaching metadata to topics for simpler retrieval +- Message bus performance improvement +- Multi-platform publish/subscribe for simpler coordination across platforms +- Drivers contributed back for SEP 2.0 and ChargePoint EV + + +VOLTTRON 4.0 Release +==================== + +- Documentation moved to ReadTheDocs +- VOLTTRON Configuration Wizard +- Configuration store to dynamically configure agents +- Aggregator agent for aggregating topics +- More reliable remote install mechanism +- UI for device configuration +- Automatic registration of VOLTTRON instances with management agent + + +VOLTTRON 3.0 Release +==================== + +- Modularize Data Historian +- Modularize Device Drivers +- Secure and accountable communication using the VIP +- Web Console for Monitoring and Administering VOLTTRON Deployments + + +VOLTTRON 2.0 Release +==================== + +- Advanced Security Features +- Guaranteed resource allocation to agents using execution contracts +- Signing and verification of agent packaging +- Agent mobility +- Admin can send agents to another platform +- Agent can request to move +- Enhanced command framework + + +VOLTTRON 1.0 – 1.2 +================== + +- Agent execution platform +- Message bus +- Modbus and BACnet drivers +- Historian +- Data logger +- Device scheduling +- Device actuation +- Multi-node communication +- Weather service diff --git a/docs/source/volttron-topics/change-log/upgrading-versions.rst b/docs/source/volttron-topics/VOLTTRON-releases/upgrading-versions.rst similarity index 63% rename from docs/source/volttron-topics/change-log/upgrading-versions.rst rename to docs/source/volttron-topics/VOLTTRON-releases/upgrading-versions.rst index c0cf0fffa0..5160e01d5c 100644 --- a/docs/source/volttron-topics/change-log/upgrading-versions.rst +++ b/docs/source/volttron-topics/VOLTTRON-releases/upgrading-versions.rst @@ -8,6 +8,48 @@ It is often recommended that users upgrade to the latest stable release of VOLTT releases include helpful new features, bug fixes, and other improvements. Please see the guides below for upgrading your existing deployment to the latest version. +VOLTTRON 8 +========== + +VOLTTRON 8 introduces three changes that require an explict upgrade step when upgrading from a earlier VOLTTRON version + + 1. Dynamic RPC authorization feature - This requires a modification to the auth file. If you have a pre-existing + instance of VOLTTRON running on an older version, the auth file will need to be updated. + 2. Historian agents now store the cache database (backup.sqlite file) in + /agents///.agent-data directory instead of + /agents// directory. In future all core agents will write data only + to the .agent-data subdirectory. This is because vctl install --force backs up and restores + only the contents of this directory. + 3. SQLHistorians (historian version 4.0.0 and above) now use a new database schema where metadata is stored in + topics table instead of separate metadata table. SQLHistorians with version >= 4.0.0 can work with existing + database with older schema however the historian agent code should be upgraded to newer version (>=4.0.0) to run + with VOLTTRON 8 core. + + +To upgrade: + + 1. If upgrading historian, make sure historians are not in auto start mode. To remove any historian from auto start + mode use the command 'vctl disable . This is necessary so that the old + sqlhistorian does not automatically start after step 5. + 2. Update volttron source code version to VOLTTRON 8 + 3. activate the volttron environment, and run ```python bootstrap.py --force```. If you have + any additional bootstrap options that you need (rabbitmq, web, drivers, etc.) include these in the above command. + 4. Run ```volttron-upgrade``` to update the auth file and move historian cache files into agent-data directory. + Note that the upgrade script will only move the backup.sqlite file and will not move sqlite historian's db file + if they are within the install directory. If using a SQLite historian, please backup the database file of + sqlite historian before upgrading to the latest historian version. + 5. Start VOLTTRON + 6. Run ```vctl install --force --vip-identity --agent-config ``` to upgrade + to the latest historian version. vctl install --force will backup the cache in .agent-data + folder, installs the latest version of the historian and restore the contents of + .agent-data folder. + +Upgrading aggregate historians +------------------------------ + +VOLTTRON 8 also comes with updated SQL aggregate historian schema. However, there is no automated upgrade path for +aggregate historian. To upgrade an existing aggregate historian please refer to the CHANGELOG.md within +SQLAggregateHistorian source directory VOLTTRON 7 ========== diff --git a/docs/source/volttron-topics/change-log/index.rst b/docs/source/volttron-topics/change-log/index.rst deleted file mode 100644 index c488332810..0000000000 --- a/docs/source/volttron-topics/change-log/index.rst +++ /dev/null @@ -1,15 +0,0 @@ -.. _Change-Log: - -========== -Change Log -========== - -This section includes individual documents describing important changes to platform components, such as the RabbitMQ -message bus implementation. For information on specific changes, please refer to the corresponding document. - - -.. toctree:: - - scalability/scalability - version-history - upgrading-versions diff --git a/docs/source/volttron-topics/change-log/version-history.rst b/docs/source/volttron-topics/change-log/version-history.rst deleted file mode 100644 index 1671bfc04d..0000000000 --- a/docs/source/volttron-topics/change-log/version-history.rst +++ /dev/null @@ -1,164 +0,0 @@ -.. _Version-History: - -=============== -Version History -=============== - -VOLTTRON 1.0 – 1.2 -================== - -- Agent execution platform -- Message bus -- Modbus and BACnet drivers -- Historian -- Data logger -- Device scheduling -- Device actuation -- Multi-node communication -- Weather service - - -VOLTTRON 2.0 -============ - -- Advanced Security Features -- Guaranteed resource allocation to agents using execution contracts -- Signing and verification of agent packaging -- Agent mobility -- Admin can send agents to another platform -- Agent can request to move -- Enhanced command framework - - -VOLTTRON 3.0 -============ - -- Modularize Data Historian -- Modularize Device Drivers -- Secure and accountable communication using the VIP -- Web Console for Monitoring and Administering VOLTTRON Deployments - - -VOLTTRON 4.0 -============ - -- Documentation moved to ReadTheDocs -- VOLTTRON Configuration Wizard -- Configuration store to dynamically configure agents -- Aggregator agent for aggregating topics -- More reliable remote install mechanism -- UI for device configuration -- Automatic registration of VOLTTRON instances with management agent - - -VOLTTRON 5.0 -============ - -- Tagging service for attaching metadata to topics for simpler retrieval -- Message bus performance improvement -- Multi-platform publish/subscribe for simpler coordination across platforms -- Drivers contributed back for SEP 2.0 and ChargePoint EV - - -VOLTTRON 6.0 -============ - -- Maintained backward compatibility with communication between zmq and rmq deployments. -- Added DarkSky Weather Agent -- Web Based Additions -- Added CSR support for multiplatform communication -- Added SSL support to the platform for secure communication -- Backported SSL support to zmq based deployments. -- Upgraded VC to use the platform login. -- Added docker support to the test environment for easier Rabbitmq testing. -- Updated volttron-config (vcfg) to support both RabbitMQ and ZMQ including https based instances. -- Added test support for RabbitMQ installations of all core agents. -- Added multiplatform (zmq and rmq based platform) testing. -- Integrated RabbitMQ documentation into the core documentation. - - -VOLTTRON 7.0rc1 -=============== - - -Python3 Upgrade ---------------- - -- Update libraries to appropriate and compatible versions -- String handling efficiency -- Encode/Decode of strings has been simplified and centralized -- Added additional test cases for frame serialization in ZMQ -- Syntax updates such difference in handling exceptions, dictionaries, sorting lists, pytest markers etc. -- Made bootstrap process simpler -- Resolved gevent monkey patch issues when using third party libraries - - -RabbitMQ Message Bus --------------------- - -- Client code for integrating non-VOLTTRON applications with the message bus - available at: https://github.com/VOLTTRON/external-clients-for-rabbitmq -- Includes support for MQTT, non-VOLTTRON Python, and Java-based RabbitMQ - clients - - -Config store secured --------------------- - -- Agents can prevent other agents from modifying their configuration store entry - - -Known Issues which will be dealt with for the final release: ------------------------------------------------------------- - -- Python 3.7 has conflicts with some libraries such as gevent -- The VOLTTRON Central agent is not fully integrated into Python3 -- CFFI library has conflicts on the Raspian OS which interferes with bootstrapping - - -VOLTTRON 7.0 Full Release -========================= - -This is a full release of the 7.0 version of VOLTTRON which has been refactored to work with Python3. This release -incorporates community feedback from the release candidate as well as new contributions and features. -Major new features and highlights since the release candidate include: - -* Added secure agent user feature which allows agents to be launched as a user separate from the platform. This - protects the platform against malformed or malicious agents accessing platform level files -* Added a driver to interface with the Ecobee smart thermostat and make data available to agents on the platform -* Updated VOLTTRON Central UI to work with Python3 -* Added web support to authenticate remote VOLTTRON ZMQ message bus-based connections -* Updated ZMQ-based multiplatform RPC with Python 3 -* To reduce installation size and complexity, fewer services are installed by default -* MasterDriver dependencies are not installed by default during bootstrap. To use MasterDriver, please use the - following command: - - .. code-block:: bash - - python3 bootstrap.py --driver - -* Web dependencies are not installed by default during bootstrap. To use the MasterWeb service, please use the - following command: - - .. code-block:: bash - - python3 bootstrap.py --web - -* Added initial version of test cases for `volttron-cfg` (`vcfg`) utility -* On all arm-based systems, `libffi` is now a required dependency, this is reflected in the installation instructions -* On arm-based systems, Raspbian >= 10 or Ubuntu >= 18.04 is required -* Updated examples and several contributed features to Python 3 -* Inclusion of docker in test handling for databases -* A new `/gs` endpoint to access platform services without using Volttron Central through Json-RPC -* A new SCPAgent to transfer files between two remote systems - -Known Issues ------------- - -* Continued documentation updates to ensure correctness -* Rainforest Eagle driver is not yet upgraded to Python3 -* A bug in the Modbus TK library prevents creating connections from 2 different masters to a single slave. -* BACnet Proxy Agent and BACnet auto configuration scripts require the version of BACPypes installed in the virtual - environment of VOLTTRON to be version 0.16.7. We have pinned it to version 0.16.7 since it does not work properly in - later versions of BACPypes. -* VOLTTRON 7.0 code base is not fully tested in Ubuntu 20.04 LTS so issues with this combination have not been addressed diff --git a/requirements.py b/requirements.py index fb6a85af2a..90263b22c7 100644 --- a/requirements.py +++ b/requirements.py @@ -37,58 +37,56 @@ # }}} -extras_require = { 'crate': ['crate==0.26.0'], - 'databases': [ 'mysql-connector-python==8.0.26', - 'bson==0.5.7', - 'pymongo==3.7.2', - 'crate==0.26.0', - 'influxdb==5.3.1', - 'psycopg2-binary==2.8.6'], - 'documentation': [ 'mock==4.0.3', - 'Sphinx==4.1.2', - 'sphinx-rtd-theme==0.5.2', - 'sphinx==3.3.0', - 'm2r2==0.3.1'], - 'drivers': [ 'pymodbus==2.5.2', - 'bacpypes==0.16.7', - 'modbus-tk==1.1.2', - 'pyserial==3.5'], - 'influxdb': ['influxdb==5.3.1'], - 'market': ['numpy==1.19.5', 'transitions==0.8.8'], - 'mongo': ['bson==0.5.7pymongo==3.7.2'], - 'mysql': ['mysql-connector-python==8.0.26'], - 'pandas': ['numpy==1.19.5', 'pandas==1.1.5'], - 'postgres': ['psycopg2-binary==2.8.6'], - 'testing': [ 'mock==4.0.3', - 'pytest==6.2.4', - 'pytest-timeout==1.4.2', - 'pytest-rerunfailures==10.1', - 'websocket-client==1.2.1', - 'deepdiff==5.5.0', - 'docker==5.0.0'], - 'weather': ['Pint==0.17'], - 'web': [ 'ws4py==0.5.1', - 'PyJWT==1.7.1', - 'Jinja2==3.0.1', - 'passlib==1.7.4', - 'argon2-cffi==20.1.0', - 'Werkzeug==2.0.1']} -install_requires = [ 'gevent==20.6.1', - 'greenlet==0.4.16', - 'grequests==0.6.0', - 'idna<3,>=2.5', - 'requests==2.23.0', - 'ply==3.11', - 'psutil==5.8.0', - 'python-dateutil==2.8.2', - 'pytz==2021.1', - 'PyYAML==5.4.1', - 'pyzmq==22.2.1', - 'setuptools==40.0.0', - 'tzlocal==2.1', - 'pyOpenSSL==19.0.0', - 'cryptography==2.3', - 'watchdog-gevent==0.1.1', - 'wheel==0.30'] +extras_require = {'crate': ['crate==0.26.0'], + 'databases': ['mysql-connector-python==8.0.26', + 'bson==0.5.7', + 'pymongo==3.7.2', + 'crate==0.26.0', + 'influxdb==5.3.1', + 'psycopg2-binary==2.8.6'], + 'documentation': ['mock==4.0.3', + 'Sphinx==4.1.2', + 'sphinx-rtd-theme==0.5.2', + 'sphinx==3.3.0', + 'm2r2==0.3.1'], + 'drivers': ['pymodbus==2.5.2', + 'bacpypes==0.16.7', + 'modbus-tk==1.1.2', + 'pyserial==3.5'], + 'influxdb': ['influxdb==5.3.1'], + 'market': ['numpy==1.19.5', 'transitions==0.8.8'], + 'mongo': ['bson==0.5.7', 'pymongo==3.7.2'], + 'mysql': ['mysql-connector-python==8.0.26'], + 'pandas': ['numpy==1.19.5', 'pandas==1.1.5'], + 'postgres': ['psycopg2-binary==2.8.6'], + 'testing': ['mock==4.0.3', + 'pytest==6.2.4', + 'pytest-timeout==1.4.2', + 'pytest-rerunfailures==10.1', + 'websocket-client==1.2.1', + 'deepdiff==5.5.0', + 'docker==5.0.0'], + 'weather': ['Pint==0.17'], + 'web': ['ws4py==0.5.1', + 'PyJWT==1.7.1', + 'Jinja2==3.0.1', + 'passlib==1.7.4', + 'argon2-cffi==20.1.0', + 'Werkzeug==2.0.1']} +install_requires = ['gevent==20.6.1', + 'greenlet==0.4.16', + 'grequests==0.6.0', + 'idna<3,>=2.5', + 'requests==2.23.0', + 'ply==3.11', + 'psutil==5.8.0', + 'python-dateutil==2.8.2', + 'pytz==2021.1', + 'PyYAML==5.4.1', + 'setuptools>=40.0.0', + 'tzlocal==2.1', + 'pyOpenSSL==19.0.0', + 'cryptography==2.3', + 'watchdog-gevent==0.1.1'] -option_requirements = [('pyzmq==22.2.1', ['--zmq=bundled'])] +option_requirements = [('wheel==0.30', []), ('pyzmq==22.2.1', ['--zmq=bundled'])] diff --git a/services/core/SQLHistorian/README.md b/services/core/SQLHistorian/README.md index 60289d7dd4..aa96ec0b18 100644 --- a/services/core/SQLHistorian/README.md +++ b/services/core/SQLHistorian/README.md @@ -7,7 +7,43 @@ inconsistent network connectivity (automatic re-connection to tcp based databases). All additions to the historian are batched and wrapped within a transaction with commit and rollback functions properly implemented. This allows the maximum throughput of data with the most -protection. +protection + +## Common Configurations +All SQLHistorians support two parameters +1. connection - This is a mandatory parameter with type indicating the type of + sql historian (ex. mysql, sqlite, etc.) and params containing the connection + parameters specific to the connecting database type. + +2. tables_def - Optional parameter to provide custom table names for + topics, data, and metadata. This is useful when you want to use more than + one instance of sqlhistorian with the same database + +Example: + +JSON format : + + { + "connection": { + # type should be sqlite + "type": "sqlite", + "params": { + "database": "data/historian.sqlite", + } + } + "tables_def": { + # prefix for data, topics, and (in version < 4.0.0 metadata tables) + # default is "" + "table_prefix": "", + # table name for time series data. default "data" + "data_table": "data", + # table name for list of topics. default "topics" + "topics_table": "topics", + # table name mapping topic to metadata. default "meta" + # In sqlhistorian version >= 4.0.0 metadata is stored in topics table + "meta_table": "meta" + } + } ## MySQL diff --git a/services/core/SQLHistorian/sqlhistorian/historian.py b/services/core/SQLHistorian/sqlhistorian/historian.py index 4a4db86b6e..bd63e2a23c 100644 --- a/services/core/SQLHistorian/sqlhistorian/historian.py +++ b/services/core/SQLHistorian/sqlhistorian/historian.py @@ -138,8 +138,6 @@ def __init__(self, connection, tables_def=None, **kwargs): self.topic_name_map = {} self.topic_meta = {} self.agg_topic_id_map = {} - database_type = self.connection['type'] - self.db_functs_class = sqlutils.get_dbfuncts_class(database_type) # Create two instance so connection is shared within a single thread. # This is because sqlite only supports sharing of connection within # a single thread. @@ -147,7 +145,7 @@ def __init__(self, connection, tables_def=None, **kwargs): # everything else happens in the MainThread # One utils class instance( hence one db connection) for main thread - self.main_thread_dbutils = self.db_functs_class(self.connection['params'], self.table_names) + self.main_thread_dbutils = self.get_dbfuncts_object() # One utils class instance( hence one db connection) for background thread # this gets initialized in the bg_thread within historian_setup self.bg_thread_dbutils = None @@ -342,7 +340,7 @@ def query_historian(self, topic, start=None, end=None, agg_type=None, agg_period def historian_setup(self): thread_name = threading.currentThread().getName() _log.info("historian_setup on Thread: {}".format(thread_name)) - self.bg_thread_dbutils = self.db_functs_class(self.connection['params'], self.table_names) + self.bg_thread_dbutils = self.get_dbfuncts_object() if not self._readonly: self.bg_thread_dbutils.setup_historian_tables() @@ -356,6 +354,10 @@ def historian_setup(self): _log.debug(f"###DEBUG Loaded topics and metadata on start. Len of topics {len(self.topic_id_map)} " f"Len of metadata: {len(self.topic_meta)}") + def get_dbfuncts_object(self): + db_functs_class = sqlutils.get_dbfuncts_class(self.connection['type']) + return db_functs_class(self.connection['params'], self.table_names) + def main(argv=sys.argv): """ diff --git a/services/core/SQLHistorian/tests/test_sqlitehistorian_unit.py b/services/core/SQLHistorian/tests/test_sqlitehistorian_unit.py index ac2bfe6251..1ce2982b9e 100644 --- a/services/core/SQLHistorian/tests/test_sqlitehistorian_unit.py +++ b/services/core/SQLHistorian/tests/test_sqlitehistorian_unit.py @@ -1,13 +1,16 @@ import os from shutil import rmtree import subprocess +from pathlib import Path import pytest from gevent import sleep from datetime import timedelta from services.core.SQLHistorian.sqlhistorian import historian -CACHE_NAME = "backup.sqlite" +agent_data_dir = os.path.join(os.getcwd(), os.path.basename(os.getcwd()) + ".agent-data") +os.makedirs(agent_data_dir, exist_ok=True) +CACHE_NAME = str(Path(agent_data_dir).joinpath("backup.sqlite")) HISTORIAN_DB = "./data/historian.sqlite" @@ -76,6 +79,8 @@ def sql_historian(): rmtree("./data") if os.path.exists(CACHE_NAME): os.remove(CACHE_NAME) + if os.path.exists(agent_data_dir): + os.rmdir(agent_data_dir) def query_db(query, db): diff --git a/setup.py b/setup.py index 2ec369bd8b..f7369da700 100644 --- a/setup.py +++ b/setup.py @@ -70,6 +70,7 @@ 'vctl = volttron.platform.control:_main', 'vpkg = volttron.platform.packaging:_main', 'vcfg = volttron.platform.config:_main', + 'volttron-upgrade = volttron.platform.upgrade.upgrade_volttron:_main', ] }, zip_safe = False, diff --git a/volttron/platform/__init__.py b/volttron/platform/__init__.py index 57ea86ab2c..2f33379f47 100644 --- a/volttron/platform/__init__.py +++ b/volttron/platform/__init__.py @@ -49,7 +49,7 @@ from urllib.parse import urlparse from ..utils.frozendict import FrozenDict -__version__ = '8.1' +__version__ = '8.1.1' _log = logging.getLogger(__name__) diff --git a/volttron/platform/agent/base_historian.py b/volttron/platform/agent/base_historian.py index 4b62dcbbdb..b540bc52a9 100644 --- a/volttron/platform/agent/base_historian.py +++ b/volttron/platform/agent/base_historian.py @@ -312,6 +312,7 @@ def add_timing_data_to_header(headers, agent_id, phase): STATUS_KEY_CACHE_FULL = "cache_full" STATUS_KEY_TIME_ERROR = "records_with_invalid_timestamp" STATUS_KEY_CACHE_ONLY = "cache_only_enabled" +STATUS_KEY_ERROR_MANAGE_DB_SIZE = "error_managing_db_size" class BaseHistorianAgent(Agent): @@ -412,7 +413,9 @@ def __init__(self, STATUS_KEY_BACKLOGGED: False, STATUS_KEY_PUBLISHING: True, STATUS_KEY_CACHE_FULL: False, - STATUS_KEY_CACHE_ONLY: False + STATUS_KEY_CACHE_ONLY: False, + STATUS_KEY_TIME_ERROR: False, + STATUS_KEY_ERROR_MANAGE_DB_SIZE: False } self._all_platforms = bool(all_platforms) self._time_tolerance = float(time_tolerance) if time_tolerance else None @@ -1062,7 +1065,8 @@ def _get_status_from_context(context): if (context.get(STATUS_KEY_BACKLOGGED) or context.get(STATUS_KEY_CACHE_FULL) or not context.get(STATUS_KEY_PUBLISHING) or - context.get(STATUS_KEY_TIME_ERROR)): + context.get(STATUS_KEY_TIME_ERROR) or + context.get(STATUS_KEY_ERROR_MANAGE_DB_SIZE)): status = STATUS_BAD return status @@ -1215,10 +1219,17 @@ def _do_process_loop(self): try: if not cache_only_enabled: self.publish_to_historian(to_publish_list) + except Exception as e: + _log.exception( + f"An unhandled exception occurred while publishing: {e}") + + try: self.manage_db_size(history_limit_timestamp, self._storage_limit_gb) - except: + self._update_status({STATUS_KEY_ERROR_MANAGE_DB_SIZE: False}) + except Exception as e: _log.exception( - "An unhandled exception occurred while publishing.") + f"An unhandled exception occurred while attempting to managing db size: {e}") + self._send_alert({STATUS_KEY_ERROR_MANAGE_DB_SIZE: True}, "error_managing_db_size") # if the success queue is empty then we need not remove # them from the database and we are probably having connection problems. @@ -1734,13 +1745,13 @@ def close(self): def _setupdb(self, check_same_thread): """ Creates a backup database for the historian if doesn't exist.""" - _log.debug("Setting up backup DB.") - if utils.is_secure_mode(): - # we want to create it in the agent-data directory since agent will not have write access to any other - # directory in secure mode - backup_db = os.path.join(os.getcwd(), os.path.basename(os.getcwd()) + ".agent-data", 'backup.sqlite') - else: - backup_db = 'backup.sqlite' + _log.debug(f"Setting up backup DB. {os.getcwd()}") + # we want to create it in the agent-data directory since agent will not have write access to any other + # directory in secure mode + # TODO - revisit logic to get agent-data directory. Refer to aip.get_agent_data() + # Also take into account dynamic agents - especially for testing + backup_db = os.path.join(os.getcwd(), os.path.basename(os.getcwd()) + ".agent-data", 'backup.sqlite') + _log.info(f"Creating backup db at {backup_db}") self._connection = sqlite3.connect( backup_db, diff --git a/volttron/platform/agent/utils.py b/volttron/platform/agent/utils.py index 911fedf481..285722a486 100644 --- a/volttron/platform/agent/utils.py +++ b/volttron/platform/agent/utils.py @@ -797,7 +797,7 @@ def execute_command(cmds, env=None, cwd=None, logger=None, err_prefix=None) -> s results.stderr) if logger: logger.exception(err_message) - raise RuntimeError() + raise RuntimeError(err_message) else: raise RuntimeError(err_message) diff --git a/volttron/platform/aip.py b/volttron/platform/aip.py index 0a9238ee4f..38f4e31da5 100644 --- a/volttron/platform/aip.py +++ b/volttron/platform/aip.py @@ -343,7 +343,7 @@ def set_agent_user_permissions(self, volttron_agent_user, # except agent-data dir. agent-data dir has rwx self.set_acl_for_path("rx", volttron_agent_user, agent_dir) # creates dir if it doesn't exist - data_dir = self._get_agent_data_dir(agent_path_with_name) + data_dir = self.get_agent_data_dir(agent_path_with_name) for (root, directories, files) in os.walk(agent_dir, topdown=True): for directory in directories: @@ -609,7 +609,7 @@ def _unauthorize_agent_keys(self, agent_uuid): publickey = self.get_agent_keystore(agent_uuid).public AuthFile().remove_by_credentials(publickey) - def _get_agent_data_dir(self, agent_path): + def get_agent_data_dir(self, agent_path): pkg = UnpackedPackage(agent_path) data_dir = os.path.join(os.path.dirname(pkg.distinfo), '{}.agent-data'.format(pkg.package_name)) @@ -618,7 +618,7 @@ def _get_agent_data_dir(self, agent_path): return data_dir def create_agent_data_dir_if_missing(self, agent_uuid): - new_agent_data_dir = self._get_agent_data_dir(self.agent_dir(agent_uuid)) + new_agent_data_dir = self.get_agent_data_dir(self.agent_dir(agent_uuid)) return new_agent_data_dir def _get_data_dir(self, agent_path, agent_name): @@ -953,7 +953,7 @@ def start_agent(self, agent_uuid): resmon = getattr(self.env, 'resmon', None) agent_user = None - data_dir = self._get_agent_data_dir(agent_path_with_name) + data_dir = self.get_agent_data_dir(agent_path_with_name) if self.secure_agent_user: _log.info("Starting agent securely...") diff --git a/volttron/platform/auth.py b/volttron/platform/auth.py index 095783524a..061b815518 100644 --- a/volttron/platform/auth.py +++ b/volttron/platform/auth.py @@ -1866,6 +1866,7 @@ def upgrade_1_2_to_1_3(allow_list): version["minor"] = 1 if version["major"] == 1 and version["minor"] == 1: allow_list = upgrade_1_1_to_1_2(allow_list) + version["minor"] = 2 if version["major"] == 1 and version["minor"] == 2: allow_list = upgrade_1_2_to_1_3(allow_list) diff --git a/volttron/platform/control.py b/volttron/platform/control.py index dd99a394f2..fe2ea7eb0f 100644 --- a/volttron/platform/control.py +++ b/volttron/platform/control.py @@ -4429,7 +4429,7 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: # else we return 0 from here. This has the added effect of # allowing us to cascade short circuit calls. if exc.args[0] != 0: - error = exc.message + error = exc else: return 0 except InstallRuntimeError as exrt: diff --git a/volttron/platform/install_agents.py b/volttron/platform/install_agents.py index db7c4998a5..0929b83f47 100644 --- a/volttron/platform/install_agents.py +++ b/volttron/platform/install_agents.py @@ -174,6 +174,7 @@ def _send_and_intialize_agent(opts, publickey, secretkey): try: if opts.start: + gevent.sleep(2) _log.debug(f"Staring agent {agent_uuid}") opts.connection.call('start_agent', agent_uuid) output_dict['starting'] = True diff --git a/volttron/platform/instance_setup.py b/volttron/platform/instance_setup.py index d00e2e4180..2061df46e7 100644 --- a/volttron/platform/instance_setup.py +++ b/volttron/platform/instance_setup.py @@ -209,7 +209,7 @@ def _cleanup_on_exit(): _shutdown_platform() -def _install_agent(agent_dir, config, tag): +def _install_agent(agent_dir, config, tag, identity): if not isinstance(config, dict): config_file = config else: @@ -217,7 +217,11 @@ def _install_agent(agent_dir, config, tag): with open(cfg.name, 'w') as fout: fout.write(jsonapi.dumps(config)) config_file = cfg.name - _cmd(['volttron-ctl', 'install', "--agent-config", config_file, "--tag", tag, "--force", agent_dir]) + cmd_array = ['volttron-ctl', 'install', "--agent-config", config_file, "--tag", tag, "--force"] + if identity: + cmd_array.extend(["--vip-identity", identity]) + cmd_array.append(agent_dir) + _cmd(cmd_array) def _is_agent_installed(tag): @@ -238,9 +242,6 @@ def wrap(config_func): global available_agents def func(*args, **kwargs): - if identity is not None: - os.environ['AGENT_VIP_IDENTITY'] = identity - print('Configuring {}.'.format(agent_dir)) config = config_func(*args, **kwargs) _update_config_file() @@ -248,7 +249,7 @@ def func(*args, **kwargs): #TODO: (potentially only starting the platform once per vcfg) _start_platform() - _install_agent(agent_dir, config, tag) + _install_agent(agent_dir, config, tag, identity) if not _is_agent_installed(tag): print(tag + ' not installed correctly!') diff --git a/volttron/platform/upgrade/__init__.py b/volttron/platform/upgrade/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/volttron/platform/upgrade/move_sqlite_files.py b/volttron/platform/upgrade/move_sqlite_files.py new file mode 100644 index 0000000000..9b41e62b8b --- /dev/null +++ b/volttron/platform/upgrade/move_sqlite_files.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- {{{ +# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# +# Copyright 2020, Battelle Memorial Institute. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This material was prepared as an account of work sponsored by an agency of +# the United States Government. Neither the United States Government nor the +# United States Department of Energy, nor Battelle, nor any of their +# employees, nor any jurisdiction or organization that has cooperated in the +# development of these materials, makes any warranty, express or +# implied, or assumes any legal liability or responsibility for the accuracy, +# completeness, or usefulness or any information, apparatus, product, +# software, or process disclosed, or represents that its use would not infringe +# privately owned rights. Reference herein to any specific commercial product, +# process, or service by trade name, trademark, manufacturer, or otherwise +# does not necessarily constitute or imply its endorsement, recommendation, or +# favoring by the United States Government or any agency thereof, or +# Battelle Memorial Institute. The views and opinions of authors expressed +# herein do not necessarily state or reflect those of the +# United States Government or any agency thereof. +# +# PACIFIC NORTHWEST NATIONAL LABORATORY operated by +# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY +# under Contract DE-AC05-76RL01830 +# }}} + +import sys +import shutil +from pathlib import Path +import glob +import re +from gevent import monkey as curious_george +curious_george.patch_all(thread=False, select=False) + +from volttron.platform import get_home +from volttron.platform.aip import AIPplatform +from volttron.platform.instance_setup import fail_if_instance_running + + +def get_aip(): + """Get AIPplatform to interface with agent directories in vhome""" + + vhome = get_home() + options = type("Options", (), dict(volttron_home=vhome)) + aip = AIPplatform(options) + return aip + + +def move_historian_cache_files(aip): + """ + Moves any keystore.json from agent-data to dist-info. + Only applies to agents in auth file. + """ + + vhome = Path(aip.env.volttron_home) + install_dir = vhome.joinpath("agents") + # pattern example - (/vhome/agents/uuid/agentname-version/)(backup.sqlite) + # pattern example - (vhome/agents/uuid/agentname-version/)(data/subdir/sqlitehistoriandb.sqlite) + pattern = "(" + str(install_dir) + "/[^/]*/[^/]*/)(.*)" + re_pattern = re.compile(pattern) + # Look for all .sqlite files in installed agents + print(f"Attempting to move backup.sqlite files in {install_dir} into corresponding agent-data directory") + # currently this is only used for backup.sqlite + # In 9.0 we could use the same code for *.sqlite files + # for example ones created by sqlitetagging, topic watcher, weather etc. when we make agent-data folder default + # agent write directory for all core agents + glob_path = str(install_dir.joinpath("**/backup.sqlite")) + for sqlite_file in glob.glob(glob_path, recursive=True): + result = re_pattern.match(sqlite_file).groups() + agent_dir = result[0] # /agents/uuid/ + source_file = result[1].split('/', 1)[0] # file or directory name to be moved to agent-data folder + source_path = str(Path(agent_dir).joinpath(source_file)) + dest_dir = aip.get_agent_data_dir(agent_path=agent_dir) + if source_path != dest_dir: + # if file is not already in agent-data dir + result = shutil.move(source_path, dest_dir) + + # print from uuid dir name so that it is easy to read + print_src = source_path.split(str(install_dir)+"/")[1] + print_dest = result.split(str(install_dir) + "/")[1] + print(f"Moved {print_src} to {print_dest}") + + +def main(): + """Upgrade auth file to function with dynamic rpc authorizations""" + + fail_if_instance_running() + aip = get_aip() + move_historian_cache_files(aip) + print("") + print("Moving historian backup files complete. " + "You can now safely upgrade historian agents other than SQLITE Historian with vctl install --force. " + "If using using SQLite historian please back up and restore sqlite historian's db manually") + + +def _main(): + """ Wrapper for main function""" + + try: + sys.exit(main()) + except KeyboardInterrupt: + sys.exit(1) + + +if __name__ == "__main__": + _main() diff --git a/volttron/platform/upgrade/update_auth_file.py b/volttron/platform/upgrade/update_auth_file.py new file mode 100644 index 0000000000..fb44b7b5fc --- /dev/null +++ b/volttron/platform/upgrade/update_auth_file.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- {{{ +# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# +# Copyright 2020, Battelle Memorial Institute. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This material was prepared as an account of work sponsored by an agency of +# the United States Government. Neither the United States Government nor the +# United States Department of Energy, nor Battelle, nor any of their +# employees, nor any jurisdiction or organization that has cooperated in the +# development of these materials, makes any warranty, express or +# implied, or assumes any legal liability or responsibility for the accuracy, +# completeness, or usefulness or any information, apparatus, product, +# software, or process disclosed, or represents that its use would not infringe +# privately owned rights. Reference herein to any specific commercial product, +# process, or service by trade name, trademark, manufacturer, or otherwise +# does not necessarily constitute or imply its endorsement, recommendation, or +# favoring by the United States Government or any agency thereof, or +# Battelle Memorial Institute. The views and opinions of authors expressed +# herein do not necessarily state or reflect those of the +# United States Government or any agency thereof. +# +# PACIFIC NORTHWEST NATIONAL LABORATORY operated by +# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY +# under Contract DE-AC05-76RL01830 +# }}} + +import sys +import shutil +from pathlib import Path + +from volttron.platform import get_home +from volttron.platform.aip import AIPplatform +from volttron.platform.auth import AuthFile +from volttron.platform.instance_setup import fail_if_instance_running +from volttron.platform.keystore import KeyStore + + +def get_aip(): + """Get AIPplatform to interface with agent directories in vhome""" + + vhome = get_home() + options = type("Options", (), dict(volttron_home=vhome)) + aip = AIPplatform(options) + return aip + + +def get_agent_path(agent_dir_path, agent_dir_suffix): + """ + Stand-alone method based off of agent_name method from AIPplatform. + Gets the path to the agent file of the specified directory if it exists. + """ + for agent_name in agent_dir_path.iterdir(): + try: + for agent_subdir in agent_name.iterdir(): + agent_dir = agent_name.joinpath( + agent_subdir.stem + f".{agent_dir_suffix}") + if agent_dir.exists(): + return agent_dir + # Ignore files that are not directories + except NotADirectoryError: + pass + + raise KeyError(agent_dir_path.stem) + + +def upgrade_old_agents(aip): + """ + Moves any keystore.json from agent-data to dist-info. + Only applies to agents in auth file. + """ + + vhome = Path(aip.env.volttron_home) + agent_map = aip.get_agent_identity_to_uuid_mapping() + + auth_file = AuthFile() + install_dir = vhome.joinpath("agents") + for agent in agent_map: + agent_path = install_dir.joinpath(agent_map[agent]) + try: + agent_data = get_agent_path(agent_path, 'agent-data') + # Skip if no agent-data exists + except KeyError as err: + print(f"agent-data not found for {err}") + continue + + keystore_path = agent_data.joinpath('keystore.json') + try: + dist_info = get_agent_path(agent_path, 'dist-info') + # Skip if no dist-info exists + except KeyError as err: + print(f"dist-info not found for {err}") + continue + keystore_dest_path = dist_info.joinpath('keystore.json') + + if keystore_path.exists(): + agent_keystore = KeyStore(keystore_path) + for entry in auth_file.read()[0]: + # Only move if agent exists in auth file + if entry.credentials == agent_keystore.public: + shutil.move(str(keystore_path), str(keystore_dest_path)) + break + return + + + +def get_identity_credentials(aip): + """Returns a dictionary containing a mapping from publickey to identity""" + + agent_map = aip.get_agent_identity_to_uuid_mapping() + agent_credential_map = {} + for agent in agent_map: + agent_credential = aip.get_agent_keystore(agent_map[agent]).public + agent_credential_map[agent_credential] = agent + return agent_credential_map + + +def set_auth_identities(agent_credential_map): + """Updates auth entries' identity field in auth file based on existing agents""" + + auth_file = AuthFile() + entries, deny_entries, groups, roles = auth_file.read() + for entry in entries: + for credential in agent_credential_map: + if entry.credentials == credential: + entry.identity = agent_credential_map[credential] + auth_file._write(entries, deny_entries, groups, roles) + return + + +def main(): + """Upgrade auth file to function with dynamic rpc authorizations""" + + fail_if_instance_running() + aip = get_aip() + upgrade_old_agents(aip) + identity_map = get_identity_credentials(aip) + set_auth_identities(identity_map) + print("Auth File Update Complete!") + + +def _main(): + """ Wrapper for main function""" + + try: + sys.exit(main()) + except KeyboardInterrupt: + sys.exit(1) + + +if __name__ == "__main__": + _main() diff --git a/volttron/platform/upgrade/upgrade_volttron.py b/volttron/platform/upgrade/upgrade_volttron.py new file mode 100644 index 0000000000..24cc837258 --- /dev/null +++ b/volttron/platform/upgrade/upgrade_volttron.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- {{{ +# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# +# Copyright 2020, Battelle Memorial Institute. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This material was prepared as an account of work sponsored by an agency of +# the United States Government. Neither the United States Government nor the +# United States Department of Energy, nor Battelle, nor any of their +# employees, nor any jurisdiction or organization that has cooperated in the +# development of these materials, makes any warranty, express or +# implied, or assumes any legal liability or responsibility for the accuracy, +# completeness, or usefulness or any information, apparatus, product, +# software, or process disclosed, or represents that its use would not infringe +# privately owned rights. Reference herein to any specific commercial product, +# process, or service by trade name, trademark, manufacturer, or otherwise +# does not necessarily constitute or imply its endorsement, recommendation, or +# favoring by the United States Government or any agency thereof, or +# Battelle Memorial Institute. The views and opinions of authors expressed +# herein do not necessarily state or reflect those of the +# United States Government or any agency thereof. +# +# PACIFIC NORTHWEST NATIONAL LABORATORY operated by +# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY +# under Contract DE-AC05-76RL01830 +# }}} + +import sys +from gevent import monkey as curious_george +curious_george.patch_all(thread=False, select=False) + +from . import update_auth_file +from . import move_sqlite_files + + +def main(): + # Upgrade auth file to function with dynamic rpc authorizations + update_auth_file.main() + print("") + # Moves backup cache of historian (backup.sqlite files) into corresponding agent-data directory so that + # historian agents other than sqlitehistorian, can be upgraded to latest version using + # vctl install --force without losing cache data. vctl install --force will backup and restore + # contents of //.agent-data directory + # If using sqlite historian manually backup and restore sqlite historian's db before upgrading to historian version + # 4.0.0 or later + move_sqlite_files.main() + + +def _main(): + """ Wrapper for main function""" + try: + sys.exit(main()) + except KeyboardInterrupt: + sys.exit(1) + +if __name__ == "__main__": + _main() diff --git a/volttron/platform/vip/agent/subsystems/auth.py b/volttron/platform/vip/agent/subsystems/auth.py index 937241f23b..26f8fae9a2 100644 --- a/volttron/platform/vip/agent/subsystems/auth.py +++ b/volttron/platform/vip/agent/subsystems/auth.py @@ -61,7 +61,7 @@ get_messagebus, ) from volttron.platform.certs import Certs -from volttron.platform.jsonrpc import RemoteError +from volttron.platform.jsonrpc import RemoteError, MethodNotFound from volttron.utils.rmq_config_params import RMQConfig from volttron.platform.keystore import KeyStore from volttron.platform.vip.agent.subsystems.health import BAD_STATUS, Status @@ -683,20 +683,30 @@ def update_rpc_method_capabilities(self): rpc_method_authorizations[method] = self.get_rpc_authorizations( method ) - updated_rpc_authorizations = ( - self._rpc() - .call( - AUTH, - "update_id_rpc_authorizations", - self._core().identity, - rpc_method_authorizations, + try: + updated_rpc_authorizations = ( + self._rpc() + .call( + AUTH, + "update_id_rpc_authorizations", + self._core().identity, + rpc_method_authorizations, + ) + .get(timeout=4) ) - .get(timeout=4) - ) + except MethodNotFound: + _log.warning("update_id_rpc_authorization method is missing from " + "AuthService! The VOLTTRON Instance you are " + "attempting to connect to is to old to support " + "dynamic RPC authorizations.") + return if updated_rpc_authorizations is None: - _log.error( + _log.warning( f"Auth entry not found for {self._core().identity}: " - f"rpc_method_authorizations not updated." + f"rpc_method_authorizations not updated. If this agent " + f"does have an auth entry, verify that the 'identity' field " + f"has been included in the auth entry. This should be set to " + f"the identity of the agent" ) return if rpc_method_authorizations != updated_rpc_authorizations: diff --git a/volttrontesting/platform/auth_tests/test_auth_control.py b/volttrontesting/platform/auth_tests/test_auth_control.py index bb58b672d6..8c7e46da78 100644 --- a/volttrontesting/platform/auth_tests/test_auth_control.py +++ b/volttrontesting/platform/auth_tests/test_auth_control.py @@ -193,7 +193,7 @@ def auth_instance(volttron_instance): # Number of tries to check if auth file is updated properly -auth_retry = 30 +auth_retry = 5 @pytest.mark.control @@ -214,8 +214,8 @@ def test_auth_add(auth_instance): print(entries) assert len(entries) > 0 i = 0 - while len(entries) < len_entries and i < auth_retry: - gevent.sleep(1) + while len(entries) <= len_entries and i < auth_retry: + gevent.sleep(i) entries = auth_list_json(platform) i += 1 @@ -234,8 +234,8 @@ def test_auth_add_cmd_line(auth_instance): print(entries) assert len(entries) > 0 i = 0 - while len(entries) < len_entries and i < auth_retry: - gevent.sleep(1) + while len(entries) <= len_entries and i < auth_retry: + gevent.sleep(i) entries = auth_list_json(platform) i += 1 assert_auth_entries_same(entries[-1], _auth_entry2.__dict__) @@ -252,8 +252,8 @@ def test_auth_update(auth_instance): print(entries) assert len(entries) > 0 i = 0 - while len(entries) < len_entries and i < auth_retry: - gevent.sleep(1) + while len(entries) <= len_entries and i < auth_retry: + gevent.sleep(i) entries = auth_list_json(platform) i += 1 auth_update(platform, len(entries) - 1, **_auth_entry4.__dict__) @@ -274,16 +274,16 @@ def test_auth_remove(auth_instance): entries = auth_list_json(platform) assert len(entries) > 0 i = 0 - while len(entries) < len_entries and i < auth_retry: - gevent.sleep(1) + while len(entries) <= len_entries and i < auth_retry: + gevent.sleep(i) entries = auth_list_json(platform) i += 1 auth_add(platform, _auth_entry6) entries = auth_list_json(platform) assert len(entries) > 0 i = 0 - while len(entries) < (len_entries + 1) and i < auth_retry: - gevent.sleep(1) + while len(entries) <= (len_entries + 1) and i < auth_retry: + gevent.sleep(i) entries = auth_list_json(platform) i += 1 print(entries) @@ -296,7 +296,7 @@ def test_auth_remove(auth_instance): assert len(entries) > 0 i = 0 while len(entries) > (len_entries + 1) and i < auth_retry: - gevent.sleep(1) + gevent.sleep(i) entries = auth_list_json(platform) i += 1 assert_auth_entries_same(entries[-1], _auth_entry5.__dict__) @@ -312,8 +312,8 @@ def test_auth_rpc_method_add(auth_instance): entries = auth_list_json(platform) assert len(entries) > 0 i = 0 - while len(entries) < len_entries and i < auth_retry: - gevent.sleep(1) + while len(entries) <= len_entries and i < auth_retry: + gevent.sleep(i) entries = auth_list_json(platform) i += 1 print(entries) @@ -324,7 +324,7 @@ def test_auth_rpc_method_add(auth_instance): i = 0 while entries[-1]['rpc_method_authorizations'] != {'test_method': ["test_auth"]} and i < auth_retry: - gevent.sleep(1) + gevent.sleep(i) entries = auth_list_json(platform) i += 1 @@ -341,8 +341,8 @@ def test_auth_rpc_method_remove(auth_instance): entries = auth_list_json(platform) assert len(entries) > 0 i = 0 - while len(entries) < len_entries and i < auth_retry: - gevent.sleep(1) + while len(entries) <= len_entries and i < auth_retry: + gevent.sleep(i) entries = auth_list_json(platform) i += 1 print(entries) @@ -353,7 +353,7 @@ def test_auth_rpc_method_remove(auth_instance): i = 0 while entries[-1]['rpc_method_authorizations'] != {'test_method': ["test_auth"]} and i < auth_retry: - gevent.sleep(1) + gevent.sleep(i) entries = auth_list_json(platform) i += 1 @@ -365,7 +365,7 @@ def test_auth_rpc_method_remove(auth_instance): i = 0 while entries[-1]['rpc_method_authorizations'] == {'test_method': ["test_auth"]} and i < auth_retry: - gevent.sleep(1) + gevent.sleep(i) entries = auth_list_json(platform) i += 1 diff --git a/volttrontesting/platform/auth_tests/test_auth_file.py b/volttrontesting/platform/auth_tests/test_auth_file.py index ec7724151a..f7d8c65e05 100644 --- a/volttrontesting/platform/auth_tests/test_auth_file.py +++ b/volttrontesting/platform/auth_tests/test_auth_file.py @@ -253,7 +253,7 @@ def test_groups_and_roles(auth_file_platform_tuple): @pytest.mark.auth -def test_upgrade_file_verison_0_to_1_2(tmpdir_factory): +def test_upgrade_file_verison_0_to_latest(tmpdir_factory): mechanism = "CURVE" publickey = "A" * 43 version0 = { @@ -275,7 +275,11 @@ def test_upgrade_file_verison_0_to_1_2(tmpdir_factory): }, "groups": { "admin": ["reader", "writer"] - } + }, + "version": { + "major": 0, + "minor": 0 + }, } filename = str(tmpdir_factory.mktemp('auth_test').join('auth.json')) @@ -293,16 +297,23 @@ def test_upgrade_file_verison_0_to_1_2(tmpdir_factory): expected["mechanism"] = mechanism expected["capabilities"] = {'can_publish_temperature': None, 'edit_config_store': {'identity': entries[0].user_id}} + expected["rpc_method_authorizations"] = {} assert_auth_entries_same(expected, vars(entries[0])) - + # RPC Method Authorizations added with 1.3 + for entry in upgraded.auth_data["allow_list"]: + assert entry["rpc_method_authorizations"] == {} @pytest.mark.auth -def test_upgrade_file_verison_0_to_1_2_minimum_entries(tmpdir_factory): +def test_upgrade_file_verison_0_to_latest_minimum_entries(tmpdir_factory): """The only required field in 'version 0' was credentials""" mechanism = "CURVE" publickey = "A" * 43 version0 = { "allow": [{"credentials": mechanism + ":" + publickey}], + "version": { + "major": 0, + "minor": 0 + }, } filename = str(tmpdir_factory.mktemp('auth_test').join('auth.json')) @@ -323,10 +334,14 @@ def test_upgrade_file_verison_0_to_1_2_minimum_entries(tmpdir_factory): expected["enabled"] = True expected["comments"] = None expected["capabilities"] = {'edit_config_store': {'identity': entries[0].user_id}} + expected["rpc_method_authorizations"] = {} expected["roles"] = [] expected["groups"] = [] assert_auth_entries_same(expected, vars(entries[0])) + # RPC Method Authorizations added with 1.3 + for entry in upgraded.auth_data["allow_list"]: + assert entry["rpc_method_authorizations"] == {} @pytest.mark.auth def test_upgrade_file_version_1_1_to_1_2(tmpdir_factory): @@ -422,7 +437,7 @@ def test_upgrade_file_version_1_1_to_1_2(tmpdir_factory): def test_upgrade_file_version_1_2_to_1_3(tmpdir_factory): """The only required field in 'version 0' was credentials""" - version1_1 = { + version1_2 = { "roles":{ "manager":[ "can_managed_platform" @@ -495,7 +510,7 @@ def test_upgrade_file_version_1_2_to_1_3(tmpdir_factory): filename = str(tmpdir_factory.mktemp('auth_test').join('auth.json')) with open(filename, 'w') as fp: - fp.write(jsonapi.dumps(version1_1, indent=2)) + fp.write(jsonapi.dumps(version1_2, indent=2)) upgraded = AuthFile(filename) entries = upgraded.read()[0] diff --git a/volttrontesting/platform/dbutils/test_backup_database.py b/volttrontesting/platform/dbutils/test_backup_database.py index a87ce62084..d3ecdc82bb 100644 --- a/volttrontesting/platform/dbutils/test_backup_database.py +++ b/volttrontesting/platform/dbutils/test_backup_database.py @@ -1,6 +1,6 @@ import os import pytest - +from pathlib import Path from gevent import subprocess from datetime import datetime from pytz import UTC @@ -9,6 +9,8 @@ SIZE_LIMIT = 1000 # the default submit_size_limit for BaseHistorianAgents +agent_data_dir = os.path.join(os.getcwd(), os.path.basename(os.getcwd()) + ".agent-data") +cache_db = str(Path(agent_data_dir).joinpath("backup.sqlite")) def test_get_outstanding_to_publish_should_return_records( backup_database, new_publish_list_unique @@ -245,13 +247,16 @@ def new_publish_list_dupes(): @pytest.fixture() def backup_database(): + os.makedirs(agent_data_dir, exist_ok=True) yield BackupDatabase(BaseHistorian(), None, 0.9) # Teardown # the backup database is an sqlite database with the name "backup.sqlite". # the db is created if it doesn't exist; see the method: BackupDatabase._setupdb(check_same_thread) for details - if os.path.exists("./backup.sqlite"): - os.remove("./backup.sqlite") + if os.path.exists(cache_db): + os.remove(cache_db) + if os.path.exists(agent_data_dir): + os.rmdir(agent_data_dir) def get_all_data(table): @@ -262,7 +267,7 @@ def get_all_data(table): def query_db(query): output = subprocess.run( - ["sqlite3", "backup.sqlite", query], text=True, capture_output=True + ["sqlite3", cache_db, query], text=True, capture_output=True ) # check_returncode() will raise a CalledProcessError if the query fails # see https://docs.python.org/3/library/subprocess.html#subprocess.CompletedProcess.returncode diff --git a/volttrontesting/services/historian/test_base_historian.py b/volttrontesting/services/historian/test_base_historian.py index 51a578e09b..4771e257a5 100644 --- a/volttrontesting/services/historian/test_base_historian.py +++ b/volttrontesting/services/historian/test_base_historian.py @@ -60,6 +60,13 @@ class Historian(BaseHistorian): + + def __init__(self, **kwargs): + self.agent_data_dir = os.path.join(os.getcwd(), os.path.basename(os.getcwd()) + ".agent-data") + os.makedirs(self.agent_data_dir, exist_ok=True) + self.backup_sqlite = Path(self.agent_data_dir).joinpath("backup.sqlite") + super(Historian, self).__init__(**kwargs) + def publish_to_historian(self, _): pass @@ -71,9 +78,9 @@ def query_historian(self, **kwargs): def remove_backup_cache_db(self): try: - abspath = Path('backup.sqlite').absolute() - if abspath.exists(): - os.remove(str(abspath)) + print(f"Removing backup cache {self.backup_sqlite}") + os.remove(str(self.backup_sqlite)) + os.remove(str(self.agent_data_dir)) except: print("Don't throw here if os.remove fails...") @@ -89,6 +96,9 @@ def listener(peer, sender, bus, topic, headers, message): class BasicHistorian(BaseHistorian): def __init__(self, **kwargs): + self.agent_data_dir = os.path.join(os.getcwd(), os.path.basename(os.getcwd()) + ".agent-data") + os.makedirs(self.agent_data_dir, exist_ok=True) + self.backup_sqlite = Path(self.agent_data_dir).joinpath("backup.sqlite") super(BasicHistorian, self).__init__(**kwargs) self.publish_fail = False self.publish_sleep = 0 @@ -108,9 +118,9 @@ def reset(self): def remove_backup_cache_db(self): try: - abspath = Path('backup.sqlite').absolute() - if abspath.exists(): - os.remove(str(abspath)) + print(f"Removing backup cache {self.backup_sqlite}") + os.remove(str(self.backup_sqlite)) + os.remove(str(self.agent_data_dir)) except: print("Don't throw here if os.remove fails...") @@ -271,7 +281,6 @@ def test_time_tolerance_check(request, volttron_instance, client_agent): enable_store=True) assert "could not convert string to float: 'invalid'" in str(e.value) print(e) - historian = volttron_instance.build_agent(agent_class=BasicHistorian, identity=identity, submit_size_limit=5, @@ -283,7 +292,7 @@ def test_time_tolerance_check(request, volttron_instance, client_agent): DEVICES_ALL_TOPIC = "devices/Building/LAB/Device/all" gevent.sleep(5) # wait for historian to be fully up historian.publish_sleep = 0 - db_file = Path('backup.sqlite').absolute() + db_file = historian.backup_sqlite assert db_file.exists() db_connection = sqlite3.connect(str(db_file)) c = db_connection.cursor() @@ -524,6 +533,9 @@ def test_health_stuff(request, volttron_instance, client_agent): class FailureHistorian(BaseHistorian): def __init__(self, **kwargs): + self.agent_data_dir = os.path.join(os.getcwd(), os.path.basename(os.getcwd()) + ".agent-data") + os.makedirs(self.agent_data_dir, exist_ok=True) + self.backup_sqlite = Path(self.agent_data_dir).joinpath("backup.sqlite") super(FailureHistorian, self).__init__(**kwargs) self.publish_fail = False self.setup_fail = False @@ -557,9 +569,9 @@ def reset(self): def remove_backup_cache_db(self): try: - abspath = Path('backup.sqlite').absolute() - if abspath.exists(): - os.remove(str(abspath)) + print(f"Removing backup cache {self.backup_sqlite}") + os.remove(str(self.backup_sqlite)) + os.remove(str(self.agent_data_dir)) except: print("Don't throw here if os.remove fails...") diff --git a/volttrontesting/testutils/test_base_historian_unit.py b/volttrontesting/testutils/test_base_historian_unit.py index 52640b13ba..6534ad6d71 100644 --- a/volttrontesting/testutils/test_base_historian_unit.py +++ b/volttrontesting/testutils/test_base_historian_unit.py @@ -3,6 +3,7 @@ import os from shutil import rmtree from time import sleep +from pathlib import Path import pytest from pytz import UTC @@ -10,7 +11,11 @@ from volttrontesting.utils.utils import AgentMock from volttron.platform.agent.base_historian import BaseHistorianAgent, Agent -CACHE_NAME = "backup.sqlite" + +agent_data_dir = os.path.join(os.getcwd(), os.path.basename(os.getcwd()) + ".agent-data") +os.makedirs(agent_data_dir, exist_ok=True) +CACHE_NAME = str(Path(agent_data_dir).joinpath("backup.sqlite")) + HISTORIAN_DB = "./data/historian.sqlite" @@ -113,3 +118,5 @@ def base_historian_agent(): rmtree("./data") if os.path.exists(CACHE_NAME): os.remove(CACHE_NAME) + if os.path.exists(agent_data_dir): + os.rmdir(agent_data_dir)