From 6a0c3342bf212a90bc0fef9666b4ec9578ed3ffa Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Fri, 31 Dec 2021 02:05:19 +0000 Subject: [PATCH 01/67] Fix more Python 3 issues --- python/examples/ola_candidate_ports.py | 24 ++++++++++++++++++------ python/examples/ola_devices.py | 4 ++++ tools/rdm/ModelCollector.py | 2 +- tools/rdm/TestDefinitions.py | 2 +- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/python/examples/ola_candidate_ports.py b/python/examples/ola_candidate_ports.py index 99337634a1..3b1d4cf683 100755 --- a/python/examples/ola_candidate_ports.py +++ b/python/examples/ola_candidate_ports.py @@ -25,6 +25,8 @@ __author__ = 'simon.marchi@polymtl.ca (Simon Marchi)' +wrapper = None + def ParseArgs(): desc = 'Show the candidate ports to patch to a universe.' @@ -52,11 +54,21 @@ def GetCandidatePortsCallback(status, devices): else: print('Error: %s' % status.message, file=sys.stderr) + global wrapper + if wrapper: + wrapper.Stop() + + +def main(): + args = ParseArgs() + universe = args.universe + + global wrapper + wrapper = ClientWrapper() + client = wrapper.Client() + client.GetCandidatePorts(GetCandidatePortsCallback, universe) + wrapper.Run() -args = ParseArgs() -universe = args.universe -wrapper = ClientWrapper() -client = wrapper.Client() -client.GetCandidatePorts(GetCandidatePortsCallback, universe) -wrapper.Run() +if __name__ == "__main__": + main() diff --git a/python/examples/ola_devices.py b/python/examples/ola_devices.py index f184ce79e7..d9832ef80e 100755 --- a/python/examples/ola_devices.py +++ b/python/examples/ola_devices.py @@ -57,3 +57,7 @@ def main(): client = wrapper.Client() client.FetchDevices(Devices) wrapper.Run() + + +if __name__ == "__main__": + main() diff --git a/tools/rdm/ModelCollector.py b/tools/rdm/ModelCollector.py index 5dfb31d835..4ea6f9a1ae 100644 --- a/tools/rdm/ModelCollector.py +++ b/tools/rdm/ModelCollector.py @@ -103,7 +103,7 @@ def _GetDevice(self): def _GetVersion(self): this_device = self._GetDevice() software_versions = this_device['software_versions'] - return software_versions[software_versions.keys()[0]] + return software_versions[list(software_versions.keys())[0]] def _GetCurrentPersonality(self): this_device = self._GetDevice() diff --git a/tools/rdm/TestDefinitions.py b/tools/rdm/TestDefinitions.py index 2f8dccb982..5e97ed16d9 100644 --- a/tools/rdm/TestDefinitions.py +++ b/tools/rdm/TestDefinitions.py @@ -3264,7 +3264,7 @@ class SetLampOnMode(TestMixins.SetMixin, OptionalParameterTestFixture): EXPECTED_FIELDS = ['mode'] REQUIRES = ['lamp_on_mode'] ALLOWED_MODES = [0, 1, 2] - ALL_MODES = ALLOWED_MODES + [3] + range(0x80, 0xe0) + ALL_MODES = ALLOWED_MODES + [3] + list(range(0x80, 0xe0)) def OldValue(self): old = self.Property('lamp_on_mode') From ed103986774edb9845f46e012fc0d95dffb12835 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Fri, 31 Dec 2021 03:33:36 +0000 Subject: [PATCH 02/67] Encode JSON output in RDM test server for Python 3 compatibility --- tools/rdm/rdm_test_server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/rdm/rdm_test_server.py b/tools/rdm/rdm_test_server.py index 2852d7bd7e..ad56766123 100755 --- a/tools/rdm/rdm_test_server.py +++ b/tools/rdm/rdm_test_server.py @@ -505,7 +505,7 @@ def HandleRequest(self, request, response): try: json_data = self.GetJson(request, response) - response.AppendData(json.dumps(json_data, sort_keys=True)) + response.AppendData(json.dumps(json_data, sort_keys=True).encode()) except ServerException as e: # For JSON requests, rather than returning 500s we return the error as # JSON @@ -514,7 +514,7 @@ def HandleRequest(self, request, response): 'status': False, 'error': str(e), } - response.AppendData(json.dumps(json_data, sort_keys=True)) + response.AppendData(json.dumps(json_data, sort_keys=True).encode()) def RaiseExceptionIfMissing(self, request, param): """Helper method to raise an exception if the param is missing.""" @@ -545,7 +545,7 @@ def HandleRequest(self, request, response): 'status': False, 'error': 'The OLA Server instance is no longer running', } - response.AppendData(json.dumps(json_data, sort_keys=True)) + response.AppendData(json.dumps(json_data, sort_keys=True).encode()) class TestDefinitionsHandler(JsonRequestHandler): From 260b12d4eb2fe69a138b0d2e4c0abefac4a1f2a0 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Mon, 3 Jan 2022 21:05:37 +0000 Subject: [PATCH 03/67] Add tests for TestRunner.GetTestClasses --- .gitignore | 1 + tools/rdm/Makefile.mk | 8 +++++++ tools/rdm/TestRunnerTest.py | 48 +++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+) create mode 100644 tools/rdm/TestRunnerTest.py diff --git a/.gitignore b/.gitignore index 0460c47421..4737a71005 100644 --- a/.gitignore +++ b/.gitignore @@ -226,6 +226,7 @@ tools/ola_trigger/FileValidateTest.sh tools/rdm/DataLocation.py tools/rdm/ResponderTestTest.sh tools/rdm/TestHelpersTest.sh +tools/rdm/TestRunnerTest.sh tools/rdm/TestStateTest.sh tools/rdmpro/rdmpro_sniffer tools/rdmpro/rdmpro_sniffer.exe diff --git a/tools/rdm/Makefile.mk b/tools/rdm/Makefile.mk index 1eebf27b87..3b092bdd47 100644 --- a/tools/rdm/Makefile.mk +++ b/tools/rdm/Makefile.mk @@ -65,6 +65,11 @@ tools/rdm/TestHelpersTest.sh: tools/rdm/Makefile.mk echo "PYTHONPATH=${top_builddir}/python $(PYTHON) ${srcdir}/tools/rdm/TestHelpersTest.py; exit \$$?" > $(top_builddir)/tools/rdm/TestHelpersTest.sh chmod +x $(top_builddir)/tools/rdm/TestHelpersTest.sh +tools/rdm/TestRunnerTest.sh: tools/rdm/Makefile.mk + mkdir -p $(top_builddir)/python/ola + echo "PYTHONPATH=${top_builddir}/python $(PYTHON) ${srcdir}/tools/rdm/TestRunnerTest.py; exit \$$?" > $(top_builddir)/tools/rdm/TestRunnerTest.sh + chmod +x $(top_builddir)/tools/rdm/TestRunnerTest.sh + tools/rdm/TestStateTest.sh: tools/rdm/Makefile.mk mkdir -p $(top_builddir)/python/ola echo "PYTHONPATH=${top_builddir}/python $(PYTHON) ${srcdir}/tools/rdm/TestStateTest.py; exit \$$?" > $(top_builddir)/tools/rdm/TestStateTest.sh @@ -73,12 +78,14 @@ tools/rdm/TestStateTest.sh: tools/rdm/Makefile.mk dist_check_SCRIPTS += \ tools/rdm/ResponderTestTest.py \ tools/rdm/TestHelpersTest.py \ + tools/rdm/TestRunnerTest.py \ tools/rdm/TestStateTest.py if BUILD_PYTHON_LIBS test_scripts += \ tools/rdm/ResponderTestTest.sh \ tools/rdm/TestHelpersTest.sh \ + tools/rdm/TestRunnerTest.sh \ tools/rdm/TestStateTest.sh endif @@ -86,6 +93,7 @@ CLEANFILES += \ tools/rdm/*.pyc \ tools/rdm/ResponderTestTest.sh \ tools/rdm/TestHelpersTest.sh \ + tools/rdm/TestRunnerTest.sh \ tools/rdm/TestStateTest.sh \ tools/rdm/__pycache__/* diff --git a/tools/rdm/TestRunnerTest.py b/tools/rdm/TestRunnerTest.py new file mode 100644 index 0000000000..517941f66b --- /dev/null +++ b/tools/rdm/TestRunnerTest.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# TestRunnerTest.py +# Copyright (C) 2022 Peter Newman + +import unittest +from TestRunner import GetTestClasses +from ola.testing.rdm import TestDefinitions + +"""Test cases for TestRunner utilities.""" + +__author__ = 'nomis52@gmail.com (Simon Newton)' + + +class TestRunnerGetTestClasses(unittest.TestCase): + def testGetTestClasses(self): + self.assertTrue(len(GetTestClasses(TestDefinitions)) > 0, + "Didn't find any test classes") + self.assertTrue(len(GetTestClasses(TestDefinitions)) > 100, + "Didn't find a realistic number of test classes") + # Check for a common test + self.assertTrue("GetDeviceInfo" in GetTestClasses(TestDefinitions)) + # Check we don't contain the base classes: + for classname in ["OptionalParameterTestFixture", + "ResponderTestFixture", + "TestFixture", + "ResponderTest.OptionalParameterTestFixture", + "ResponderTest.ResponderTestFixture", + "ResponderTest.TestFixture"]: + self.assertTrue(classname not in GetTestClasses(TestDefinitions), + "Class %s found in list of test classes" % classname) + + +if __name__ == '__main__': + unittest.main() From f3f67043f6594db5b6c7a9a1b38f1212e8afe399 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Mon, 3 Jan 2022 21:26:04 +0000 Subject: [PATCH 04/67] Cause rdm_responder_test to die if there were no tests found --- tools/rdm/rdm_responder_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/rdm/rdm_responder_test.py b/tools/rdm/rdm_responder_test.py index 3e24d1672c..eccc5d99a4 100755 --- a/tools/rdm/rdm_responder_test.py +++ b/tools/rdm/rdm_responder_test.py @@ -248,6 +248,9 @@ def main(): options = ParseOptions() test_classes = TestRunner.GetTestClasses(TestDefinitions) + if len(test_classes) <= 0: + print('Failed to find any tests to run') + sys.exit(2) if options.list_tests: for test_name in sorted(c.__name__ for c in test_classes): print(test_name) From ef7494d51bdb3c3e4af4a0b724d1c3ed7788b87e Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Mon, 3 Jan 2022 21:51:57 +0000 Subject: [PATCH 05/67] Change how we do the import to try and fix the tests --- tools/rdm/TestRunnerTest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/rdm/TestRunnerTest.py b/tools/rdm/TestRunnerTest.py index 517941f66b..669ffff71d 100644 --- a/tools/rdm/TestRunnerTest.py +++ b/tools/rdm/TestRunnerTest.py @@ -17,8 +17,8 @@ # Copyright (C) 2022 Peter Newman import unittest +import TestDefinitions from TestRunner import GetTestClasses -from ola.testing.rdm import TestDefinitions """Test cases for TestRunner utilities.""" From 19663dad453ef93107842fe9a2aeb517d73e29fc Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Tue, 4 Jan 2022 00:49:05 +0000 Subject: [PATCH 06/67] Another way to try and work round the path issues for the Python test --- .gitignore | 2 ++ tools/rdm/Makefile.mk | 5 ++++- tools/rdm/TestRunnerTest.py | 11 +++++------ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 4737a71005..d34790ace7 100644 --- a/.gitignore +++ b/.gitignore @@ -195,6 +195,8 @@ python/ola/PidStoreTest.sh python/ola/RDMTest.sh python/ola/Version.py python/ola/rpc/SimpleRpcControllerTest.sh +python/ola/testing/ +python/ola/testing/__init__.py slp/slp_client slp/slp_client.exe slp/slp_server diff --git a/tools/rdm/Makefile.mk b/tools/rdm/Makefile.mk index 3b092bdd47..3ed33c3c57 100644 --- a/tools/rdm/Makefile.mk +++ b/tools/rdm/Makefile.mk @@ -66,7 +66,9 @@ tools/rdm/TestHelpersTest.sh: tools/rdm/Makefile.mk chmod +x $(top_builddir)/tools/rdm/TestHelpersTest.sh tools/rdm/TestRunnerTest.sh: tools/rdm/Makefile.mk - mkdir -p $(top_builddir)/python/ola + mkdir -p $(top_builddir)/python/ola/testing + echo "" > $(top_builddir)/python/ola/testing/__init__.py + $(LN_S) -f ../../../tools/rdm $(top_builddir)/python/ola/testing/rdm echo "PYTHONPATH=${top_builddir}/python $(PYTHON) ${srcdir}/tools/rdm/TestRunnerTest.py; exit \$$?" > $(top_builddir)/tools/rdm/TestRunnerTest.sh chmod +x $(top_builddir)/tools/rdm/TestRunnerTest.sh @@ -90,6 +92,7 @@ test_scripts += \ endif CLEANFILES += \ + python/ola/testing/__init__.py \ tools/rdm/*.pyc \ tools/rdm/ResponderTestTest.sh \ tools/rdm/TestHelpersTest.sh \ diff --git a/tools/rdm/TestRunnerTest.py b/tools/rdm/TestRunnerTest.py index 669ffff71d..4bec5d9b50 100644 --- a/tools/rdm/TestRunnerTest.py +++ b/tools/rdm/TestRunnerTest.py @@ -17,8 +17,7 @@ # Copyright (C) 2022 Peter Newman import unittest -import TestDefinitions -from TestRunner import GetTestClasses +from ola.testing.rdm import TestDefinitions, TestRunner """Test cases for TestRunner utilities.""" @@ -27,12 +26,12 @@ class TestRunnerGetTestClasses(unittest.TestCase): def testGetTestClasses(self): - self.assertTrue(len(GetTestClasses(TestDefinitions)) > 0, + self.assertTrue(len(TestRunner.GetTestClasses(TestDefinitions)) > 0, "Didn't find any test classes") - self.assertTrue(len(GetTestClasses(TestDefinitions)) > 100, + self.assertTrue(len(TestRunner.GetTestClasses(TestDefinitions)) > 100, "Didn't find a realistic number of test classes") # Check for a common test - self.assertTrue("GetDeviceInfo" in GetTestClasses(TestDefinitions)) + self.assertTrue("GetDeviceInfo" in TestRunner.GetTestClasses(TestDefinitions)) # Check we don't contain the base classes: for classname in ["OptionalParameterTestFixture", "ResponderTestFixture", @@ -40,7 +39,7 @@ def testGetTestClasses(self): "ResponderTest.OptionalParameterTestFixture", "ResponderTest.ResponderTestFixture", "ResponderTest.TestFixture"]: - self.assertTrue(classname not in GetTestClasses(TestDefinitions), + self.assertTrue(classname not in TestRunner.GetTestClasses(TestDefinitions), "Class %s found in list of test classes" % classname) From 8996884085c4dd23bf8f39cb942acf2e5ddc2c5c Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Tue, 4 Jan 2022 01:28:16 +0000 Subject: [PATCH 07/67] Fix the flake8 errors, try fixing the build on Travis --- tools/rdm/Makefile.mk | 4 ++-- tools/rdm/TestRunnerTest.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/rdm/Makefile.mk b/tools/rdm/Makefile.mk index 3ed33c3c57..1b64018959 100644 --- a/tools/rdm/Makefile.mk +++ b/tools/rdm/Makefile.mk @@ -67,8 +67,8 @@ tools/rdm/TestHelpersTest.sh: tools/rdm/Makefile.mk tools/rdm/TestRunnerTest.sh: tools/rdm/Makefile.mk mkdir -p $(top_builddir)/python/ola/testing - echo "" > $(top_builddir)/python/ola/testing/__init__.py - $(LN_S) -f ../../../tools/rdm $(top_builddir)/python/ola/testing/rdm + touch $(top_builddir)/python/ola/testing/__init__.py + $(LN_S) -f $(top_builddir)/tools/rdm $(top_builddir)/python/ola/testing/rdm echo "PYTHONPATH=${top_builddir}/python $(PYTHON) ${srcdir}/tools/rdm/TestRunnerTest.py; exit \$$?" > $(top_builddir)/tools/rdm/TestRunnerTest.sh chmod +x $(top_builddir)/tools/rdm/TestRunnerTest.sh diff --git a/tools/rdm/TestRunnerTest.py b/tools/rdm/TestRunnerTest.py index 4bec5d9b50..0a7c22b2e1 100644 --- a/tools/rdm/TestRunnerTest.py +++ b/tools/rdm/TestRunnerTest.py @@ -31,7 +31,8 @@ def testGetTestClasses(self): self.assertTrue(len(TestRunner.GetTestClasses(TestDefinitions)) > 100, "Didn't find a realistic number of test classes") # Check for a common test - self.assertTrue("GetDeviceInfo" in TestRunner.GetTestClasses(TestDefinitions)) + self.assertTrue("GetDeviceInfo" in + TestRunner.GetTestClasses(TestDefinitions)) # Check we don't contain the base classes: for classname in ["OptionalParameterTestFixture", "ResponderTestFixture", @@ -39,7 +40,8 @@ def testGetTestClasses(self): "ResponderTest.OptionalParameterTestFixture", "ResponderTest.ResponderTestFixture", "ResponderTest.TestFixture"]: - self.assertTrue(classname not in TestRunner.GetTestClasses(TestDefinitions), + self.assertTrue(classname not in + TestRunner.GetTestClasses(TestDefinitions), "Class %s found in list of test classes" % classname) From 370ebe6b84a549245da39237bd6c1c1183a39c05 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Tue, 4 Jan 2022 18:29:24 +0000 Subject: [PATCH 08/67] Fix the TestRunnerTest on Python 2.7 --- tools/rdm/Makefile.mk | 2 +- tools/rdm/TestRunnerTest.py | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tools/rdm/Makefile.mk b/tools/rdm/Makefile.mk index 1b64018959..4306509f2a 100644 --- a/tools/rdm/Makefile.mk +++ b/tools/rdm/Makefile.mk @@ -68,7 +68,7 @@ tools/rdm/TestHelpersTest.sh: tools/rdm/Makefile.mk tools/rdm/TestRunnerTest.sh: tools/rdm/Makefile.mk mkdir -p $(top_builddir)/python/ola/testing touch $(top_builddir)/python/ola/testing/__init__.py - $(LN_S) -f $(top_builddir)/tools/rdm $(top_builddir)/python/ola/testing/rdm + $(LN_S) -f ../../../tools/rdm $(top_builddir)/python/ola/testing/rdm echo "PYTHONPATH=${top_builddir}/python $(PYTHON) ${srcdir}/tools/rdm/TestRunnerTest.py; exit \$$?" > $(top_builddir)/tools/rdm/TestRunnerTest.sh chmod +x $(top_builddir)/tools/rdm/TestRunnerTest.sh diff --git a/tools/rdm/TestRunnerTest.py b/tools/rdm/TestRunnerTest.py index 0a7c22b2e1..aab3a907df 100644 --- a/tools/rdm/TestRunnerTest.py +++ b/tools/rdm/TestRunnerTest.py @@ -18,6 +18,9 @@ import unittest from ola.testing.rdm import TestDefinitions, TestRunner +from ola.testing.rdm.ResponderTest import OptionalParameterTestFixture +from ola.testing.rdm.ResponderTest import ResponderTestFixture, TestFixture +from ola.testing.rdm.TestDefinitions import GetDeviceInfo """Test cases for TestRunner utilities.""" @@ -31,15 +34,13 @@ def testGetTestClasses(self): self.assertTrue(len(TestRunner.GetTestClasses(TestDefinitions)) > 100, "Didn't find a realistic number of test classes") # Check for a common test - self.assertTrue("GetDeviceInfo" in - TestRunner.GetTestClasses(TestDefinitions)) + self.assertTrue(GetDeviceInfo in + TestRunner.GetTestClasses(TestDefinitions), + "GetDeviceInfo missing from list of test classes") # Check we don't contain the base classes: - for classname in ["OptionalParameterTestFixture", - "ResponderTestFixture", - "TestFixture", - "ResponderTest.OptionalParameterTestFixture", - "ResponderTest.ResponderTestFixture", - "ResponderTest.TestFixture"]: + for classname in [OptionalParameterTestFixture, + ResponderTestFixture, + TestFixture]: self.assertTrue(classname not in TestRunner.GetTestClasses(TestDefinitions), "Class %s found in list of test classes" % classname) From f6de4c24831180eb6629bbfb90c65b22ab58d5ba Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Fri, 18 Nov 2022 02:05:20 +0000 Subject: [PATCH 09/67] Add test cases for ExpectedResults classes. --- .gitignore | 1 + tools/rdm/ExpectedResults.py | 12 +++- tools/rdm/ExpectedResultsTest.py | 96 ++++++++++++++++++++++++++++++++ tools/rdm/Makefile.mk | 8 +++ tools/rdm/ResponderTestTest.py | 1 - 5 files changed, 114 insertions(+), 4 deletions(-) create mode 100644 tools/rdm/ExpectedResultsTest.py diff --git a/.gitignore b/.gitignore index d34790ace7..51cfc8e891 100644 --- a/.gitignore +++ b/.gitignore @@ -226,6 +226,7 @@ tools/ola_trigger/ola_trigger tools/ola_trigger/ola_trigger.exe tools/ola_trigger/FileValidateTest.sh tools/rdm/DataLocation.py +tools/rdm/ExpectedResultsTest.sh tools/rdm/ResponderTestTest.sh tools/rdm/TestHelpersTest.sh tools/rdm/TestRunnerTest.sh diff --git a/tools/rdm/ExpectedResults.py b/tools/rdm/ExpectedResults.py index d72232a9b9..aa4ae97ce0 100644 --- a/tools/rdm/ExpectedResults.py +++ b/tools/rdm/ExpectedResults.py @@ -18,12 +18,18 @@ # Expected result classes are broken down as follows: # # BaseExpectedResult - the base class -# BroadcastResult - expects the request to be broadcast +# BroadcastResult - expects the request to be broadcast, hence no response +# TimeoutResult - expects the response to be a timeout +# InvalidResult - expects the response to be invalid +# UnsupportedResult - expects RDM Discovery to be unsupported +# DUBResult - expects an RDM DUB response # SuccessfulResult - expects a well formed response from the device # NackResult - parent NACK class +# NackDiscoveryResult - expects DISCOVERY_COMMAND_RESPONSE with a NACK # NackGetResult - expects GET_COMMAND_RESPONSE with a NACK # NackSetResult - expects SET_COMMAND_RESPONSE with a NACK # AckResult - parent ACK class +# AckDiscoveryResult - expects DISCOVERY_COMMAND_RESPONSE with an ACK # AckGetResult - expects GET_COMMAND_RESPONSE with an ACK # AckSetResult - expects SET_COMMAND_RESPONSE with an ACK # QueuedMessageResult - expects an ACK or NACK for any PID other than @@ -134,8 +140,8 @@ class SuccessfulResult(BaseExpectedResult): """This checks that we received a valid response from the device. This doesn't check that the response was a certain type, but simply that the - message was formed correctly. Other classes inherit from this an perform more - specific checking. + message was formed correctly. Other classes inherit from this and perform + more specific checking. """ def __str__(self): return 'RDM_COMPLETED_OK' diff --git a/tools/rdm/ExpectedResultsTest.py b/tools/rdm/ExpectedResultsTest.py new file mode 100644 index 0000000000..c30544813b --- /dev/null +++ b/tools/rdm/ExpectedResultsTest.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# ExpectedResultsTest.py +# Copyright (C) 2021 Peter Newman + +import unittest +from ExpectedResults import (BroadcastResult, DUBResult, InvalidResponse, + TimeoutResult, SuccessfulResult, UnsupportedResult) +from ola.OlaClient import OlaClient +from collections import namedtuple + +"""Test cases for ExpectedResults classes.""" + +__author__ = 'nomis52@gmail.com (Simon Newton)' + + +class ExpectedResultsTest(unittest.TestCase): + MockResponse = namedtuple('MockResponse', ['response_code']) + MockBroadcastResponse = MockResponse(OlaClient.RDM_WAS_BROADCAST) + MockTimeoutResponse = MockResponse(OlaClient.RDM_TIMEOUT) + MockInvalidResponse = MockResponse(OlaClient.RDM_INVALID_RESPONSE) + MockUnsupportedResult = MockResponse( + OlaClient.RDM_PLUGIN_DISCOVERY_NOT_SUPPORTED) + MockDUBResult = MockResponse(OlaClient.RDM_DUB_RESPONSE) + MockSuccessfulResult = MockResponse(OlaClient.RDM_COMPLETED_OK) + + def testBroadcastResult(self): + br = BroadcastResult() + self.assertTrue(br.Matches(self.MockBroadcastResponse, {})) + self.assertFalse(br.Matches(self.MockTimeoutResponse, {})) + self.assertFalse(br.Matches(self.MockInvalidResponse, {})) + self.assertFalse(br.Matches(self.MockUnsupportedResult, {})) + self.assertFalse(br.Matches(self.MockDUBResult, {})) + self.assertFalse(br.Matches(self.MockSuccessfulResult, {})) + + def testTimeoutResult(self): + tr = TimeoutResult() + self.assertFalse(tr.Matches(self.MockBroadcastResponse, {})) + self.assertTrue(tr.Matches(self.MockTimeoutResponse, {})) + self.assertFalse(tr.Matches(self.MockInvalidResponse, {})) + self.assertFalse(tr.Matches(self.MockUnsupportedResult, {})) + self.assertFalse(tr.Matches(self.MockDUBResult, {})) + self.assertFalse(tr.Matches(self.MockSuccessfulResult, {})) + + def testInvalidResponse(self): + ir = InvalidResponse() + self.assertFalse(ir.Matches(self.MockBroadcastResponse, {})) + self.assertFalse(ir.Matches(self.MockTimeoutResponse, {})) + self.assertTrue(ir.Matches(self.MockInvalidResponse, {})) + self.assertFalse(ir.Matches(self.MockUnsupportedResult, {})) + self.assertFalse(ir.Matches(self.MockDUBResult, {})) + self.assertFalse(ir.Matches(self.MockSuccessfulResult, {})) + + def testUnsupportedResult(self): + ur = UnsupportedResult() + self.assertFalse(ur.Matches(self.MockBroadcastResponse, {})) + self.assertFalse(ur.Matches(self.MockTimeoutResponse, {})) + self.assertFalse(ur.Matches(self.MockInvalidResponse, {})) + self.assertTrue(ur.Matches(self.MockUnsupportedResult, {})) + self.assertFalse(ur.Matches(self.MockDUBResult, {})) + self.assertFalse(ur.Matches(self.MockSuccessfulResult, {})) + + def testDUBResult(self): + dr = DUBResult() + self.assertFalse(dr.Matches(self.MockBroadcastResponse, {})) + self.assertFalse(dr.Matches(self.MockTimeoutResponse, {})) + self.assertFalse(dr.Matches(self.MockInvalidResponse, {})) + self.assertFalse(dr.Matches(self.MockUnsupportedResult, {})) + self.assertTrue(dr.Matches(self.MockDUBResult, {})) + self.assertFalse(dr.Matches(self.MockSuccessfulResult, {})) + + def testSuccessfulResult(self): + sr = SuccessfulResult() + self.assertFalse(sr.Matches(self.MockBroadcastResponse, {})) + self.assertFalse(sr.Matches(self.MockTimeoutResponse, {})) + self.assertFalse(sr.Matches(self.MockInvalidResponse, {})) + self.assertFalse(sr.Matches(self.MockUnsupportedResult, {})) + self.assertFalse(sr.Matches(self.MockDUBResult, {})) + self.assertTrue(sr.Matches(self.MockSuccessfulResult, {})) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/rdm/Makefile.mk b/tools/rdm/Makefile.mk index 4306509f2a..bb2f90f2ed 100644 --- a/tools/rdm/Makefile.mk +++ b/tools/rdm/Makefile.mk @@ -55,6 +55,11 @@ launcher_files = \ EXTRA_DIST += $(launcher_files) +tools/rdm/ExpectedResultsTest.sh: tools/rdm/Makefile.mk + mkdir -p $(top_builddir)/python/ola + echo "PYTHONPATH=${top_builddir}/python $(PYTHON) ${srcdir}/tools/rdm/ExpectedResultsTest.py; exit \$$?" > $(top_builddir)/tools/rdm/ExpectedResultsTest.sh + chmod +x $(top_builddir)/tools/rdm/ExpectedResultsTest.sh + tools/rdm/ResponderTestTest.sh: tools/rdm/Makefile.mk mkdir -p $(top_builddir)/python/ola echo "PYTHONPATH=${top_builddir}/python $(PYTHON) ${srcdir}/tools/rdm/ResponderTestTest.py; exit \$$?" > $(top_builddir)/tools/rdm/ResponderTestTest.sh @@ -78,6 +83,7 @@ tools/rdm/TestStateTest.sh: tools/rdm/Makefile.mk chmod +x $(top_builddir)/tools/rdm/TestStateTest.sh dist_check_SCRIPTS += \ + tools/rdm/ExpectedResultsTest.py \ tools/rdm/ResponderTestTest.py \ tools/rdm/TestHelpersTest.py \ tools/rdm/TestRunnerTest.py \ @@ -85,6 +91,7 @@ dist_check_SCRIPTS += \ if BUILD_PYTHON_LIBS test_scripts += \ + tools/rdm/ExpectedResultsTest.sh \ tools/rdm/ResponderTestTest.sh \ tools/rdm/TestHelpersTest.sh \ tools/rdm/TestRunnerTest.sh \ @@ -94,6 +101,7 @@ endif CLEANFILES += \ python/ola/testing/__init__.py \ tools/rdm/*.pyc \ + tools/rdm/ExpectedResultsTest.sh \ tools/rdm/ResponderTestTest.sh \ tools/rdm/TestHelpersTest.sh \ tools/rdm/TestRunnerTest.sh \ diff --git a/tools/rdm/ResponderTestTest.py b/tools/rdm/ResponderTestTest.py index c0d7f16039..3881667904 100644 --- a/tools/rdm/ResponderTestTest.py +++ b/tools/rdm/ResponderTestTest.py @@ -33,7 +33,6 @@ class ZTestFixture(TestFixture): class TestFixtureTest(unittest.TestCase): - def testCmp(self): base = TestFixture({}, 2, 123, None) base2 = TestFixture({}, 3, 456, None) From 97c4b21a7083573e35f88906c83ebc2f5308bf27 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Fri, 18 Nov 2022 14:07:12 +0000 Subject: [PATCH 10/67] Make more build steps verbose so we hopefully see test failures when we need to --- .travis-ci.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.travis-ci.sh b/.travis-ci.sh index eda7b31dce..fc9bd59e79 100755 --- a/.travis-ci.sh +++ b/.travis-ci.sh @@ -53,7 +53,7 @@ if [[ $TASK = 'lint' ]]; then # the following is a bit of a hack to build the files normally built during # the build, so they are present for linting to run against travis_fold start "make_builtfiles" - make builtfiles; + make builtfiles VERBOSE=1; travis_fold end "make_builtfiles" # first check we've not got any generic NOLINTs # count the number of generic NOLINTs @@ -67,7 +67,7 @@ if [[ $TASK = 'lint' ]]; then echo "Found $nolints generic NOLINTs" fi; # run the cpplint tool, fetching it if necessary - make cpplint + make cpplint VERBOSE=1 elif [[ $TASK = 'check-licences' ]]; then # check licences only if it is the requested task travis_fold start "autoreconf" @@ -79,7 +79,7 @@ elif [[ $TASK = 'check-licences' ]]; then # the following is a bit of a hack to build the files normally built during # the build, so they are present for licence checking to run against travis_fold start "make_builtfiles" - make builtfiles; + make builtfiles VERBOSE=1; travis_fold end "make_builtfiles" ./scripts/enforce_licence.py if [[ $? -ne 0 ]]; then @@ -96,7 +96,7 @@ elif [[ $TASK = 'spellintian' ]]; then # the following is a bit of a hack to build the files normally built during # the build, so they are present for spellintian to run against travis_fold start "make_builtfiles" - make builtfiles; + make builtfiles VERBOSE=1; travis_fold end "make_builtfiles" spellingfiles=$(eval "find ./ -type f -and ! \( \ $SPELLINGBLACKLIST \ @@ -122,7 +122,7 @@ elif [[ $TASK = 'spellintian-duplicates' ]]; then # the following is a bit of a hack to build the files normally built during # the build, so they are present for spellintian to run against travis_fold start "make_builtfiles" - make builtfiles; + make builtfiles VERBOSE=1; travis_fold end "make_builtfiles" spellingfiles=$(eval "find ./ -type f -and ! \( \ $SPELLINGBLACKLIST \ @@ -148,7 +148,7 @@ elif [[ $TASK = 'codespell' ]]; then # the following is a bit of a hack to build the files normally built during # the build, so they are present for codespell to run against travis_fold start "make_builtfiles" - make builtfiles; + make builtfiles VERBOSE=1; travis_fold end "make_builtfiles" spellingfiles=$(eval "find ./ -type f -and ! \( \ $SPELLINGBLACKLIST \ @@ -175,13 +175,13 @@ elif [[ $TASK = 'doxygen' ]]; then # the following is a bit of a hack to build the files normally built during # the build, so they are present for Doxygen to run against travis_fold start "make_builtfiles" - make builtfiles; + make builtfiles VERBOSE=1; travis_fold end "make_builtfiles" # count the number of warnings warnings=$(make doxygen-doc 2>&1 >/dev/null | wc -l) if [[ $warnings -ne 0 ]]; then # print the output for info - make doxygen-doc + make doxygen-doc VERBOSE=1 echo "Found $warnings doxygen warnings" exit 1; else @@ -227,7 +227,7 @@ elif [[ $TASK = 'flake8' ]]; then # the following is a bit of a hack to build the files normally built during # the build, so they are present for flake8 to run against travis_fold start "make_builtfiles" - make builtfiles; + make builtfiles VERBOSE=1; travis_fold end "make_builtfiles" make flake8 else @@ -245,7 +245,7 @@ else ./configure $DISTCHECK_CONFIGURE_FLAGS; travis_fold end "configure" travis_fold start "make_distcheck" - make distcheck; + make distcheck VERBOSE=1; travis_fold end "make_distcheck" travis_fold start "make_dist" make dist; From 67a5a9d9512f853750aa815bdfb6b9408fc4ea79 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Fri, 18 Nov 2022 14:21:12 +0000 Subject: [PATCH 11/67] Short-circuit the main build while we're troubleshooting --- .travis-ci.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis-ci.sh b/.travis-ci.sh index fc9bd59e79..c7b9669e9a 100755 --- a/.travis-ci.sh +++ b/.travis-ci.sh @@ -244,6 +244,8 @@ else travis_fold start "configure" ./configure $DISTCHECK_CONFIGURE_FLAGS; travis_fold end "configure" + make tools/rdm/TestRunnerTest.sh + ./tools/rdm/TestRunnerTest.sh travis_fold start "make_distcheck" make distcheck VERBOSE=1; travis_fold end "make_distcheck" From c2ab8fcac3547a248c4f5029905ae33493b94748 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Fri, 18 Nov 2022 14:24:26 +0000 Subject: [PATCH 12/67] Build more files to fix the testing --- .travis-ci.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis-ci.sh b/.travis-ci.sh index c7b9669e9a..05ed6a07f7 100755 --- a/.travis-ci.sh +++ b/.travis-ci.sh @@ -244,6 +244,7 @@ else travis_fold start "configure" ./configure $DISTCHECK_CONFIGURE_FLAGS; travis_fold end "configure" + make builtfiles VERBOSE=1; make tools/rdm/TestRunnerTest.sh ./tools/rdm/TestRunnerTest.sh travis_fold start "make_distcheck" From c92bf06e07ce57531d0936f898745be32e346a5d Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Fri, 18 Nov 2022 16:09:24 +0000 Subject: [PATCH 13/67] Add more missing symlinks to try and fix the tests --- configure.ac | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configure.ac b/configure.ac index 48a9620c59..d5b762f165 100644 --- a/configure.ac +++ b/configure.ac @@ -973,6 +973,8 @@ AM_CONDITIONAL([FOUND_CPPLINT], [test "x$cpplint" = xyes]) # srcdir and set PYTHONPATH=${top_builddir}/python in data/rdm/Makefile.am AC_CONFIG_LINKS([python/ola/__init__.py:python/ola/__init__.py python/ola/ClientWrapper.py:python/ola/ClientWrapper.py + python/ola/DMXConstants.py:python/ola/DMXConstants.py + python/ola/DUBDecoder.py:python/ola/DUBDecoder.py python/ola/MACAddress.py:python/ola/MACAddress.py python/ola/OlaClient.py:python/ola/OlaClient.py python/ola/PidStore.py:python/ola/PidStore.py @@ -984,7 +986,10 @@ AC_CONFIG_LINKS([python/ola/__init__.py:python/ola/__init__.py python/ola/rpc/SimpleRpcController.py:python/ola/rpc/SimpleRpcController.py python/ola/rpc/StreamRpcChannel.py:python/ola/rpc/StreamRpcChannel.py tools/rdm/ExpectedResults.py:tools/rdm/ExpectedResults.py + tools/rdm/ResponderTest.py:tools/rdm/ResponderTest.py tools/rdm/TestCategory.py:tools/rdm/TestCategory.py + tools/rdm/TestDefinitions.py:tools/rdm/TestDefinitions.py + tools/rdm/TestRunner.py:tools/rdm/TestRunner.py tools/rdm/TestState.py:tools/rdm/TestState.py tools/rdm/TimingStats.py:tools/rdm/TimingStats.py]) From 6a56b549b4baa6edbba60ea5f3db040aa1d049f1 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Fri, 18 Nov 2022 16:10:26 +0000 Subject: [PATCH 14/67] Add another Python symlink for the tests --- configure.ac | 1 + 1 file changed, 1 insertion(+) diff --git a/configure.ac b/configure.ac index d5b762f165..980204648b 100644 --- a/configure.ac +++ b/configure.ac @@ -985,6 +985,7 @@ AC_CONFIG_LINKS([python/ola/__init__.py:python/ola/__init__.py python/ola/rpc/__init__.py:python/ola/rpc/__init__.py python/ola/rpc/SimpleRpcController.py:python/ola/rpc/SimpleRpcController.py python/ola/rpc/StreamRpcChannel.py:python/ola/rpc/StreamRpcChannel.py + tools/rdm/__init__.py:tools/rdm/__init__.py tools/rdm/ExpectedResults.py:tools/rdm/ExpectedResults.py tools/rdm/ResponderTest.py:tools/rdm/ResponderTest.py tools/rdm/TestCategory.py:tools/rdm/TestCategory.py From c90596a4f98676818fb1edd3cc634d781ea24094 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Fri, 18 Nov 2022 16:28:06 +0000 Subject: [PATCH 15/67] Ignore any distcheck folders --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 51cfc8e891..169ab16a4f 100644 --- a/.gitignore +++ b/.gitignore @@ -175,6 +175,7 @@ libs/acn/e131_loadtest libs/acn/e131_loadtest.exe libs/acn/e131_transmit_test libs/acn/e131_transmit_test.exe +ola-*/ ola.spec olad/olad olad/olad.exe From 5af94cdb4796ef51b9e5b5a9ec7fec22d777a4a4 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Fri, 18 Nov 2022 16:36:30 +0000 Subject: [PATCH 16/67] Also clean compiled pyc files from testing folder --- tools/rdm/Makefile.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/rdm/Makefile.mk b/tools/rdm/Makefile.mk index bb2f90f2ed..256795f486 100644 --- a/tools/rdm/Makefile.mk +++ b/tools/rdm/Makefile.mk @@ -99,6 +99,7 @@ test_scripts += \ endif CLEANFILES += \ + python/ola/testing/*.pyc \ python/ola/testing/__init__.py \ tools/rdm/*.pyc \ tools/rdm/ExpectedResultsTest.sh \ From 322d41e15452602cf0d8c996e9c32a5f8d389caa Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Fri, 18 Nov 2022 17:12:37 +0000 Subject: [PATCH 17/67] Add a comment about one of the test hacks --- tools/rdm/Makefile.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/rdm/Makefile.mk b/tools/rdm/Makefile.mk index 256795f486..b72addd982 100644 --- a/tools/rdm/Makefile.mk +++ b/tools/rdm/Makefile.mk @@ -73,6 +73,7 @@ tools/rdm/TestHelpersTest.sh: tools/rdm/Makefile.mk tools/rdm/TestRunnerTest.sh: tools/rdm/Makefile.mk mkdir -p $(top_builddir)/python/ola/testing touch $(top_builddir)/python/ola/testing/__init__.py + # This link is relative within the builddir $(LN_S) -f ../../../tools/rdm $(top_builddir)/python/ola/testing/rdm echo "PYTHONPATH=${top_builddir}/python $(PYTHON) ${srcdir}/tools/rdm/TestRunnerTest.py; exit \$$?" > $(top_builddir)/tools/rdm/TestRunnerTest.sh chmod +x $(top_builddir)/tools/rdm/TestRunnerTest.sh From 621c75db4de973bf22ef6a8b255160ad47b86bc3 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Fri, 18 Nov 2022 17:13:04 +0000 Subject: [PATCH 18/67] More NEWS --- NEWS | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index d1e4868414..40c66de27c 100644 --- a/NEWS +++ b/NEWS @@ -1,14 +1,17 @@ -x/y/2020 ola-0.10.9 +x/y/2022 ola-0.10.9 Features: - * + * API: - * + * Python: Add a fetch current DMX example. RDM Tests: * Bugs: + * Fix some tests not working on Big Endian machines because our RPC is encoded + with native format and those tests used sample data from a Little Endian + machine * Renamed EndpointNoticationEvent(sic) to EndpointNotificationEvent in the E1.33 EndpointManager code From 81b0e45bfbefa477722e7fc759649e50cf131f1d Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Fri, 18 Nov 2022 17:13:49 +0000 Subject: [PATCH 19/67] Revert "Short-circuit the main build while we're troubleshooting" This reverts commit 67a5a9d9512f853750aa815bdfb6b9408fc4ea79. --- .travis-ci.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis-ci.sh b/.travis-ci.sh index 05ed6a07f7..fc9bd59e79 100755 --- a/.travis-ci.sh +++ b/.travis-ci.sh @@ -244,9 +244,6 @@ else travis_fold start "configure" ./configure $DISTCHECK_CONFIGURE_FLAGS; travis_fold end "configure" - make builtfiles VERBOSE=1; - make tools/rdm/TestRunnerTest.sh - ./tools/rdm/TestRunnerTest.sh travis_fold start "make_distcheck" make distcheck VERBOSE=1; travis_fold end "make_distcheck" From 1bf606ac0b95f14fa790dec6b9c5f79abcdbaba5 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Tue, 22 Nov 2022 05:35:42 +0000 Subject: [PATCH 20/67] Fix the RDM test suite discovery code to work with Python 3 --- tools/rdm/TestRunner.py | 13 +++++++++---- tools/rdm/TestRunnerTest.py | 15 ++++++++++++--- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/tools/rdm/TestRunner.py b/tools/rdm/TestRunner.py index a611c9a984..d07e31fc74 100644 --- a/tools/rdm/TestRunner.py +++ b/tools/rdm/TestRunner.py @@ -194,15 +194,20 @@ def GetTestClasses(module): cls = getattr(module, symbol) if not inspect.isclass(cls): continue + # Test for dynamic versions of these due to issues with Python 3 base_classes = [ - ResponderTest.OptionalParameterTestFixture, - ResponderTest.ResponderTestFixture, - ResponderTest.TestFixture + # Original + getattr(module, "OptionalParameterTestFixture"), + getattr(module, "ResponderTestFixture"), + getattr(module, "TestFixture") ] if cls in base_classes: continue - if issubclass(cls, ResponderTest.TestFixture): + # This seems to confuse Python 3 if we compare it to + # ResponderTest.TestFixture, some sort of diamond inheritance issue? + # So test for the dynamic version of it instead + if issubclass(cls, getattr(module, "TestFixture")): classes.append(cls) return classes diff --git a/tools/rdm/TestRunnerTest.py b/tools/rdm/TestRunnerTest.py index aab3a907df..7b6de3de63 100644 --- a/tools/rdm/TestRunnerTest.py +++ b/tools/rdm/TestRunnerTest.py @@ -18,8 +18,9 @@ import unittest from ola.testing.rdm import TestDefinitions, TestRunner -from ola.testing.rdm.ResponderTest import OptionalParameterTestFixture -from ola.testing.rdm.ResponderTest import ResponderTestFixture, TestFixture +from ola.testing.rdm import ResponderTest +from ola.testing.rdm.ResponderTest import OptionalParameterTestFixture, \ + ResponderTestFixture, TestFixture from ola.testing.rdm.TestDefinitions import GetDeviceInfo """Test cases for TestRunner utilities.""" @@ -38,9 +39,17 @@ def testGetTestClasses(self): TestRunner.GetTestClasses(TestDefinitions), "GetDeviceInfo missing from list of test classes") # Check we don't contain the base classes: + # Test for various versions of them due to issues with Python 3 + # The static versions are probably a better test for the test run for classname in [OptionalParameterTestFixture, ResponderTestFixture, - TestFixture]: + TestFixture, + ResponderTest.OptionalParameterTestFixture, + ResponderTest.ResponderTestFixture, + ResponderTest.TestFixture, + TestDefinitions.OptionalParameterTestFixture, + TestDefinitions.ResponderTestFixture, + TestDefinitions.TestFixture]: self.assertTrue(classname not in TestRunner.GetTestClasses(TestDefinitions), "Class %s found in list of test classes" % classname) From 1cf9e36663eb7be6390a4a6ad4df0aa206c0678d Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Tue, 22 Nov 2022 06:26:40 +0000 Subject: [PATCH 21/67] Try and fix more encoding issues for further RDM tests --- tools/rdm/ResponderTest.py | 6 ++-- tools/rdm/TestDefinitions.py | 53 ++++++++++++++++++------------------ tools/rdm/TestMixins.py | 8 +++--- 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/tools/rdm/ResponderTest.py b/tools/rdm/ResponderTest.py index 2cad724c95..9b54866475 100644 --- a/tools/rdm/ResponderTest.py +++ b/tools/rdm/ResponderTest.py @@ -374,7 +374,7 @@ def SendDirectedDiscovery(self, uid, sub_device, pid, args=[]): args, include_frames=True) - def SendRawDiscovery(self, sub_device, pid, data=""): + def SendRawDiscovery(self, sub_device, pid, data=b''): """Send a raw Discovery request. Args: @@ -424,7 +424,7 @@ def SendDirectedGet(self, uid, sub_device, pid, args=[]): include_frames=True) return ret_code - def SendRawGet(self, sub_device, pid, data=""): + def SendRawGet(self, sub_device, pid, data=b''): """Send a raw GET request. Args: @@ -476,7 +476,7 @@ def SendDirectedSet(self, uid, sub_device, pid, args=[]): self.SleepAfterBroadcastSet() return ret_code - def SendRawSet(self, sub_device, pid, data=""): + def SendRawSet(self, sub_device, pid, data=b''): """Send a raw SET request. Args: diff --git a/tools/rdm/TestDefinitions.py b/tools/rdm/TestDefinitions.py index 5e97ed16d9..84e594a899 100644 --- a/tools/rdm/TestDefinitions.py +++ b/tools/rdm/TestDefinitions.py @@ -94,7 +94,7 @@ def Test(self): TimeoutResult(), UnsupportedResult() ]) - self.SendRawDiscovery(ROOT_DEVICE, self.pid, 'x') + self.SendRawDiscovery(ROOT_DEVICE, self.pid, b'x') class UnMuteDevice(ResponderTestFixture): @@ -133,7 +133,7 @@ def Test(self): TimeoutResult(), UnsupportedResult() ]) - self.SendRawDiscovery(ROOT_DEVICE, self.pid, 'x') + self.SendRawDiscovery(ROOT_DEVICE, self.pid, b'x') class RequestsWhileUnmuted(ResponderTestFixture): @@ -583,7 +583,7 @@ def Test(self): field_values=self.FIELD_VALUES, warning='Get %s with data returned an ack' % self.pid.name) ]) - self.SendRawGet(ROOT_DEVICE, self.pid, 'x') + self.SendRawGet(ROOT_DEVICE, self.pid, b'x') def VerifyResult(self, response, fields): self.SetProperty('supports_over_sized_pdl', True) @@ -608,7 +608,7 @@ def Test(self): advisory='Responder timed out to a command with PDL of %d' % self.MAX_PDL), ]) - self.SendRawGet(ROOT_DEVICE, self.pid, 'x' * self.MAX_PDL) + self.SendRawGet(ROOT_DEVICE, self.pid, b'x' * self.MAX_PDL) def VerifyResult(self, response, fields): ok = response.response_code not in [OlaClient.RDM_INVALID_RESPONSE, @@ -645,7 +645,7 @@ def SendGet(self): InvalidResponse(action=self.GetFailed), TimeoutResult(action=self.GetFailed), ]) - self.SendRawGet(ROOT_DEVICE, self.pid, 'x' * self._current) + self.SendRawGet(ROOT_DEVICE, self.pid, b'x' * self._current) def GetPassed(self): self._lower = self._current @@ -855,7 +855,7 @@ def Test(self): ]) else: self.AddExpectedResults(self.NackGetResult(RDMNack.NR_UNKNOWN_PID)) - self.SendRawGet(ROOT_DEVICE, self.pid, 'foo') + self.SendRawGet(ROOT_DEVICE, self.pid, b'foo') class SetSupportedParameters(ResponderTestFixture): @@ -1104,7 +1104,7 @@ def Test(self): if self.Property('manufacturer_parameters'): results = self.NackGetResult(RDMNack.NR_FORMAT_ERROR) self.AddExpectedResults(results) - self.SendRawGet(ROOT_DEVICE, self.pid, 'foo') + self.SendRawGet(ROOT_DEVICE, self.pid, b'foo') class SetParamDescription(TestMixins.UnsupportedSetMixin, @@ -1317,7 +1317,7 @@ class SetDeviceModelDescriptionWithData(TestMixins.UnsupportedSetMixin, """SET the device model description with data.""" CATEGORY = TestCategory.ERROR_CONDITIONS PID = 'DEVICE_MODEL_DESCRIPTION' - DATA = 'FOO BAR' + DATA = b'FOO BAR' class AllSubDevicesGetModelDescription(TestMixins.AllSubDevicesGetMixin, @@ -1357,7 +1357,7 @@ class SetManufacturerLabelWithData(TestMixins.UnsupportedSetMixin, """SET the manufacturer label with data.""" CATEGORY = TestCategory.ERROR_CONDITIONS PID = 'MANUFACTURER_LABEL' - DATA = 'FOO BAR' + DATA = b'FOO BAR' class AllSubDevicesGetManufacturerLabel(TestMixins.AllSubDevicesGetMixin, @@ -1997,7 +1997,7 @@ class SetOversizedPersonality(OptionalParameterTestFixture): def Test(self): self.AddIfSetSupported(self.NackSetResult(RDMNack.NR_FORMAT_ERROR)) - self.SendRawSet(ROOT_DEVICE, self.pid, 'foo') + self.SendRawSet(ROOT_DEVICE, self.pid, b'foo') class AllSubDevicesGetPersonality(TestMixins.AllSubDevicesGetMixin, @@ -2066,7 +2066,7 @@ def Test(self): warning='Get %s with data returned an ack' % self.pid.name), ] self.AddExpectedResults(results) - self.SendRawGet(PidStore.ROOT_DEVICE, self.pid, 'foo') + self.SendRawGet(PidStore.ROOT_DEVICE, self.pid, b'foo') class SetStartAddress(TestMixins.SetStartAddressMixin, ResponderTestFixture): @@ -2181,7 +2181,7 @@ def Test(self): self.NackSetResult(RDMNack.NR_UNSUPPORTED_COMMAND_CLASS), self.NackSetResult(RDMNack.NR_FORMAT_ERROR), ]) - self.SendRawSet(ROOT_DEVICE, self.pid, 'foo') + self.SendRawSet(ROOT_DEVICE, self.pid, b'foo') class AllSubDevicesGetStartAddress(TestMixins.AllSubDevicesGetMixin, @@ -2294,7 +2294,7 @@ class GetSlotDescriptionWithTooMuchData(OptionalParameterTestFixture): def Test(self): self.AddIfGetSupported(self.NackGetResult(RDMNack.NR_FORMAT_ERROR)) - self.SendRawGet(ROOT_DEVICE, self.pid, 'foo') + self.SendRawGet(ROOT_DEVICE, self.pid, b'foo') class GetUndefinedSlotDefinitionDescriptions(OptionalParameterTestFixture): @@ -2645,7 +2645,7 @@ class GetSensorDefinitionWithTooMuchData(OptionalParameterTestFixture): def Test(self): self.AddIfGetSupported(self.NackGetResult(RDMNack.NR_FORMAT_ERROR)) - self.SendRawGet(ROOT_DEVICE, self.pid, 'foo') + self.SendRawGet(ROOT_DEVICE, self.pid, b'foo') class GetInvalidSensorDefinition(OptionalParameterTestFixture): @@ -2922,7 +2922,7 @@ def Test(self): self.NackSetResult(RDMNack.NR_FORMAT_ERROR), self.NackSetResult(RDMNack.NR_UNSUPPORTED_COMMAND_CLASS), ]) - self.SendRawSet(ROOT_DEVICE, self.pid, '') + self.SendRawSet(ROOT_DEVICE, self.pid, b'') class AllSubDevicesGetSensorValue(TestMixins.AllSubDevicesGetMixin, @@ -3069,7 +3069,7 @@ def Test(self): else: expected_result = RDMNack.NR_UNSUPPORTED_COMMAND_CLASS self.AddIfSetSupported(self.NackSetResult(expected_result)) - self.SendRawSet(ROOT_DEVICE, self.pid, '') + self.SendRawSet(ROOT_DEVICE, self.pid, b'') class AllSubDevicesGetDeviceHours(TestMixins.AllSubDevicesGetMixin, @@ -3128,7 +3128,7 @@ def Test(self): else: expected_result = RDMNack.NR_UNSUPPORTED_COMMAND_CLASS self.AddIfSetSupported(self.NackSetResult(expected_result)) - self.SendRawSet(ROOT_DEVICE, self.pid, '') + self.SendRawSet(ROOT_DEVICE, self.pid, b'') class AllSubDevicesGetLampHours(TestMixins.AllSubDevicesGetMixin, @@ -3186,7 +3186,7 @@ def Test(self): else: expected_result = RDMNack.NR_UNSUPPORTED_COMMAND_CLASS self.AddIfSetSupported(self.NackSetResult(expected_result)) - self.SendRawSet(ROOT_DEVICE, self.pid, '') + self.SendRawSet(ROOT_DEVICE, self.pid, b'') class AllSubDevicesGetLampStrikes(TestMixins.AllSubDevicesGetMixin, @@ -3369,7 +3369,7 @@ def Test(self): else: expected_result = RDMNack.NR_UNSUPPORTED_COMMAND_CLASS self.AddIfSetSupported(self.NackSetResult(expected_result)) - self.SendRawSet(ROOT_DEVICE, self.pid, '') + self.SendRawSet(ROOT_DEVICE, self.pid, b'') class AllSubDevicesGetDevicePowerCycles(TestMixins.AllSubDevicesGetMixin, @@ -3486,6 +3486,7 @@ class AllSubDevicesGetDisplayLevel(TestMixins.AllSubDevicesGetMixin, class GetPanInvert(TestMixins.GetMixin, OptionalParameterTestFixture): """GET the pan invert setting.""" CATEGORY = TestCategory.CONFIGURATION + PID = 'PAN_INVERT' EXPECTED_FIELDS = ['invert'] PROVIDES = ['pan_invert'] @@ -3624,7 +3625,7 @@ class GetRealTimeClock(OptionalParameterTestFixture): def Test(self): self.AddIfGetSupported( - self.AckGetResult(field_names=self.ALLOWED_RANGES.keys() + ['second'])) + self.AckGetResult(field_names=list(self.ALLOWED_RANGES.keys()) + ['second'])) self.SendGet(ROOT_DEVICE, self.pid) def VerifyResult(self, response, fields): @@ -3671,7 +3672,7 @@ def Test(self): self.NackSetResult(RDMNack.NR_UNSUPPORTED_COMMAND_CLASS), self.NackSetResult(RDMNack.NR_FORMAT_ERROR), ]) - self.SendRawSet(ROOT_DEVICE, self.pid, '') + self.SendRawSet(ROOT_DEVICE, self.pid, b'') class AllSubDevicesGetRealTimeClock(TestMixins.AllSubDevicesGetMixin, @@ -3795,7 +3796,7 @@ class SetIdentifyDeviceWithNoData(ResponderTestFixture): def Test(self): self.AddExpectedResults(self.NackSetResult(RDMNack.NR_FORMAT_ERROR)) - self.SendRawSet(ROOT_DEVICE, self.pid, '') + self.SendRawSet(ROOT_DEVICE, self.pid, b'') def ResetState(self): self.SendSet(ROOT_DEVICE, self.pid, [self.Property('identify_state')]) @@ -5116,7 +5117,7 @@ class GetLockPinWithData(TestMixins.GetWithDataMixin, """Get LOCK_PIN with data.""" CATEGORY = TestCategory.ERROR_CONDITIONS PID = 'LOCK_PIN' - DATA = 'foo' + DATA = b'foo' # Some responders may not let you GET the pin. ALLOWED_NACKS = [RDMNack.NR_UNSUPPORTED_COMMAND_CLASS] @@ -6817,7 +6818,7 @@ class GetIPv4DefaultRouteWithData(TestMixins.GetWithDataMixin, # """SET the IPv4 default route with data.""" # CATEGORY = TestCategory.ERROR_CONDITIONS # PID = 'IPV4_DEFAULT_ROUTE' -# DATA = 'FOOBAR' +# DATA = b'FOOBAR' class AllSubDevicesGetIPv4DefaultRoute(TestMixins.AllSubDevicesGetMixin, @@ -6853,7 +6854,7 @@ class GetInterfaceLabelWithTooMuchData(OptionalParameterTestFixture): def Test(self): self.AddIfGetSupported(self.NackGetResult(RDMNack.NR_FORMAT_ERROR)) - self.SendRawGet(ROOT_DEVICE, self.pid, 'foobar') + self.SendRawGet(ROOT_DEVICE, self.pid, b'foobar') class GetZeroInterfaceLabel(TestMixins.GetZeroUInt32Mixin, @@ -6874,7 +6875,7 @@ class SetInterfaceLabelWithData(TestMixins.UnsupportedSetMixin, """SET the interface label with data.""" CATEGORY = TestCategory.ERROR_CONDITIONS PID = 'INTERFACE_LABEL' - DATA = 'FOO BAR' + DATA = b'FOO BAR' # Cross check the control fields with various other properties diff --git a/tools/rdm/TestMixins.py b/tools/rdm/TestMixins.py index c8af6607c1..4d29606222 100644 --- a/tools/rdm/TestMixins.py +++ b/tools/rdm/TestMixins.py @@ -185,7 +185,7 @@ class GetWithDataMixin(object): If ALLOWED_NACKS is non-empty, this adds a custom NackGetResult to the list of allowed results for each entry. """ - DATA = 'foo' + DATA = b'foo' ALLOWED_NACKS = [] def Test(self): @@ -202,7 +202,7 @@ def Test(self): class GetMandatoryPIDWithDataMixin(object): """GET a mandatory PID with junk param data.""" - DATA = 'foo' + DATA = b'foo' def Test(self): # PID must return something as this PID is required (can't return @@ -248,7 +248,7 @@ def Test(self): class SetWithDataMixin(ResponderTestFixture): """SET a PID with random param data.""" - DATA = 'foo' + DATA = b'foo' def Test(self): self.AddIfSetSupported([ @@ -263,7 +263,7 @@ class SetWithNoDataMixin(object): """Attempt a set with no data.""" def Test(self): self.AddIfSetSupported(self.NackSetResult(RDMNack.NR_FORMAT_ERROR)) - self.SendRawSet(PidStore.ROOT_DEVICE, self.pid, '') + self.SendRawSet(PidStore.ROOT_DEVICE, self.pid, b'') # TODO(simon): add a method to check this didn't change the value From 8b6441b28a5ff55eaabf8b4dee315664ebf5aec0 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Tue, 22 Nov 2022 06:52:51 +0000 Subject: [PATCH 22/67] Clean more pycache files --- tools/rdm/Makefile.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/rdm/Makefile.mk b/tools/rdm/Makefile.mk index b72addd982..e0a330d4d4 100644 --- a/tools/rdm/Makefile.mk +++ b/tools/rdm/Makefile.mk @@ -101,6 +101,7 @@ endif CLEANFILES += \ python/ola/testing/*.pyc \ + python/ola/testing/__pycache__/* \ python/ola/testing/__init__.py \ tools/rdm/*.pyc \ tools/rdm/ExpectedResultsTest.sh \ From fa2350625011739fbc3b3153c93266f06da15373 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Tue, 22 Nov 2022 07:09:24 +0000 Subject: [PATCH 23/67] Fix a few more RDM bugs in both the PidStore and a TestMixin --- python/ola/PidStore.py | 2 +- tools/rdm/TestMixins.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ola/PidStore.py b/python/ola/PidStore.py index 7812d60807..a62eb0f7b5 100644 --- a/python/ola/PidStore.py +++ b/python/ola/PidStore.py @@ -877,7 +877,7 @@ def Unpack(self, data): 'Too many repeated group_count for %s, limit is %d, found %d' % (self.name, self.max, group_count)) - if self.max is not None and group_count < self.min: + if self.min is not None and group_count < self.min: raise UnpackException( 'Too few repeated group_count for %s, limit is %d, found %d' % (self.name, self.min, group_count)) diff --git a/tools/rdm/TestMixins.py b/tools/rdm/TestMixins.py index 4d29606222..3ff395e94d 100644 --- a/tools/rdm/TestMixins.py +++ b/tools/rdm/TestMixins.py @@ -345,7 +345,7 @@ def Test(self): class SetOversizedLabelMixin(object): """Send an over-sized SET label command.""" - LONG_STRING = 'this is a string which is more than 32 characters' + LONG_STRING = b'this is a string which is more than 32 characters' def Test(self): self.verify_result = False From 2bdfa820f6cb21473583b830ef9fcf10a8ebd857 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Fri, 25 Nov 2022 14:03:02 +0000 Subject: [PATCH 24/67] Add a test for packing non-ascii characters, currently fails on Python 3 --- python/ola/PidStoreTest.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/ola/PidStoreTest.py b/python/ola/PidStoreTest.py index 9b6a832a0d..1b00250f57 100755 --- a/python/ola/PidStoreTest.py +++ b/python/ola/PidStoreTest.py @@ -278,6 +278,13 @@ def testPackRanges(self): args = ["enx"] blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] + # test packing some non-ascii characters + args = ["\x0e\xc0"] + blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] + self.assertEqual(blob, binascii.unhexlify("0ec0")) + decoded = pid.Unpack(blob, PidStore.RDM_GET) + self.assertEqual(decoded, {'languages': [{'language': '\\x0e\\xc0'}]}) + # valid empty string pid = store.GetName("STATUS_ID_DESCRIPTION") args = [""] From ec0fe8a2c2655ea0eb52ba2de8e0593fcd1b2bdc Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Fri, 25 Nov 2022 14:07:33 +0000 Subject: [PATCH 25/67] Fix the new non-ASCII PidStore test on Python 2 --- python/ola/PidStoreTest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ola/PidStoreTest.py b/python/ola/PidStoreTest.py index 1b00250f57..dd15668749 100755 --- a/python/ola/PidStoreTest.py +++ b/python/ola/PidStoreTest.py @@ -283,7 +283,7 @@ def testPackRanges(self): blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] self.assertEqual(blob, binascii.unhexlify("0ec0")) decoded = pid.Unpack(blob, PidStore.RDM_GET) - self.assertEqual(decoded, {'languages': [{'language': '\\x0e\\xc0'}]}) + self.assertEqual(decoded, {'languages': [{'language': '\x0e\xc0'}]}) # valid empty string pid = store.GetName("STATUS_ID_DESCRIPTION") From 17818945cb0e3383d00f7f71019ece7a9b145d64 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Fri, 25 Nov 2022 14:20:34 +0000 Subject: [PATCH 26/67] Switch back to the same non-ascii as the responder test --- python/ola/PidStoreTest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ola/PidStoreTest.py b/python/ola/PidStoreTest.py index dd15668749..1ad8ac40c0 100755 --- a/python/ola/PidStoreTest.py +++ b/python/ola/PidStoreTest.py @@ -279,11 +279,11 @@ def testPackRanges(self): blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] # test packing some non-ascii characters - args = ["\x0e\xc0"] + args = ["\x0d\xc0"] blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] self.assertEqual(blob, binascii.unhexlify("0ec0")) decoded = pid.Unpack(blob, PidStore.RDM_GET) - self.assertEqual(decoded, {'languages': [{'language': '\x0e\xc0'}]}) + self.assertEqual(decoded, {'languages': [{'language': '\x0d\xc0'}]}) # valid empty string pid = store.GetName("STATUS_ID_DESCRIPTION") From d0e772b86ca2656beba86eb07c7a9492dcfcfabb Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Fri, 25 Nov 2022 15:03:48 +0000 Subject: [PATCH 27/67] Fix even more tests on Python 3 --- tools/rdm/TestDefinitions.py | 15 ++++++++------- tools/rdm/TestMixins.py | 4 ++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tools/rdm/TestDefinitions.py b/tools/rdm/TestDefinitions.py index 84e594a899..5527f03128 100644 --- a/tools/rdm/TestDefinitions.py +++ b/tools/rdm/TestDefinitions.py @@ -889,7 +889,7 @@ class GetSubDeviceSupportedParameters(ResponderTestFixture): 'IDENTIFY_DEVICE'] def Test(self): - self._sub_devices = self.Property('sub_device_addresses').keys() + self._sub_devices = list(self.Property('sub_device_addresses').keys()) self._sub_devices.reverse() self._params = {} self._GetSupportedParams() @@ -1664,7 +1664,7 @@ class GetSubDeviceSoftwareVersionLabel(ResponderTestFixture): REQUIRES = ['sub_device_addresses'] def Test(self): - self._sub_devices = self.Property('sub_device_addresses').keys() + self._sub_devices = list(self.Property('sub_device_addresses').keys()) self._sub_devices.reverse() self._GetSoftwareVersion() @@ -2688,7 +2688,7 @@ class GetSensorValues(OptionalParameterTestFixture): def Test(self): # The head of the list is the current sensor we're querying - self._sensors = self.Property('sensor_definitions').values() + self._sensors = list(self.Property('sensor_definitions').values()) self._sensor_values = [] if self._sensors: @@ -2826,7 +2826,7 @@ class ResetSensorValue(OptionalParameterTestFixture): def Test(self): # The head of the list is the current sensor we're querying - self._sensors = self.Property('sensor_definitions').values() + self._sensors = list(self.Property('sensor_definitions').values()) self._sensor_values = [] if self._sensors: @@ -2952,7 +2952,7 @@ class RecordSensorValues(OptionalParameterTestFixture): def Test(self): # The head of the list is the current sensor we're querying - self._sensors = self.Property('sensor_definitions').values() + self._sensors = list(self.Property('sensor_definitions').values()) self._sensor_values = [] if self._sensors: @@ -3625,7 +3625,8 @@ class GetRealTimeClock(OptionalParameterTestFixture): def Test(self): self.AddIfGetSupported( - self.AckGetResult(field_names=list(self.ALLOWED_RANGES.keys()) + ['second'])) + self.AckGetResult(field_names=list(self.ALLOWED_RANGES.keys()) + + ['second'])) self.SendGet(ROOT_DEVICE, self.pid) def VerifyResult(self, response, fields): @@ -3817,7 +3818,7 @@ class GetSubDeviceIdentifyDevice(ResponderTestFixture): REQUIRES = ['sub_device_addresses'] def Test(self): - self._sub_devices = self.Property('sub_device_addresses').keys() + self._sub_devices = list(self.Property('sub_device_addresses').keys()) self._sub_devices.reverse() self._GetIdentifyDevice() diff --git a/tools/rdm/TestMixins.py b/tools/rdm/TestMixins.py index 3ff395e94d..c7c45150f8 100644 --- a/tools/rdm/TestMixins.py +++ b/tools/rdm/TestMixins.py @@ -1042,8 +1042,8 @@ def ListOfSettings(self): if self.NumberOfSettings() is None: return [] else: - return range(self.FIRST_INDEX_OFFSET, - self.NumberOfSettings() + self.FIRST_INDEX_OFFSET) + return list(range(self.FIRST_INDEX_OFFSET, + self.NumberOfSettings() + self.FIRST_INDEX_OFFSET)) class GetSettingDescriptionsListMixin(GetSettingDescriptionsMixin): From f0c0b8d664893292ec2d84cfc9596d5a1baa7224 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sat, 26 Nov 2022 02:02:36 +0000 Subject: [PATCH 28/67] Add a StringEscape function wrapper --- python/ola/Makefile.mk | 2 ++ python/ola/StringUtils.py | 41 +++++++++++++++++++++++++++++++++++ python/ola/StringUtilsTest.py | 40 ++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 python/ola/StringUtils.py create mode 100755 python/ola/StringUtilsTest.py diff --git a/python/ola/Makefile.mk b/python/ola/Makefile.mk index e282ef187b..19d7e395da 100644 --- a/python/ola/Makefile.mk +++ b/python/ola/Makefile.mk @@ -85,6 +85,7 @@ dist_check_SCRIPTS += \ python/ola/OlaClientTest.py \ python/ola/PidStoreTest.py \ python/ola/RDMTest.py \ + python/ola/StringUtilsTest.py \ python/ola/TestUtils.py \ python/ola/UIDTest.py @@ -96,6 +97,7 @@ test_scripts += \ python/ola/OlaClientTest.sh \ python/ola/PidStoreTest.sh \ python/ola/RDMTest.sh \ + python/ola/StringUtilsTest.py \ python/ola/UIDTest.py endif diff --git a/python/ola/StringUtils.py b/python/ola/StringUtils.py new file mode 100644 index 0000000000..f70add56c5 --- /dev/null +++ b/python/ola/StringUtils.py @@ -0,0 +1,41 @@ +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# StringUtils.py +# Copyright (C) 2013 Peter Newman + +"""Common utils for OLA Python string handling""" + +import sys + +if sys.version_info >= (3, 0): + try: + unicode + except NameError: + unicode = str + +__author__ = 'nomis52@gmail.com (Simon Newton)' + + +def StringEscape(s): + """Escape unprintable characters in a string.""" + # TODO(Peter): How does this interact with the E1.20 Unicode flag? + # We don't use sys.version_info.major to support Python 2.6. + if sys.version_info[0] == 2 and type(s) == str: + return s.encode('string-escape') + elif sys.version_info[0] == 2 and type(s) == unicode: + return s.encode('unicode-escape') + elif type(s) == str: + # All strings in Python 3 are unicode + return s.encode('unicode-escape') diff --git a/python/ola/StringUtilsTest.py b/python/ola/StringUtilsTest.py new file mode 100755 index 0000000000..8ef4d7239b --- /dev/null +++ b/python/ola/StringUtilsTest.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# StringUtilsTest.py +# Copyright (C) 2022 Peter Newman + +import unittest +from StringUtils import StringEscape + +"""Test cases for StringUtils utilities.""" + +__author__ = 'nomis52@gmail.com (Simon Newton)' + + +class StringUtilsTest(unittest.TestCase): + def testStringEscape(self): + self.assertEqual(b'foo', StringEscape("foo")) + self.assertEqual(b'bar', StringEscape("bar")) + self.assertEqual(b'bar[]', StringEscape("bar[]")) + self.assertEqual(b'foo-bar', StringEscape(u'foo-bar')) + self.assertEqual(b'foo\\x00bar', StringEscape("foo\x00bar")) + # TODO(Peter): How does this interact with the E1.20 Unicode flag? + self.assertEqual(b'caf\\xe9', StringEscape(u'caf\xe9')) + self.assertEqual(b'foo\\u2014bar', StringEscape(u'foo\u2014bar')) + + +if __name__ == '__main__': + unittest.main() From f340f99255a2251902e35a136e7ebe02b7c20ab8 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sat, 26 Nov 2022 02:09:05 +0000 Subject: [PATCH 29/67] Switch ContainsUnprintable to use our new StringEscape function --- python/ola/StringUtilsTest.py | 2 +- tools/rdm/TestHelpers.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/python/ola/StringUtilsTest.py b/python/ola/StringUtilsTest.py index 8ef4d7239b..cd6b1853f3 100755 --- a/python/ola/StringUtilsTest.py +++ b/python/ola/StringUtilsTest.py @@ -17,7 +17,7 @@ # Copyright (C) 2022 Peter Newman import unittest -from StringUtils import StringEscape +from ola.StringUtils import StringEscape """Test cases for StringUtils utilities.""" diff --git a/tools/rdm/TestHelpers.py b/tools/rdm/TestHelpers.py index daa12c2099..dab74eb2a0 100644 --- a/tools/rdm/TestHelpers.py +++ b/tools/rdm/TestHelpers.py @@ -16,6 +16,7 @@ # Copyright (C) 2013 Peter Newman import sys +from ola.StringUtils import StringEscape if sys.version_info >= (3, 0): try: @@ -30,12 +31,10 @@ def ContainsUnprintable(s): """Check if a string s contain unprintable characters.""" # TODO(Peter): How does this interact with the E1.20 Unicode flag? # We don't use sys.version_info.major to support Python 2.6. - if sys.version_info[0] == 2 and type(s) == str: - return s != s.encode('string-escape') - elif sys.version_info[0] == 2 and type(s) == unicode: - return s != s.encode('unicode-escape') + if sys.version_info[0] == 2 and (type(s) == str or type(s) == unicode): + return s != StringEscape(s) elif type(s) == str: # All strings in Python 3 are unicode - return s.encode() != s.encode('unicode-escape') + return s.encode() != StringEscape(s) else: return False From 3c7f6dec8a0631436dac1f2d799ef0b4bdec7fee Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sat, 26 Nov 2022 17:48:34 +0000 Subject: [PATCH 30/67] Catch and raise an exception if StringEscape is used on things other than strings --- python/ola/StringUtils.py | 2 ++ python/ola/StringUtilsTest.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/python/ola/StringUtils.py b/python/ola/StringUtils.py index f70add56c5..1c670abfb6 100644 --- a/python/ola/StringUtils.py +++ b/python/ola/StringUtils.py @@ -39,3 +39,5 @@ def StringEscape(s): elif type(s) == str: # All strings in Python 3 are unicode return s.encode('unicode-escape') + else: + raise TypeError('Only strings are supported') diff --git a/python/ola/StringUtilsTest.py b/python/ola/StringUtilsTest.py index cd6b1853f3..14c36bbee5 100755 --- a/python/ola/StringUtilsTest.py +++ b/python/ola/StringUtilsTest.py @@ -35,6 +35,9 @@ def testStringEscape(self): self.assertEqual(b'caf\\xe9', StringEscape(u'caf\xe9')) self.assertEqual(b'foo\\u2014bar', StringEscape(u'foo\u2014bar')) + with self.assertRaises(TypeError): + result = StringEscape(42) + if __name__ == '__main__': unittest.main() From 4f1da3f4c6800f0d1da41dfbcead305c650a21d2 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sat, 26 Nov 2022 17:53:39 +0000 Subject: [PATCH 31/67] Catch and raise an exception if ContainsUnprintable is used on things other than strings --- tools/rdm/TestHelpers.py | 2 +- tools/rdm/TestHelpersTest.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/rdm/TestHelpers.py b/tools/rdm/TestHelpers.py index dab74eb2a0..2c115a2fb3 100644 --- a/tools/rdm/TestHelpers.py +++ b/tools/rdm/TestHelpers.py @@ -37,4 +37,4 @@ def ContainsUnprintable(s): # All strings in Python 3 are unicode return s.encode() != StringEscape(s) else: - return False + raise TypeError('Only strings are supported') diff --git a/tools/rdm/TestHelpersTest.py b/tools/rdm/TestHelpersTest.py index 98434bd27e..5b1dc64c10 100644 --- a/tools/rdm/TestHelpersTest.py +++ b/tools/rdm/TestHelpersTest.py @@ -31,10 +31,14 @@ def testContainsUnprintable(self): self.assertFalse(ContainsUnprintable("bar[]")) self.assertFalse(ContainsUnprintable(u'foo-bar')) self.assertTrue(ContainsUnprintable("foo\x00bar")) - # TODO(Peter): How does this interact with the E1.20 Unicode flag? + # TODO(Peter): How do these interact with the E1.20 Unicode flag? self.assertTrue(ContainsUnprintable(u'caf\xe9')) + self.assertTrue(ContainsUnprintable(u'c\xc0f\xe9')) self.assertTrue(ContainsUnprintable(u'foo\u2014bar')) + with self.assertRaises(TypeError): + result = ContainsUnprintable(42) + if __name__ == '__main__': unittest.main() From c8c6ee51de07ba922bc0ba9cf135d8b32894b4b4 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sat, 26 Nov 2022 17:56:13 +0000 Subject: [PATCH 32/67] Fix string escaping on Python 3 too --- tools/rdm/rdm_responder_test.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tools/rdm/rdm_responder_test.py b/tools/rdm/rdm_responder_test.py index eccc5d99a4..c6f6e6a521 100755 --- a/tools/rdm/rdm_responder_test.py +++ b/tools/rdm/rdm_responder_test.py @@ -21,6 +21,7 @@ from ola.testing.rdm.DMXSender import DMXSender from ola.testing.rdm.TestState import TestState from ola.testing.rdm.TimingStats import TimingStats +from ola.StringUtils import StringEscape import datetime import logging import re @@ -200,17 +201,15 @@ def DisplaySummary(options, runner, tests, device): manufacturer_label = getattr(device, 'manufacturer_label', None) if manufacturer_label: - logging.info('Manufacturer: %s' % - manufacturer_label.encode('string-escape')) + logging.info('Manufacturer: %s' % StringEscape(manufacturer_label)) model_description = getattr(device, 'model_description', None) if model_description: - logging.info('Model Description: %s' % - model_description.encode('string-escape')) + logging.info('Model Description: %s' % StringEscape(model_description)) software_version = getattr(device, 'software_version', None) if software_version: - logging.info('Software Version: %s' % software_version) + logging.info('Software Version: %s' % StringEscape(software_version)) if options.timing: timing_stats = runner.TimingStats() From fcb9fa69261b8280b600d4e640cae9d4b057abfe Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sat, 26 Nov 2022 19:43:42 +0000 Subject: [PATCH 33/67] Fix StringEscape so it still returns a string in Python 3, also add some tests to check that --- python/ola/StringUtils.py | 4 +++- python/ola/StringUtilsTest.py | 26 +++++++++++++++++++------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/python/ola/StringUtils.py b/python/ola/StringUtils.py index 1c670abfb6..9b9b744361 100644 --- a/python/ola/StringUtils.py +++ b/python/ola/StringUtils.py @@ -38,6 +38,8 @@ def StringEscape(s): return s.encode('unicode-escape') elif type(s) == str: # All strings in Python 3 are unicode - return s.encode('unicode-escape') + # This encode/decode pair gets us an escaped string + return s.encode('unicode-escape').decode(encoding="ascii", + errors="backslashreplace") else: raise TypeError('Only strings are supported') diff --git a/python/ola/StringUtilsTest.py b/python/ola/StringUtilsTest.py index 14c36bbee5..6f7e5bef29 100755 --- a/python/ola/StringUtilsTest.py +++ b/python/ola/StringUtilsTest.py @@ -26,15 +26,27 @@ class StringUtilsTest(unittest.TestCase): def testStringEscape(self): - self.assertEqual(b'foo', StringEscape("foo")) - self.assertEqual(b'bar', StringEscape("bar")) - self.assertEqual(b'bar[]', StringEscape("bar[]")) - self.assertEqual(b'foo-bar', StringEscape(u'foo-bar')) - self.assertEqual(b'foo\\x00bar', StringEscape("foo\x00bar")) + # Test we escape properly + self.assertEqual('foo', StringEscape("foo")) + self.assertEqual('bar', StringEscape("bar")) + self.assertEqual('bar[]', StringEscape("bar[]")) + self.assertEqual('foo-bar', StringEscape(u'foo-bar')) + self.assertEqual('foo\\x00bar', StringEscape("foo\x00bar")) # TODO(Peter): How does this interact with the E1.20 Unicode flag? - self.assertEqual(b'caf\\xe9', StringEscape(u'caf\xe9')) - self.assertEqual(b'foo\\u2014bar', StringEscape(u'foo\u2014bar')) + self.assertEqual('caf\\xe9', StringEscape(u'caf\xe9')) + self.assertEqual('foo\\u2014bar', StringEscape(u'foo\u2014bar')) + + # Test that we display nicely in a string + self.assertEqual('foo', ("%s" % StringEscape("foo"))) + self.assertEqual('bar[]', ("%s" % StringEscape("bar[]"))) + self.assertEqual('foo-bar', ("%s" % StringEscape(u'foo-bar'))) + self.assertEqual('foo\\x00bar', ("%s" % StringEscape("foo\x00bar"))) + # TODO(Peter): How does this interact with the E1.20 Unicode flag? + self.assertEqual('caf\\xe9', ("%s" % StringEscape(u'caf\xe9'))) + self.assertEqual('foo\\u2014bar', ("%s" % StringEscape(u'foo\u2014bar'))) + # Confirm we throw an exception if we pass in a number or something else + # that's not a string with self.assertRaises(TypeError): result = StringEscape(42) From 7dfe97150de97a7452ea503a7b20f675aec710ea Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sat, 26 Nov 2022 19:50:58 +0000 Subject: [PATCH 34/67] Fix the ContainsUnprintable test given our StringEscape changes --- tools/rdm/TestHelpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/rdm/TestHelpers.py b/tools/rdm/TestHelpers.py index 2c115a2fb3..9a9c4a71a1 100644 --- a/tools/rdm/TestHelpers.py +++ b/tools/rdm/TestHelpers.py @@ -35,6 +35,6 @@ def ContainsUnprintable(s): return s != StringEscape(s) elif type(s) == str: # All strings in Python 3 are unicode - return s.encode() != StringEscape(s) + return s != StringEscape(s) else: raise TypeError('Only strings are supported') From 9d0c68b04fe3a03cff7ab63c5c377f26feaa9172 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sat, 26 Nov 2022 20:19:25 +0000 Subject: [PATCH 35/67] Use our escape function and write the pickle data as binary --- tools/rdm/TestLogger.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/rdm/TestLogger.py b/tools/rdm/TestLogger.py index e53e6baa71..9f8bf2c252 100644 --- a/tools/rdm/TestLogger.py +++ b/tools/rdm/TestLogger.py @@ -21,6 +21,7 @@ import re from ola import Version +from ola.StringUtils import StringEscape from ola.testing.rdm.TestState import TestState @@ -93,7 +94,9 @@ def SaveLog(self, uid, timestamp, end_time, tests, device, test_parameters): filename = os.path.join(self._log_dir, filename) try: - log_file = open(filename, 'w') + # We need to write as binary because pickle on Python 3 generates binary + # data + log_file = open(filename, 'wb') except IOError as e: raise TestLoggerException( 'Failed to write to %s: %s' % (filename, e.message)) @@ -211,17 +214,17 @@ def _FormatData(self, test_data, requested_category, requested_test_state, manufacturer_label = test_data['properties'].get('manufacturer_label', None) if manufacturer_label: - results_log.append('Manufacturer: %s' % - manufacturer_label.encode('string-escape')) + results_log.append('Manufacturer: %s' % StringEscape(manufacturer_label)) model_description = test_data['properties'].get('model_description', None) if model_description: results_log.append('Model Description: %s' % - model_description.encode('string-escape')) + StringEscape(model_description)) software_version = test_data['properties'].get('software_version', None) if software_version: - results_log.append('Software Version: %s' % software_version) + results_log.append('Software Version: %s' % + StringEscape(software_version)) results_log.append("------------------- Warnings --------------------") results_log.extend(warnings) From fc9db0db4f1e05bbbd06f0e4034d5bea9b017e0b Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sat, 26 Nov 2022 21:12:25 +0000 Subject: [PATCH 36/67] Use our new StringEscape function rather than duplicating code --- tools/rdm/ResponderTest.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tools/rdm/ResponderTest.py b/tools/rdm/ResponderTest.py index 9b54866475..e284cfe633 100644 --- a/tools/rdm/ResponderTest.py +++ b/tools/rdm/ResponderTest.py @@ -34,6 +34,7 @@ from TimingStats import TimingStats from ola import PidStore from ola.OlaClient import OlaClient, RDMNack +from ola.StringUtils import StringEscape if sys.version_info >= (3, 0): try: @@ -616,17 +617,8 @@ def _EscapeData(self, data): # We can't escape the key as then it may become a new key d[k] = self._EscapeData(v) return d - # TODO(Peter): How does this interact with the E1.20 Unicode flag? - # We don't use sys.version_info.major to support Python 2.6. - elif sys.version_info[0] == 2 and type(data) == str: - return data.encode('string-escape') - elif sys.version_info[0] == 2 and type(data) == unicode: - return data.encode('unicode-escape') - elif type(data) == str: - # All strings in Python 3 are unicode - # This encode/decode pair gets us an escaped string - return data.encode('unicode-escape').decode(encoding="ascii", - errors="backslashreplace") + elif type(data) == str or type(data) == unicode: + return StringEscape(data) else: return data From 6451ad5ac0225087972d9990683369d9e7d097fa Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sat, 26 Nov 2022 21:23:59 +0000 Subject: [PATCH 37/67] Encode more response data and decode the post data so it behaves as it used to --- tools/rdm/rdm_test_server.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/rdm/rdm_test_server.py b/tools/rdm/rdm_test_server.py index ad56766123..46604b3690 100755 --- a/tools/rdm/rdm_test_server.py +++ b/tools/rdm/rdm_test_server.py @@ -403,7 +403,10 @@ def PostParam(self, param, default=None): request_body = self._environ['wsgi.input'].read(request_body_size) post_params = urlparse.parse_qs(request_body) for p in post_params: - self._post_params[p] = post_params[p][0] + # In Python 3, the param name and value is a bytestring not a string, + # so convert it for backwards compatibility and sanity + self._post_params[p.decode('utf-8')] = + post_params[p][0].decode('utf-8') return self._post_params.get(param, default) @@ -632,7 +635,6 @@ class DownloadModelDataHandler(RequestHandler): """Take the data in the form and return it as a downloadable file.""" def HandleRequest(self, request, response): - print(dir(request)) model_data = request.PostParam('model_data') or '' logging.info(model_data) @@ -642,7 +644,7 @@ def HandleRequest(self, request, response): 'attachment; filename="%s"' % filename) response.SetHeader('Content-type', 'text/plain') response.SetHeader('Content-length', '%d' % len(model_data)) - response.AppendData(model_data) + response.AppendData(model_data.encode()) class DownloadResultsHandler(RequestHandler): @@ -678,7 +680,7 @@ def HandleRequest(self, request, response): 'attachment; filename="%s"' % filename) response.SetHeader('Content-type', 'text/plain') response.SetHeader('Content-length', '%d' % len(output)) - response.AppendData(output) + response.AppendData(output.encode()) class RunTestsHandler(OLAServerRequestHandler): From e5cefeb9a78e8eb9076fb866fe49b8ca538ade4e Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sat, 26 Nov 2022 21:28:16 +0000 Subject: [PATCH 38/67] Use our StringEscape function everywhere else --- tools/rdm/TestMixins.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tools/rdm/TestMixins.py b/tools/rdm/TestMixins.py index c7c45150f8..90384592e3 100644 --- a/tools/rdm/TestMixins.py +++ b/tools/rdm/TestMixins.py @@ -28,6 +28,7 @@ from ola.OlaClient import OlaClient, RDMNack from ola.PidStore import ROOT_DEVICE from ola.RDMConstants import RDM_MAX_STRING_LENGTH +from ola.StringUtils import StringEscape from ola.UID import UID '''Mixins used by the test definitions. @@ -108,7 +109,7 @@ def VerifyResult(self, response, fields): self.AddAdvisory( '%s field in %s contains unprintable characters, was %s' % (self.EXPECTED_FIELDS[0].capitalize(), self.PID, - string_field.encode('string-escape'))) + StringEscape(string_field))) if self.MIN_LENGTH and len(string_field) < self.MIN_LENGTH: self.SetFailed( @@ -164,7 +165,7 @@ def VerifyResult(self, response, fields): self.AddAdvisory( '%s field in %s contains unprintable characters, was %s' % (self.EXPECTED_FIELDS[0].capitalize(), self.PID, - string_field.encode('string-escape'))) + StringEscape(string_field))) if self.MIN_LENGTH and len(string_field) < self.MIN_LENGTH: self.SetFailed( @@ -317,8 +318,8 @@ def VerifyResult(self, response, fields): (self.pid, len(new_label))) else: self.SetFailed('Labels didn\'t match, expected "%s", got "%s"' % - (self.TEST_LABEL.encode('string-escape'), - new_label.encode('string-escape'))) + (StringEscape(self.TEST_LABEL), + StringEscape(new_label))) def ResetState(self): if not self.OldValue(): @@ -1023,7 +1024,7 @@ def VerifyResult(self, response, fields): self.PID, self.DESCRIPTION_FIELD, self.current_item, - fields[self.DESCRIPTION_FIELD].encode('string-escape'))) + StringEscape(fields[self.DESCRIPTION_FIELD]))) class GetSettingDescriptionsRangeMixin(GetSettingDescriptionsMixin): From aedf7ec522bbcb514dfa55411a98a44acdecf1bd Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sat, 26 Nov 2022 21:45:02 +0000 Subject: [PATCH 39/67] Make the exception more useful --- python/ola/StringUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ola/StringUtils.py b/python/ola/StringUtils.py index 9b9b744361..ffb053bcf8 100644 --- a/python/ola/StringUtils.py +++ b/python/ola/StringUtils.py @@ -42,4 +42,4 @@ def StringEscape(s): return s.encode('unicode-escape').decode(encoding="ascii", errors="backslashreplace") else: - raise TypeError('Only strings are supported') + raise TypeError('Only strings are supported not %s' % type(s)) From 315f0715d3dcb027d0d1af5c38f3edc66a09f766 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sat, 26 Nov 2022 21:45:38 +0000 Subject: [PATCH 40/67] Software version doesn't need escaping as it's just an int --- tools/rdm/TestLogger.py | 3 +-- tools/rdm/rdm_responder_test.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/rdm/TestLogger.py b/tools/rdm/TestLogger.py index 9f8bf2c252..17dd716350 100644 --- a/tools/rdm/TestLogger.py +++ b/tools/rdm/TestLogger.py @@ -223,8 +223,7 @@ def _FormatData(self, test_data, requested_category, requested_test_state, software_version = test_data['properties'].get('software_version', None) if software_version: - results_log.append('Software Version: %s' % - StringEscape(software_version)) + results_log.append('Software Version: %s' % software_version) results_log.append("------------------- Warnings --------------------") results_log.extend(warnings) diff --git a/tools/rdm/rdm_responder_test.py b/tools/rdm/rdm_responder_test.py index c6f6e6a521..aa310d87ea 100755 --- a/tools/rdm/rdm_responder_test.py +++ b/tools/rdm/rdm_responder_test.py @@ -209,7 +209,7 @@ def DisplaySummary(options, runner, tests, device): software_version = getattr(device, 'software_version', None) if software_version: - logging.info('Software Version: %s' % StringEscape(software_version)) + logging.info('Software Version: %s' % software_version) if options.timing: timing_stats = runner.TimingStats() From de53080b3d421b4871553ff7d3850f94237fee3e Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sat, 26 Nov 2022 21:51:26 +0000 Subject: [PATCH 41/67] Escape using our function. Add separate non-printable and non-ASCII tests and some other language edge cases --- tools/rdm/TestDefinitions.py | 86 +++++++++++++++++++++++++++++++----- 1 file changed, 74 insertions(+), 12 deletions(-) diff --git a/tools/rdm/TestDefinitions.py b/tools/rdm/TestDefinitions.py index 5527f03128..6fc620ce6f 100644 --- a/tools/rdm/TestDefinitions.py +++ b/tools/rdm/TestDefinitions.py @@ -36,6 +36,7 @@ RDM_MANUFACTURER_SD_MIN, RDM_MANUFACTURER_SD_MAX) from ola.OlaClient import OlaClient, RDMNack from ola.PidStore import ROOT_DEVICE +from ola.StringUtils import StringEscape from ola.UID import UID from TestHelpers import ContainsUnprintable import TestMixins @@ -1062,7 +1063,7 @@ def VerifyResult(self, response, fields): if ContainsUnprintable(fields['description']): self.AddAdvisory( 'Description field in %s contains unprintable characters, was %s' % - (self.PID, fields['description'].encode('string-escape'))) + (self.PID, StringEscape(fields['description']))) class GetParamDescriptionForNonManufacturerPid(ResponderTestFixture): @@ -1446,13 +1447,33 @@ def OldValue(self): return self.Property('device_label') +class SetNonPrintableAsciiDeviceLabel(TestMixins.SetLabelMixin, + OptionalParameterTestFixture): + """SET the device label to something that contains non-printable ASCII characters.""" + CATEGORY = TestCategory.PRODUCT_INFORMATION + PID = 'DEVICE_LABEL' + REQUIRES = ['device_label'] + TEST_LABEL = 'str w\x0d non\x1bprint ASCII\x7f' + + def ExpectedResults(self): + return [ + self.NackSetResult(RDMNack.NR_DATA_OUT_OF_RANGE), + self.NackSetResult(RDMNack.NR_FORMAT_ERROR), + self.NackSetResult(RDMNack.NR_UNSUPPORTED_COMMAND_CLASS), + self.AckSetResult(action=self.VerifySet) + ] + + def OldValue(self): + return self.Property('device_label') + + class SetNonAsciiDeviceLabel(TestMixins.SetLabelMixin, OptionalParameterTestFixture): - """SET the device label to something that contains non-ascii data.""" + """SET the device label to something that contains non-ASCII data.""" CATEGORY = TestCategory.PRODUCT_INFORMATION PID = 'DEVICE_LABEL' REQUIRES = ['device_label'] - TEST_LABEL = 'string with\x0d non ascii\xc0' + TEST_LABEL = 'string with\x0d non ASCII\xc0' def ExpectedResults(self): return [ @@ -1465,6 +1486,14 @@ def ExpectedResults(self): def OldValue(self): return self.Property('device_label') + def Test(self): + # We have to override test here as this has to be raw as we can't encode it + # on Python 3 as it turns it to UTF-8 + # It's also technically out of spec for E1.20 unless it's sent as UTF-8 + self._test_state = self.SET + self.AddIfSetSupported(self.ExpectedResults()) + self.SendRawSet(PidStore.ROOT_DEVICE, self.pid, self.TEST_LABEL.encode()) + class SetEmptyDeviceLabel(TestMixins.SetLabelMixin, OptionalParameterTestFixture): @@ -1520,7 +1549,7 @@ def VerifyResult(self, response, fields): if ContainsUnprintable(language): self.AddAdvisory( 'Language name in languague capabilities contains unprintable ' - 'characters, was %s' % language.encode('string-escape')) + 'characters, was %s' % StringEscape(language)) self.SetProperty('languages_capabilities', language_set) @@ -1590,14 +1619,48 @@ def VerifySet(self): self.SendGet(ROOT_DEVICE, self.pid) +class SetNumericLanguage(OptionalParameterTestFixture): + """Try to set the language to ASCII numeric characters.""" + CATEGORY = TestCategory.PRODUCT_INFORMATION + PID = 'LANGUAGE' + + def Test(self): + self.AddIfSetSupported(self.NackSetResult(RDMNack.NR_DATA_OUT_OF_RANGE)) + self.SendSet(ROOT_DEVICE, self.pid, ['01']) + + +class SetNullLanguage(OptionalParameterTestFixture): + """Try to set the language to two null ASCII characters.""" + CATEGORY = TestCategory.PRODUCT_INFORMATION + PID = 'LANGUAGE' + + def Test(self): + self.AddIfSetSupported(self.NackSetResult(RDMNack.NR_DATA_OUT_OF_RANGE)) + self.SendSet(ROOT_DEVICE, self.pid, ['\x00\x00']) + + +class SetNonPrintableAsciiLanguage(OptionalParameterTestFixture): + """Try to set the language to non-printable ASCII characters.""" + CATEGORY = TestCategory.PRODUCT_INFORMATION + PID = 'LANGUAGE' + + def Test(self): + self.AddIfSetSupported(self.NackSetResult(RDMNack.NR_DATA_OUT_OF_RANGE)) + self.SendSet(ROOT_DEVICE, self.pid, ['\x1b\x7f']) + + class SetNonAsciiLanguage(OptionalParameterTestFixture): - """Try to set the language to non-ascii characters.""" + """Try to set the language to non-ASCII characters.""" CATEGORY = TestCategory.PRODUCT_INFORMATION PID = 'LANGUAGE' def Test(self): self.AddIfSetSupported(self.NackSetResult(RDMNack.NR_DATA_OUT_OF_RANGE)) - self.SendSet(ROOT_DEVICE, self.pid, ['\x0d\xc0']) + # This has to be raw as we can't encode it on Python 3 as it turns it to + # UTF-8 and too many characters + # It's also technically out of spec for E1.20 unless it's sent as UTF-8 at + # which point we're back at square one + self.SendRawSet(ROOT_DEVICE, self.pid, b'\x0d\xc0') class SetUnsupportedLanguage(OptionalParameterTestFixture): @@ -1797,7 +1860,7 @@ def VerifyResult(self, response, fields): if ContainsUnprintable(fields['name']): self.AddAdvisory( 'Name field in %s contains unprintable characters, was %s' % - (self.PID, fields['name'].encode('string-escape'))) + (self.PID, StringEscape(fields['name']))) class GetPersonality(OptionalParameterTestFixture): @@ -1878,7 +1941,7 @@ def VerifyResult(self, response, fields): if ContainsUnprintable(fields['name']): self.AddAdvisory( 'Name field in %s contains unprintable characters, was %s' % - (self.PID, fields['name'].encode('string-escape'))) + (self.PID, StringEscape(fields['name']))) class SetPersonality(OptionalParameterTestFixture): @@ -2620,7 +2683,7 @@ def VerifyResult(self, response, fields): self.AddAdvisory( 'Name field in sensor definition for sensor %d contains unprintable' ' characters, was %s' % (self._current_index, - fields['name'].encode('string-escape'))) + StringEscape(fields['name']))) def CheckCondition(self, sensor_number, fields, lhs, predicate_str, rhs): """Check for a condition and add a warning if it isn't true.""" @@ -3973,7 +4036,7 @@ def VerifyResult(self, response, fields): self.AddAdvisory( 'Description field in self test description for test number %d ' 'contains unprintable characters, was %s' % - (1, fields['description'].encode('string-escape'))) + (1, StringEscape(fields['description']))) class GetSelfTestDescriptionWithNoData(TestMixins.GetWithNoDataMixin, @@ -4026,8 +4089,7 @@ def VerifyResult(self, response, fields): self.AddAdvisory( 'Description field in self test description for test number %d ' 'contains unprintable characters, was %s' % - (fields['test_number'], - fields['description'].encode('string-escape'))) + (fields['test_number'], StringEscape(fields['description']))) class AllSubDevicesGetSelfTestDescription(TestMixins.AllSubDevicesGetMixin, From a405bb2d397d0719bc7d291ef0958ef1499d9093 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sat, 26 Nov 2022 21:55:51 +0000 Subject: [PATCH 42/67] Fix a syntax issue --- tools/rdm/rdm_test_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/rdm/rdm_test_server.py b/tools/rdm/rdm_test_server.py index 46604b3690..6faaa2e91d 100755 --- a/tools/rdm/rdm_test_server.py +++ b/tools/rdm/rdm_test_server.py @@ -405,8 +405,8 @@ def PostParam(self, param, default=None): for p in post_params: # In Python 3, the param name and value is a bytestring not a string, # so convert it for backwards compatibility and sanity - self._post_params[p.decode('utf-8')] = - post_params[p][0].decode('utf-8') + self._post_params[p.decode('utf-8')] = ( + post_params[p][0].decode('utf-8')) return self._post_params.get(param, default) From 63a3105786bf03ed18c923d8b84745e0bbe821e6 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sun, 27 Nov 2022 00:02:29 +0000 Subject: [PATCH 43/67] Fix the flake8 issues --- python/ola/StringUtilsTest.py | 1 + tools/rdm/TestDefinitions.py | 4 +++- tools/rdm/TestHelpersTest.py | 1 + tools/rdm/TestRunner.py | 1 - 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/python/ola/StringUtilsTest.py b/python/ola/StringUtilsTest.py index 6f7e5bef29..b400b8ebf3 100755 --- a/python/ola/StringUtilsTest.py +++ b/python/ola/StringUtilsTest.py @@ -49,6 +49,7 @@ def testStringEscape(self): # that's not a string with self.assertRaises(TypeError): result = StringEscape(42) + self.assertNone(result) if __name__ == '__main__': diff --git a/tools/rdm/TestDefinitions.py b/tools/rdm/TestDefinitions.py index 6fc620ce6f..ba0198b6bd 100644 --- a/tools/rdm/TestDefinitions.py +++ b/tools/rdm/TestDefinitions.py @@ -1449,7 +1449,9 @@ def OldValue(self): class SetNonPrintableAsciiDeviceLabel(TestMixins.SetLabelMixin, OptionalParameterTestFixture): - """SET the device label to something that contains non-printable ASCII characters.""" + """SET the device label to something that contains non-printable ASCII + characters. + """ CATEGORY = TestCategory.PRODUCT_INFORMATION PID = 'DEVICE_LABEL' REQUIRES = ['device_label'] diff --git a/tools/rdm/TestHelpersTest.py b/tools/rdm/TestHelpersTest.py index 5b1dc64c10..ba28d242bd 100644 --- a/tools/rdm/TestHelpersTest.py +++ b/tools/rdm/TestHelpersTest.py @@ -38,6 +38,7 @@ def testContainsUnprintable(self): with self.assertRaises(TypeError): result = ContainsUnprintable(42) + self.assertNone(result) if __name__ == '__main__': diff --git a/tools/rdm/TestRunner.py b/tools/rdm/TestRunner.py index d07e31fc74..18f5db7170 100644 --- a/tools/rdm/TestRunner.py +++ b/tools/rdm/TestRunner.py @@ -23,7 +23,6 @@ from ola import PidStore from ola.OlaClient import OlaClient, RDMNack from ola.RDMAPI import RDMAPI -from ola.testing.rdm import ResponderTest __author__ = 'nomis52@gmail.com (Simon Newton)' From 02af38ebd2a19129d5115eceaeb8687bf750b9fa Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sun, 27 Nov 2022 00:18:10 +0000 Subject: [PATCH 44/67] Switch to some alternative test characters for the String PID test code due to UTF-8 behaviour --- python/ola/PidStore.py | 4 ++++ python/ola/PidStoreTest.py | 20 ++++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/python/ola/PidStore.py b/python/ola/PidStore.py index a62eb0f7b5..31e6f4b50c 100644 --- a/python/ola/PidStore.py +++ b/python/ola/PidStore.py @@ -636,6 +636,10 @@ def Pack(self, args): arg = args[0] arg_size = len(arg) + # Handle the fact a UTF-8 character could be multi-byte + if sys.version_info >= (3, 2): + arg_size = max(arg_size, len(bytes(arg, 'utf8'))) + if self.max is not None and arg_size > self.max: raise ArgsValidationError('%s can be at most %d,' % (self.name, self.max)) diff --git a/python/ola/PidStoreTest.py b/python/ola/PidStoreTest.py index 1ad8ac40c0..ee55c208d1 100755 --- a/python/ola/PidStoreTest.py +++ b/python/ola/PidStoreTest.py @@ -278,12 +278,24 @@ def testPackRanges(self): args = ["enx"] blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] - # test packing some non-ascii characters - args = ["\x0d\xc0"] + # test packing some non-printable characters + args = ["\x0d\x7f"] blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] - self.assertEqual(blob, binascii.unhexlify("0ec0")) + self.assertEqual(blob, binascii.unhexlify("0d7f")) decoded = pid.Unpack(blob, PidStore.RDM_GET) - self.assertEqual(decoded, {'languages': [{'language': '\x0d\xc0'}]}) + self.assertEqual(decoded, {'languages': [{'language': '\x0d\x7f'}]}) + + # test packing some non-ascii characters, this fails on Python 3 as the + # LATIN CAPITAL LETTER A WITH GRAVE, unicode U+00C0 gets encoded as two + # bytes (\xc3\x80) so the total length is three bytes and it doesn't fit! + with self.assertRaises(PidStore.ArgsValidationError): + args = ["\x0d\xc0"] + blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] + + # It works on it's own as it's short enough... + with self.assertRaises(PidStore.ArgsValidationError): + args = ["\xc0"] + blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] # valid empty string pid = store.GetName("STATUS_ID_DESCRIPTION") From c4ca6ca01355d6e424704694b5fe12920684df35 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sun, 27 Nov 2022 01:35:27 +0000 Subject: [PATCH 45/67] Handle utf8 throughout the Python string class. Conversely it will now correctly only allow ASCII 7 bit characters --- python/ola/PidStore.py | 4 +++- python/ola/PidStoreTest.py | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/python/ola/PidStore.py b/python/ola/PidStore.py index 31e6f4b50c..7593c3c69d 100644 --- a/python/ola/PidStore.py +++ b/python/ola/PidStore.py @@ -639,6 +639,8 @@ def Pack(self, args): # Handle the fact a UTF-8 character could be multi-byte if sys.version_info >= (3, 2): arg_size = max(arg_size, len(bytes(arg, 'utf8'))) + else: + arg_size = max(arg_size, len(arg.encode('utf8'))) if self.max is not None and arg_size > self.max: raise ArgsValidationError('%s can be at most %d,' % @@ -652,7 +654,7 @@ def Pack(self, args): if sys.version_info >= (3, 2): data = struct.unpack('%ds' % arg_size, bytes(arg, 'utf8')) else: - data = struct.unpack('%ds' % arg_size, arg) + data = struct.unpack('%ds' % arg_size, arg.encode('utf8')) except struct.error as e: raise ArgsValidationError("Can't pack data: %s" % e) return data[0], 1 diff --git a/python/ola/PidStoreTest.py b/python/ola/PidStoreTest.py index ee55c208d1..3dc5c12b17 100755 --- a/python/ola/PidStoreTest.py +++ b/python/ola/PidStoreTest.py @@ -293,9 +293,12 @@ def testPackRanges(self): blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] # It works on it's own as it's short enough... - with self.assertRaises(PidStore.ArgsValidationError): - args = ["\xc0"] - blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] + args = ["\xc0"] + blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] + self.assertEqual(blob, binascii.unhexlify("c380")) + decoded = pid.Unpack(blob, PidStore.RDM_GET) + # This is the unicode code point for it + self.assertEqual(decoded, {'languages': [{'language': '\u00c0'}]}) # valid empty string pid = store.GetName("STATUS_ID_DESCRIPTION") From 00a273dec1a7442706753c4317723ab9df6dce0c Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sun, 27 Nov 2022 02:24:39 +0000 Subject: [PATCH 46/67] Fix the UTF-8 related tests on both Python versions, also sort unpack --- python/ola/PidStore.py | 13 +++++-------- python/ola/PidStoreTest.py | 6 +++--- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/python/ola/PidStore.py b/python/ola/PidStore.py index 7593c3c69d..b088c07b66 100644 --- a/python/ola/PidStore.py +++ b/python/ola/PidStore.py @@ -638,9 +638,9 @@ def Pack(self, args): # Handle the fact a UTF-8 character could be multi-byte if sys.version_info >= (3, 2): - arg_size = max(arg_size, len(bytes(arg, 'utf8'))) + arg_size = max(arg_size, len(bytes(arg, 'utf-8'))) else: - arg_size = max(arg_size, len(arg.encode('utf8'))) + arg_size = max(arg_size, len(arg.encode('utf-8'))) if self.max is not None and arg_size > self.max: raise ArgsValidationError('%s can be at most %d,' % @@ -652,9 +652,9 @@ def Pack(self, args): try: if sys.version_info >= (3, 2): - data = struct.unpack('%ds' % arg_size, bytes(arg, 'utf8')) + data = struct.unpack('%ds' % arg_size, bytes(arg, 'utf-8')) else: - data = struct.unpack('%ds' % arg_size, arg.encode('utf8')) + data = struct.unpack('%ds' % arg_size, arg.encode('utf-8')) except struct.error as e: raise ArgsValidationError("Can't pack data: %s" % e) return data[0], 1 @@ -674,10 +674,7 @@ def Unpack(self, data): except struct.error as e: raise UnpackException(e) - if sys.version_info >= (3, 2): - return value[0].rstrip(b'\x00').decode('utf-8') - else: - return value[0].rstrip(b'\x00') + return value[0].rstrip(b'\x00').decode('utf-8') def GetDescription(self, indent=0): indent = ' ' * indent diff --git a/python/ola/PidStoreTest.py b/python/ola/PidStoreTest.py index 3dc5c12b17..579e55c01d 100755 --- a/python/ola/PidStoreTest.py +++ b/python/ola/PidStoreTest.py @@ -289,16 +289,16 @@ def testPackRanges(self): # LATIN CAPITAL LETTER A WITH GRAVE, unicode U+00C0 gets encoded as two # bytes (\xc3\x80) so the total length is three bytes and it doesn't fit! with self.assertRaises(PidStore.ArgsValidationError): - args = ["\x0d\xc0"] + args = [u"\x0d\xc0"] blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] # It works on it's own as it's short enough... - args = ["\xc0"] + args = [u"\u00c0"] blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] self.assertEqual(blob, binascii.unhexlify("c380")) decoded = pid.Unpack(blob, PidStore.RDM_GET) # This is the unicode code point for it - self.assertEqual(decoded, {'languages': [{'language': '\u00c0'}]}) + self.assertEqual(decoded, {'languages': [{'language': u'\u00c0'}]}) # valid empty string pid = store.GetName("STATUS_ID_DESCRIPTION") From abf91a57b58d50c0f28a5ded23a132fb07622589 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sun, 27 Nov 2022 04:04:09 +0000 Subject: [PATCH 47/67] Fix our SetNonAsciiDeviceLabel test so it sends the expected bytes --- tools/rdm/TestDefinitions.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/rdm/TestDefinitions.py b/tools/rdm/TestDefinitions.py index ba0198b6bd..cb505f1ece 100644 --- a/tools/rdm/TestDefinitions.py +++ b/tools/rdm/TestDefinitions.py @@ -1475,7 +1475,8 @@ class SetNonAsciiDeviceLabel(TestMixins.SetLabelMixin, CATEGORY = TestCategory.PRODUCT_INFORMATION PID = 'DEVICE_LABEL' REQUIRES = ['device_label'] - TEST_LABEL = 'string with\x0d non ASCII\xc0' + # Store directly as bytes so we don't try and decode as UTF-8 + TEST_LABEL = b'string with\x0d non ASCII\xc0' def ExpectedResults(self): return [ @@ -1490,11 +1491,11 @@ def OldValue(self): def Test(self): # We have to override test here as this has to be raw as we can't encode it - # on Python 3 as it turns it to UTF-8 + # on Python 3 as it turns it to UTF-8 or escapes it # It's also technically out of spec for E1.20 unless it's sent as UTF-8 self._test_state = self.SET self.AddIfSetSupported(self.ExpectedResults()) - self.SendRawSet(PidStore.ROOT_DEVICE, self.pid, self.TEST_LABEL.encode()) + self.SendRawSet(PidStore.ROOT_DEVICE, self.pid, self.TEST_LABEL) class SetEmptyDeviceLabel(TestMixins.SetLabelMixin, From 1342f577e58754838e5bdc9957a83c9f9a152dbd Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sun, 27 Nov 2022 04:04:57 +0000 Subject: [PATCH 48/67] Catch, and test, for Unicode decode errors on incoming data --- python/ola/PidStore.py | 7 ++++++- python/ola/PidStoreTest.py | 8 ++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/python/ola/PidStore.py b/python/ola/PidStore.py index b088c07b66..fc3518db18 100644 --- a/python/ola/PidStore.py +++ b/python/ola/PidStore.py @@ -674,7 +674,12 @@ def Unpack(self, data): except struct.error as e: raise UnpackException(e) - return value[0].rstrip(b'\x00').decode('utf-8') + try: + value = value[0].rstrip(b'\x00').decode('utf-8') + except UnicodeDecodeError as e: + raise UnpackException(e) + + return value def GetDescription(self, indent=0): indent = ' ' * indent diff --git a/python/ola/PidStoreTest.py b/python/ola/PidStoreTest.py index 579e55c01d..5b855d6b19 100755 --- a/python/ola/PidStoreTest.py +++ b/python/ola/PidStoreTest.py @@ -221,6 +221,10 @@ def testPackUnpack(self): self.assertEqual(decoded['slots_required'], 7) self.assertEqual(decoded['name'], "UnpackTest") + with self.assertRaises(PidStore.UnpackException): + blob = binascii.unhexlify("2a0007556e7061636bc054657374") + decoded = pid.Unpack(blob, PidStore.RDM_GET) + def testPackRanges(self): store = PidStore.PidStore() store.Load([os.path.join(path, "test_pids.proto")]) @@ -285,9 +289,9 @@ def testPackRanges(self): decoded = pid.Unpack(blob, PidStore.RDM_GET) self.assertEqual(decoded, {'languages': [{'language': '\x0d\x7f'}]}) - # test packing some non-ascii characters, this fails on Python 3 as the + # test packing some non-ascii characters, as the # LATIN CAPITAL LETTER A WITH GRAVE, unicode U+00C0 gets encoded as two - # bytes (\xc3\x80) so the total length is three bytes and it doesn't fit! + # bytes (\xc3\x80) the total length is three bytes and it doesn't fit! with self.assertRaises(PidStore.ArgsValidationError): args = [u"\x0d\xc0"] blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] From ecf842c0e622b898f9a3658babd2ce1c962c4d4a Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sun, 27 Nov 2022 14:48:57 +0000 Subject: [PATCH 49/67] Add a basic null handling test --- python/ola/PidStoreTest.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python/ola/PidStoreTest.py b/python/ola/PidStoreTest.py index 5b855d6b19..a5c5275c85 100755 --- a/python/ola/PidStoreTest.py +++ b/python/ola/PidStoreTest.py @@ -221,6 +221,16 @@ def testPackUnpack(self): self.assertEqual(decoded['slots_required'], 7) self.assertEqual(decoded['name'], "UnpackTest") + # Test null handling, trailing null should be truncated on the way back in + args = ["42", "7", "Foo\0"] + blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] + # Not truncated here + self.assertEqual(blob, binascii.unhexlify("2a0007466f6f00")) + decoded = pid.Unpack(blob, PidStore.RDM_GET) + self.assertEqual(decoded['personality'], 42) + self.assertEqual(decoded['slots_required'], 7) + self.assertEqual(decoded['name'], "Foo") + with self.assertRaises(PidStore.UnpackException): blob = binascii.unhexlify("2a0007556e7061636bc054657374") decoded = pid.Unpack(blob, PidStore.RDM_GET) From 73d2f395ff53c2111a6236ff3eef5a45fe661973 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sun, 27 Nov 2022 15:10:54 +0000 Subject: [PATCH 50/67] Sort the Python imports with isort --- .github/workflows/isort.yml | 12 ++++++++ data/rdm/PidDataTest.py | 1 + include/ola/gen_callbacks.py | 1 + python/examples/ola_artnet_params.py | 5 +++- python/examples/ola_candidate_ports.py | 4 ++- python/examples/ola_devices.py | 4 ++- python/examples/ola_fetch_dmx.py | 6 ++-- python/examples/ola_patch_unpatch.py | 6 ++-- python/examples/ola_plugin_info.py | 6 ++-- python/examples/ola_rdm_discover.py | 4 ++- python/examples/ola_rdm_get.py | 5 +++- python/examples/ola_recv_dmx.py | 3 +- python/examples/ola_send_dmx.py | 4 ++- python/examples/ola_simple_fade.py | 5 ++-- python/examples/ola_universe_info.py | 4 ++- python/examples/rdm_compare.py | 1 + python/examples/rdm_snapshot.py | 4 ++- python/ola/ClientWrapper.py | 1 + python/ola/ClientWrapperTest.py | 5 ++-- python/ola/DUBDecoder.py | 1 + python/ola/DUBDecoderTest.py | 1 + python/ola/MACAddressTest.py | 3 +- python/ola/OlaClient.py | 6 ++-- python/ola/OlaClientTest.py | 5 ++-- python/ola/PidStore.py | 7 +++-- python/ola/PidStoreTest.py | 3 +- python/ola/RDMAPI.py | 3 ++ python/ola/RDMTest.py | 5 ++-- python/ola/StringUtilsTest.py | 1 + python/ola/UIDTest.py | 3 +- python/ola/rpc/SimpleRpcControllerTest.py | 1 + python/ola/rpc/StreamRpcChannel.py | 1 + scripts/enforce_licence.py | 1 + scripts/verify_trees.py | 4 +-- tools/ola_mon/ola_mon.py | 6 ++-- tools/rdm/ExpectedResultsTest.py | 6 ++-- tools/rdm/ModelCollector.py | 5 +++- tools/rdm/ResponderTest.py | 6 ++-- tools/rdm/ResponderTestTest.py | 3 +- tools/rdm/TestDefinitions.py | 35 ++++++++++++----------- tools/rdm/TestHelpers.py | 1 + tools/rdm/TestHelpersTest.py | 1 + tools/rdm/TestLogger.py | 2 +- tools/rdm/TestMixins.py | 12 ++++---- tools/rdm/TestRunner.py | 6 ++-- tools/rdm/TestRunnerTest.py | 8 +++--- tools/rdm/TestStateTest.py | 3 +- tools/rdm/TimingStats.py | 1 + tools/rdm/launch_tests.py | 6 ++-- tools/rdm/rdm_model_collector.py | 7 +++-- tools/rdm/rdm_responder_test.py | 18 ++++++------ tools/rdm/rdm_test_server.py | 14 ++++----- tools/rdm/setup_patch.py | 2 ++ 53 files changed, 177 insertions(+), 91 deletions(-) create mode 100644 .github/workflows/isort.yml diff --git a/.github/workflows/isort.yml b/.github/workflows/isort.yml new file mode 100644 index 0000000000..f314e5b555 --- /dev/null +++ b/.github/workflows/isort.yml @@ -0,0 +1,12 @@ +name: isort + +on: + - push + - pull_request + +jobs: + isort: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: isort/isort-action@v1 diff --git a/data/rdm/PidDataTest.py b/data/rdm/PidDataTest.py index 5268286719..f92a53f029 100755 --- a/data/rdm/PidDataTest.py +++ b/data/rdm/PidDataTest.py @@ -20,6 +20,7 @@ import os import unittest + from ola import PidStore __author__ = 'nomis52@gmail.com (Simon Newton)' diff --git a/include/ola/gen_callbacks.py b/include/ola/gen_callbacks.py index ba530c929f..18f149df89 100755 --- a/include/ola/gen_callbacks.py +++ b/include/ola/gen_callbacks.py @@ -18,6 +18,7 @@ from __future__ import print_function + import textwrap diff --git a/python/examples/ola_artnet_params.py b/python/examples/ola_artnet_params.py index af0955e082..5aad994fdd 100755 --- a/python/examples/ola_artnet_params.py +++ b/python/examples/ola_artnet_params.py @@ -19,9 +19,12 @@ """Fetch some ArtNet parameters.""" from __future__ import print_function + +import sys + from ola.ClientWrapper import ClientWrapper + from ola import ArtNetConfigMessages_pb2 -import sys __author__ = 'nomis52@gmail.com (Simon Newton)' diff --git a/python/examples/ola_candidate_ports.py b/python/examples/ola_candidate_ports.py index 3b1d4cf683..6ca56bcdf4 100755 --- a/python/examples/ola_candidate_ports.py +++ b/python/examples/ola_candidate_ports.py @@ -19,10 +19,12 @@ """List candidate ports for patching.""" from __future__ import print_function -from ola.ClientWrapper import ClientWrapper + import argparse import sys +from ola.ClientWrapper import ClientWrapper + __author__ = 'simon.marchi@polymtl.ca (Simon Marchi)' wrapper = None diff --git a/python/examples/ola_devices.py b/python/examples/ola_devices.py index d9832ef80e..ead9cc8634 100755 --- a/python/examples/ola_devices.py +++ b/python/examples/ola_devices.py @@ -19,9 +19,11 @@ """Lists the devices / ports.""" from __future__ import print_function -from ola.ClientWrapper import ClientWrapper + import sys +from ola.ClientWrapper import ClientWrapper + __author__ = 'nomis52@gmail.com (Simon Newton)' wrapper = None diff --git a/python/examples/ola_fetch_dmx.py b/python/examples/ola_fetch_dmx.py index 1b166b0176..cb05c0a626 100755 --- a/python/examples/ola_fetch_dmx.py +++ b/python/examples/ola_fetch_dmx.py @@ -19,10 +19,12 @@ """Gets a current frame of DMX for a universe.""" from __future__ import print_function -from ola.ClientWrapper import ClientWrapper + import getopt -import textwrap import sys +import textwrap + +from ola.ClientWrapper import ClientWrapper __author__ = 'nomis52@gmail.com (Simon Newton)' diff --git a/python/examples/ola_patch_unpatch.py b/python/examples/ola_patch_unpatch.py index a2a9ec4ff6..19aa2667e7 100755 --- a/python/examples/ola_patch_unpatch.py +++ b/python/examples/ola_patch_unpatch.py @@ -19,11 +19,13 @@ """Patch and unpatch ports.""" from __future__ import print_function -from ola.ClientWrapper import ClientWrapper -from ola.OlaClient import OlaClient + import argparse import sys +from ola.ClientWrapper import ClientWrapper +from ola.OlaClient import OlaClient + __author__ = 'simon.marchi@polymtl.ca (Simon Marchi)' wrapper = None diff --git a/python/examples/ola_plugin_info.py b/python/examples/ola_plugin_info.py index b85506f96f..74aa6dd935 100755 --- a/python/examples/ola_plugin_info.py +++ b/python/examples/ola_plugin_info.py @@ -19,10 +19,12 @@ """Lists the loaded plugins.""" from __future__ import print_function -from ola.ClientWrapper import ClientWrapper + import getopt -import textwrap import sys +import textwrap + +from ola.ClientWrapper import ClientWrapper __author__ = 'nomis52@gmail.com (Simon Newton)' diff --git a/python/examples/ola_rdm_discover.py b/python/examples/ola_rdm_discover.py index 4c84a04ebf..32846428d1 100755 --- a/python/examples/ola_rdm_discover.py +++ b/python/examples/ola_rdm_discover.py @@ -19,9 +19,11 @@ '''Show the UIDs for a universe.''' from __future__ import print_function + import getopt -import textwrap import sys +import textwrap + from ola.ClientWrapper import ClientWrapper __author__ = 'nomis52@gmail.com (Simon Newton)' diff --git a/python/examples/ola_rdm_get.py b/python/examples/ola_rdm_get.py index 009ea7da8e..606b584ac6 100755 --- a/python/examples/ola_rdm_get.py +++ b/python/examples/ola_rdm_get.py @@ -19,18 +19,21 @@ '''Get a PID from a UID.''' from __future__ import print_function + import cmd import getopt import os.path import readline import sys import textwrap -from ola import PidStore + from ola.ClientWrapper import ClientWrapper from ola.OlaClient import OlaClient, RDMNack from ola.RDMAPI import RDMAPI from ola.UID import UID +from ola import PidStore + __author__ = 'nomis52@gmail.com (Simon Newton)' diff --git a/python/examples/ola_recv_dmx.py b/python/examples/ola_recv_dmx.py index 6f1abdf100..9fb578b4c8 100755 --- a/python/examples/ola_recv_dmx.py +++ b/python/examples/ola_recv_dmx.py @@ -19,8 +19,9 @@ """Receive DMX data.""" import getopt -import textwrap import sys +import textwrap + from ola.ClientWrapper import ClientWrapper __author__ = 'nomis52@gmail.com (Simon Newton)' diff --git a/python/examples/ola_send_dmx.py b/python/examples/ola_send_dmx.py index 3adeb0ec78..df43dae624 100755 --- a/python/examples/ola_send_dmx.py +++ b/python/examples/ola_send_dmx.py @@ -19,10 +19,12 @@ """Send some DMX data.""" from __future__ import print_function -from ola.ClientWrapper import ClientWrapper + import array import sys +from ola.ClientWrapper import ClientWrapper + __author__ = 'nomis52@gmail.com (Simon Newton)' wrapper = None diff --git a/python/examples/ola_simple_fade.py b/python/examples/ola_simple_fade.py index d9ef3184df..e2d93c4c4e 100755 --- a/python/examples/ola_simple_fade.py +++ b/python/examples/ola_simple_fade.py @@ -17,9 +17,10 @@ # Copyright (C) 2014 Sean Sill from array import array + from ola.ClientWrapper import ClientWrapper -from ola.DMXConstants import DMX_MIN_SLOT_VALUE, DMX_MAX_SLOT_VALUE, \ - DMX_UNIVERSE_SIZE +from ola.DMXConstants import (DMX_MAX_SLOT_VALUE, DMX_MIN_SLOT_VALUE, + DMX_UNIVERSE_SIZE) __author__ = 'Sean Sill' diff --git a/python/examples/ola_universe_info.py b/python/examples/ola_universe_info.py index cc62175e53..7fb7c87379 100755 --- a/python/examples/ola_universe_info.py +++ b/python/examples/ola_universe_info.py @@ -19,9 +19,11 @@ """Lists the active universes.""" from __future__ import print_function + +import sys + from ola.ClientWrapper import ClientWrapper from ola.OlaClient import Universe -import sys __author__ = 'nomis52@gmail.com (Simon Newton)' diff --git a/python/examples/rdm_compare.py b/python/examples/rdm_compare.py index 6aba073be0..633459e001 100755 --- a/python/examples/rdm_compare.py +++ b/python/examples/rdm_compare.py @@ -23,6 +23,7 @@ import tempfile import textwrap import webbrowser + from ola.UID import UID '''Compare the RDM configurations saves with rdm_snapshot.py''' diff --git a/python/examples/rdm_snapshot.py b/python/examples/rdm_snapshot.py index 088dd70f9c..30ba4e9694 100755 --- a/python/examples/rdm_snapshot.py +++ b/python/examples/rdm_snapshot.py @@ -22,12 +22,14 @@ import pprint import sys import textwrap -from ola import PidStore + from ola.ClientWrapper import ClientWrapper from ola.OlaClient import OlaClient, RDMNack from ola.RDMAPI import RDMAPI from ola.UID import UID +from ola import PidStore + '''Quick script to collect the settings from a rig.''' __author__ = 'nomis52@gmail.com (Simon Newton)' diff --git a/python/ola/ClientWrapper.py b/python/ola/ClientWrapper.py index e9fc3309ad..f302e0318c 100644 --- a/python/ola/ClientWrapper.py +++ b/python/ola/ClientWrapper.py @@ -25,6 +25,7 @@ import termios import threading import traceback + from ola.OlaClient import OlaClient """A simple client wrapper for the OlaClient.""" diff --git a/python/ola/ClientWrapperTest.py b/python/ola/ClientWrapperTest.py index 00c86caf7f..d4252c7bb7 100644 --- a/python/ola/ClientWrapperTest.py +++ b/python/ola/ClientWrapperTest.py @@ -22,10 +22,9 @@ import socket # import timeout_decorator import unittest -from ola.ClientWrapper import ClientWrapper -from ola.ClientWrapper import _Event -from ola.TestUtils import handleRPCByteOrder +from ola.ClientWrapper import ClientWrapper, _Event +from ola.TestUtils import handleRPCByteOrder """Test cases for the Event and Event loop of ClientWrapper class.""" diff --git a/python/ola/DUBDecoder.py b/python/ola/DUBDecoder.py index 28f78e0819..736aae1c22 100644 --- a/python/ola/DUBDecoder.py +++ b/python/ola/DUBDecoder.py @@ -16,6 +16,7 @@ # Copyright (C) 2012 Simon Newton import itertools + from ola.UID import UID """Decodes a DUB response.""" diff --git a/python/ola/DUBDecoderTest.py b/python/ola/DUBDecoderTest.py index a93d753949..a102945458 100755 --- a/python/ola/DUBDecoderTest.py +++ b/python/ola/DUBDecoderTest.py @@ -17,6 +17,7 @@ # Copyright (C) Simon Newton import unittest + from ola.DUBDecoder import DecodeResponse """Test cases for the DUBDecoder class.""" diff --git a/python/ola/MACAddressTest.py b/python/ola/MACAddressTest.py index 26b9ff670c..6bd05a6769 100755 --- a/python/ola/MACAddressTest.py +++ b/python/ola/MACAddressTest.py @@ -18,8 +18,9 @@ import sys import unittest + from ola.MACAddress import MACAddress -from ola.TestUtils import allNotEqual, allHashNotEqual +from ola.TestUtils import allHashNotEqual, allNotEqual """Test cases for the MACAddress class.""" diff --git a/python/ola/OlaClient.py b/python/ola/OlaClient.py index 6d1fb32c6d..47e8dfa41c 100644 --- a/python/ola/OlaClient.py +++ b/python/ola/OlaClient.py @@ -19,11 +19,13 @@ import socket import struct import sys -from ola.rpc.StreamRpcChannel import StreamRpcChannel + from ola.rpc.SimpleRpcController import SimpleRpcController -from ola import Ola_pb2 +from ola.rpc.StreamRpcChannel import StreamRpcChannel from ola.UID import UID +from ola import Ola_pb2 + """The client used to communicate with the Ola Server.""" __author__ = 'nomis52@gmail.com (Simon Newton)' diff --git a/python/ola/OlaClientTest.py b/python/ola/OlaClientTest.py index 9e7dfebab5..9cbe098d33 100644 --- a/python/ola/OlaClientTest.py +++ b/python/ola/OlaClientTest.py @@ -17,8 +17,9 @@ # Copyright (C) 2019 Bruce Lowekamp import unittest -from ola.OlaClient import Plugin, Device, Port, Universe, RDMNack -from ola.TestUtils import allNotEqual, allHashNotEqual + +from ola.OlaClient import Device, Plugin, Port, RDMNack, Universe +from ola.TestUtils import allHashNotEqual, allNotEqual """Test cases for data structures of OlaClient. SendDMX is tested with ClientWrapper.""" diff --git a/python/ola/PidStore.py b/python/ola/PidStore.py index fc3518db18..5a6f1fdec4 100644 --- a/python/ola/PidStore.py +++ b/python/ola/PidStore.py @@ -17,19 +17,20 @@ # Holds all the information about RDM PIDs from __future__ import print_function + import binascii import math import os import socket import struct import sys + from google.protobuf import text_format -from ola import PidStoreLocation -from ola import Pids_pb2 from ola.MACAddress import MACAddress -from ola import RDMConstants from ola.UID import UID +from ola import Pids_pb2, PidStoreLocation, RDMConstants + """The PID Store.""" __author__ = 'nomis52@gmail.com (Simon Newton)' diff --git a/python/ola/PidStoreTest.py b/python/ola/PidStoreTest.py index a5c5275c85..13deb782eb 100755 --- a/python/ola/PidStoreTest.py +++ b/python/ola/PidStoreTest.py @@ -19,8 +19,9 @@ import binascii import os import unittest + import ola.PidStore as PidStore -from ola.TestUtils import allNotEqual, allHashNotEqual +from ola.TestUtils import allHashNotEqual, allNotEqual """Test cases for the PidStore class. Relies on the PID data from RDM tests in the directory diff --git a/python/ola/RDMAPI.py b/python/ola/RDMAPI.py index 2686dcacb8..e02ca7077c 100644 --- a/python/ola/RDMAPI.py +++ b/python/ola/RDMAPI.py @@ -16,8 +16,11 @@ # Copyright (C) 2010 Simon Newton from __future__ import print_function + import sys + from ola.OlaClient import OlaClient + from ola import PidStore """The Python RDM API.""" diff --git a/python/ola/RDMTest.py b/python/ola/RDMTest.py index cfbf486e3c..36bd74ff3a 100644 --- a/python/ola/RDMTest.py +++ b/python/ola/RDMTest.py @@ -21,13 +21,14 @@ import socket # import timeout_decorator import unittest -from ola import PidStore + from ola.ClientWrapper import ClientWrapper from ola.OlaClient import RDMNack from ola.RDMAPI import RDMAPI -from ola.UID import UID from ola.TestUtils import handleRPCByteOrder +from ola.UID import UID +from ola import PidStore """Test cases for RDM device commands.""" diff --git a/python/ola/StringUtilsTest.py b/python/ola/StringUtilsTest.py index b400b8ebf3..3d1b95df02 100755 --- a/python/ola/StringUtilsTest.py +++ b/python/ola/StringUtilsTest.py @@ -17,6 +17,7 @@ # Copyright (C) 2022 Peter Newman import unittest + from ola.StringUtils import StringEscape """Test cases for StringUtils utilities.""" diff --git a/python/ola/UIDTest.py b/python/ola/UIDTest.py index 9b7f0b6017..0e71af2ce1 100755 --- a/python/ola/UIDTest.py +++ b/python/ola/UIDTest.py @@ -17,8 +17,9 @@ # Copyright (C) 2005 Simon Newton import unittest + +from ola.TestUtils import allHashNotEqual, allNotEqual from ola.UID import UID, UIDOutOfRangeException -from ola.TestUtils import allNotEqual, allHashNotEqual """Test cases for the UID class.""" diff --git a/python/ola/rpc/SimpleRpcControllerTest.py b/python/ola/rpc/SimpleRpcControllerTest.py index 78dc91bff1..7f1d2988d7 100755 --- a/python/ola/rpc/SimpleRpcControllerTest.py +++ b/python/ola/rpc/SimpleRpcControllerTest.py @@ -17,6 +17,7 @@ # Copyright (C) 2005 Simon Newton import unittest + from SimpleRpcController import SimpleRpcController """Test cases for the SimpleRpcController.""" diff --git a/python/ola/rpc/StreamRpcChannel.py b/python/ola/rpc/StreamRpcChannel.py index d4ddd9178b..ffbfe71ba6 100644 --- a/python/ola/rpc/StreamRpcChannel.py +++ b/python/ola/rpc/StreamRpcChannel.py @@ -18,6 +18,7 @@ import binascii import logging import struct + from google.protobuf import service from ola.rpc import Rpc_pb2 from ola.rpc.SimpleRpcController import SimpleRpcController diff --git a/scripts/enforce_licence.py b/scripts/enforce_licence.py index 9c930f73fe..b15d157681 100755 --- a/scripts/enforce_licence.py +++ b/scripts/enforce_licence.py @@ -17,6 +17,7 @@ # Copyright (C) 2013 Simon Newton from __future__ import print_function + import difflib import getopt import glob diff --git a/scripts/verify_trees.py b/scripts/verify_trees.py index c4a8d114e2..b1cc9b2c8b 100755 --- a/scripts/verify_trees.py +++ b/scripts/verify_trees.py @@ -18,10 +18,10 @@ from __future__ import print_function -import os import fnmatch -import textwrap +import os import sys +import textwrap # File & directory patterns that differ between what's in the git repo and # what's in the tarball. diff --git a/tools/ola_mon/ola_mon.py b/tools/ola_mon/ola_mon.py index 00e1b81907..cc9cac53ee 100755 --- a/tools/ola_mon/ola_mon.py +++ b/tools/ola_mon/ola_mon.py @@ -20,15 +20,17 @@ # doesn't stop from __future__ import print_function + import getopt -import rrdtool -import time import os.path import re import socket import sys import textwrap import threading +import time + +import rrdtool if sys.version_info >= (3, 0): try: diff --git a/tools/rdm/ExpectedResultsTest.py b/tools/rdm/ExpectedResultsTest.py index c30544813b..27b6512853 100644 --- a/tools/rdm/ExpectedResultsTest.py +++ b/tools/rdm/ExpectedResultsTest.py @@ -17,10 +17,12 @@ # Copyright (C) 2021 Peter Newman import unittest +from collections import namedtuple + from ExpectedResults import (BroadcastResult, DUBResult, InvalidResponse, - TimeoutResult, SuccessfulResult, UnsupportedResult) + SuccessfulResult, TimeoutResult, + UnsupportedResult) from ola.OlaClient import OlaClient -from collections import namedtuple """Test cases for ExpectedResults classes.""" diff --git a/tools/rdm/ModelCollector.py b/tools/rdm/ModelCollector.py index 4ea6f9a1ae..9d0b761ab5 100644 --- a/tools/rdm/ModelCollector.py +++ b/tools/rdm/ModelCollector.py @@ -16,12 +16,15 @@ # Copyright (C) 2011 Simon Newton from __future__ import print_function + import logging + import ola.RDMConstants -from ola import PidStore from ola.OlaClient import OlaClient, RDMNack from ola.RDMAPI import RDMAPI +from ola import PidStore + '''Quick script to collect information about responders.''' __author__ = 'nomis52@gmail.com (Simon Newton)' diff --git a/tools/rdm/ResponderTest.py b/tools/rdm/ResponderTest.py index e284cfe633..e51a076fe3 100644 --- a/tools/rdm/ResponderTest.py +++ b/tools/rdm/ResponderTest.py @@ -27,14 +27,16 @@ import logging import sys import time + from ExpectedResults import (AckDiscoveryResult, AckGetResult, AckSetResult, NackDiscoveryResult, NackGetResult, NackSetResult) +from ola.OlaClient import OlaClient, RDMNack +from ola.StringUtils import StringEscape from TestCategory import TestCategory from TestState import TestState from TimingStats import TimingStats + from ola import PidStore -from ola.OlaClient import OlaClient, RDMNack -from ola.StringUtils import StringEscape if sys.version_info >= (3, 0): try: diff --git a/tools/rdm/ResponderTestTest.py b/tools/rdm/ResponderTestTest.py index 8bd771666d..0caa499c3a 100644 --- a/tools/rdm/ResponderTestTest.py +++ b/tools/rdm/ResponderTestTest.py @@ -17,7 +17,8 @@ # Copyright (C) 2019 Bruce Lowekamp import unittest -from ResponderTest import TestFixture, ResponderTestFixture + +from ResponderTest import ResponderTestFixture, TestFixture """Test cases for sorting TestFixtures.""" diff --git a/tools/rdm/TestDefinitions.py b/tools/rdm/TestDefinitions.py index cb505f1ece..ae00954ef8 100644 --- a/tools/rdm/TestDefinitions.py +++ b/tools/rdm/TestDefinitions.py @@ -18,30 +18,31 @@ import datetime import operator import struct -from ExpectedResults import (AckGetResult, BroadcastResult, NackGetResult, - InvalidResponse, TimeoutResult, UnsupportedResult, - RDM_GET, RDM_SET) -from ResponderTest import ResponderTestFixture, TestFixture -from ResponderTest import OptionalParameterTestFixture -from TestCategory import TestCategory -from ola import PidStore -from ola import RDMConstants -from ola.RDMConstants import (RDM_MIN_HOSTNAME_LENGTH, RDM_MAX_HOSTNAME_LENGTH, - RDM_MAX_DOMAIN_NAME_LENGTH, - RDM_MANUFACTURER_PID_MIN, - RDM_MANUFACTURER_PID_MAX, RDM_INTERFACE_INDEX_MIN, - RDM_INTERFACE_INDEX_MAX, - INTERFACE_HARDWARE_TYPE_ETHERNET, - RDM_ZERO_FOOTPRINT_DMX_ADDRESS, - RDM_MANUFACTURER_SD_MIN, RDM_MANUFACTURER_SD_MAX) + +import TestMixins +from ExpectedResults import (RDM_GET, RDM_SET, AckGetResult, BroadcastResult, + InvalidResponse, NackGetResult, TimeoutResult, + UnsupportedResult) from ola.OlaClient import OlaClient, RDMNack from ola.PidStore import ROOT_DEVICE +from ola.RDMConstants import (INTERFACE_HARDWARE_TYPE_ETHERNET, + RDM_INTERFACE_INDEX_MAX, RDM_INTERFACE_INDEX_MIN, + RDM_MANUFACTURER_PID_MAX, + RDM_MANUFACTURER_PID_MIN, + RDM_MANUFACTURER_SD_MAX, RDM_MANUFACTURER_SD_MIN, + RDM_MAX_DOMAIN_NAME_LENGTH, + RDM_MAX_HOSTNAME_LENGTH, RDM_MIN_HOSTNAME_LENGTH, + RDM_ZERO_FOOTPRINT_DMX_ADDRESS) from ola.StringUtils import StringEscape from ola.UID import UID +from ResponderTest import (OptionalParameterTestFixture, ResponderTestFixture, + TestFixture) +from TestCategory import TestCategory from TestHelpers import ContainsUnprintable -import TestMixins from TestMixins import MAX_DMX_ADDRESS +from ola import PidStore, RDMConstants + '''This defines all the tests for RDM responders.''' __author__ = 'nomis52@gmail.com (Simon Newton)' diff --git a/tools/rdm/TestHelpers.py b/tools/rdm/TestHelpers.py index 9a9c4a71a1..e07934b224 100644 --- a/tools/rdm/TestHelpers.py +++ b/tools/rdm/TestHelpers.py @@ -16,6 +16,7 @@ # Copyright (C) 2013 Peter Newman import sys + from ola.StringUtils import StringEscape if sys.version_info >= (3, 0): diff --git a/tools/rdm/TestHelpersTest.py b/tools/rdm/TestHelpersTest.py index ba28d242bd..150841813b 100644 --- a/tools/rdm/TestHelpersTest.py +++ b/tools/rdm/TestHelpersTest.py @@ -17,6 +17,7 @@ # Copyright (C) 2021 Peter Newman import unittest + from TestHelpers import ContainsUnprintable """Test cases for TestHelpers utilities.""" diff --git a/tools/rdm/TestLogger.py b/tools/rdm/TestLogger.py index 17dd716350..dbbae91ff2 100644 --- a/tools/rdm/TestLogger.py +++ b/tools/rdm/TestLogger.py @@ -20,10 +20,10 @@ import pickle import re -from ola import Version from ola.StringUtils import StringEscape from ola.testing.rdm.TestState import TestState +from ola import Version __author__ = 'nomis52@gmail.com (Simon Newton)' diff --git a/tools/rdm/TestMixins.py b/tools/rdm/TestMixins.py index 90384592e3..117e96cc0f 100644 --- a/tools/rdm/TestMixins.py +++ b/tools/rdm/TestMixins.py @@ -16,13 +16,10 @@ # Copyright (C) 2010 Simon Newton import struct -from ExpectedResults import (AckGetResult, AckDiscoveryResult, BroadcastResult, + +from ExpectedResults import (AckDiscoveryResult, AckGetResult, BroadcastResult, DUBResult, NackSetResult, TimeoutResult, UnsupportedResult) -from ResponderTest import ResponderTestFixture -from TestCategory import TestCategory -from TestHelpers import ContainsUnprintable -from ola import PidStore from ola.DMXConstants import DMX_UNIVERSE_SIZE from ola.DUBDecoder import DecodeResponse from ola.OlaClient import OlaClient, RDMNack @@ -30,6 +27,11 @@ from ola.RDMConstants import RDM_MAX_STRING_LENGTH from ola.StringUtils import StringEscape from ola.UID import UID +from ResponderTest import ResponderTestFixture +from TestCategory import TestCategory +from TestHelpers import ContainsUnprintable + +from ola import PidStore '''Mixins used by the test definitions. diff --git a/tools/rdm/TestRunner.py b/tools/rdm/TestRunner.py index 18f5db7170..5b28435812 100644 --- a/tools/rdm/TestRunner.py +++ b/tools/rdm/TestRunner.py @@ -19,10 +19,12 @@ import inspect import logging import time -from TimingStats import TimingStats -from ola import PidStore + from ola.OlaClient import OlaClient, RDMNack from ola.RDMAPI import RDMAPI +from TimingStats import TimingStats + +from ola import PidStore __author__ = 'nomis52@gmail.com (Simon Newton)' diff --git a/tools/rdm/TestRunnerTest.py b/tools/rdm/TestRunnerTest.py index 7b6de3de63..70c73a0f85 100644 --- a/tools/rdm/TestRunnerTest.py +++ b/tools/rdm/TestRunnerTest.py @@ -17,10 +17,10 @@ # Copyright (C) 2022 Peter Newman import unittest -from ola.testing.rdm import TestDefinitions, TestRunner -from ola.testing.rdm import ResponderTest -from ola.testing.rdm.ResponderTest import OptionalParameterTestFixture, \ - ResponderTestFixture, TestFixture + +from ola.testing.rdm import ResponderTest, TestDefinitions, TestRunner +from ola.testing.rdm.ResponderTest import (OptionalParameterTestFixture, + ResponderTestFixture, TestFixture) from ola.testing.rdm.TestDefinitions import GetDeviceInfo """Test cases for TestRunner utilities.""" diff --git a/tools/rdm/TestStateTest.py b/tools/rdm/TestStateTest.py index 636d548306..9e3aa26962 100644 --- a/tools/rdm/TestStateTest.py +++ b/tools/rdm/TestStateTest.py @@ -17,8 +17,9 @@ # Copyright (C) 2019 Bruce Lowekamp import unittest + +from ola.TestUtils import allHashNotEqual, allNotEqual from TestState import TestState -from ola.TestUtils import allNotEqual, allHashNotEqual """Test cases for sorting TestState.""" diff --git a/tools/rdm/TimingStats.py b/tools/rdm/TimingStats.py index 1fcc0bbf42..67974b6de2 100644 --- a/tools/rdm/TimingStats.py +++ b/tools/rdm/TimingStats.py @@ -16,6 +16,7 @@ # Copyright (C) 2015 Simon Newton import logging + import numpy from ola.OlaClient import OlaClient diff --git a/tools/rdm/launch_tests.py b/tools/rdm/launch_tests.py index 2ef9aacc93..b1b8a16405 100755 --- a/tools/rdm/launch_tests.py +++ b/tools/rdm/launch_tests.py @@ -17,10 +17,9 @@ # Copyright (C) 2012 Simon Newton from __future__ import print_function -from optparse import OptionParser + import logging import os -import setup_patch # The Port Autopatcher import shutil import signal import subprocess @@ -28,6 +27,9 @@ import tempfile import textwrap import time +from optparse import OptionParser + +import setup_patch # The Port Autopatcher """ Launch the OLA RDM test environment. diff --git a/tools/rdm/rdm_model_collector.py b/tools/rdm/rdm_model_collector.py index f9d6b7ca74..c37e1bbd4e 100755 --- a/tools/rdm/rdm_model_collector.py +++ b/tools/rdm/rdm_model_collector.py @@ -17,14 +17,17 @@ # Copyright (C) 2011 Simon Newton from __future__ import print_function + import getopt import logging import pprint import sys import textwrap -from ola import PidStore -from ola.testing.rdm.ModelCollector import ModelCollector + from ola.ClientWrapper import ClientWrapper +from ola.testing.rdm.ModelCollector import ModelCollector + +from ola import PidStore '''Quick script to collect information about responders.''' diff --git a/tools/rdm/rdm_responder_test.py b/tools/rdm/rdm_responder_test.py index aa310d87ea..6433499d04 100755 --- a/tools/rdm/rdm_responder_test.py +++ b/tools/rdm/rdm_responder_test.py @@ -17,22 +17,24 @@ # Copyright (C) 2010 Simon Newton from __future__ import print_function -from ola.testing.rdm import TestDefinitions, TestRunner -from ola.testing.rdm.DMXSender import DMXSender -from ola.testing.rdm.TestState import TestState -from ola.testing.rdm.TimingStats import TimingStats -from ola.StringUtils import StringEscape + import datetime import logging import re import sys import textwrap import time -from ola import PidStore -from ola import Version +from optparse import OptionParser + from ola.ClientWrapper import ClientWrapper +from ola.StringUtils import StringEscape +from ola.testing.rdm import TestDefinitions, TestRunner +from ola.testing.rdm.DMXSender import DMXSender +from ola.testing.rdm.TestState import TestState +from ola.testing.rdm.TimingStats import TimingStats from ola.UID import UID -from optparse import OptionParser + +from ola import PidStore, Version if sys.version_info >= (3, 0): try: diff --git a/tools/rdm/rdm_test_server.py b/tools/rdm/rdm_test_server.py index 6faaa2e91d..742768d991 100755 --- a/tools/rdm/rdm_test_server.py +++ b/tools/rdm/rdm_test_server.py @@ -17,6 +17,7 @@ # Copyright (C) 2012 Ravindra Nath Kakarla & Simon Newton from __future__ import print_function + import json import logging import mimetypes @@ -27,23 +28,22 @@ import sys import textwrap import traceback - from datetime import datetime from optparse import OptionParser from threading import Condition, Event, Lock, Thread from time import time from wsgiref.simple_server import make_server -from ola.UID import UID + from ola.ClientWrapper import ClientWrapper, SelectServer from ola.OlaClient import OlaClient, OLADNotRunningException -from ola import PidStore +from ola.testing.rdm import (DataLocation, TestDefinitions, TestLogger, + TestRunner) from ola.testing.rdm.DMXSender import DMXSender -from ola.testing.rdm import DataLocation -from ola.testing.rdm import TestDefinitions -from ola.testing.rdm import TestLogger -from ola.testing.rdm import TestRunner from ola.testing.rdm.ModelCollector import ModelCollector from ola.testing.rdm.TestState import TestState +from ola.UID import UID + +from ola import PidStore try: import urllib.parse as urlparse diff --git a/tools/rdm/setup_patch.py b/tools/rdm/setup_patch.py index 2b0c6c6a38..051505508b 100755 --- a/tools/rdm/setup_patch.py +++ b/tools/rdm/setup_patch.py @@ -17,7 +17,9 @@ # Copyright (C) 2012 Simon Newton from __future__ import print_function + import logging + from ola.ClientWrapper import ClientWrapper from ola.OlaClient import OlaClient, Plugin From 5ba161547204b3964bb8c8eb5afafe2c69199af6 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sun, 27 Nov 2022 15:30:52 +0000 Subject: [PATCH 51/67] Switch to fully qualified imports for Python Python 3 treats TestState and ola.testing.rdm.TestState as different and not instances of each other which breaks some functionality, so import them all fully qualified to avoid that. --- tools/rdm/ExpectedResultsTest.py | 6 +++--- tools/rdm/ResponderTest.py | 11 ++++++----- tools/rdm/TestDefinitions.py | 19 ++++++++++--------- tools/rdm/TestHelpersTest.py | 2 +- tools/rdm/TestMixins.py | 13 +++++++------ tools/rdm/TestRunner.py | 2 +- tools/rdm/TestStateTest.py | 2 +- 7 files changed, 29 insertions(+), 26 deletions(-) diff --git a/tools/rdm/ExpectedResultsTest.py b/tools/rdm/ExpectedResultsTest.py index 27b6512853..fd01cbadb2 100644 --- a/tools/rdm/ExpectedResultsTest.py +++ b/tools/rdm/ExpectedResultsTest.py @@ -19,10 +19,10 @@ import unittest from collections import namedtuple -from ExpectedResults import (BroadcastResult, DUBResult, InvalidResponse, - SuccessfulResult, TimeoutResult, - UnsupportedResult) from ola.OlaClient import OlaClient +from ola.testing.rdm.ExpectedResults import (BroadcastResult, DUBResult, + InvalidResponse, SuccessfulResult, + TimeoutResult, UnsupportedResult) """Test cases for ExpectedResults classes.""" diff --git a/tools/rdm/ResponderTest.py b/tools/rdm/ResponderTest.py index e51a076fe3..93490306ff 100644 --- a/tools/rdm/ResponderTest.py +++ b/tools/rdm/ResponderTest.py @@ -28,13 +28,14 @@ import sys import time -from ExpectedResults import (AckDiscoveryResult, AckGetResult, AckSetResult, - NackDiscoveryResult, NackGetResult, NackSetResult) from ola.OlaClient import OlaClient, RDMNack from ola.StringUtils import StringEscape -from TestCategory import TestCategory -from TestState import TestState -from TimingStats import TimingStats +from ola.testing.rdm.ExpectedResults import (AckDiscoveryResult, AckGetResult, + AckSetResult, NackDiscoveryResult, + NackGetResult, NackSetResult) +from ola.testing.rdm.TestCategory import TestCategory +from ola.testing.rdm.TestState import TestState +from ola.testing.rdm.TimingStats import TimingStats from ola import PidStore diff --git a/tools/rdm/TestDefinitions.py b/tools/rdm/TestDefinitions.py index ae00954ef8..69ad416e9c 100644 --- a/tools/rdm/TestDefinitions.py +++ b/tools/rdm/TestDefinitions.py @@ -19,10 +19,6 @@ import operator import struct -import TestMixins -from ExpectedResults import (RDM_GET, RDM_SET, AckGetResult, BroadcastResult, - InvalidResponse, NackGetResult, TimeoutResult, - UnsupportedResult) from ola.OlaClient import OlaClient, RDMNack from ola.PidStore import ROOT_DEVICE from ola.RDMConstants import (INTERFACE_HARDWARE_TYPE_ETHERNET, @@ -34,12 +30,17 @@ RDM_MAX_HOSTNAME_LENGTH, RDM_MIN_HOSTNAME_LENGTH, RDM_ZERO_FOOTPRINT_DMX_ADDRESS) from ola.StringUtils import StringEscape +from ola.testing.rdm import TestMixins +from ola.testing.rdm.ExpectedResults import (RDM_GET, RDM_SET, AckGetResult, + BroadcastResult, InvalidResponse, + NackGetResult, TimeoutResult, + UnsupportedResult) +from ola.testing.rdm.ResponderTest import (OptionalParameterTestFixture, + ResponderTestFixture, TestFixture) +from ola.testing.rdm.TestCategory import TestCategory +from ola.testing.rdm.TestHelpers import ContainsUnprintable +from ola.testing.rdm.TestMixins import MAX_DMX_ADDRESS from ola.UID import UID -from ResponderTest import (OptionalParameterTestFixture, ResponderTestFixture, - TestFixture) -from TestCategory import TestCategory -from TestHelpers import ContainsUnprintable -from TestMixins import MAX_DMX_ADDRESS from ola import PidStore, RDMConstants diff --git a/tools/rdm/TestHelpersTest.py b/tools/rdm/TestHelpersTest.py index 150841813b..856cf27d98 100644 --- a/tools/rdm/TestHelpersTest.py +++ b/tools/rdm/TestHelpersTest.py @@ -18,7 +18,7 @@ import unittest -from TestHelpers import ContainsUnprintable +from ola.testing.rdm.TestHelpers import ContainsUnprintable """Test cases for TestHelpers utilities.""" diff --git a/tools/rdm/TestMixins.py b/tools/rdm/TestMixins.py index 117e96cc0f..1762713935 100644 --- a/tools/rdm/TestMixins.py +++ b/tools/rdm/TestMixins.py @@ -17,19 +17,20 @@ import struct -from ExpectedResults import (AckDiscoveryResult, AckGetResult, BroadcastResult, - DUBResult, NackSetResult, TimeoutResult, - UnsupportedResult) from ola.DMXConstants import DMX_UNIVERSE_SIZE from ola.DUBDecoder import DecodeResponse from ola.OlaClient import OlaClient, RDMNack from ola.PidStore import ROOT_DEVICE from ola.RDMConstants import RDM_MAX_STRING_LENGTH from ola.StringUtils import StringEscape +from ola.testing.rdm.ExpectedResults import (AckDiscoveryResult, AckGetResult, + BroadcastResult, DUBResult, + NackSetResult, TimeoutResult, + UnsupportedResult) +from ola.testing.rdm.ResponderTest import ResponderTestFixture +from ola.testing.rdm.TestCategory import TestCategory +from ola.testing.rdm.TestHelpers import ContainsUnprintable from ola.UID import UID -from ResponderTest import ResponderTestFixture -from TestCategory import TestCategory -from TestHelpers import ContainsUnprintable from ola import PidStore diff --git a/tools/rdm/TestRunner.py b/tools/rdm/TestRunner.py index 5b28435812..fcf53ff3f5 100644 --- a/tools/rdm/TestRunner.py +++ b/tools/rdm/TestRunner.py @@ -22,7 +22,7 @@ from ola.OlaClient import OlaClient, RDMNack from ola.RDMAPI import RDMAPI -from TimingStats import TimingStats +from ola.testing.rdm.TimingStats import TimingStats from ola import PidStore diff --git a/tools/rdm/TestStateTest.py b/tools/rdm/TestStateTest.py index 9e3aa26962..f57e005444 100644 --- a/tools/rdm/TestStateTest.py +++ b/tools/rdm/TestStateTest.py @@ -18,8 +18,8 @@ import unittest +from ola.testing.rdm.TestState import TestState from ola.TestUtils import allHashNotEqual, allNotEqual -from TestState import TestState """Test cases for sorting TestState.""" From 9f50218033d9905ba73d69045f0d1e9b0ffdbfb7 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sun, 27 Nov 2022 15:33:10 +0000 Subject: [PATCH 52/67] Keep GitHub Actions up to date with Dependabot --- .github/dependabot.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..fd83b118ad --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +# See: https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#about-the-dependabotyml-file +version: 2 + +updates: + # Configure check for outdated GitHub Actions actions in workflows. + # See: https://docs.github.com/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot + - package-ecosystem: github-actions + directory: / # Check the repository's workflows under /.github/workflows/ + schedule: + interval: daily From 46476ee76cb86ae3a38bb9593431bf4e2a596a65 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sun, 27 Nov 2022 15:47:07 +0000 Subject: [PATCH 53/67] Simplify the test discovery code now we've changed the imports and got to the bottom of the issue --- tools/rdm/ResponderTestTest.py | 2 +- tools/rdm/TestRunner.py | 12 ++++++------ tools/rdm/TestRunnerTest.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/rdm/ResponderTestTest.py b/tools/rdm/ResponderTestTest.py index 0caa499c3a..96fd2fa949 100644 --- a/tools/rdm/ResponderTestTest.py +++ b/tools/rdm/ResponderTestTest.py @@ -18,7 +18,7 @@ import unittest -from ResponderTest import ResponderTestFixture, TestFixture +from ola.testing.rdm.ResponderTest import ResponderTestFixture, TestFixture """Test cases for sorting TestFixtures.""" diff --git a/tools/rdm/TestRunner.py b/tools/rdm/TestRunner.py index fcf53ff3f5..c1bbd0fef2 100644 --- a/tools/rdm/TestRunner.py +++ b/tools/rdm/TestRunner.py @@ -22,6 +22,8 @@ from ola.OlaClient import OlaClient, RDMNack from ola.RDMAPI import RDMAPI +from ola.testing.rdm.ResponderTest import (OptionalParameterTestFixture, + ResponderTestFixture, TestFixture) from ola.testing.rdm.TimingStats import TimingStats from ola import PidStore @@ -195,12 +197,10 @@ def GetTestClasses(module): cls = getattr(module, symbol) if not inspect.isclass(cls): continue - # Test for dynamic versions of these due to issues with Python 3 base_classes = [ - # Original - getattr(module, "OptionalParameterTestFixture"), - getattr(module, "ResponderTestFixture"), - getattr(module, "TestFixture") + OptionalParameterTestFixture, + ResponderTestFixture, + TestFixture ] if cls in base_classes: @@ -208,7 +208,7 @@ def GetTestClasses(module): # This seems to confuse Python 3 if we compare it to # ResponderTest.TestFixture, some sort of diamond inheritance issue? # So test for the dynamic version of it instead - if issubclass(cls, getattr(module, "TestFixture")): + if issubclass(cls, TestFixture): classes.append(cls) return classes diff --git a/tools/rdm/TestRunnerTest.py b/tools/rdm/TestRunnerTest.py index 70c73a0f85..436e1278f0 100644 --- a/tools/rdm/TestRunnerTest.py +++ b/tools/rdm/TestRunnerTest.py @@ -39,7 +39,7 @@ def testGetTestClasses(self): TestRunner.GetTestClasses(TestDefinitions), "GetDeviceInfo missing from list of test classes") # Check we don't contain the base classes: - # Test for various versions of them due to issues with Python 3 + # Test for various versions of them due to potential issues with Python 3 # The static versions are probably a better test for the test run for classname in [OptionalParameterTestFixture, ResponderTestFixture, From d4cf464bfa2aaa7c4394394a4818a5fbb85030cd Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sun, 27 Nov 2022 16:01:25 +0000 Subject: [PATCH 54/67] Fix the licence for the new test --- python/ola/StringUtilsTest.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/python/ola/StringUtilsTest.py b/python/ola/StringUtilsTest.py index 3d1b95df02..b7191111c1 100755 --- a/python/ola/StringUtilsTest.py +++ b/python/ola/StringUtilsTest.py @@ -1,17 +1,17 @@ #!/usr/bin/env python -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. # -# This program is distributed in the hope that it will be useful, +# This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA # # StringUtilsTest.py # Copyright (C) 2022 Peter Newman From 05a59450ad43f1b2d2dfde991ad528f85ce58bc3 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sun, 27 Nov 2022 16:05:18 +0000 Subject: [PATCH 55/67] Symlink more files required for the distcheck build --- configure.ac | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configure.ac b/configure.ac index 980204648b..8245a44fb1 100644 --- a/configure.ac +++ b/configure.ac @@ -980,6 +980,7 @@ AC_CONFIG_LINKS([python/ola/__init__.py:python/ola/__init__.py python/ola/PidStore.py:python/ola/PidStore.py python/ola/RDMAPI.py:python/ola/RDMAPI.py python/ola/RDMConstants.py:python/ola/RDMConstants.py + python/ola/StringUtils.py:python/ola/StringUtils.py python/ola/TestUtils.py:python/ola/TestUtils.py python/ola/UID.py:python/ola/UID.py python/ola/rpc/__init__.py:python/ola/rpc/__init__.py @@ -990,6 +991,7 @@ AC_CONFIG_LINKS([python/ola/__init__.py:python/ola/__init__.py tools/rdm/ResponderTest.py:tools/rdm/ResponderTest.py tools/rdm/TestCategory.py:tools/rdm/TestCategory.py tools/rdm/TestDefinitions.py:tools/rdm/TestDefinitions.py + tools/rdm/TestHelpers.py:tools/rdm/TestHelpers.py tools/rdm/TestRunner.py:tools/rdm/TestRunner.py tools/rdm/TestState.py:tools/rdm/TestState.py tools/rdm/TimingStats.py:tools/rdm/TimingStats.py]) From a95d4fafba1c3d31ab743d40d286e929b286a67a Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sun, 27 Nov 2022 16:19:05 +0000 Subject: [PATCH 56/67] Hopefully add the last hacky symlink --- configure.ac | 1 + 1 file changed, 1 insertion(+) diff --git a/configure.ac b/configure.ac index 8245a44fb1..c2703567f3 100644 --- a/configure.ac +++ b/configure.ac @@ -992,6 +992,7 @@ AC_CONFIG_LINKS([python/ola/__init__.py:python/ola/__init__.py tools/rdm/TestCategory.py:tools/rdm/TestCategory.py tools/rdm/TestDefinitions.py:tools/rdm/TestDefinitions.py tools/rdm/TestHelpers.py:tools/rdm/TestHelpers.py + tools/rdm/TestMixins.py:tools/rdm/TestMixins.py tools/rdm/TestRunner.py:tools/rdm/TestRunner.py tools/rdm/TestState.py:tools/rdm/TestState.py tools/rdm/TimingStats.py:tools/rdm/TimingStats.py]) From dfe877f9e9aa22aa23e32438c0eda600e3d1940a Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sun, 27 Nov 2022 18:53:09 +0000 Subject: [PATCH 57/67] Simplify the ContainsUnprintable logic --- tools/rdm/TestHelpers.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tools/rdm/TestHelpers.py b/tools/rdm/TestHelpers.py index e07934b224..4e35c22230 100644 --- a/tools/rdm/TestHelpers.py +++ b/tools/rdm/TestHelpers.py @@ -31,11 +31,8 @@ def ContainsUnprintable(s): """Check if a string s contain unprintable characters.""" # TODO(Peter): How does this interact with the E1.20 Unicode flag? - # We don't use sys.version_info.major to support Python 2.6. - if sys.version_info[0] == 2 and (type(s) == str or type(s) == unicode): - return s != StringEscape(s) - elif type(s) == str: - # All strings in Python 3 are unicode + if type(s) == str or type(s) == unicode: + # All strings in Python 3 are unicode, Python 2 ones might not be return s != StringEscape(s) else: raise TypeError('Only strings are supported') From b69d42470cfb4a76db625af15651882a5613f010 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sun, 27 Nov 2022 18:57:12 +0000 Subject: [PATCH 58/67] Fix a copyright year --- python/ola/StringUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ola/StringUtils.py b/python/ola/StringUtils.py index ffb053bcf8..a0f51cfafd 100644 --- a/python/ola/StringUtils.py +++ b/python/ola/StringUtils.py @@ -13,7 +13,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA # # StringUtils.py -# Copyright (C) 2013 Peter Newman +# Copyright (C) 2022 Peter Newman """Common utils for OLA Python string handling""" From 84ffa627c3a9b8822096af31a3d78882f71020cb Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sun, 27 Nov 2022 19:23:55 +0000 Subject: [PATCH 59/67] Improve some comments and exception messages --- python/ola/PidStoreTest.py | 2 ++ tools/rdm/TestHelpers.py | 2 +- tools/rdm/TestRunner.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python/ola/PidStoreTest.py b/python/ola/PidStoreTest.py index 13deb782eb..600f1dd389 100755 --- a/python/ola/PidStoreTest.py +++ b/python/ola/PidStoreTest.py @@ -232,6 +232,8 @@ def testPackUnpack(self): self.assertEqual(decoded['slots_required'], 7) self.assertEqual(decoded['name'], "Foo") + # Confirm we raise an error if we try and unpack a non-ASCII, non-UTF-8 + # containing packet (0xc0) with self.assertRaises(PidStore.UnpackException): blob = binascii.unhexlify("2a0007556e7061636bc054657374") decoded = pid.Unpack(blob, PidStore.RDM_GET) diff --git a/tools/rdm/TestHelpers.py b/tools/rdm/TestHelpers.py index 4e35c22230..51eb606de7 100644 --- a/tools/rdm/TestHelpers.py +++ b/tools/rdm/TestHelpers.py @@ -35,4 +35,4 @@ def ContainsUnprintable(s): # All strings in Python 3 are unicode, Python 2 ones might not be return s != StringEscape(s) else: - raise TypeError('Only strings are supported') + raise TypeError('Only strings are supported not %s' % type(s)) diff --git a/tools/rdm/TestRunner.py b/tools/rdm/TestRunner.py index c1bbd0fef2..b4399884e4 100644 --- a/tools/rdm/TestRunner.py +++ b/tools/rdm/TestRunner.py @@ -207,7 +207,7 @@ def GetTestClasses(module): continue # This seems to confuse Python 3 if we compare it to # ResponderTest.TestFixture, some sort of diamond inheritance issue? - # So test for the dynamic version of it instead + # So test for the base version of it instead if issubclass(cls, TestFixture): classes.append(cls) return classes From 85da44b05168835818fcd958eb264b20377fb305 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sun, 27 Nov 2022 20:52:39 +0000 Subject: [PATCH 60/67] Simplify some of the RDM unit testing --- tools/rdm/ExpectedResultsTest.py | 8 +++++--- tools/rdm/Makefile.mk | 5 ++++- tools/rdm/TestHelpersTest.py | 3 ++- tools/rdm/TestStateTest.py | 4 +++- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tools/rdm/ExpectedResultsTest.py b/tools/rdm/ExpectedResultsTest.py index fd01cbadb2..6b034ef345 100644 --- a/tools/rdm/ExpectedResultsTest.py +++ b/tools/rdm/ExpectedResultsTest.py @@ -20,9 +20,11 @@ from collections import namedtuple from ola.OlaClient import OlaClient -from ola.testing.rdm.ExpectedResults import (BroadcastResult, DUBResult, - InvalidResponse, SuccessfulResult, - TimeoutResult, UnsupportedResult) + +# Keep this import relative to simplify the testing +from ExpectedResults import (BroadcastResult, DUBResult, InvalidResponse, + SuccessfulResult, TimeoutResult, + UnsupportedResult) """Test cases for ExpectedResults classes.""" diff --git a/tools/rdm/Makefile.mk b/tools/rdm/Makefile.mk index e0a330d4d4..afc6b72ba6 100644 --- a/tools/rdm/Makefile.mk +++ b/tools/rdm/Makefile.mk @@ -61,7 +61,10 @@ tools/rdm/ExpectedResultsTest.sh: tools/rdm/Makefile.mk chmod +x $(top_builddir)/tools/rdm/ExpectedResultsTest.sh tools/rdm/ResponderTestTest.sh: tools/rdm/Makefile.mk - mkdir -p $(top_builddir)/python/ola + mkdir -p $(top_builddir)/python/ola/testing + touch $(top_builddir)/python/ola/testing/__init__.py + # This link is relative within the builddir + $(LN_S) -f ../../../tools/rdm $(top_builddir)/python/ola/testing/rdm echo "PYTHONPATH=${top_builddir}/python $(PYTHON) ${srcdir}/tools/rdm/ResponderTestTest.py; exit \$$?" > $(top_builddir)/tools/rdm/ResponderTestTest.sh chmod +x $(top_builddir)/tools/rdm/ResponderTestTest.sh diff --git a/tools/rdm/TestHelpersTest.py b/tools/rdm/TestHelpersTest.py index 856cf27d98..f02805577a 100644 --- a/tools/rdm/TestHelpersTest.py +++ b/tools/rdm/TestHelpersTest.py @@ -18,7 +18,8 @@ import unittest -from ola.testing.rdm.TestHelpers import ContainsUnprintable +# Keep this import relative to simplify the testing +from TestHelpers import ContainsUnprintable """Test cases for TestHelpers utilities.""" diff --git a/tools/rdm/TestStateTest.py b/tools/rdm/TestStateTest.py index f57e005444..dfb1885dd5 100644 --- a/tools/rdm/TestStateTest.py +++ b/tools/rdm/TestStateTest.py @@ -18,9 +18,11 @@ import unittest -from ola.testing.rdm.TestState import TestState from ola.TestUtils import allHashNotEqual, allNotEqual +# Keep this import relative to simplify the testing +from TestState import TestState + """Test cases for sorting TestState.""" __author__ = 'bruce@lowekamp.net (Bruce Lowekamp)' From 1ca6d300fdcabe781c49f96ca06b0018da3cb480 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sun, 27 Nov 2022 20:59:50 +0000 Subject: [PATCH 61/67] Remove some errant whitespace --- tools/rdm/TestDefinitions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/rdm/TestDefinitions.py b/tools/rdm/TestDefinitions.py index 69ad416e9c..04088a38bf 100644 --- a/tools/rdm/TestDefinitions.py +++ b/tools/rdm/TestDefinitions.py @@ -3554,7 +3554,6 @@ class AllSubDevicesGetDisplayLevel(TestMixins.AllSubDevicesGetMixin, class GetPanInvert(TestMixins.GetMixin, OptionalParameterTestFixture): """GET the pan invert setting.""" CATEGORY = TestCategory.CONFIGURATION - PID = 'PAN_INVERT' EXPECTED_FIELDS = ['invert'] PROVIDES = ['pan_invert'] From eab7a44baf47060678460ab9e342d7e2e857720b Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sun, 27 Nov 2022 21:34:16 +0000 Subject: [PATCH 62/67] Sort the imports as isort likes --- tools/rdm/ExpectedResultsTest.py | 3 +-- tools/rdm/TestStateTest.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/rdm/ExpectedResultsTest.py b/tools/rdm/ExpectedResultsTest.py index 6b034ef345..9fb1ab997a 100644 --- a/tools/rdm/ExpectedResultsTest.py +++ b/tools/rdm/ExpectedResultsTest.py @@ -19,12 +19,11 @@ import unittest from collections import namedtuple -from ola.OlaClient import OlaClient - # Keep this import relative to simplify the testing from ExpectedResults import (BroadcastResult, DUBResult, InvalidResponse, SuccessfulResult, TimeoutResult, UnsupportedResult) +from ola.OlaClient import OlaClient """Test cases for ExpectedResults classes.""" diff --git a/tools/rdm/TestStateTest.py b/tools/rdm/TestStateTest.py index dfb1885dd5..c2b37a149a 100644 --- a/tools/rdm/TestStateTest.py +++ b/tools/rdm/TestStateTest.py @@ -19,7 +19,6 @@ import unittest from ola.TestUtils import allHashNotEqual, allNotEqual - # Keep this import relative to simplify the testing from TestState import TestState From 45145ba6e9ddfda5a0ed8c92b9dfefbb5ca29313 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Tue, 14 Feb 2023 16:57:59 +0000 Subject: [PATCH 63/67] Comment out the SetNonAsciiDeviceLabel test for now as it breaks when we try to decode its invalid ASCII and UTF-8 values, a TODO for another day... --- tools/rdm/TestDefinitions.py | 57 +++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/tools/rdm/TestDefinitions.py b/tools/rdm/TestDefinitions.py index 04088a38bf..456b6b3aea 100644 --- a/tools/rdm/TestDefinitions.py +++ b/tools/rdm/TestDefinitions.py @@ -1471,33 +1471,36 @@ def OldValue(self): return self.Property('device_label') -class SetNonAsciiDeviceLabel(TestMixins.SetLabelMixin, - OptionalParameterTestFixture): - """SET the device label to something that contains non-ASCII data.""" - CATEGORY = TestCategory.PRODUCT_INFORMATION - PID = 'DEVICE_LABEL' - REQUIRES = ['device_label'] - # Store directly as bytes so we don't try and decode as UTF-8 - TEST_LABEL = b'string with\x0d non ASCII\xc0' - - def ExpectedResults(self): - return [ - self.NackSetResult(RDMNack.NR_DATA_OUT_OF_RANGE), - self.NackSetResult(RDMNack.NR_FORMAT_ERROR), - self.NackSetResult(RDMNack.NR_UNSUPPORTED_COMMAND_CLASS), - self.AckSetResult(action=self.VerifySet) - ] - - def OldValue(self): - return self.Property('device_label') - - def Test(self): - # We have to override test here as this has to be raw as we can't encode it - # on Python 3 as it turns it to UTF-8 or escapes it - # It's also technically out of spec for E1.20 unless it's sent as UTF-8 - self._test_state = self.SET - self.AddIfSetSupported(self.ExpectedResults()) - self.SendRawSet(PidStore.ROOT_DEVICE, self.pid, self.TEST_LABEL) +# TODO(Peter): Get this test to work where we just compare the returned string +# as bytes so we don't try and fail to decode it as ASCII/UTF-8 +# class SetNonAsciiDeviceLabel(TestMixins.SetLabelMixin, +# OptionalParameterTestFixture): +# """SET the device label to something that contains non-ASCII data.""" +# CATEGORY = TestCategory.PRODUCT_INFORMATION +# PID = 'DEVICE_LABEL' +# REQUIRES = ['device_label'] +# # Store directly as bytes so we don't try and decode as UTF-8 +# TEST_LABEL = b'string with\x0d non ASCII\xc0' +# +# def ExpectedResults(self): +# return [ +# self.NackSetResult(RDMNack.NR_DATA_OUT_OF_RANGE), +# self.NackSetResult(RDMNack.NR_FORMAT_ERROR), +# self.NackSetResult(RDMNack.NR_UNSUPPORTED_COMMAND_CLASS), +# self.AckSetResult(action=self.VerifySet) +# ] +# +# def OldValue(self): +# return self.Property('device_label') +# +# def Test(self): +# # We have to override test here as this has to be raw as we can't encode +# # it +# # on Python 3 as it turns it to UTF-8 or escapes it +# # It's also technically out of spec for E1.20 unless it's sent as UTF-8 +# self._test_state = self.SET +# self.AddIfSetSupported(self.ExpectedResults()) +# self.SendRawSet(PidStore.ROOT_DEVICE, self.pid, self.TEST_LABEL) class SetEmptyDeviceLabel(TestMixins.SetLabelMixin, From c41589de0b67015f5d94eafb061eba511801ef69 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sat, 25 Feb 2023 12:37:06 +0000 Subject: [PATCH 64/67] Revert "Remove some statements that won't be true until another PR has been merged" This reverts commit 0328b639a2ec3fa2af0036dab74ac0ec7818a943. --- NEWS | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index e0d0e7706c..8011a16f95 100644 --- a/NEWS +++ b/NEWS @@ -1,12 +1,14 @@ x/y/2023 ola-0.10.9 Features: - * Further improvements on Python 3 compatibility #1506 + * Python 3 compatibility across the board (including the RDM Responder Tests)! + #1506 * Support for the JMS USB2DMX PRO V2.1 device via the FTDI plugin #1728 API: * Python: Add a fetch current DMX example. RDM Tests: + * Python 3 compatibility of the RDM Tests #1599 * Fix a longstanding bug in the GetMaxPacketSize RDM test around timeouts Bugs: From 7a55db7277a69f72ff6a66a820122e8b9e97bb25 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sat, 25 Feb 2023 12:39:18 +0000 Subject: [PATCH 65/67] Remove some duplicate codespell ignore lines --- .codespellignorelines | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.codespellignorelines b/.codespellignorelines index 460b8f21b0..a1834989ca 100644 --- a/.codespellignorelines +++ b/.codespellignorelines @@ -17,7 +17,6 @@ /(?:([0-9]{1,3})(?:\s+THRU\s+([0-9]{0,3}))?)(?:\s+@\s+([0-9]{0,3}))?$/); str = str.replace('>', 'THRU'); ' THRU ' + ola.common.DmxConstants.MAX_CHANNEL_NUMBER); - ' THRU ' + ola.common.DmxConstants.MAX_CHANNEL_NUMBER); // If it's the T or > keys, autocomplete 'THRU' case 'U': // THRU var values = ['7', '8', '9', ' THRU ', '4', '5', '6', ' @ ', '1', '2', '3', @@ -118,9 +117,6 @@ class AsyncronousLibUsbAdaptor : public BaseLibUsbAdaptor { OLA_ASSERT_EQ(expected, JsonWriter::AsString(uint_value)); * Test the uint item " \"type\": \"uint\",\n" - " \"type\": \"uint\",\n" - " \"type\": \"uint\",\n" - " \"type\": \"uint\",\n" std::map m_uint_map_variables; if (message.uint_offset < MAX_UINT_FIELDS) { message.uint16_fields[message.uint_offset++] = field->Value(); @@ -131,10 +127,8 @@ class AsyncronousLibUsbAdaptor : public BaseLibUsbAdaptor { status_message() : uint_offset(0), int_offset(0), status_type(0), std::string Type() const { return "uint"; } if (items[i]['type'] == 'uint') { - if (items[i]['type'] == 'uint') { if (type == 'string' || type == 'uint' || type == 'hidden') { const char RDMHTTPModule::GENERIC_UINT_FIELD[] = "int"; - section.AddItem(new HiddenItem("1", GENERIC_UINT_FIELD)); section.AddItem(new HiddenItem("1", GENERIC_UINT_FIELD)); SelectItem *item = new SelectItem("Personality", GENERIC_UINT_FIELD); string personality_str = request->GetParameter(GENERIC_UINT_FIELD); From f9175882d22748253eec1cf81e27be70b43fa1fe Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sat, 25 Feb 2023 13:53:13 +0000 Subject: [PATCH 66/67] Ignore more codespell lines --- .codespellignorelines | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.codespellignorelines b/.codespellignorelines index a1834989ca..10832fd44e 100644 --- a/.codespellignorelines +++ b/.codespellignorelines @@ -192,4 +192,6 @@ import java.nio.ByteOrder; "{'a': 'caf\\\\xe9'}") # self.assertEqual('%s' % rtf._EscapeData({"caf\xe9": "bar"}), # "{'caf\xe9': 'bar'}") + self.assertEqual('caf\\xe9', StringEscape(u'caf\xe9')) + self.assertEqual('caf\\xe9', ("%s" % StringEscape(u'caf\xe9'))) "forin": true, From f5be4e75cc5f32bac60dd7ecd438b5be6c574914 Mon Sep 17 00:00:00 2001 From: Peter Newman Date: Sat, 25 Feb 2023 23:14:32 +0000 Subject: [PATCH 67/67] Fix a comment --- python/ola/PidStore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ola/PidStore.py b/python/ola/PidStore.py index 5a6f1fdec4..3028af709f 100644 --- a/python/ola/PidStore.py +++ b/python/ola/PidStore.py @@ -45,7 +45,7 @@ MAX_VALID_SUB_DEVICE = 0x0200 ALL_SUB_DEVICES = 0xffff -# The two types of commands classes +# The different types of commands classes RDM_GET, RDM_SET, RDM_DISCOVERY = range(3)