From 6c1a7d8f498bc1f640ce33a6a2edef9f4a311045 Mon Sep 17 00:00:00 2001 From: Rob Van Dam Date: Wed, 13 Sep 2023 14:39:10 -0600 Subject: [PATCH 1/3] Allow per item validators to be specified by string --- docs/source/item-validation.rst | 11 +++++++++++ spidermon/contrib/scrapy/pipelines.py | 2 +- tests/contrib/scrapy/test_pipelines.py | 10 ++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/source/item-validation.rst b/docs/source/item-validation.rst index 8f3a0606..4d617a6c 100644 --- a/docs/source/item-validation.rst +++ b/docs/source/item-validation.rst @@ -141,6 +141,17 @@ as a `dict`: OtherItem: '/path/to/otheritem_schema.json', } +Keys of `dict` can also be strings matching item class names: + +.. code-block:: python + + SPIDERMON_VALIDATION_SCHEMAS = { + 'DummyItem': '/path/to/dummyitem_schema.json', + 'OtherItem': '/path/to/otheritem_schema.json', + } + + + Validation in Monitors ---------------------- diff --git a/spidermon/contrib/scrapy/pipelines.py b/spidermon/contrib/scrapy/pipelines.py index d86fcdc6..286f84fb 100644 --- a/spidermon/contrib/scrapy/pipelines.py +++ b/spidermon/contrib/scrapy/pipelines.py @@ -56,7 +56,7 @@ def set_validators(loader, schema): if type(schema) in (list, tuple): schema = {Item: schema} for obj, paths in schema.items(): - key = obj.__name__ + key = obj if type(obj) in (str) else obj.__name__ paths = paths if type(paths) in (list, tuple) else [paths] objects = [loader(v) for v in paths] validators[key].extend(objects) diff --git a/tests/contrib/scrapy/test_pipelines.py b/tests/contrib/scrapy/test_pipelines.py index d2a7ff99..e089dd55 100644 --- a/tests/contrib/scrapy/test_pipelines.py +++ b/tests/contrib/scrapy/test_pipelines.py @@ -203,6 +203,16 @@ class PipelineJSONSchemaValidator(PipelineTest): assert_type_in_stats(TreeItem), ], ), + DataTest( + name="validators specified by str rather than class", + item=TreeItem(), + settings={SETTING_SCHEMAS: {"TestItem": test_schema, "TreeItem": tree_schema}}, + cases=[ + f"{{stats}}['{STATS_MISSINGS}'] is 1", + assert_type_in_stats(TestItem), + assert_type_in_stats(TreeItem), + ], + ), ] From a75e6fbba34db20232ae7290a7f04c324959246e Mon Sep 17 00:00:00 2001 From: Rob Van Dam Date: Thu, 14 Sep 2023 16:06:53 -0600 Subject: [PATCH 2/3] Safer test for class vs string --- spidermon/contrib/scrapy/pipelines.py | 2 +- tests/contrib/scrapy/test_pipelines.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/spidermon/contrib/scrapy/pipelines.py b/spidermon/contrib/scrapy/pipelines.py index 286f84fb..96ad572a 100644 --- a/spidermon/contrib/scrapy/pipelines.py +++ b/spidermon/contrib/scrapy/pipelines.py @@ -56,7 +56,7 @@ def set_validators(loader, schema): if type(schema) in (list, tuple): schema = {Item: schema} for obj, paths in schema.items(): - key = obj if type(obj) in (str) else obj.__name__ + key = obj.__name__ if hasattr(obj, "__name__") else str(obj) paths = paths if type(paths) in (list, tuple) else [paths] objects = [loader(v) for v in paths] validators[key].extend(objects) diff --git a/tests/contrib/scrapy/test_pipelines.py b/tests/contrib/scrapy/test_pipelines.py index e089dd55..896eaa99 100644 --- a/tests/contrib/scrapy/test_pipelines.py +++ b/tests/contrib/scrapy/test_pipelines.py @@ -206,7 +206,9 @@ class PipelineJSONSchemaValidator(PipelineTest): DataTest( name="validators specified by str rather than class", item=TreeItem(), - settings={SETTING_SCHEMAS: {"TestItem": test_schema, "TreeItem": tree_schema}}, + settings={ + SETTING_SCHEMAS: {"TestItem": test_schema, "TreeItem": tree_schema} + }, cases=[ f"{{stats}}['{STATS_MISSINGS}'] is 1", assert_type_in_stats(TestItem), From 5acd876b225e7868f95e1831d1080e90cfdc3fdb Mon Sep 17 00:00:00 2001 From: Rob Van Dam Date: Fri, 15 Sep 2023 08:01:33 -0600 Subject: [PATCH 3/3] Update docs/source/item-validation.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/source/item-validation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/item-validation.rst b/docs/source/item-validation.rst index 4d617a6c..5d3d7c02 100644 --- a/docs/source/item-validation.rst +++ b/docs/source/item-validation.rst @@ -141,7 +141,7 @@ as a `dict`: OtherItem: '/path/to/otheritem_schema.json', } -Keys of `dict` can also be strings matching item class names: +Keys of ``dict`` can also be strings matching item class names: .. code-block:: python