From 6e098898189f6587d1b645607611bdc4f4267495 Mon Sep 17 00:00:00 2001 From: Victor Ruiz Date: Thu, 9 May 2024 12:32:23 +0200 Subject: [PATCH] fix traverse_nested when value is not compatible with ItemAdapter --- spidermon/contrib/utils/attributes.py | 5 ++++- tests/contrib/utils/test_attributes.py | 8 +++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/spidermon/contrib/utils/attributes.py b/spidermon/contrib/utils/attributes.py index 7d3d3a9d..99cd0223 100644 --- a/spidermon/contrib/utils/attributes.py +++ b/spidermon/contrib/utils/attributes.py @@ -1,4 +1,5 @@ from typing import Any, List + from itemadapter import ItemAdapter @@ -14,7 +15,9 @@ def traverse_nested(obj: ItemAdapter, keys: List[str]) -> ItemAdapter: # Traverse next level of item object key = keys.pop(0) current_obj = ItemAdapter(current_obj[key]) - except KeyError: + # KeyError: Key does not exist + # TypeError: Key is not compatible with ItemAdapter (None or unsupported type) + except (KeyError, TypeError): raise KeyError(f'Invalid key "{key}" for {current_obj} in {obj}') return current_obj diff --git a/tests/contrib/utils/test_attributes.py b/tests/contrib/utils/test_attributes.py index cfdf43fd..4bd8c8b3 100644 --- a/tests/contrib/utils/test_attributes.py +++ b/tests/contrib/utils/test_attributes.py @@ -1,7 +1,8 @@ -import pytest from dataclasses import dataclass +import pytest from itemadapter import ItemAdapter + from spidermon.contrib.utils.attributes import ( get_nested_attribute, set_nested_attribute, @@ -19,6 +20,11 @@ def test_get_nested_attribute(): with pytest.raises(KeyError): get_nested_attribute(item, "attr1.missing_attribute.attr2") + # Intermiddle attribute is None + item = ItemAdapter({"foo": None}) + with pytest.raises(KeyError): + get_nested_attribute(item, "foo.missing_attribute") + def test_set_nested_attribute(): item = ItemAdapter({"foo": None, "attr1": {"attr2": {"attr3": None}}})