From fadbea4b17e75767f87094f26dfa2131988eb1aa Mon Sep 17 00:00:00 2001 From: Sheng Yu Date: Tue, 9 Apr 2024 14:39:32 -0400 Subject: [PATCH] feat(grammar)!: generalized grammar type class --- craft_grammar/models.py | 293 ++++++++++++++------------------------ tests/unit/test_models.py | 130 +++-------------- 2 files changed, 127 insertions(+), 296 deletions(-) diff --git a/craft_grammar/models.py b/craft_grammar/models.py index 93f0b2e..57df977 100644 --- a/craft_grammar/models.py +++ b/craft_grammar/models.py @@ -18,7 +18,10 @@ import abc import re -from typing import Any, Dict, List, Union +from typing import Any, Dict, List, get_args, get_origin + +from pydantic import BaseConfig, PydanticTypeError +from pydantic.validators import find_validators from overrides import overrides @@ -33,6 +36,8 @@ _GrammarType = Dict[str, Any] +_CONFIG = BaseConfig() + class _GrammarBase(abc.ABC): @classmethod @@ -53,196 +58,106 @@ def _grammar_append(cls, entry: List, item: Any) -> None: _mark_and_append(entry, {key: cls.validate(value)}) -# Public types for grammar-enabled attributes -class GrammarBool(_GrammarBase): - """Grammar-enabled bool field.""" - - __root__: Union[bool, _GrammarType] - - @classmethod - @overrides - def validate(cls, entry): - # GrammarBool entry can be a list if it contains clauses - if isinstance(entry, list): - new_entry = [] - for item in entry: - if _is_grammar_clause(item): - cls._grammar_append(new_entry, item) - else: - raise TypeError(f"value must be a list of bool: {entry!r}") - return new_entry - - if isinstance(entry, bool): - return entry - - raise TypeError(f"value must be a bool: {entry!r}") - - -class GrammarInt(_GrammarBase): - """Grammar-enabled integer field.""" - - __root__: Union[int, _GrammarType] - - @classmethod - @overrides - def validate(cls, entry): - # GrammarInt entry can be a list if it contains clauses - if isinstance(entry, list): - new_entry = [] - for item in entry: - if _is_grammar_clause(item): - cls._grammar_append(new_entry, item) - else: - raise TypeError(f"value must be a list of integer: {entry!r}") - return new_entry - - if isinstance(entry, int): - return entry - - raise TypeError(f"value must be a integer: {entry!r}") - - -class GrammarFloat(_GrammarBase): - """Grammar-enabled float field.""" - - __root__: Union[float, _GrammarType] - - @classmethod - @overrides - def validate(cls, entry): - # GrammarFloat entry can be a list if it contains clauses - if isinstance(entry, list): - new_entry = [] - for item in entry: - if _is_grammar_clause(item): - cls._grammar_append(new_entry, item) - else: - raise TypeError(f"value must be a list of float: {entry!r}") - return new_entry - - if isinstance(entry, (int, float)): - return float(entry) - - raise TypeError(f"value must be a float: {entry!r}") - - -class GrammarStr(_GrammarBase): - """Grammar-enabled string field.""" - - __root__: Union[str, _GrammarType] - - @classmethod - @overrides - def validate(cls, entry): - # GrammarStr entry can be a list if it contains clauses - if isinstance(entry, list): - new_entry = [] - for item in entry: - if _is_grammar_clause(item): - cls._grammar_append(new_entry, item) - else: - raise TypeError(f"value must be a string: {entry!r}") - return new_entry - - if isinstance(entry, str): - return entry - - raise TypeError(f"value must be a string: {entry!r}") - - -class GrammarStrList(_GrammarBase): - """Grammar-enabled list of strings field.""" - - __root__: Union[List[Union[str, _GrammarType]], _GrammarType] - - @classmethod - @overrides - def validate(cls, entry): - # GrammarStrList will always be a list - if isinstance(entry, list): - new_entry = [] - for item in entry: - if _is_grammar_clause(item): - cls._grammar_append(new_entry, item) - elif isinstance(item, str): - new_entry.append(item) - else: - raise TypeError(f"value must be a list of string: {entry!r}") - return new_entry - - raise TypeError(f"value must be a list of string: {entry!r}") - - -class GrammarSingleEntryDictList(_GrammarBase): - """Grammar-enabled list of dictionaries field.""" - - __root__: Union[List[Dict[str, Any]], _GrammarType] - - @classmethod - @overrides - def validate(cls, entry): - # GrammarSingleEntryDictList will always be a list - if isinstance(entry, list): - new_entry = [] - for item in entry: - if _is_grammar_clause(item): - cls._grammar_append(new_entry, item) - elif isinstance(item, dict) and len(item) == 1: - new_entry.append(item) - else: - raise TypeError( - f"value must be a list of single-entry dictionaries: {entry!r}" - ) - return new_entry - - raise TypeError(f"value must be a list of single-entry dictionaries: {entry!r}") - - -class GrammarDict(_GrammarBase): - """Grammar-enabled dictionary field.""" - - __root__: Union[Dict[str, Any], _GrammarType] - - @classmethod - @overrides - def validate(cls, entry): - # GrammarDict entry can be a list if it contains clauses - if isinstance(entry, list): - new_entry = [] - for item in entry: - if _is_grammar_clause(item): - cls._grammar_append(new_entry, item) - else: - raise TypeError(f"value must be a list of dictionaries: {entry!r}") - return new_entry - - if isinstance(entry, dict): - return entry +def _format_type_error(type_: type, entry: Any) -> str: + """Format a type error message.""" + origin = get_origin(type_) + args = get_args(type_) - raise TypeError(f"value must be a dictionary: {entry!r}") + # handle primitive types which origin is None + if not origin: + origin = type_ - -class GrammarDictList(_GrammarBase): - """Grammar-enabled list of dictionary field.""" - - __root__: Union[List[Dict[str, Any]], _GrammarType] - - @classmethod - @overrides - def validate(cls, entry): - # GrammarDictList will always be a list - if isinstance(entry, list): - new_entry = [] - for item in entry: - if _is_grammar_clause(item): - cls._grammar_append(new_entry, item) - elif isinstance(item, dict): - new_entry.append(item) - else: - raise TypeError(f"value must be a list of dictionaries: {entry!r}") - return new_entry - - raise TypeError(f"value must be a list of dictionary: {entry!r}") + if issubclass(origin, list): + if args: + return f"value must be a list of {args[0].__name__}: {entry!r}" + else: + return f"value must be a list: {entry!r}" + elif issubclass(origin, dict): + if len(args) == 2: + return f"value must be a dict of {args[0].__name__} and {args[1].__name__}: {entry!r}" + else: + return f"value must be a dict: {entry!r}" + else: + return f"value must be a {type_.__name__}: {entry!r}" + + +class GrammarGeneratorMetaClass(type): + # Define __getitem__ method to be able to use index + def __getitem__(self, type_): + # Define Main Class + class GrammarScalar(_GrammarBase): + + _type = type_ + + @classmethod + @overrides + def validate(cls, entry): + # Grammar[T] entry can be a list if it contains clauses + if isinstance(entry, list): + # Check if the type_ supposed to be a list + sub_type = get_args(cls._type) + + # handle typed list + if sub_type: + sub_type = sub_type[0] + if sub_type is Any: + sub_type = None + + new_entry = [] + for item in entry: + # Check if the item is a valid grammar clause + if _is_grammar_clause(item): + cls._grammar_append(new_entry, item) + else: + # Check if the item is a valid type if not a grammar clause + if sub_type and isinstance(item, sub_type): + new_entry.append(item) + else: + raise TypeError(_format_type_error(cls._type, entry)) + + return new_entry + + # Not a valid grammar, check if it is a dict + if isinstance(entry, dict): + # Check if the type_ supposed to be a dict + if get_origin(cls._type) is not dict: + raise TypeError(_format_type_error(cls._type, entry)) + + sub_type = get_args(cls._type) + # The dict is not a typed dict + if not sub_type: + return entry + + sub_key_type = sub_type[0] if sub_type else Any + sub_value_type = sub_type[1] if sub_type else Any + + # validate the dict + for k, v in entry.items(): + if (sub_key_type is Any or isinstance(k, sub_key_type)) and ( + sub_value_type is Any or isinstance(v, sub_value_type) + ): + # we do not need the return value if it is a valid dict + pass + else: + raise TypeError(_format_type_error(cls._type, entry)) + + return entry + + # handle standard types with pydantic validators + try: + for validator in find_validators(cls._type, _CONFIG): + # we do not need the return value of the validator + validator(entry) + except PydanticTypeError: + raise TypeError(_format_type_error(cls._type, entry)) + + return entry + + return GrammarScalar + + +class GrammarGenerator(metaclass=GrammarGeneratorMetaClass): + pass def _ensure_selector_valid(selector: str, *, clause: str) -> None: diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py index 868cc6c..a2b2b36 100644 --- a/tests/unit/test_models.py +++ b/tests/unit/test_models.py @@ -20,16 +20,10 @@ import pydantic import pytest import yaml +from typing import Any from craft_grammar.models import ( - GrammarBool, - GrammarDict, - GrammarDictList, - GrammarFloat, - GrammarInt, - GrammarSingleEntryDictList, - GrammarStr, - GrammarStrList, + GrammarGenerator, ) @@ -37,14 +31,13 @@ class ValidationTest(pydantic.BaseModel): """A test model containing all types of grammar-aware types.""" control: str - grammar_bool: GrammarBool - grammar_int: GrammarInt - grammar_float: GrammarFloat - grammar_str: GrammarStr - grammar_strlist: GrammarStrList - grammar_dict: GrammarDict - grammar_single_entry_dictlist: GrammarSingleEntryDictList - grammar_dictlist: GrammarDictList + grammar_bool: GrammarGenerator[bool] + grammar_int: GrammarGenerator[int] + grammar_float: GrammarGenerator[float] + grammar_str: GrammarGenerator[str] + grammar_strlist: GrammarGenerator[list[str]] + grammar_dict: GrammarGenerator[dict[str, Any]] + grammar_dictlist: GrammarGenerator[list[dict]] def test_validate_grammar_trivial(): @@ -63,9 +56,6 @@ def test_validate_grammar_trivial(): grammar_dict: key: value other_key: other_value - grammar_single_entry_dictlist: - - key: value - - other_key: other_value grammar_dictlist: - key: value other_key: other_value @@ -83,10 +73,6 @@ def test_validate_grammar_trivial(): assert v.grammar_str == "another string" assert v.grammar_strlist == ["a", "string", "list"] assert v.grammar_dict == {"key": "value", "other_key": "other_value"} - assert v.grammar_single_entry_dictlist == [ - {"key": "value"}, - {"other_key": "other_value"}, - ] assert v.grammar_dictlist == [ {"key": "value", "other_key": "other_value"}, {"key2": "value", "other_key2": "other_value"}, @@ -121,11 +107,6 @@ def test_validate_grammar_simple(): key: value other_key: other_value - else fail - grammar_single_entry_dictlist: - - on arch: - - key: value - - other_key: other_value - - else fail grammar_dictlist: - on arch: - key: value @@ -163,10 +144,6 @@ def test_validate_grammar_simple(): {"*on amd64": {"key": "value", "other_key": "other_value"}}, "*else fail", ] - assert v.grammar_single_entry_dictlist == [ - {"*on arch": [{"key": "value"}, {"other_key": "other_value"}]}, - "*else fail", - ] assert v.grammar_dictlist == [ { "*on arch": [ @@ -231,16 +208,6 @@ def test_validate_grammar_recursive(): yet_another_key: yet_another_value - else fail - else fail - grammar_single_entry_dictlist: - - on arch,other_arch: - - on other_arch: - - to yet_another_arch: - - key: value - - other_key: other_value - - else fail - - else: - - yet_another_key: yet_another_value - - else fail grammar_dictlist: - on arch,other_arch: - on other_arch: @@ -321,26 +288,6 @@ def test_validate_grammar_recursive(): }, ] - assert v.grammar_single_entry_dictlist == [ - { - "*on arch,other_arch": [ - { - "*on other_arch": [ - { - "*to yet_another_arch": [ - {"key": "value"}, - {"other_key": "other_value"}, - ] - }, - "*else fail", - ] - }, - {"*else": [{"yet_another_key": "yet_another_value"}]}, - ] - }, - "*else fail", - ] - assert v.grammar_dictlist == [ { "*on arch,other_arch": [ @@ -373,13 +320,13 @@ def test_validate_grammar_recursive(): @pytest.mark.parametrize( "value", - [23, True, ["foo"], {"x"}, [{"a": "b"}]], + [["foo"], {"x"}, [{"a": "b"}]], ) def test_grammar_str_error(value): class GrammarValidation(pydantic.BaseModel): """Test validation of grammar-enabled types.""" - x: GrammarStr + x: GrammarGenerator[str] with pytest.raises(pydantic.ValidationError) as raised: GrammarValidation(x=value) # type: ignore @@ -388,7 +335,7 @@ class GrammarValidation(pydantic.BaseModel): assert len(err) == 1 assert err[0]["loc"] == ("x",) assert err[0]["type"] == "type_error" - assert err[0]["msg"] == f"value must be a string: {value!r}" + assert err[0]["msg"] == f"value must be a str: {value!r}" @pytest.mark.parametrize( @@ -399,7 +346,7 @@ def test_grammar_strlist_error(value): class GrammarValidation(pydantic.BaseModel): """Test validation of grammar-enabled types.""" - x: GrammarStrList + x: GrammarGenerator[list[str]] with pytest.raises(pydantic.ValidationError) as raised: GrammarValidation(x=value) # type: ignore @@ -408,55 +355,33 @@ class GrammarValidation(pydantic.BaseModel): assert len(err) == 1 assert err[0]["loc"] == ("x",) assert err[0]["type"] == "type_error" - assert err[0]["msg"] == f"value must be a list of string: {value!r}" - - -@pytest.mark.parametrize( - "value", - [23, "string", [{"a": 42}, "foo"], [{"a": 42, "b": 43}]], -) -def test_grammar_single_entry_dictlist_error(value): - class GrammarValidation(pydantic.BaseModel): - """Test validation of grammar-enabled types.""" - - x: GrammarSingleEntryDictList - - with pytest.raises(pydantic.ValidationError) as raised: - GrammarValidation(x=value) # type: ignore - - err = raised.value.errors() - assert len(err) == 1 - assert err[0]["loc"] == ("x",) - assert err[0]["type"] == "type_error" - assert err[0]["msg"] == ( - f"value must be a list of single-entry dictionaries: {value!r}" - ) + assert err[0]["msg"] == f"value must be a list of str: {value!r}" def test_grammar_nested_error(): class GrammarValidation(pydantic.BaseModel): """Test validation of grammar-enabled types.""" - x: GrammarStr + x: GrammarGenerator[str] with pytest.raises(pydantic.ValidationError) as raised: GrammarValidation( x=[ - {"on arm64,amd64": [{"on arm64": "foo"}, {"else": 35}]}, + {"on arm64,amd64": [{"on arm64": "foo"}, {"else": [35]}]}, ] # type: ignore ) err = raised.value.errors() assert len(err) == 1 assert err[0]["loc"] == ("x",) assert err[0]["type"] == "type_error" - assert err[0]["msg"] == "value must be a string: 35" + assert err[0]["msg"] == "value must be a str: [35]" def test_grammar_str_elsefail(): class GrammarValidation(pydantic.BaseModel): """Test validation of grammar-enabled types.""" - x: GrammarStr + x: GrammarGenerator[str] GrammarValidation(x=[{"on arch": "foo"}, "else fail"]) # type: ignore @@ -465,25 +390,16 @@ def test_grammar_strlist_elsefail(): class GrammarValidation(pydantic.BaseModel): """Test validation of grammar-enabled types.""" - x: GrammarStrList + x: GrammarGenerator[list[str]] GrammarValidation(x=[{"on arch": ["foo"]}, "else fail"]) # type: ignore -def test_grammar_single_entry_dictlist_elsefail(): - class GrammarValidation(pydantic.BaseModel): - """Test validation of grammar-enabled types.""" - - x: GrammarSingleEntryDictList - - GrammarValidation(x=[{"on arch": [{"foo": "bar"}]}, "else fail"]) # type: ignore - - def test_grammar_try(): class GrammarValidation(pydantic.BaseModel): """Test validation of grammar-enabled types.""" - x: GrammarStr + x: GrammarGenerator[str] with pytest.raises(pydantic.ValidationError) as raised: GrammarValidation(x=[{"try": "foo"}]) # type: ignore @@ -498,13 +414,13 @@ class GrammarValidation(pydantic.BaseModel): @pytest.mark.parametrize( "clause,err_msg", [ - ("on", "value must be a string: [{'on': 'foo'}]"), + ("on", "value must be a str: [{'on': 'foo'}]"), ("on ,", "syntax error in 'on' selector"), ("on ,arch", "syntax error in 'on' selector"), ("on arch,", "syntax error in 'on' selector"), ("on arch,,arch", "syntax error in 'on' selector"), ("on arch, arch", "spaces are not allowed in 'on' selector"), - ("to", "value must be a string: [{'to': 'foo'}]"), + ("to", "value must be a str: [{'to': 'foo'}]"), ("to ,", "syntax error in 'to' selector"), ("to ,arch", "syntax error in 'to' selector"), ("to arch,", "syntax error in 'to' selector"), @@ -526,7 +442,7 @@ def test_grammar_errors(clause, err_msg): class GrammarValidation(pydantic.BaseModel): """Test validation of grammar-enabled types.""" - x: GrammarStr + x: GrammarGenerator[str] with pytest.raises(pydantic.ValidationError) as raised: GrammarValidation(x=[{clause: "foo"}]) # type: ignore