Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

propagate ignore_metadataflag #133

Merged
merged 2 commits into from
Jul 31, 2024
Merged

propagate ignore_metadataflag #133

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4f304a5
propagate ignore_metadataflag
jana-starkova Jul 29, 2024
ea57d81
fix lint
jana-starkova Jul 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions chispa/schema_comparer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def inner(s1, s2, ignore_nullable, ignore_metadata):
t = PrettyTable(["schema1", "schema2"])
zipped = list(zip_longest(s1, s2))
for sf1, sf2 in zipped:
if are_structfields_equal(sf1, sf2, True):
if are_structfields_equal(sf1, sf2, ignore_nullable, ignore_metadata):
t.add_row([blue(sf1), blue(sf2)])
else:
t.add_row([sf1, sf2])
Expand Down Expand Up @@ -70,12 +70,12 @@ def assert_schema_equality_ignore_nullable(s1, s2):


# deprecate this. ignore_nullable should be a flag.
def are_schemas_equal_ignore_nullable(s1, s2):
def are_schemas_equal_ignore_nullable(s1, s2, ignore_metadata=False):
if len(s1) != len(s2):
return False
zipped = list(zip_longest(s1, s2))
for sf1, sf2 in zipped:
if not are_structfields_equal(sf1, sf2, True):
if not are_structfields_equal(sf1, sf2, True, ignore_metadata):
return False
return True

Expand All @@ -95,11 +95,11 @@ def are_structfields_equal(sf1, sf2, ignore_nullability=False, ignore_metadata=F
if not ignore_metadata and sf1.metadata != sf2.metadata:
return False
else:
return are_datatypes_equal_ignore_nullable(sf1.dataType, sf2.dataType)
return are_datatypes_equal_ignore_nullable(sf1.dataType, sf2.dataType, ignore_metadata)


# deprecate this
def are_datatypes_equal_ignore_nullable(dt1, dt2):
def are_datatypes_equal_ignore_nullable(dt1, dt2, ignore_metadata=False):
"""Checks if datatypes are equal, descending into structs and arrays to
ignore nullability.
"""
Expand All @@ -108,7 +108,7 @@ def are_datatypes_equal_ignore_nullable(dt1, dt2):
if dt1.typeName() == "array":
return are_datatypes_equal_ignore_nullable(dt1.elementType, dt2.elementType)
elif dt1.typeName() == "struct":
return are_schemas_equal_ignore_nullable(dt1, dt2)
return are_schemas_equal_ignore_nullable(dt1, dt2, ignore_metadata)
else:
return True
else:
Expand Down
5 changes: 5 additions & 0 deletions tests/test_structfield_comparer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,3 +55,8 @@ def it_returns_true_when_nested_types_have_different_nullability_with_ignore_nul
sf1 = StructField("hi", StructType([StructField("world", IntegerType(), False)]), False)
sf2 = StructField("hi", StructType([StructField("world", IntegerType(), True)]), False)
assert are_structfields_equal(sf1, sf2, ignore_nullability=True) is True

def it_returns_true_when_inner_metadata_is_different_but_ignored():
sf1 = StructField("hi", StructType([StructField("world", IntegerType(), False)]), False)
sf2 = StructField("hi", StructType([StructField("world", IntegerType(), False, {"a": "b"})]), False)
assert are_structfields_equal(sf1, sf2, ignore_metadata=True) is True
Loading