Skip to content

Commit

Permalink
removed l2ss-py-autotest test
Browse files Browse the repository at this point in the history
  • Loading branch information
James Wood authored and James Wood committed Apr 24, 2024
1 parent dcd7bb8 commit e260bc8
Showing 1 changed file with 0 additions and 142 deletions.
142 changes: 0 additions & 142 deletions tests/verify_collection.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,148 +337,6 @@ def get_lat_lon_var_names(dataset: xarray.Dataset, file_to_subset: str, collecti
pytest.fail(f"Unable to find latitude and longitude variables.")


@pytest.mark.timeout(600)
@pytest.mark.skip
def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
harmony_env, tmp_path: pathlib.Path, bearer_token):
test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"

logging.info("Using granule %s for test", granule_json['meta']['concept-id'])

# Compute a box that is smaller than the granule extent bounding box
north, south, east, west = get_bounding_box(granule_json)
east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)

# Build harmony request
harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
request_collection = harmony.Collection(id=collection_concept_id)
harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
granule_id=[granule_json['meta']['concept-id']])
logging.info("Sending harmony request %s", harmony_client.request_as_curl(harmony_request))

# Submit harmony request and download result
job_id = harmony_client.submit(harmony_request)
logging.info("Submitted harmony job %s", job_id)
harmony_client.wait_for_processing(job_id, show_progress=True)
subsetted_filepath = None
for filename in [file_future.result()
for file_future
in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
logging.info(f'Downloaded: %s', filename)
subsetted_filepath = pathlib.Path(filename)

# Verify spatial subset worked
subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
group = None
# Try to read group in file
lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)

with netCDF4.Dataset(subsetted_filepath) as f:
group_list = []
def group_walk(groups, nc_d, current_group):
global subsetted_ds_new
subsetted_ds_new = None
# check if the top group has lat or lon variable
if lat_var_name in list(nc_d.variables.keys()):
subsetted_ds_new = subsetted_ds
else:
# if not then we'll need to keep track of the group layers
group_list.append(current_group)

# loop through the groups in the current layer
for g in groups:
# end the loop if we've already found latitude
if subsetted_ds_new:
break
# check if the groups have latitude, define the dataset and end the loop if found
if lat_var_name in list(nc_d.groups[g].variables.keys()):
group_list.append(g)
g = '/'.join(group_list)
subsetted_ds_new = xarray.open_dataset(subsetted_filepath, group=g, decode_times=False)
break
# recall the function on a group that has groups in it and didn't find latitude
# this is going 'deeper' into the groups
if len(list(nc_d.groups[g].groups.keys())) > 0:
group_walk(nc_d.groups[g].groups, nc_d.groups[g], g)
else:
continue

group_walk(f.groups, f, '')

assert lat_var_name and lon_var_name

var_ds = None
msk = None

if science_vars := get_science_vars(collection_variables):
for idx, value in enumerate(science_vars):
science_var_name = science_vars[idx]['umm']['Name']
try:
var_ds = subsetted_ds_new[science_var_name]
msk = np.logical_not(np.isnan(var_ds.data.squeeze()))
break
except Exception:
try:
# if the variable couldn't be found because the name includes a group, e.g.,
# `geolocation/relative_azimuth_angle`,
# then try to access the variable after removing the group name.
var_ds = subsetted_ds_new[science_var_name.rsplit("/", 1)[-1]]
msk = np.logical_not(np.isnan(var_ds.data.squeeze()))
break
except Exception:
var_ds = None
msk = None

if var_ds is None and msk is None:
pytest.fail(f"Unable to find variable from umm-v to use as science variable.")

else:
# Can't find a science var in UMM-V, just pick one

science_var_name = next(iter([v for v in subsetted_ds_new.variables if
str(v) not in lat_var_name and str(v) not in lon_var_name and 'time' not in str(v)]))

var_ds = subsetted_ds_new[science_var_name]

try:
msk = np.logical_not(np.isnan(var_ds.data.squeeze()))
llat = subsetted_ds_new[lat_var_name].where(msk)
llon = subsetted_ds_new[lon_var_name].where(msk)
except ValueError:

llat = subsetted_ds_new[lat_var_name]
llon = subsetted_ds_new[lon_var_name]

lat_max = llat.max()
lat_min = llat.min()

lon_min = llon.min()
lon_max = llon.max()

lon_min = (lon_min + 180) % 360 - 180
lon_max = (lon_max + 180) % 360 - 180

lat_var_fill_value = subsetted_ds_new[lat_var_name].encoding.get('_FillValue')
lon_var_fill_value = subsetted_ds_new[lon_var_name].encoding.get('_FillValue')

if lat_var_fill_value:
if (lat_max <= north or np.isclose(lat_max, north)) and (lat_min >= south or np.isclose(lat_min, south)):
logging.info("Successful Latitude subsetting")
elif np.isnan(lat_max) and np.isnan(lat_min):
logging.info("Partial Lat Success - no Data")
else:
assert False

if lon_var_fill_value:
if (lon_max <= east or np.isclose(lon_max, east)) and (lon_min >= west or np.isclose(lon_min, west)):
logging.info("Successful Longitude subsetting")
elif np.isnan(lon_max) and np.isnan(lon_min):
logging.info("Partial Lon Success - no Data")
else:
assert False


def verify_dims(merged_group, origin_group, both_merged):
for dim in origin_group.dimensions:
if both_merged:
Expand Down

0 comments on commit e260bc8

Please sign in to comment.