Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add fix for swot temporal subsetting by checking fill value #200

Merged
merged 1 commit into from
Sep 18, 2023
Merged

add fix for swot temporal subsetting by checking fill value #200

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 8 additions & 29 deletions podaac/subsetter/subset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import os
from itertools import zip_longest
from typing import List, Optional, Tuple, Union
import traceback
import dateutil
from dateutil import parser

Expand Down Expand Up @@ -1073,33 +1072,6 @@ def decode_cf_datetime(num_dates, units, calendar=None, use_cftime=None):
xarray.coding.times.decode_cf_datetime = decode_cf_datetime


def open_dataset_test(file, args):
"""
Open a NetCDF dataset using xarray, handling specific exceptions.

This function attempts to open a NetCDF dataset using the provided arguments.
If an OverflowError with a specific message is encountered, it modifies the
'mask_and_scale' argument to True and retries opening the dataset.

Args:
file (str): Path to the NetCDF file.
args (dict): Dictionary of arguments to pass to xr.open_dataset.

Returns:
None: The function modifies the 'args' dictionary in place.

"""
try:
test_xr_open = xr.open_dataset(file, **args)
test_xr_open.close()
except Exception: # pylint: disable=broad-except
traceback_str = traceback.format_exc()

# Check for the specific OverflowError message
if "Python int too large to convert to C long" in traceback_str and "Failed to decode variable 'time': unable to decode time units" in traceback_str:
args["mask_and_scale"] = True


def subset(file_to_subset: str, bbox: np.ndarray, output_file: str,
variables: Union[List[str], str, None] = (),
# pylint: disable=too-many-branches, disable=too-many-statements
Expand Down Expand Up @@ -1197,7 +1169,14 @@ def subset(file_to_subset: str, bbox: np.ndarray, output_file: str,

if min_time or max_time:
args['decode_times'] = True
open_dataset_test(file_to_subset, args)
# check fill value and dtype, we know that this will cause an integer Overflow with xarray
if 'time' in nc_dataset.variables.keys():
try:
if nc_dataset['time'].getncattr('_FillValue') == nc.default_fillvals.get('f8') and \
nc_dataset['time'].dtype == 'float64':
args['mask_and_scale'] = True
except AttributeError:
pass

with xr.open_dataset(
xr.backends.NetCDF4DataStore(nc_dataset),
Expand Down
Loading