From 833c9356f58cbdba0e4885c1267ecae2ff2dd523 Mon Sep 17 00:00:00 2001 From: Kaitlin Naughten Date: Fri, 8 Jun 2018 16:42:07 +0100 Subject: [PATCH 1/2] Ignore iterations which only have pickup files. --- xmitgcm/mds_store.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/xmitgcm/mds_store.py b/xmitgcm/mds_store.py index 0f9d7bec..18e3fb30 100644 --- a/xmitgcm/mds_store.py +++ b/xmitgcm/mds_store.py @@ -141,7 +141,6 @@ def open_mdsdataset(data_dir, grid_dir=None, these_prefixes = _get_all_matching_prefixes( data_dir, iternum, prefix ) - # don't care about order if set(these_prefixes) != set(first_prefixes): raise IOError("Could not find the expected file " "prefixes %s at iternum %g. (Instead " @@ -750,7 +749,7 @@ def _concat_dicts(list_of_dicts): def _get_all_iternums(data_dir, file_prefixes=None, - file_format='*.??????????.data'): + file_format='*.??????????.data', ignore_pickup=True): """Scan a directory for all iteration number suffixes.""" iternums = set() all_datafiles = glob(os.path.join(data_dir, file_format)) @@ -759,6 +758,9 @@ def _get_all_iternums(data_dir, file_prefixes=None, for f in all_datafiles: iternum = int(f[istart:iend]) prefix = os.path.split(f[:istart-1])[-1] + # Only process iternums which have something more than pickups + if ignore_pickup and _is_pickup_prefix(prefix): + continue if file_prefixes is None: iternums.add(iternum) else: From f4e5ff488022b96d0d239e55653984bd9dc4546c Mon Sep 17 00:00:00 2001 From: Kaitlin Naughten Date: Fri, 8 Jun 2018 16:45:41 +0100 Subject: [PATCH 2/2] Reinstated an accidentally deleted comment --- xmitgcm/mds_store.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xmitgcm/mds_store.py b/xmitgcm/mds_store.py index 18e3fb30..e69e6aa6 100644 --- a/xmitgcm/mds_store.py +++ b/xmitgcm/mds_store.py @@ -141,6 +141,7 @@ def open_mdsdataset(data_dir, grid_dir=None, these_prefixes = _get_all_matching_prefixes( data_dir, iternum, prefix ) + # don't care about order if set(these_prefixes) != set(first_prefixes): raise IOError("Could not find the expected file " "prefixes %s at iternum %g. (Instead "