From a8bc837bb9a1a920bdb4c0be1c6daf07680c991c Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Tue, 14 May 2024 15:21:40 -0600 Subject: [PATCH 01/16] oisst test --- recipes/feedstock/recipe.py | 55 ++++++++++++++++++++++++++++++ recipes/feedstock/requirements.txt | 9 +++++ 2 files changed, 64 insertions(+) create mode 100644 recipes/feedstock/recipe.py create mode 100644 recipes/feedstock/requirements.txt diff --git a/recipes/feedstock/recipe.py b/recipes/feedstock/recipe.py new file mode 100644 index 0000000000..78534dfad6 --- /dev/null +++ b/recipes/feedstock/recipe.py @@ -0,0 +1,55 @@ +import apache_beam as beam +import pandas as pd +import s3fs +from beam_pyspark_runner.pyspark_runner import PySparkRunner + +from pangeo_forge_recipes.patterns import ConcatDim, FilePattern +from pangeo_forge_recipes.storage import FSSpecTarget +from pangeo_forge_recipes.transforms import ( + ConsolidateMetadata, + OpenURLWithFSSpec, + OpenWithXarray, + StoreToZarr, +) + +dates = pd.date_range('1981-09-01', '1981-09-03', freq='D') + +URL_FORMAT = ( + 'https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/' + 'v2.1/access/avhrr/{time:%Y%m}/oisst-avhrr-v02r01.{time:%Y%m%d}.nc' +) + + +def make_url(time): + return URL_FORMAT.format(time=time) + + +time_concat_dim = ConcatDim('time', dates, nitems_per_file=1) +pattern = FilePattern(make_url, time_concat_dim) + + +# NOTE: source uses the EMR serverless execution role (veda-data-reader-dev) +source_fsspec_kwargs = { + 'anon': False, + 'client_kwargs': {'region_name': 'us-west-2'}, +} + +# NOTE: target uses the EMR serverless execution role (veda-data-reader-dev) +target_fsspec_kwargs = {'anon': False, 'client_kwargs': {'region_name': 'us-west-2'}} +fs_target = s3fs.S3FileSystem(**target_fsspec_kwargs) +target_root = FSSpecTarget(fs_target, 's3://veda-pforge-emr-outputs-v4') + + +with beam.Pipeline(runner=PySparkRunner()) as p: + ( + p + | beam.Create(pattern.items()) + | OpenURLWithFSSpec(open_kwargs=source_fsspec_kwargs) + | OpenWithXarray(file_type=pattern.file_type) + | StoreToZarr( + target_root=target_root, + store_name='oisst_test.zarr', + combine_dims=pattern.combine_dim_keys, + ) + | ConsolidateMetadata() + ) diff --git a/recipes/feedstock/requirements.txt b/recipes/feedstock/requirements.txt new file mode 100644 index 0000000000..4724b677bd --- /dev/null +++ b/recipes/feedstock/requirements.txt @@ -0,0 +1,9 @@ +s3fs +boto3 +requests +git+https://github.com/ranchodeluxe/beam-pyspark-runner@patch-2 +apache-beam==2.53.0 +pangeo-forge-recipes>=0.10.5 +xarray>=2024.1.1 +zarr==2.16.1 +rioxarray From eb046c6caebb4d40d4a2399783e9d7e2bddcea06 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Tue, 14 May 2024 15:31:28 -0600 Subject: [PATCH 02/16] updated repo struct --- {recipes/feedstock => feedstock}/recipe.py | 0 {recipes/feedstock => feedstock}/requirements.txt | 0 recipes/README.md | 7 ------- 3 files changed, 7 deletions(-) rename {recipes/feedstock => feedstock}/recipe.py (100%) rename {recipes/feedstock => feedstock}/requirements.txt (100%) delete mode 100644 recipes/README.md diff --git a/recipes/feedstock/recipe.py b/feedstock/recipe.py similarity index 100% rename from recipes/feedstock/recipe.py rename to feedstock/recipe.py diff --git a/recipes/feedstock/requirements.txt b/feedstock/requirements.txt similarity index 100% rename from recipes/feedstock/requirements.txt rename to feedstock/requirements.txt diff --git a/recipes/README.md b/recipes/README.md deleted file mode 100644 index c72452b5e3..0000000000 --- a/recipes/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Welcome to `staged-recipes`! - -This directory is deliberately left empty. - -Please refer to the -[Recipe Contribution docs](https://pangeo-forge.readthedocs.io/en/latest/pangeo_forge_cloud/recipe_contribution.html) -to learn how to contribute a recipe to Pangeo Forge Cloud. From 6cb9059953569483b37377fdfb9cd4e96d126ee2 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Tue, 14 May 2024 15:37:41 -0600 Subject: [PATCH 03/16] removed source s3 specific kwargs --- feedstock/recipe.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/feedstock/recipe.py b/feedstock/recipe.py index 78534dfad6..32ffb1a4aa 100644 --- a/feedstock/recipe.py +++ b/feedstock/recipe.py @@ -30,8 +30,7 @@ def make_url(time): # NOTE: source uses the EMR serverless execution role (veda-data-reader-dev) source_fsspec_kwargs = { - 'anon': False, - 'client_kwargs': {'region_name': 'us-west-2'}, + 'anon': True, } # NOTE: target uses the EMR serverless execution role (veda-data-reader-dev) From 9082e23e3a17b6ee93490b8387731c4819f61b82 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Tue, 14 May 2024 15:41:34 -0600 Subject: [PATCH 04/16] remove anon --- feedstock/recipe.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/feedstock/recipe.py b/feedstock/recipe.py index 32ffb1a4aa..85040dfe0c 100644 --- a/feedstock/recipe.py +++ b/feedstock/recipe.py @@ -28,11 +28,6 @@ def make_url(time): pattern = FilePattern(make_url, time_concat_dim) -# NOTE: source uses the EMR serverless execution role (veda-data-reader-dev) -source_fsspec_kwargs = { - 'anon': True, -} - # NOTE: target uses the EMR serverless execution role (veda-data-reader-dev) target_fsspec_kwargs = {'anon': False, 'client_kwargs': {'region_name': 'us-west-2'}} fs_target = s3fs.S3FileSystem(**target_fsspec_kwargs) @@ -43,7 +38,7 @@ def make_url(time): ( p | beam.Create(pattern.items()) - | OpenURLWithFSSpec(open_kwargs=source_fsspec_kwargs) + | OpenURLWithFSSpec() | OpenWithXarray(file_type=pattern.file_type) | StoreToZarr( target_root=target_root, From 12af2076a8847424b5103ef60bddf6efc50cc12c Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Tue, 14 May 2024 16:04:41 -0600 Subject: [PATCH 05/16] retool for parquet kerchunk test 20 years --- feedstock/recipe.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/feedstock/recipe.py b/feedstock/recipe.py index 85040dfe0c..1dc1f38360 100644 --- a/feedstock/recipe.py +++ b/feedstock/recipe.py @@ -8,11 +8,13 @@ from pangeo_forge_recipes.transforms import ( ConsolidateMetadata, OpenURLWithFSSpec, + OpenWithKerchunk, OpenWithXarray, StoreToZarr, + WriteCombinedReference, ) -dates = pd.date_range('1981-09-01', '1981-09-03', freq='D') +dates = pd.date_range('1981-09-01', '2000-09-01', freq='D') URL_FORMAT = ( 'https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/' @@ -28,22 +30,22 @@ def make_url(time): pattern = FilePattern(make_url, time_concat_dim) -# NOTE: target uses the EMR serverless execution role (veda-data-reader-dev) +# # NOTE: target uses the EMR serverless execution role (veda-data-reader-dev) target_fsspec_kwargs = {'anon': False, 'client_kwargs': {'region_name': 'us-west-2'}} fs_target = s3fs.S3FileSystem(**target_fsspec_kwargs) target_root = FSSpecTarget(fs_target, 's3://veda-pforge-emr-outputs-v4') - with beam.Pipeline(runner=PySparkRunner()) as p: ( p | beam.Create(pattern.items()) | OpenURLWithFSSpec() - | OpenWithXarray(file_type=pattern.file_type) - | StoreToZarr( + | OpenWithKerchunk(file_type=pattern.file_type) + | WriteCombinedReference( + identical_dims=['lat', 'lon', 'zlev'], target_root=target_root, - store_name='oisst_test.zarr', - combine_dims=pattern.combine_dim_keys, + store_name='oisst_kerchunk_20_years', + concat_dims=['time'], + output_file_name='combined_oisst.parquet', ) - | ConsolidateMetadata() ) From 3444dcf50dd0b50ddc88d4424ed1d20448acfc2f Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Tue, 14 May 2024 16:41:41 -0600 Subject: [PATCH 06/16] 1 year parquet oisst --- feedstock/recipe.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/feedstock/recipe.py b/feedstock/recipe.py index 1dc1f38360..ee0dd1b683 100644 --- a/feedstock/recipe.py +++ b/feedstock/recipe.py @@ -14,7 +14,7 @@ WriteCombinedReference, ) -dates = pd.date_range('1981-09-01', '2000-09-01', freq='D') +dates = pd.date_range('1981-09-01', '1982-09-01', freq='D') URL_FORMAT = ( 'https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/' @@ -44,8 +44,8 @@ def make_url(time): | WriteCombinedReference( identical_dims=['lat', 'lon', 'zlev'], target_root=target_root, - store_name='oisst_kerchunk_20_years', + store_name='oisst_kerchunk_1_year', concat_dims=['time'], - output_file_name='combined_oisst.parquet', + output_file_name='combined_oisst_1year.parquet', ) ) From 7106a6a5c390df0f5ca410715410adc8a1c8f3df Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Tue, 14 May 2024 16:51:21 -0600 Subject: [PATCH 07/16] 3day --- feedstock/recipe.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/feedstock/recipe.py b/feedstock/recipe.py index ee0dd1b683..021578ef5a 100644 --- a/feedstock/recipe.py +++ b/feedstock/recipe.py @@ -14,7 +14,7 @@ WriteCombinedReference, ) -dates = pd.date_range('1981-09-01', '1982-09-01', freq='D') +dates = pd.date_range('1981-09-01', '1981-09-04', freq='D') URL_FORMAT = ( 'https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/' @@ -44,8 +44,8 @@ def make_url(time): | WriteCombinedReference( identical_dims=['lat', 'lon', 'zlev'], target_root=target_root, - store_name='oisst_kerchunk_1_year', + store_name='oisst_kerchunk_3_day', concat_dims=['time'], - output_file_name='combined_oisst_1year.parquet', + output_file_name='combined_oisst_3day.parquet', ) ) From 850f8e555a87d43a9b2c5ddc32c6ce3014219f85 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 29 May 2024 13:58:16 -0700 Subject: [PATCH 08/16] 1 month --- feedstock/recipe.py | 44 ++++++++++++++++++++++++-------------- feedstock/requirements.txt | 1 + 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/feedstock/recipe.py b/feedstock/recipe.py index 021578ef5a..1c3fda76c1 100644 --- a/feedstock/recipe.py +++ b/feedstock/recipe.py @@ -1,20 +1,17 @@ +from dataclasses import dataclass + import apache_beam as beam import pandas as pd import s3fs +import xarray as xr from beam_pyspark_runner.pyspark_runner import PySparkRunner +from pangeo_forge_ndpyramid.transforms import StoreToPyramid from pangeo_forge_recipes.patterns import ConcatDim, FilePattern from pangeo_forge_recipes.storage import FSSpecTarget -from pangeo_forge_recipes.transforms import ( - ConsolidateMetadata, - OpenURLWithFSSpec, - OpenWithKerchunk, - OpenWithXarray, - StoreToZarr, - WriteCombinedReference, -) +from pangeo_forge_recipes.transforms import OpenURLWithFSSpec, OpenWithXarray -dates = pd.date_range('1981-09-01', '1981-09-04', freq='D') +dates = pd.date_range('1981-09-01', '1981-10-01', freq='D') URL_FORMAT = ( 'https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/' @@ -30,22 +27,37 @@ def make_url(time): pattern = FilePattern(make_url, time_concat_dim) -# # NOTE: target uses the EMR serverless execution role (veda-data-reader-dev) +# NOTE: target uses the EMR serverless execution role (veda-data-reader-dev) target_fsspec_kwargs = {'anon': False, 'client_kwargs': {'region_name': 'us-west-2'}} fs_target = s3fs.S3FileSystem(**target_fsspec_kwargs) target_root = FSSpecTarget(fs_target, 's3://veda-pforge-emr-outputs-v4') + +@dataclass +class SelectSingleZlev(beam.PTransform): + def select_single_zlev(self, ds: xr.Dataset) -> xr.Dataset: + return ds.isel(zlev=0).drop('zlev') + + def expand(self, pcoll): + return pcoll | 'Select single zlev' >> beam.MapTuple( + lambda k, v: (k, self.select_single_zlev(v)) + ) + + with beam.Pipeline(runner=PySparkRunner()) as p: ( p | beam.Create(pattern.items()) | OpenURLWithFSSpec() - | OpenWithKerchunk(file_type=pattern.file_type) - | WriteCombinedReference( - identical_dims=['lat', 'lon', 'zlev'], + | OpenWithXarray(file_type=pattern.file_type) + | SelectSingleZlev() + | 'Write Pyramid Levels' + >> StoreToPyramid( target_root=target_root, - store_name='oisst_kerchunk_3_day', - concat_dims=['time'], - output_file_name='combined_oisst_3day.parquet', + store_name='oisst_pyramid_2_lvl_1_month_pyramid.zarr', + epsg_code='4326', + rename_spatial_dims={'lon': 'longitude', 'lat': 'latitude'}, + levels=2, + combine_dims=pattern.combine_dim_keys, ) ) diff --git a/feedstock/requirements.txt b/feedstock/requirements.txt index 4724b677bd..4b8e157887 100644 --- a/feedstock/requirements.txt +++ b/feedstock/requirements.txt @@ -2,6 +2,7 @@ s3fs boto3 requests git+https://github.com/ranchodeluxe/beam-pyspark-runner@patch-2 +git+https://github.com/carbonplan/pangeo-forge-ndpyramid apache-beam==2.53.0 pangeo-forge-recipes>=0.10.5 xarray>=2024.1.1 From ad54b2e2dff137f298b8fd64d9f536b4688dbfe3 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 29 May 2024 14:31:41 -0700 Subject: [PATCH 09/16] new store path --- feedstock/recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feedstock/recipe.py b/feedstock/recipe.py index 1c3fda76c1..3c82d92125 100644 --- a/feedstock/recipe.py +++ b/feedstock/recipe.py @@ -54,7 +54,7 @@ def expand(self, pcoll): | 'Write Pyramid Levels' >> StoreToPyramid( target_root=target_root, - store_name='oisst_pyramid_2_lvl_1_month_pyramid.zarr', + store_name='oisst_pyramid_2_lvl_1_month_pyramid_attempt2.zarr', epsg_code='4326', rename_spatial_dims={'lon': 'longitude', 'lat': 'latitude'}, levels=2, From f2ac36427efd59cb02f3e7b69dbba17b5fd5058e Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 29 May 2024 15:08:48 -0700 Subject: [PATCH 10/16] 3day job --- feedstock/recipe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/feedstock/recipe.py b/feedstock/recipe.py index 3c82d92125..feb3a3be24 100644 --- a/feedstock/recipe.py +++ b/feedstock/recipe.py @@ -11,7 +11,7 @@ from pangeo_forge_recipes.storage import FSSpecTarget from pangeo_forge_recipes.transforms import OpenURLWithFSSpec, OpenWithXarray -dates = pd.date_range('1981-09-01', '1981-10-01', freq='D') +dates = pd.date_range('1981-09-01', '1981-09-03', freq='D') URL_FORMAT = ( 'https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/' @@ -54,7 +54,7 @@ def expand(self, pcoll): | 'Write Pyramid Levels' >> StoreToPyramid( target_root=target_root, - store_name='oisst_pyramid_2_lvl_1_month_pyramid_attempt2.zarr', + store_name='oisst_pyramid_2_lvl_3_day_pyramid.zarr', epsg_code='4326', rename_spatial_dims={'lon': 'longitude', 'lat': 'latitude'}, levels=2, From 6c25d5c0854cd7bd87d0da9c63e945938f07093a Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 29 May 2024 15:14:31 -0700 Subject: [PATCH 11/16] 1week --- feedstock/recipe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/feedstock/recipe.py b/feedstock/recipe.py index feb3a3be24..ff7ef95819 100644 --- a/feedstock/recipe.py +++ b/feedstock/recipe.py @@ -11,7 +11,7 @@ from pangeo_forge_recipes.storage import FSSpecTarget from pangeo_forge_recipes.transforms import OpenURLWithFSSpec, OpenWithXarray -dates = pd.date_range('1981-09-01', '1981-09-03', freq='D') +dates = pd.date_range('1981-09-01', '1981-09-07', freq='D') URL_FORMAT = ( 'https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/' @@ -54,7 +54,7 @@ def expand(self, pcoll): | 'Write Pyramid Levels' >> StoreToPyramid( target_root=target_root, - store_name='oisst_pyramid_2_lvl_3_day_pyramid.zarr', + store_name='oisst_pyramid_2_lvl_1_week_pyramid.zarr', epsg_code='4326', rename_spatial_dims={'lon': 'longitude', 'lat': 'latitude'}, levels=2, From 539b3b43541a1841ff05b2322ab340612a57a244 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 29 May 2024 15:50:54 -0700 Subject: [PATCH 12/16] updated to optional-sync-http branch --- feedstock/recipe.py | 6 +++--- feedstock/requirements.txt | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/feedstock/recipe.py b/feedstock/recipe.py index ff7ef95819..88ea39ccee 100644 --- a/feedstock/recipe.py +++ b/feedstock/recipe.py @@ -11,7 +11,7 @@ from pangeo_forge_recipes.storage import FSSpecTarget from pangeo_forge_recipes.transforms import OpenURLWithFSSpec, OpenWithXarray -dates = pd.date_range('1981-09-01', '1981-09-07', freq='D') +dates = pd.date_range('1981-09-01', '1981-10-01', freq='D') URL_FORMAT = ( 'https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/' @@ -48,13 +48,13 @@ def expand(self, pcoll): ( p | beam.Create(pattern.items()) - | OpenURLWithFSSpec() + | OpenURLWithFSSpec(fsspec_sync_patch=True) | OpenWithXarray(file_type=pattern.file_type) | SelectSingleZlev() | 'Write Pyramid Levels' >> StoreToPyramid( target_root=target_root, - store_name='oisst_pyramid_2_lvl_1_week_pyramid.zarr', + store_name='oisst_pyramid_2_lvl_3_week_pyramid_sync_a2.zarr', epsg_code='4326', rename_spatial_dims={'lon': 'longitude', 'lat': 'latitude'}, levels=2, diff --git a/feedstock/requirements.txt b/feedstock/requirements.txt index 4b8e157887..caadc03785 100644 --- a/feedstock/requirements.txt +++ b/feedstock/requirements.txt @@ -4,7 +4,8 @@ requests git+https://github.com/ranchodeluxe/beam-pyspark-runner@patch-2 git+https://github.com/carbonplan/pangeo-forge-ndpyramid apache-beam==2.53.0 -pangeo-forge-recipes>=0.10.5 +git+https://github.com/pangeo-forge/pangeo-forge-recipes@feature/optional-sync-http xarray>=2024.1.1 zarr==2.16.1 rioxarray +httpfs_sync From 9f4b413780024b72f8964c21524876b7e4b8743a Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 29 May 2024 15:51:29 -0700 Subject: [PATCH 13/16] 1 month, sync PR of PGF --- feedstock/recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feedstock/recipe.py b/feedstock/recipe.py index 88ea39ccee..281be1887e 100644 --- a/feedstock/recipe.py +++ b/feedstock/recipe.py @@ -54,7 +54,7 @@ def expand(self, pcoll): | 'Write Pyramid Levels' >> StoreToPyramid( target_root=target_root, - store_name='oisst_pyramid_2_lvl_3_week_pyramid_sync_a2.zarr', + store_name='oisst_pyramid_2_lvl_1_month_pyramid_sync.zarr', epsg_code='4326', rename_spatial_dims={'lon': 'longitude', 'lat': 'latitude'}, levels=2, From c0a12678be86458d1ae7ea7d960c2f2f886976b2 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 29 May 2024 16:14:38 -0700 Subject: [PATCH 14/16] 1yr --- feedstock/recipe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/feedstock/recipe.py b/feedstock/recipe.py index 281be1887e..4a28ef171f 100644 --- a/feedstock/recipe.py +++ b/feedstock/recipe.py @@ -11,7 +11,7 @@ from pangeo_forge_recipes.storage import FSSpecTarget from pangeo_forge_recipes.transforms import OpenURLWithFSSpec, OpenWithXarray -dates = pd.date_range('1981-09-01', '1981-10-01', freq='D') +dates = pd.date_range('1981-09-01', '1982-09-01', freq='D') URL_FORMAT = ( 'https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/' @@ -54,7 +54,7 @@ def expand(self, pcoll): | 'Write Pyramid Levels' >> StoreToPyramid( target_root=target_root, - store_name='oisst_pyramid_2_lvl_1_month_pyramid_sync.zarr', + store_name='oisst_pyramid_2_lvl_1_year_pyramid_sync.zarr', epsg_code='4326', rename_spatial_dims={'lon': 'longitude', 'lat': 'latitude'}, levels=2, From c52c688ad1d7f2ac18625f7f89f30e3cfe67abb3 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 29 May 2024 16:18:18 -0700 Subject: [PATCH 15/16] 10yr --- feedstock/recipe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/feedstock/recipe.py b/feedstock/recipe.py index 4a28ef171f..b10d8afd86 100644 --- a/feedstock/recipe.py +++ b/feedstock/recipe.py @@ -11,7 +11,7 @@ from pangeo_forge_recipes.storage import FSSpecTarget from pangeo_forge_recipes.transforms import OpenURLWithFSSpec, OpenWithXarray -dates = pd.date_range('1981-09-01', '1982-09-01', freq='D') +dates = pd.date_range('1981-09-01', '1991-09-01', freq='D') URL_FORMAT = ( 'https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/' @@ -54,7 +54,7 @@ def expand(self, pcoll): | 'Write Pyramid Levels' >> StoreToPyramid( target_root=target_root, - store_name='oisst_pyramid_2_lvl_1_year_pyramid_sync.zarr', + store_name='oisst_pyramid_2_lvl_10_year_pyramid_sync.zarr', epsg_code='4326', rename_spatial_dims={'lon': 'longitude', 'lat': 'latitude'}, levels=2, From 33c5f253f350648c13874ef9ac7c81f193f13801 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Thu, 30 May 2024 09:17:47 -0700 Subject: [PATCH 16/16] 3lvl, 3day --- feedstock/recipe.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/feedstock/recipe.py b/feedstock/recipe.py index b10d8afd86..5a849bf4d2 100644 --- a/feedstock/recipe.py +++ b/feedstock/recipe.py @@ -11,7 +11,7 @@ from pangeo_forge_recipes.storage import FSSpecTarget from pangeo_forge_recipes.transforms import OpenURLWithFSSpec, OpenWithXarray -dates = pd.date_range('1981-09-01', '1991-09-01', freq='D') +dates = pd.date_range('1981-09-01', '1981-09-03', freq='D') URL_FORMAT = ( 'https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/' @@ -54,10 +54,10 @@ def expand(self, pcoll): | 'Write Pyramid Levels' >> StoreToPyramid( target_root=target_root, - store_name='oisst_pyramid_2_lvl_10_year_pyramid_sync.zarr', + store_name='oisst_pyramid_3_lvl_3_day_pyramid_sync.zarr', epsg_code='4326', rename_spatial_dims={'lon': 'longitude', 'lat': 'latitude'}, - levels=2, + levels=3, combine_dims=pattern.combine_dim_keys, ) )