From 26adfbd286d100a045ba3977eed1742d33d35ac6 Mon Sep 17 00:00:00 2001 From: dabhicusp Date: Tue, 12 Mar 2024 06:00:42 +0000 Subject: [PATCH 01/12] Retry logic for the ee.data.computePixels update. --- xee/ext.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/xee/ext.py b/xee/ext.py index 6b8b203..f5d4a97 100644 --- a/xee/ext.py +++ b/xee/ext.py @@ -147,6 +147,8 @@ def open( ee_init_kwargs: Optional[Dict[str, Any]] = None, ee_init_if_necessary: bool = False, executor_kwargs: Optional[Dict[str, Any]] = None, + compute_pixels_max_retries: Optional[int] = None, + compute_pixels_initial_delay: Optional[int] = None, ) -> 'EarthEngineStore': if mode != 'r': raise ValueError( @@ -168,6 +170,8 @@ def open( ee_init_kwargs=ee_init_kwargs, ee_init_if_necessary=ee_init_if_necessary, executor_kwargs=executor_kwargs, + compute_pixels_max_retries=compute_pixels_max_retries, + compute_pixels_initial_delay=compute_pixels_initial_delay, ) def __init__( @@ -186,6 +190,8 @@ def __init__( ee_init_kwargs: Optional[Dict[str, Any]] = None, ee_init_if_necessary: bool = False, executor_kwargs: Optional[Dict[str, Any]] = None, + compute_pixels_max_retries: Optional[int] = None, + compute_pixels_initial_delay: Optional[int] = None, ): self.ee_init_kwargs = ee_init_kwargs self.ee_init_if_necessary = ee_init_if_necessary @@ -195,6 +201,17 @@ def __init__( executor_kwargs = {} self.executor_kwargs = executor_kwargs + # Here 6 & 500 is default value. + # (https://github.com/pydata/xarray/blob/main/xarray/backends/common.py#L181). + self.compute_pixels_max_retries = ( + 6 if compute_pixels_max_retries is None else compute_pixels_max_retries + ) + self.compute_pixels_initial_delay = ( + 500 + if compute_pixels_initial_delay is None + else compute_pixels_initial_delay + ) + self.image_collection = image_collection if n_images != -1: self.image_collection = image_collection.limit(n_images) @@ -482,7 +499,11 @@ def image_to_array( **kwargs, } raw = common.robust_getitem( - pixels_getter, params, catch=ee.ee_exception.EEException + pixels_getter, + params, + catch=ee.ee_exception.EEException, + max_retries=self.compute_pixels_max_retries, + initial_delay=self.compute_pixels_initial_delay, ) # Extract out the shape information from EE response. @@ -965,6 +986,8 @@ def open_dataset( ee_init_if_necessary: bool = False, ee_init_kwargs: Optional[Dict[str, Any]] = None, executor_kwargs: Optional[Dict[str, Any]] = None, + compute_pixels_max_retries: Optional[int] = None, + compute_pixels_initial_delay: Optional[int] = None, ) -> xarray.Dataset: # type: ignore """Open an Earth Engine ImageCollection as an Xarray Dataset. @@ -1037,6 +1060,10 @@ def open_dataset( executor_kwargs (optional): A dictionary of keyword arguments to pass to the ThreadPoolExecutor that handles the parallel computation of pixels i.e. {'max_workers': 2}. + compute_pixels_max_retries (optional): The maximum number of retry + attempts for calling ee.data.computePixels(). + compute_pixels_initial_delay (optional): The initial delay in milliseconds + before the first retry of calling ee.data.computePixels(). Returns: An xarray.Dataset that streams in remote data from Earth Engine. @@ -1067,6 +1094,8 @@ def open_dataset( ee_init_kwargs=ee_init_kwargs, ee_init_if_necessary=ee_init_if_necessary, executor_kwargs=executor_kwargs, + compute_pixels_max_retries=compute_pixels_max_retries, + compute_pixels_initial_delay=compute_pixels_initial_delay, ) store_entrypoint = backends_store.StoreBackendEntrypoint() From 2853dc7204cece9cec51a6011f84e4f81c9b239a Mon Sep 17 00:00:00 2001 From: dabhicusp Date: Tue, 12 Mar 2024 06:13:11 +0000 Subject: [PATCH 02/12] Lint error fixed. --- xee/ext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xee/ext.py b/xee/ext.py index f5d4a97..c4906b2 100644 --- a/xee/ext.py +++ b/xee/ext.py @@ -1060,7 +1060,7 @@ def open_dataset( executor_kwargs (optional): A dictionary of keyword arguments to pass to the ThreadPoolExecutor that handles the parallel computation of pixels i.e. {'max_workers': 2}. - compute_pixels_max_retries (optional): The maximum number of retry + compute_pixels_max_retries (optional): The maximum number of retry attempts for calling ee.data.computePixels(). compute_pixels_initial_delay (optional): The initial delay in milliseconds before the first retry of calling ee.data.computePixels(). From 4f814e78d3eeee83d5c41a85d502596d2fc69893 Mon Sep 17 00:00:00 2001 From: dabhicusp Date: Tue, 12 Mar 2024 08:24:18 +0000 Subject: [PATCH 03/12] Declare retries as a default arguments. --- xee/ext.py | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/xee/ext.py b/xee/ext.py index c4906b2..888fe20 100644 --- a/xee/ext.py +++ b/xee/ext.py @@ -147,8 +147,8 @@ def open( ee_init_kwargs: Optional[Dict[str, Any]] = None, ee_init_if_necessary: bool = False, executor_kwargs: Optional[Dict[str, Any]] = None, - compute_pixels_max_retries: Optional[int] = None, - compute_pixels_initial_delay: Optional[int] = None, + compute_pixels_max_retries: int = 6, + compute_pixels_initial_delay: int = 500, ) -> 'EarthEngineStore': if mode != 'r': raise ValueError( @@ -190,8 +190,8 @@ def __init__( ee_init_kwargs: Optional[Dict[str, Any]] = None, ee_init_if_necessary: bool = False, executor_kwargs: Optional[Dict[str, Any]] = None, - compute_pixels_max_retries: Optional[int] = None, - compute_pixels_initial_delay: Optional[int] = None, + compute_pixels_max_retries: int = 6, + compute_pixels_initial_delay: int = 500, ): self.ee_init_kwargs = ee_init_kwargs self.ee_init_if_necessary = ee_init_if_necessary @@ -201,16 +201,8 @@ def __init__( executor_kwargs = {} self.executor_kwargs = executor_kwargs - # Here 6 & 500 is default value. - # (https://github.com/pydata/xarray/blob/main/xarray/backends/common.py#L181). - self.compute_pixels_max_retries = ( - 6 if compute_pixels_max_retries is None else compute_pixels_max_retries - ) - self.compute_pixels_initial_delay = ( - 500 - if compute_pixels_initial_delay is None - else compute_pixels_initial_delay - ) + self.compute_pixels_max_retries = compute_pixels_max_retries + self.compute_pixels_initial_delay = compute_pixels_initial_delay self.image_collection = image_collection if n_images != -1: @@ -986,8 +978,8 @@ def open_dataset( ee_init_if_necessary: bool = False, ee_init_kwargs: Optional[Dict[str, Any]] = None, executor_kwargs: Optional[Dict[str, Any]] = None, - compute_pixels_max_retries: Optional[int] = None, - compute_pixels_initial_delay: Optional[int] = None, + compute_pixels_max_retries: int = 6, + compute_pixels_initial_delay: int = 500, ) -> xarray.Dataset: # type: ignore """Open an Earth Engine ImageCollection as an Xarray Dataset. @@ -1060,10 +1052,11 @@ def open_dataset( executor_kwargs (optional): A dictionary of keyword arguments to pass to the ThreadPoolExecutor that handles the parallel computation of pixels i.e. {'max_workers': 2}. - compute_pixels_max_retries (optional): The maximum number of retry - attempts for calling ee.data.computePixels(). - compute_pixels_initial_delay (optional): The initial delay in milliseconds - before the first retry of calling ee.data.computePixels(). + compute_pixels_max_retries (int): The maximum number of retry + attempts for calling ee.data.computePixels(). defaults to 6. + # (https://github.com/pydata/xarray/blob/main/xarray/backends/common.py#L181). + compute_pixels_initial_delay (int): The initial delay in milliseconds + before the first retry of calling ee.data.computePixels(). defaults to 500. Returns: An xarray.Dataset that streams in remote data from Earth Engine. From 9154d15667a2ed75d29a84df402ec744157c0b30 Mon Sep 17 00:00:00 2001 From: dabhicusp Date: Tue, 12 Mar 2024 09:00:52 +0000 Subject: [PATCH 04/12] nit changes done. --- xee/ext.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xee/ext.py b/xee/ext.py index 888fe20..152769d 100644 --- a/xee/ext.py +++ b/xee/ext.py @@ -1053,10 +1053,10 @@ def open_dataset( the ThreadPoolExecutor that handles the parallel computation of pixels i.e. {'max_workers': 2}. compute_pixels_max_retries (int): The maximum number of retry - attempts for calling ee.data.computePixels(). defaults to 6. + attempts for calling ee.data.computePixels(). By default, it is 6. # (https://github.com/pydata/xarray/blob/main/xarray/backends/common.py#L181). compute_pixels_initial_delay (int): The initial delay in milliseconds - before the first retry of calling ee.data.computePixels(). defaults to 500. + before the first retry of calling ee.data.computePixels(). By default, it is 500. Returns: An xarray.Dataset that streams in remote data from Earth Engine. From 4d6385c3d7b61340a8cd305aac181b5b8ebfa6dc Mon Sep 17 00:00:00 2001 From: dabhicusp Date: Thu, 14 Mar 2024 06:31:11 +0000 Subject: [PATCH 05/12] Tile_fetch_kwargs added. --- xee/ext.py | 46 +++++++++++++++++++++---------------- xee/ext_integration_test.py | 10 ++++++++ 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/xee/ext.py b/xee/ext.py index 152769d..aead28b 100644 --- a/xee/ext.py +++ b/xee/ext.py @@ -147,8 +147,10 @@ def open( ee_init_kwargs: Optional[Dict[str, Any]] = None, ee_init_if_necessary: bool = False, executor_kwargs: Optional[Dict[str, Any]] = None, - compute_pixels_max_retries: int = 6, - compute_pixels_initial_delay: int = 500, + tile_fetch_kwargs: Dict[str, int] = { + 'max_retries': 6, + 'initial_delay': 500, + }, ) -> 'EarthEngineStore': if mode != 'r': raise ValueError( @@ -170,8 +172,7 @@ def open( ee_init_kwargs=ee_init_kwargs, ee_init_if_necessary=ee_init_if_necessary, executor_kwargs=executor_kwargs, - compute_pixels_max_retries=compute_pixels_max_retries, - compute_pixels_initial_delay=compute_pixels_initial_delay, + tile_fetch_kwargs=tile_fetch_kwargs, ) def __init__( @@ -190,8 +191,10 @@ def __init__( ee_init_kwargs: Optional[Dict[str, Any]] = None, ee_init_if_necessary: bool = False, executor_kwargs: Optional[Dict[str, Any]] = None, - compute_pixels_max_retries: int = 6, - compute_pixels_initial_delay: int = 500, + tile_fetch_kwargs: Dict[str, int] = { + 'max_retries': 6, + 'initial_delay': 500, + }, ): self.ee_init_kwargs = ee_init_kwargs self.ee_init_if_necessary = ee_init_if_necessary @@ -201,8 +204,8 @@ def __init__( executor_kwargs = {} self.executor_kwargs = executor_kwargs - self.compute_pixels_max_retries = compute_pixels_max_retries - self.compute_pixels_initial_delay = compute_pixels_initial_delay + self.tile_fetch_max_retries = tile_fetch_kwargs['max_retries'] + self.tile_fetch_initial_delay = tile_fetch_kwargs['initial_delay'] self.image_collection = image_collection if n_images != -1: @@ -494,8 +497,8 @@ def image_to_array( pixels_getter, params, catch=ee.ee_exception.EEException, - max_retries=self.compute_pixels_max_retries, - initial_delay=self.compute_pixels_initial_delay, + max_retries=self.tile_fetch_max_retries, + initial_delay=self.tile_fetch_initial_delay, ) # Extract out the shape information from EE response. @@ -978,8 +981,10 @@ def open_dataset( ee_init_if_necessary: bool = False, ee_init_kwargs: Optional[Dict[str, Any]] = None, executor_kwargs: Optional[Dict[str, Any]] = None, - compute_pixels_max_retries: int = 6, - compute_pixels_initial_delay: int = 500, + tile_fetch_kwargs: Dict[str, int] = { + 'max_retries': 6, + 'initial_delay': 500, + }, ) -> xarray.Dataset: # type: ignore """Open an Earth Engine ImageCollection as an Xarray Dataset. @@ -1052,12 +1057,14 @@ def open_dataset( executor_kwargs (optional): A dictionary of keyword arguments to pass to the ThreadPoolExecutor that handles the parallel computation of pixels i.e. {'max_workers': 2}. - compute_pixels_max_retries (int): The maximum number of retry - attempts for calling ee.data.computePixels(). By default, it is 6. - # (https://github.com/pydata/xarray/blob/main/xarray/backends/common.py#L181). - compute_pixels_initial_delay (int): The initial delay in milliseconds - before the first retry of calling ee.data.computePixels(). By default, it is 500. - + tile_fetch_kwargs (Dict): The necessary kwargs like `max_retries`, + `initial_delay` which helps while fetching data through calling + ee.data.computePixels(). i.e. {'max_retries' : 6, 'initial_delay': 500}. + - max_retries is maximum number of retry attempts for calling + ee.data.computePixels().By default, it is 6. + - initial_delay is the initial delay in milliseconds before the first + retry of calling ee.data.computePixels(). By default, it is 500. + (https://github.com/pydata/xarray/blob/main/xarray/backends/common.py#L181). Returns: An xarray.Dataset that streams in remote data from Earth Engine. """ @@ -1087,8 +1094,7 @@ def open_dataset( ee_init_kwargs=ee_init_kwargs, ee_init_if_necessary=ee_init_if_necessary, executor_kwargs=executor_kwargs, - compute_pixels_max_retries=compute_pixels_max_retries, - compute_pixels_initial_delay=compute_pixels_initial_delay, + tile_fetch_kwargs=tile_fetch_kwargs, ) store_entrypoint = backends_store.StoreBackendEntrypoint() diff --git a/xee/ext_integration_test.py b/xee/ext_integration_test.py index ea821df..3ac2bd2 100644 --- a/xee/ext_integration_test.py +++ b/xee/ext_integration_test.py @@ -69,6 +69,7 @@ def setUp(self): '2017-01-01', '2017-01-03' ), n_images=64, + tile_fetch_kwargs={'max_retries': 10, 'initial_delay': 1500}, ) self.lnglat_store = xee.EarthEngineStore( ee.ImageCollection.fromImages([ee.Image.pixelLonLat()]), @@ -255,6 +256,15 @@ def __getitem__(self, params): self.assertEqual(getter.count, 3) + def test_tile_fetch_kwargs(self): + arr = xee.EarthEngineBackendArray('B4', self.store) + self.assertEqual(arr.store.tile_fetch_initial_delay, 1500) + self.assertEqual(arr.store.tile_fetch_max_retries, 10) + + arr1 = xee.EarthEngineBackendArray('longitude', self.lnglat_store) + self.assertEqual(arr1.store.tile_fetch_initial_delay, 500) + self.assertEqual(arr1.store.tile_fetch_max_retries, 6) + class EEBackendEntrypointTest(absltest.TestCase): From d91a9e48c5bc8e7eacb6954aa5503b3227bd7215 Mon Sep 17 00:00:00 2001 From: dabhicusp Date: Thu, 14 Mar 2024 07:14:20 +0000 Subject: [PATCH 06/12] Pop added for working on single value, too. --- xee/ext.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/xee/ext.py b/xee/ext.py index aead28b..af4d429 100644 --- a/xee/ext.py +++ b/xee/ext.py @@ -204,8 +204,9 @@ def __init__( executor_kwargs = {} self.executor_kwargs = executor_kwargs - self.tile_fetch_max_retries = tile_fetch_kwargs['max_retries'] - self.tile_fetch_initial_delay = tile_fetch_kwargs['initial_delay'] + # Default value: (https://github.com/pydata/xarray/blob/main/xarray/backends/common.py#L181). + self.tile_fetch_max_retries = tile_fetch_kwargs.pop('max_retries', 6) + self.tile_fetch_initial_delay = tile_fetch_kwargs.pop('initial_delay', 500) self.image_collection = image_collection if n_images != -1: From 41b12e2a0de12d45df3440bd2485469b9d3348d6 Mon Sep 17 00:00:00 2001 From: dabhicusp Date: Thu, 14 Mar 2024 10:31:53 +0000 Subject: [PATCH 07/12] code updated after suggestions. --- xee/ext.py | 27 ++++++++++++++------------- xee/ext_integration_test.py | 12 ++++++++---- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/xee/ext.py b/xee/ext.py index af4d429..51f36a6 100644 --- a/xee/ext.py +++ b/xee/ext.py @@ -102,6 +102,11 @@ class EarthEngineStore(common.AbstractDataStore): 'height': 256, } + TILE_FETCH_KWARGS: Dict[str, int] = { + 'max_retries': 6, + 'initial_delay': 500, + } + SCALE_UNITS: Dict[str, int] = { 'degree': 1, 'metre': 10_000, @@ -147,10 +152,7 @@ def open( ee_init_kwargs: Optional[Dict[str, Any]] = None, ee_init_if_necessary: bool = False, executor_kwargs: Optional[Dict[str, Any]] = None, - tile_fetch_kwargs: Dict[str, int] = { - 'max_retries': 6, - 'initial_delay': 500, - }, + tile_fetch_kwargs: Dict[str, int] = TILE_FETCH_KWARGS, ) -> 'EarthEngineStore': if mode != 'r': raise ValueError( @@ -191,10 +193,7 @@ def __init__( ee_init_kwargs: Optional[Dict[str, Any]] = None, ee_init_if_necessary: bool = False, executor_kwargs: Optional[Dict[str, Any]] = None, - tile_fetch_kwargs: Dict[str, int] = { - 'max_retries': 6, - 'initial_delay': 500, - }, + tile_fetch_kwargs: Dict[str, int] = TILE_FETCH_KWARGS, ): self.ee_init_kwargs = ee_init_kwargs self.ee_init_if_necessary = ee_init_if_necessary @@ -204,9 +203,7 @@ def __init__( executor_kwargs = {} self.executor_kwargs = executor_kwargs - # Default value: (https://github.com/pydata/xarray/blob/main/xarray/backends/common.py#L181). - self.tile_fetch_max_retries = tile_fetch_kwargs.pop('max_retries', 6) - self.tile_fetch_initial_delay = tile_fetch_kwargs.pop('initial_delay', 500) + self.tile_fetch_kwargs = tile_fetch_kwargs self.image_collection = image_collection if n_images != -1: @@ -498,8 +495,12 @@ def image_to_array( pixels_getter, params, catch=ee.ee_exception.EEException, - max_retries=self.tile_fetch_max_retries, - initial_delay=self.tile_fetch_initial_delay, + max_retries=self.tile_fetch_kwargs.get( + 'max_retries', self.TILE_FETCH_KWARGS.get('max_retries') + ), + initial_delay=self.tile_fetch_kwargs.get( + 'initial_delay', self.TILE_FETCH_KWARGS.get('initial_delay') + ), ) # Extract out the shape information from EE response. diff --git a/xee/ext_integration_test.py b/xee/ext_integration_test.py index 3ac2bd2..5ad9378 100644 --- a/xee/ext_integration_test.py +++ b/xee/ext_integration_test.py @@ -81,6 +81,7 @@ def setUp(self): '2020-03-30', '2020-04-01' ), n_images=64, + tile_fetch_kwargs={'max_retries': 9}, ) self.all_img_store = xee.EarthEngineStore( ee.ImageCollection('LANDSAT/LC08/C01/T1').filterDate( @@ -258,13 +259,16 @@ def __getitem__(self, params): def test_tile_fetch_kwargs(self): arr = xee.EarthEngineBackendArray('B4', self.store) - self.assertEqual(arr.store.tile_fetch_initial_delay, 1500) - self.assertEqual(arr.store.tile_fetch_max_retries, 10) + self.assertEqual(arr.store.tile_fetch_kwargs['initial_delay'], 1500) + self.assertEqual(arr.store.tile_fetch_kwargs['max_retries'], 10) arr1 = xee.EarthEngineBackendArray('longitude', self.lnglat_store) - self.assertEqual(arr1.store.tile_fetch_initial_delay, 500) - self.assertEqual(arr1.store.tile_fetch_max_retries, 6) + self.assertEqual(arr1.store.tile_fetch_kwargs['initial_delay'], 500) + self.assertEqual(arr1.store.tile_fetch_kwargs['max_retries'], 6) + arr1 = xee.EarthEngineBackendArray('spi2y', self.conus_store) + self.assertNotIn('initial_delay', arr1.store.tile_fetch_kwargs) + self.assertEqual(arr1.store.tile_fetch_kwargs['max_retries'], 9) class EEBackendEntrypointTest(absltest.TestCase): From 4b5447ae786723845032373f909ce2478741b1db Mon Sep 17 00:00:00 2001 From: dabhicusp Date: Thu, 14 Mar 2024 10:34:42 +0000 Subject: [PATCH 08/12] Lint changes done. --- xee/ext_integration_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xee/ext_integration_test.py b/xee/ext_integration_test.py index 5ad9378..a94f524 100644 --- a/xee/ext_integration_test.py +++ b/xee/ext_integration_test.py @@ -270,6 +270,7 @@ def test_tile_fetch_kwargs(self): self.assertNotIn('initial_delay', arr1.store.tile_fetch_kwargs) self.assertEqual(arr1.store.tile_fetch_kwargs['max_retries'], 9) + class EEBackendEntrypointTest(absltest.TestCase): def setUp(self): From d9aa0c0403c10bb091cd9c7edd590185e40044b2 Mon Sep 17 00:00:00 2001 From: dabhicusp Date: Thu, 14 Mar 2024 11:54:12 +0000 Subject: [PATCH 09/12] keyword arguments changed to optional. --- xee/ext.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/xee/ext.py b/xee/ext.py index 51f36a6..9df545e 100644 --- a/xee/ext.py +++ b/xee/ext.py @@ -152,7 +152,7 @@ def open( ee_init_kwargs: Optional[Dict[str, Any]] = None, ee_init_if_necessary: bool = False, executor_kwargs: Optional[Dict[str, Any]] = None, - tile_fetch_kwargs: Dict[str, int] = TILE_FETCH_KWARGS, + tile_fetch_kwargs: Optional[Dict[str, int]] = None, ) -> 'EarthEngineStore': if mode != 'r': raise ValueError( @@ -193,7 +193,7 @@ def __init__( ee_init_kwargs: Optional[Dict[str, Any]] = None, ee_init_if_necessary: bool = False, executor_kwargs: Optional[Dict[str, Any]] = None, - tile_fetch_kwargs: Dict[str, int] = TILE_FETCH_KWARGS, + tile_fetch_kwargs: Optional[Dict[str, int]] = None, ): self.ee_init_kwargs = ee_init_kwargs self.ee_init_if_necessary = ee_init_if_necessary @@ -203,7 +203,11 @@ def __init__( executor_kwargs = {} self.executor_kwargs = executor_kwargs - self.tile_fetch_kwargs = tile_fetch_kwargs + self.tile_fetch_kwargs = ( + self.TILE_FETCH_KWARGS + if tile_fetch_kwargs is None + else tile_fetch_kwargs + ) self.image_collection = image_collection if n_images != -1: @@ -983,10 +987,7 @@ def open_dataset( ee_init_if_necessary: bool = False, ee_init_kwargs: Optional[Dict[str, Any]] = None, executor_kwargs: Optional[Dict[str, Any]] = None, - tile_fetch_kwargs: Dict[str, int] = { - 'max_retries': 6, - 'initial_delay': 500, - }, + tile_fetch_kwargs: Optional[Dict[str, int]] = None, ) -> xarray.Dataset: # type: ignore """Open an Earth Engine ImageCollection as an Xarray Dataset. @@ -1059,7 +1060,7 @@ def open_dataset( executor_kwargs (optional): A dictionary of keyword arguments to pass to the ThreadPoolExecutor that handles the parallel computation of pixels i.e. {'max_workers': 2}. - tile_fetch_kwargs (Dict): The necessary kwargs like `max_retries`, + tile_fetch_kwargs (optional): The necessary kwargs like `max_retries`, `initial_delay` which helps while fetching data through calling ee.data.computePixels(). i.e. {'max_retries' : 6, 'initial_delay': 500}. - max_retries is maximum number of retry attempts for calling From 99a3d4519e5be55003b9d33794c902f592effb67 Mon Sep 17 00:00:00 2001 From: dabhicusp Date: Fri, 15 Mar 2024 06:35:07 +0000 Subject: [PATCH 10/12] tile_fetch_kwargs is updated with getitem_kwargs. --- xee/ext.py | 32 ++++++++++++-------------------- xee/ext_integration_test.py | 20 ++++++++++---------- 2 files changed, 22 insertions(+), 30 deletions(-) diff --git a/xee/ext.py b/xee/ext.py index 9df545e..1414e80 100644 --- a/xee/ext.py +++ b/xee/ext.py @@ -102,7 +102,7 @@ class EarthEngineStore(common.AbstractDataStore): 'height': 256, } - TILE_FETCH_KWARGS: Dict[str, int] = { + GETITEM_KWARGS: Dict[str, int] = { 'max_retries': 6, 'initial_delay': 500, } @@ -152,7 +152,7 @@ def open( ee_init_kwargs: Optional[Dict[str, Any]] = None, ee_init_if_necessary: bool = False, executor_kwargs: Optional[Dict[str, Any]] = None, - tile_fetch_kwargs: Optional[Dict[str, int]] = None, + getitem_kwargs: Optional[Dict[str, int]] = None, ) -> 'EarthEngineStore': if mode != 'r': raise ValueError( @@ -174,7 +174,7 @@ def open( ee_init_kwargs=ee_init_kwargs, ee_init_if_necessary=ee_init_if_necessary, executor_kwargs=executor_kwargs, - tile_fetch_kwargs=tile_fetch_kwargs, + getitem_kwargs=getitem_kwargs, ) def __init__( @@ -193,7 +193,7 @@ def __init__( ee_init_kwargs: Optional[Dict[str, Any]] = None, ee_init_if_necessary: bool = False, executor_kwargs: Optional[Dict[str, Any]] = None, - tile_fetch_kwargs: Optional[Dict[str, int]] = None, + getitem_kwargs: Optional[Dict[str, int]] = None, ): self.ee_init_kwargs = ee_init_kwargs self.ee_init_if_necessary = ee_init_if_necessary @@ -203,11 +203,7 @@ def __init__( executor_kwargs = {} self.executor_kwargs = executor_kwargs - self.tile_fetch_kwargs = ( - self.TILE_FETCH_KWARGS - if tile_fetch_kwargs is None - else tile_fetch_kwargs - ) + self.getitem_kwargs = {**self.GETITEM_KWARGS, **(getitem_kwargs or {})} self.image_collection = image_collection if n_images != -1: @@ -499,12 +495,8 @@ def image_to_array( pixels_getter, params, catch=ee.ee_exception.EEException, - max_retries=self.tile_fetch_kwargs.get( - 'max_retries', self.TILE_FETCH_KWARGS.get('max_retries') - ), - initial_delay=self.tile_fetch_kwargs.get( - 'initial_delay', self.TILE_FETCH_KWARGS.get('initial_delay') - ), + max_retries=self.getitem_kwargs['max_retries'], + initial_delay=self.getitem_kwargs['initial_delay'], ) # Extract out the shape information from EE response. @@ -987,7 +979,7 @@ def open_dataset( ee_init_if_necessary: bool = False, ee_init_kwargs: Optional[Dict[str, Any]] = None, executor_kwargs: Optional[Dict[str, Any]] = None, - tile_fetch_kwargs: Optional[Dict[str, int]] = None, + getitem_kwargs: Optional[Dict[str, int]] = None, ) -> xarray.Dataset: # type: ignore """Open an Earth Engine ImageCollection as an Xarray Dataset. @@ -1060,9 +1052,9 @@ def open_dataset( executor_kwargs (optional): A dictionary of keyword arguments to pass to the ThreadPoolExecutor that handles the parallel computation of pixels i.e. {'max_workers': 2}. - tile_fetch_kwargs (optional): The necessary kwargs like `max_retries`, - `initial_delay` which helps while fetching data through calling - ee.data.computePixels(). i.e. {'max_retries' : 6, 'initial_delay': 500}. + getitem_kwargs (optional): Exponential backoff kwargs passed into + the xarray function to index the array (`robust_getitem`). + i.e. {'max_retries' : 6, 'initial_delay': 500}. - max_retries is maximum number of retry attempts for calling ee.data.computePixels().By default, it is 6. - initial_delay is the initial delay in milliseconds before the first @@ -1097,7 +1089,7 @@ def open_dataset( ee_init_kwargs=ee_init_kwargs, ee_init_if_necessary=ee_init_if_necessary, executor_kwargs=executor_kwargs, - tile_fetch_kwargs=tile_fetch_kwargs, + getitem_kwargs=getitem_kwargs, ) store_entrypoint = backends_store.StoreBackendEntrypoint() diff --git a/xee/ext_integration_test.py b/xee/ext_integration_test.py index a94f524..771ab78 100644 --- a/xee/ext_integration_test.py +++ b/xee/ext_integration_test.py @@ -69,7 +69,7 @@ def setUp(self): '2017-01-01', '2017-01-03' ), n_images=64, - tile_fetch_kwargs={'max_retries': 10, 'initial_delay': 1500}, + getitem_kwargs={'max_retries': 10, 'initial_delay': 1500}, ) self.lnglat_store = xee.EarthEngineStore( ee.ImageCollection.fromImages([ee.Image.pixelLonLat()]), @@ -81,7 +81,7 @@ def setUp(self): '2020-03-30', '2020-04-01' ), n_images=64, - tile_fetch_kwargs={'max_retries': 9}, + getitem_kwargs={'max_retries': 9}, ) self.all_img_store = xee.EarthEngineStore( ee.ImageCollection('LANDSAT/LC08/C01/T1').filterDate( @@ -257,18 +257,18 @@ def __getitem__(self, params): self.assertEqual(getter.count, 3) - def test_tile_fetch_kwargs(self): + def test_getitem_kwargs(self): arr = xee.EarthEngineBackendArray('B4', self.store) - self.assertEqual(arr.store.tile_fetch_kwargs['initial_delay'], 1500) - self.assertEqual(arr.store.tile_fetch_kwargs['max_retries'], 10) + self.assertEqual(arr.store.getitem_kwargs['initial_delay'], 1500) + self.assertEqual(arr.store.getitem_kwargs['max_retries'], 10) arr1 = xee.EarthEngineBackendArray('longitude', self.lnglat_store) - self.assertEqual(arr1.store.tile_fetch_kwargs['initial_delay'], 500) - self.assertEqual(arr1.store.tile_fetch_kwargs['max_retries'], 6) + self.assertEqual(arr1.store.getitem_kwargs['initial_delay'], 500) + self.assertEqual(arr1.store.getitem_kwargs['max_retries'], 6) - arr1 = xee.EarthEngineBackendArray('spi2y', self.conus_store) - self.assertNotIn('initial_delay', arr1.store.tile_fetch_kwargs) - self.assertEqual(arr1.store.tile_fetch_kwargs['max_retries'], 9) + arr2 = xee.EarthEngineBackendArray('spi2y', self.conus_store) + self.assertEqual(arr2.store.getitem_kwargs['initial_delay'], 500) + self.assertEqual(arr2.store.getitem_kwargs['max_retries'], 9) class EEBackendEntrypointTest(absltest.TestCase): From e3edcec8b25eb466dfe8acab5469728362fa2d67 Mon Sep 17 00:00:00 2001 From: dabhicusp Date: Fri, 15 Mar 2024 14:48:22 +0000 Subject: [PATCH 11/12] Nit changes done. --- xee/ext.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/xee/ext.py b/xee/ext.py index 1414e80..00b3dbf 100644 --- a/xee/ext.py +++ b/xee/ext.py @@ -495,8 +495,7 @@ def image_to_array( pixels_getter, params, catch=ee.ee_exception.EEException, - max_retries=self.getitem_kwargs['max_retries'], - initial_delay=self.getitem_kwargs['initial_delay'], + **self.getitem_kwargs, ) # Extract out the shape information from EE response. @@ -1055,10 +1054,9 @@ def open_dataset( getitem_kwargs (optional): Exponential backoff kwargs passed into the xarray function to index the array (`robust_getitem`). i.e. {'max_retries' : 6, 'initial_delay': 500}. - - max_retries is maximum number of retry attempts for calling - ee.data.computePixels().By default, it is 6. - - initial_delay is the initial delay in milliseconds before the first - retry of calling ee.data.computePixels(). By default, it is 500. + - max_retries, the maximum number of retry attempts.By default, it is 6. + - initial_delay, the initial delay in milliseconds before the first + retry. By default, it is 500. (https://github.com/pydata/xarray/blob/main/xarray/backends/common.py#L181). Returns: An xarray.Dataset that streams in remote data from Earth Engine. From f13b6a2e01aac8f6e6799c6f4d01c710b9fa4664 Mon Sep 17 00:00:00 2001 From: Nathaniel Schmitz Date: Fri, 15 Mar 2024 15:10:32 +0000 Subject: [PATCH 12/12] Update getitem_kwargs documentation comment. --- xee/ext.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/xee/ext.py b/xee/ext.py index 00b3dbf..51976bc 100644 --- a/xee/ext.py +++ b/xee/ext.py @@ -1051,13 +1051,11 @@ def open_dataset( executor_kwargs (optional): A dictionary of keyword arguments to pass to the ThreadPoolExecutor that handles the parallel computation of pixels i.e. {'max_workers': 2}. - getitem_kwargs (optional): Exponential backoff kwargs passed into - the xarray function to index the array (`robust_getitem`). - i.e. {'max_retries' : 6, 'initial_delay': 500}. - - max_retries, the maximum number of retry attempts.By default, it is 6. - - initial_delay, the initial delay in milliseconds before the first - retry. By default, it is 500. - (https://github.com/pydata/xarray/blob/main/xarray/backends/common.py#L181). + getitem_kwargs (optional): Exponential backoff kwargs passed into the + xarray function to index the array (`robust_getitem`). + - 'max_retries', the maximum number of retry attempts. Defaults to 6. + - 'initial_delay', the initial delay in milliseconds before the first + retry. Defaults to 500. Returns: An xarray.Dataset that streams in remote data from Earth Engine. """