-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathSD_ESAtools.py
190 lines (169 loc) · 7.13 KB
/
SD_ESAtools.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
#!/usr/bin/env python
"""
Tools to batch process OLCI and MSI with SeaDAS default atmospheric correction.
Input files must be in the format downloaded with getOC.py
MIT License
Copyright (c) 2021 Guillaume Bourdin
"""
from datetime import datetime, timedelta
from time import sleep
import random
from subprocess import check_call
import os
import shutil
import zipfile
import sys
# Setup path to OCSSW / OCSSW_RUNNER / modules directory in OCSSW
PATH_OCSSW = '/home/bjiang/ocssw'
OCSSW_RUNNER = PATH_OCSSW + '/bin/ocssw_runner' ## Seadas8.00 architecture
sys.path.insert(0, os.path.join(PATH_OCSSW, 'bin')) ## Seadas8.00 architecture
import seadasutils.anc_utils as ga ## Seadas8.00 architecture
from seadasutils.setupenv import env## Seadas8.00 architecture
# OCSSW_RUNNER = PATH_OCSSW + '/scripts/ocssw_runner' ## Seadas version < 8.00 architecture
# sys.path.insert(0, os.path.join(PATH_OCSSW, 'scripts')) ## Seadas version < 8.00 architecture
# import modules.anc_utils as ga ## Seadas version < 8.00 architecture
# from modules.setupenv import env ## Seadas version < 8.00 architecture
def unzipS(ref, suffix):
# Decompress zip
if not os.path.isdir(ref + suffix):
print('Decompress zip ' + ref + suffix + ' ...')
if os.path.isfile(ref + suffix + '.zip'):
os.mkdir(ref + suffix)
zf = zipfile.ZipFile(ref + suffix + '.zip')
zf.extractall(path=os.path.dirname(ref + suffix))
else:
print(ref + suffix + '.zip not found')
return -1
else:
print('Decompress zip: Skip')
return None
def check_unzip(ref, suffix):
# check unzip size
MAX_RETRIES = 30
j = 0
while len(os.listdir(ref + suffix)) == 0 and j < MAX_RETRIES:
print("Empty directory, unzip attempt [" + str(j+1) + "/" + str(MAX_RETRIES) + "] failed, restarting")
shutil.rmtree(ref + suffix)
unzipS(ref, suffix)
j += 1
if j+1 == MAX_RETRIES:
print(str(MAX_RETRIES) + ' unzip attempts failed, process aborted.')
return -1
def get_ancillaries(sensor, reference, path_to_data, path_to_anc, start_dt=None, stop_dt=None):
# based on SeaDAS/Ocssw/getanc.py
# Need to be run through a SeaDAS Virtual Env and ocssw_runner:
# ./ocssw/scripts/ocssw_runner --ocsswroot /home/gbourdin/ocssw/ /home/gbourdin/.conda/envs/SeaDAS/bin/python process.py
# add random sleep time to avoid overload OBPG server
# sleep(random.uniform(0, 10))
ref = os.path.join(path_to_data, reference)
# Start and Stop dt must be YYYYDDDHHMMSS strings
if reference is not None and start_dt is None and stop_dt is None:
# g = ga.getanc(file=ref, # for seadas version < 8.00
g = ga.getanc(filename=ref, # for seadas version = 8.00
sensor=sensor,
ancdir=os.path.join(path_to_anc, reference),
ancdb=os.path.join(path_to_anc, 'ancillary_data_' + reference + '.db'),
opt_flag=5,
timeout=60,
verbose=True)
elif start_dt is not None and stop_dt is not None:
# g = ga.getanc(file=ref, # for seadas version < 8.00
g = ga.getanc(filename=ref, # for seadas version = 8.00
start=start_dt,
stop=stop_dt,
sensor=sensor,
ancdir=os.path.join(path_to_anc, reference),
ancdb=os.path.join(path_to_anc, 'ancillary_data_' + reference + '.db'),
opt_flag=5,
timeout=60,
verbose=True)
env(g) # This line check for environment variables
g.chk()
if g.finddb():
g.setup()
else:
g.setup()
g.findweb()
g.locate()
g.write_anc_par()
g.cleanup()
return g.files
def process_SENT3_L1_to_L2(path_to_data, reference, ancil_list, instrument='OLCI', suite='OC', l2_prod=None, get_anc=True, path_to_anc=None, force=False):
# Process OLCI Image from compressed L1 to L2_SEN3 using default Nasa Ocean Color parameters
# the function will change the current working directory
ref = os.path.join(path_to_data, reference)
if force:
if os.path.isdir(ref + '.SEN3'):
shutil.rmtree(ref + '.SEN3')
if os.path.isfile(ref + '.L2_SEN3.nc'):
os.remove(ref + '.L2_SEN3.nc')
if os.path.isfile(ref + '.L2_SEN3_temp.nc'):
os.remove(ref + '.L2_SEN3_temp.nc')
# Decompress zip
if not os.path.isdir(ref + '.SEN3'):
unzipS(ref, '.SEN3')
# check unzip
check_unzip(ref, '.SEN3')
# Process L2
if not os.path.isfile(ref + '.L2_SEN3.nc'):
print('Process L2 ...')
#os.chdir(ref + '.SEN3/')
cmd = [OCSSW_RUNNER, '--ocsswroot', PATH_OCSSW, 'l2gen',
'suite=' + suite,
'ifile=' + ref + '.SEN3/Oa01_radiance.nc',
'ofile=' + ref + '.L2_SEN3_temp.nc',
'maskland=0',
'maskhilt=0']
if get_anc:
foo = filter(lambda x: x != "", ancil_list.split('<>')) # remove empty string
anc_key = list(foo)
for key in anc_key:
cmd.append(key) # append each ancillary file to cmd
if l2_prod is not None:
cmd.append('l2prod=' + l2_prod)
check_call(cmd)
os.rename(ref + '.L2_SEN3_temp.nc', ref + '.L2_SEN3.nc')
else:
print('Process L2: Skip')
def process_MSI_L1_to_L2(path_to_data, reference, ancil_list, suite='OC', l2_prod=None, get_anc=True, path_to_anc=None, force=False):
# Process OLCI Image from compressed L1 to L2_SEN3 using default Nasa Ocean Color parameters
# the function will change the current working directory
ref = os.path.join(path_to_data, reference)
if force:
if os.path.isdir(ref + '.SAFE'):
shutil.rmtree(ref + '.SAFE')
if os.path.isfile(ref + '.L2_SEN2.nc'):
os.remove(ref + '.L2_SEN2.nc')
if os.path.isfile(ref + '.L2_SEN2_temp.nc'):
os.remove(ref + '.L2_SEN2_temp.nc')
# Decompress zip
if not os.path.isdir(ref + '.SAFE'):
unzipS(ref, '.SAFE')
# check unzip
check_unzip(ref, '.SAFE')
# Process L2
if not os.path.isfile(ref + '.L2_SEN2.nc'):
print('Process L2 ...')
#os.chdir(ref + '.SAFE/')
cmd = [OCSSW_RUNNER, '--ocsswroot', PATH_OCSSW, 'l2gen',
'suite=OC',
'ifile=' + ref + '.SAFE/manifest.safe',
'ofile=' + ref + '.L2_SEN2_temp.nc',
'brdf_opt=1',
'aer_opt=-2',
'cirrus_opt=false',
'cloud_thresh=0.018',
'cloud_wave=2130.0',
'maskland=0',
'maskhilt=0']
if get_anc:
foo = filter(lambda x: x != "", ancil_list.split('<>')) # remove empty string
anc_key = list(foo)
for key in anc_key:
cmd.append(key) # append each ancillary file to cmd
if l2_prod is not None:
cmd.append('l2prod=' + l2_prod)
check_call(cmd)
os.rename(ref + '.L2_SEN2_temp.nc', ref + '.L2_SEN2.nc')
else:
print('Process L2: Skip')