From 0f087a22a9775b29ff344e763d5f5c9a300cd4d0 Mon Sep 17 00:00:00 2001 From: landmanbester Date: Fri, 30 Aug 2024 11:46:20 +0200 Subject: [PATCH] add pass_missing_as_none policies to all workers --- pfb/deconv/clark.py | 3 +-- pfb/parser/degrid.yaml | 3 +++ pfb/parser/fluxmop.yaml | 3 +++ pfb/parser/fwdbwd.yaml | 3 +++ pfb/parser/grid.yaml | 16 ++++++++-------- pfb/parser/hci.yaml | 3 +++ pfb/parser/init.yaml | 11 +++++++---- pfb/parser/klean.yaml | 3 +++ pfb/parser/model2comps.yaml | 3 +++ pfb/parser/restore.yaml | 3 +++ pfb/parser/sara.yaml | 8 +++----- pfb/parser/schemas.py | 24 ++++++++++++++---------- pfb/parser/smoovie.yaml | 3 +++ pfb/parser/spotless.yaml | 3 +++ pfb/parser/uncabbedcabs.yml | 9 --------- pfb/utils/misc.py | 1 - pfb/workers/init.py | 6 ++++-- pfb/workers/main.py | 2 -- pfb/workers/sara.py | 1 - tests/test_klean.py | 1 - 20 files changed, 64 insertions(+), 45 deletions(-) diff --git a/pfb/deconv/clark.py b/pfb/deconv/clark.py index 28cf1c352..3f5af657e 100644 --- a/pfb/deconv/clark.py +++ b/pfb/deconv/clark.py @@ -73,7 +73,6 @@ def subminor(A, psf, Ip, Iq, model, wsums, gamma=0.05, th=0.0, maxit=10000): # right_y = q + Iq < ny # mask = (left_x & right_x) & (left_y & right_y) - # import ipdb; ipdb.set_trace() mask = (np.abs(Idelp) <= nxo2) & (np.abs(Idelq) <= nyo2) # Ipp, Iqq = psf[:, nxo2 - Ip[mask], nyo2 - Iq[mask]] A = subtract(A[:, mask], psf, @@ -138,7 +137,7 @@ def clark(ID, gamma=gamma, th=subth, maxit=submaxit) - # import ipdb; ipdb.set_trace() + # subtract from full image (as in major cycle) psf_convolve_cube( xpad, diff --git a/pfb/parser/degrid.yaml b/pfb/parser/degrid.yaml index 72759453c..f7d673919 100644 --- a/pfb/parser/degrid.yaml +++ b/pfb/parser/degrid.yaml @@ -55,3 +55,6 @@ inputs: outputs: {} + +policies: + pass_missing_as_none: true diff --git a/pfb/parser/fluxmop.yaml b/pfb/parser/fluxmop.yaml index 1bf0222a4..4aaf56b97 100644 --- a/pfb/parser/fluxmop.yaml +++ b/pfb/parser/fluxmop.yaml @@ -71,3 +71,6 @@ inputs: outputs: {} + +policies: + pass_missing_as_none: true diff --git a/pfb/parser/fwdbwd.yaml b/pfb/parser/fwdbwd.yaml index aa734ecf9..eaa125640 100644 --- a/pfb/parser/fwdbwd.yaml +++ b/pfb/parser/fwdbwd.yaml @@ -109,3 +109,6 @@ inputs: outputs: {} + +policies: + pass_missing_as_none: true diff --git a/pfb/parser/grid.yaml b/pfb/parser/grid.yaml index afa7b7e86..4dcc7e1f3 100644 --- a/pfb/parser/grid.yaml +++ b/pfb/parser/grid.yaml @@ -111,17 +111,17 @@ inputs: A sensible value for this parameter depends on the level of RFI in the data. Small values (eg. 2) result in aggressive reweighting and should be avoided if the model is still incomplete. - mf-weighting: - dtype: bool - default: false - info: - Base the imaging weights on the uniform weights computed over all - data instead of per band. This ensures that the MFS image can truly - be uniformly weighted. _include: - (.)gridding.yml - (.)dist.yml - (.)out.yml outputs: - {} + cds: + dtype: Directory + info: output cube dataset + mkdir: false + + +policies: + pass_missing_as_none: true diff --git a/pfb/parser/hci.yaml b/pfb/parser/hci.yaml index 726c27f24..ed4abdc65 100644 --- a/pfb/parser/hci.yaml +++ b/pfb/parser/hci.yaml @@ -177,3 +177,6 @@ inputs: outputs: {} + +policies: + pass_missing_as_none: true diff --git a/pfb/parser/init.yaml b/pfb/parser/init.yaml index 31ae6a37e..2ff1ed91c 100644 --- a/pfb/parser/init.yaml +++ b/pfb/parser/init.yaml @@ -6,22 +6,22 @@ inputs: info: Path to measurement set scans: - dtype: List[int] + dtype: Optional[List[int]] info: List of SCAN_NUMBERS to image. Defaults to all. Input as comma separated list 0,2 if running from CLI ddids: - dtype: List[int] + dtype: Optional[List[int]] info: List of DATA_DESC_ID's to images. Defaults to all. Input as comma separated list 0,2 if running from CLI fields: - dtype: List[int] + dtype: Optional[List[int]] info: List of FIELD_ID's to image. Defaults to all. Input as comma separated list 0,2 if running from CLI freq-range: - dtype: str + dtype: Optional[str] info: Frequency range to image in Hz. Specify as a string with colon delimiter eg. '1e9:1.1e9' @@ -133,3 +133,6 @@ inputs: outputs: {} + +policies: + pass_missing_as_none: true diff --git a/pfb/parser/klean.yaml b/pfb/parser/klean.yaml index 6effe9a69..f3cdae2bf 100644 --- a/pfb/parser/klean.yaml +++ b/pfb/parser/klean.yaml @@ -134,3 +134,6 @@ inputs: outputs: {} + +policies: + pass_missing_as_none: true diff --git a/pfb/parser/model2comps.yaml b/pfb/parser/model2comps.yaml index 2c4227f8c..1acaf8cb7 100644 --- a/pfb/parser/model2comps.yaml +++ b/pfb/parser/model2comps.yaml @@ -77,3 +77,6 @@ inputs: outputs: {} + +policies: + pass_missing_as_none: true diff --git a/pfb/parser/restore.yaml b/pfb/parser/restore.yaml index 05b8e0627..e3c15b71d 100644 --- a/pfb/parser/restore.yaml +++ b/pfb/parser/restore.yaml @@ -54,3 +54,6 @@ inputs: outputs: {} + +policies: + pass_missing_as_none: true diff --git a/pfb/parser/sara.yaml b/pfb/parser/sara.yaml index dd64652b6..12cf3184e 100644 --- a/pfb/parser/sara.yaml +++ b/pfb/parser/sara.yaml @@ -101,11 +101,6 @@ inputs: info: Will terminate the algorithm if the rms increases this many times. Set to larger than niter to disable this check. - do-res: - dtype: bool - default: false - info: - Deconvolve PSF to add intrinsic model resolution to dds skip-model: dtype: bool default: false @@ -125,3 +120,6 @@ inputs: outputs: {} + +policies: + pass_missing_as_none: true diff --git a/pfb/parser/schemas.py b/pfb/parser/schemas.py index d1f23b533..efba6bc3d 100644 --- a/pfb/parser/schemas.py +++ b/pfb/parser/schemas.py @@ -1,18 +1,24 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field import os.path import glob from typing import * from scabha import configuratt +from collections import OrderedDict from scabha.cargo import Parameter, _UNSET_DEFAULT from omegaconf.omegaconf import OmegaConf +def EmptyDictDefault(): + return field(default_factory=lambda:OrderedDict()) schema = None @dataclass class _CabInputsOutputs(object): - inputs: Dict[str, Parameter] - outputs: Dict[str, Parameter] + # inputs: Dict[str, Parameter] + # outputs: Dict[str, Parameter] + inputs: Dict[str, Parameter] = EmptyDictDefault() + outputs: Dict[str, Parameter] = EmptyDictDefault() + policies: Optional[Dict[str, Any]] = None # load schema files if schema is None: @@ -32,11 +38,9 @@ class _CabInputsOutputs(object): # and a set containing locations of .yaml configs for pfb workers schema = OmegaConf.create(tmp[0]) - # is this still necessary? - for worker in schema.keys(): - for param in schema[worker]['inputs']: - if schema[worker]['inputs'][param]['default'] == _UNSET_DEFAULT: - schema[worker]['inputs'][param]['default'] = None - - + # # is this still necessary? + # for worker in schema.keys(): + # for param in schema[worker]['inputs']: + # if schema[worker]['inputs'][param]['default'] == _UNSET_DEFAULT: + # schema[worker]['inputs'][param]['default'] = None diff --git a/pfb/parser/smoovie.yaml b/pfb/parser/smoovie.yaml index 885a4658d..1989a266a 100644 --- a/pfb/parser/smoovie.yaml +++ b/pfb/parser/smoovie.yaml @@ -164,3 +164,6 @@ inputs: outputs: {} + +policies: + pass_missing_as_none: true diff --git a/pfb/parser/spotless.yaml b/pfb/parser/spotless.yaml index 0ea2cfb2f..2fa7f8221 100644 --- a/pfb/parser/spotless.yaml +++ b/pfb/parser/spotless.yaml @@ -166,3 +166,6 @@ inputs: outputs: {} + +policies: + pass_missing_as_none: true diff --git a/pfb/parser/uncabbedcabs.yml b/pfb/parser/uncabbedcabs.yml index abf63fa47..91590c247 100644 --- a/pfb/parser/uncabbedcabs.yml +++ b/pfb/parser/uncabbedcabs.yml @@ -43,15 +43,6 @@ pfb.restore: _include: - (.)restore.yaml -pfb.fwdbwd: - command: pfb.workers.fwdbwd.fwdbwd - flavour: python - policies: - pass_missing_as_none: true - - _include: - - (.)fwdbwd.yaml - pfb.fluxmop: command: pfb.workers.fluxmop.fluxmop flavour: python diff --git a/pfb/utils/misc.py b/pfb/utils/misc.py index f09cb1eb9..b3f19dbd0 100644 --- a/pfb/utils/misc.py +++ b/pfb/utils/misc.py @@ -197,7 +197,6 @@ def convolve2gaussres(image, xx, yy, gaussparf, nthreads=1, gausspari=None, ax = (1, 2) # axes over which to perform fft lastsize = ny + np.sum(padding[-1]) - # import ipdb; ipdb.set_trace() padding = ((0,0),) + padding image = np.pad(image, padding, mode='constant') imhat = r2c(iFs(image, axes=ax), axes=ax, forward=True, nthreads=nthreads, diff --git a/pfb/workers/init.py b/pfb/workers/init.py index 530fdcaf2..8f403b58f 100644 --- a/pfb/workers/init.py +++ b/pfb/workers/init.py @@ -75,6 +75,7 @@ def init(**kw): for key in opts.keys(): print(' %25s = %s' % (key, opts[key]), file=log) + quit() basename = f'{basedir}/{oname}' from pfb import set_envs @@ -183,11 +184,12 @@ def _init(**kw): operator=None columns = (dc1, - opts.flag_column, 'UVW', 'ANTENNA1', 'ANTENNA2', 'TIME', 'INTERVAL', 'FLAG_ROW') schema = {} - schema[opts.flag_column] = {'dims': ('chan', 'corr')} + if opts.flag_column != 'None': + columns += (opts.flag_column) + schema[opts.flag_column] = {'dims': ('chan', 'corr')} schema[dc1] = {'dims': ('chan', 'corr')} if dc2 is not None: columns += (dc2,) diff --git a/pfb/workers/main.py b/pfb/workers/main.py index 6cad30beb..f2d400e3a 100644 --- a/pfb/workers/main.py +++ b/pfb/workers/main.py @@ -13,6 +13,4 @@ def cli(): fluxmop, hci, smoovie, sara) if __name__ == '__main__': - from pfb.workers.spotless import spotless - cli.add_command(spotless) cli() diff --git a/pfb/workers/sara.py b/pfb/workers/sara.py index 17d553b52..9e790bfed 100644 --- a/pfb/workers/sara.py +++ b/pfb/workers/sara.py @@ -435,7 +435,6 @@ def _sara(**kw): mattrs[key] = pd_tolf else: mattrs[key] = val - # import ipdb; ipdb.set_trace() coeff_dataset = xr.Dataset(data_vars=data_vars, coords=coords, diff --git a/tests/test_klean.py b/tests/test_klean.py index d27f98f99..4f35d9658 100644 --- a/tests/test_klean.py +++ b/tests/test_klean.py @@ -252,7 +252,6 @@ def test_klean(do_gains, ms_name): dds = xds_from_url(dds_name) model_inferred = np.zeros((nchan, nx, ny)) for ds in dds: - # import ipdb; ipdb.set_trace() b = int(ds.bandid) model_inferred[b] = ds.MODEL.values