add pass_missing_as_none policies to all workers

pfb/deconv/clark.py

@@ -73,7 +73,6 @@ def subminor(A, psf, Ip, Iq, model, wsums, gamma=0.05, th=0.0, maxit=10000):
         # right_y = q + Iq < ny
         # mask = (left_x & right_x) & (left_y & right_y)
 
-        # import ipdb; ipdb.set_trace()
         mask = (np.abs(Idelp) <= nxo2) & (np.abs(Idelq) <= nyo2)
         # Ipp, Iqq = psf[:, nxo2 - Ip[mask], nyo2 - Iq[mask]]
         A = subtract(A[:, mask], psf,
@@ -138,7 +137,7 @@ def clark(ID,
                          gamma=gamma,
                          th=subth,
                          maxit=submaxit)
-        # import ipdb; ipdb.set_trace()
+
         # subtract from full image (as in major cycle)
         psf_convolve_cube(
                         xpad,

pfb/parser/degrid.yaml

@@ -55,3 +55,6 @@ inputs:
 
 outputs:
   {}
+
+policies:
+  pass_missing_as_none: true

pfb/parser/fluxmop.yaml

@@ -71,3 +71,6 @@ inputs:
 
 outputs:
   {}
+
+policies:
+  pass_missing_as_none: true

pfb/parser/fwdbwd.yaml

@@ -109,3 +109,6 @@ inputs:
 
 outputs:
   {}
+
+policies:
+  pass_missing_as_none: true

pfb/parser/grid.yaml

@@ -111,17 +111,17 @@ inputs:
       A sensible value for this parameter depends on the level of RFI in the data.
       Small values (eg. 2) result in aggressive reweighting and should be avoided
       if the model is still incomplete.
-  mf-weighting:
-    dtype: bool
-    default: false
-    info:
-      Base the imaging weights on the uniform weights computed over all
-      data instead of per band. This ensures that the MFS image can truly
-      be uniformly weighted.
   _include:
     - (.)gridding.yml
     - (.)dist.yml
     - (.)out.yml
 
 outputs:
-  {}
+  cds:
+    dtype: Directory
+    info: output cube dataset
+    mkdir: false
+
+
+policies:
+  pass_missing_as_none: true

pfb/parser/hci.yaml

@@ -177,3 +177,6 @@ inputs:
 
 outputs:
     {}
+
+policies:
+  pass_missing_as_none: true

pfb/parser/init.yaml

@@ -6,22 +6,22 @@ inputs:
     info:
       Path to measurement set
   scans:
-    dtype: List[int]
+    dtype: Optional[List[int]]
     info:
       List of SCAN_NUMBERS to image. Defaults to all.
       Input as comma separated list 0,2 if running from CLI
   ddids:
-    dtype: List[int]
+    dtype: Optional[List[int]]
     info:
       List of DATA_DESC_ID's to images. Defaults to all.
       Input as comma separated list 0,2 if running from CLI
   fields:
-    dtype: List[int]
+    dtype: Optional[List[int]]
     info:
       List of FIELD_ID's to image. Defaults to all.
       Input as comma separated list 0,2 if running from CLI
   freq-range:
-    dtype: str
+    dtype: Optional[str]
     info:
       Frequency range to image in Hz.
       Specify as a string with colon delimiter eg. '1e9:1.1e9'
@@ -133,3 +133,6 @@ inputs:
 
 outputs:
     {}
+
+policies:
+  pass_missing_as_none: true

pfb/parser/klean.yaml

@@ -134,3 +134,6 @@ inputs:
 
 outputs:
   {}
+
+policies:
+  pass_missing_as_none: true

pfb/parser/model2comps.yaml

@@ -77,3 +77,6 @@ inputs:
 
 outputs:
   {}
+
+policies:
+  pass_missing_as_none: true

pfb/parser/restore.yaml

@@ -54,3 +54,6 @@ inputs:
 
 outputs:
     {}
+
+policies:
+  pass_missing_as_none: true

pfb/parser/sara.yaml

@@ -101,11 +101,6 @@ inputs:
     info:
       Will terminate the algorithm if the rms increases this many times.
       Set to larger than niter to disable this check.
-  do-res:
-    dtype: bool
-    default: false
-    info:
-      Deconvolve PSF to add intrinsic model resolution to dds
   skip-model:
     dtype: bool
     default: false
@@ -125,3 +120,6 @@ inputs:
 
 outputs:
   {}
+
+policies:
+  pass_missing_as_none: true

pfb/parser/schemas.py

@@ -1,18 +1,24 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 import os.path
 import glob
 from typing import *
 from scabha import configuratt
+from collections import OrderedDict
 from scabha.cargo import Parameter, _UNSET_DEFAULT
 from omegaconf.omegaconf import OmegaConf
 
+def EmptyDictDefault():
+    return field(default_factory=lambda:OrderedDict())
 
 schema = None
 
 @dataclass
 class _CabInputsOutputs(object):
-    inputs: Dict[str, Parameter]
-    outputs: Dict[str, Parameter]
+    # inputs: Dict[str, Parameter]
+    # outputs: Dict[str, Parameter]
+    inputs: Dict[str, Parameter] = EmptyDictDefault()
+    outputs: Dict[str, Parameter] = EmptyDictDefault()
+    policies: Optional[Dict[str, Any]] = None
 
 # load schema files
 if schema is None:
@@ -32,11 +38,9 @@ class _CabInputsOutputs(object):
     # and a set containing locations of .yaml configs for pfb workers
     schema = OmegaConf.create(tmp[0])
 
-    # is this still necessary?
-    for worker in schema.keys():
-        for param in schema[worker]['inputs']:
-            if schema[worker]['inputs'][param]['default'] == _UNSET_DEFAULT:
-                schema[worker]['inputs'][param]['default'] = None
-
-
+    # # is this still necessary?
+    # for worker in schema.keys():
+    #     for param in schema[worker]['inputs']:
+    #         if schema[worker]['inputs'][param]['default'] == _UNSET_DEFAULT:
+    #             schema[worker]['inputs'][param]['default'] = None
 

pfb/parser/smoovie.yaml

@@ -164,3 +164,6 @@ inputs:
 
 outputs:
     {}
+
+policies:
+  pass_missing_as_none: true

pfb/parser/spotless.yaml

@@ -166,3 +166,6 @@ inputs:
 
 outputs:
   {}
+
+policies:
+  pass_missing_as_none: true

pfb/parser/uncabbedcabs.yml

@@ -43,15 +43,6 @@ pfb.restore:
   _include:
     - (.)restore.yaml
 
-pfb.fwdbwd:
-  command: pfb.workers.fwdbwd.fwdbwd
-  flavour: python
-  policies:
-    pass_missing_as_none: true
-
-  _include:
-    - (.)fwdbwd.yaml
-
 pfb.fluxmop:
   command: pfb.workers.fluxmop.fluxmop
   flavour: python

pfb/utils/misc.py

@@ -197,7 +197,6 @@ def convolve2gaussres(image, xx, yy, gaussparf, nthreads=1, gausspari=None,
     ax = (1, 2)  # axes over which to perform fft
     lastsize = ny + np.sum(padding[-1])
 
-    # import ipdb; ipdb.set_trace()
     padding = ((0,0),) + padding
     image = np.pad(image, padding, mode='constant')
     imhat = r2c(iFs(image, axes=ax), axes=ax, forward=True, nthreads=nthreads,

pfb/workers/init.py

@@ -75,6 +75,7 @@ def init(**kw):
     for key in opts.keys():
         print('     %25s = %s' % (key, opts[key]), file=log)
 
+    quit()
     basename = f'{basedir}/{oname}'
 
     from pfb import set_envs
@@ -183,11 +184,12 @@ def _init(**kw):
         operator=None
 
     columns = (dc1,
-               opts.flag_column,
                'UVW', 'ANTENNA1',
                'ANTENNA2', 'TIME', 'INTERVAL', 'FLAG_ROW')
     schema = {}
-    schema[opts.flag_column] = {'dims': ('chan', 'corr')}
+    if opts.flag_column != 'None':
+        columns += (opts.flag_column)
+        schema[opts.flag_column] = {'dims': ('chan', 'corr')}
     schema[dc1] = {'dims': ('chan', 'corr')}
     if dc2 is not None:
         columns += (dc2,)

pfb/workers/main.py

@@ -13,6 +13,4 @@ def cli():
                          fluxmop, hci, smoovie, sara)
 
 if __name__ == '__main__':
-    from pfb.workers.spotless import spotless
-    cli.add_command(spotless)
     cli()

pfb/workers/sara.py

@@ -435,7 +435,6 @@ def _sara(**kw):
                     mattrs[key] = pd_tolf
                 else:
                     mattrs[key] = val
-            # import ipdb; ipdb.set_trace()
 
             coeff_dataset = xr.Dataset(data_vars=data_vars,
                                coords=coords,

tests/test_klean.py

@@ -252,7 +252,6 @@ def test_klean(do_gains, ms_name):
     dds = xds_from_url(dds_name)
     model_inferred = np.zeros((nchan, nx, ny))
     for ds in dds:
-        # import ipdb; ipdb.set_trace()
         b = int(ds.bandid)
         model_inferred[b] = ds.MODEL.values