Merge branch 'master' into montage_mosaic

examples/simulation_pipeline.py

@@ -46,7 +46,7 @@
              output=OUTPUT,                             # Output folder
              label="Creating MS",                       # Process label
              cpus=2.5,
-             memory_limit="2gb") 
+             memory_limit="2gb")
 
 
 # 2: Simulate visibilities into it

stimela/cargo/base/meqtrees/Dockerfile

@@ -1,6 +1,6 @@
 FROM stimela/base:1.0.1
 RUN docker-apt-install meqtrees time
-RUN pip install -U pip
+RUN pip install -U pip setuptools
 RUN pip install astropy pywcs
-RUN pip install owlcat
+RUN pip install -U git+https://github.com/ska-sa/owlcat
 ENV MEQTREES_CATTERY_PATH /usr/lib/python2.7/dist-packages/Cattery

stimela/cargo/base/montage/Dockerfile

@@ -1,4 +1,4 @@
 FROM stimela/astropy:1.0.1
 RUN docker-apt-install montage
 RUN pip install --upgrade pip setuptools
-RUN pip install montage-wrapper==0.9.9 astroquery==0.3.8
+RUN pip install -U montage-wrapper>=0.9.9 astroquery>=0.3.8 montage_mosaic>=0.0.1

stimela/cargo/base/ragavi/Dockerfile

@@ -3,4 +3,4 @@ RUN docker-apt-install curl
 RUN curl -sL https://deb.nodesource.com/setup_8.x | bash -
 RUN docker-apt-install nodejs
 RUN pip install -U pip setuptools
-RUN pip install ragavi
+RUN pip install git+https://github.com/ratt-ru/ragavi

stimela/cargo/base/sharpener/Dockerfile

@@ -0,0 +1,8 @@
+FROM stimela/base:1.0.1
+RUN docker-apt-install texmaker \
+                       dvipng \
+                       python-tk \
+                       python-numpy
+RUN apt-get update
+RUN pip install --upgrade pip
+RUN pip install sharpener

stimela/cargo/cab/flagms/Dockerfile

@@ -1,4 +1,4 @@
-FROM stimela/meqtrees:1.0.1
+FROM stimela/meqtrees:1.1.3
 MAINTAINER <[email protected]>
 ADD src /scratch/code
 ENV LOGFILE ${OUTPUT}/logfile.txt

stimela/cargo/cab/flagms/parameters.json

@@ -1,7 +1,7 @@
 {
     "task": "flagms", 
     "base": "stimela/meqtrees", 
-    "tag": "1.0.1", 
+    "tag": "1.1.3", 
     "description": "This program will execute an RFI strategy as can be created with the RFI gui and executes it on one or several   observations.", 
     "prefix": "--", 
     "binary": "flag-ms.py", 
@@ -216,4 +216,4 @@
             "name": "chunk-size"
         }
     ]
-}
+}

stimela/cargo/cab/pycasacore/Dockerfile

@@ -1,4 +1,4 @@
-FROM stimela/casa:1.0.1
+FROM stimela/meqtrees:1.1.3
 MAINTAINER <[email protected]>
 ADD src /scratch/code
 ENV LOGFILE ${OUTPUT}/logfile.txt

stimela/cargo/cab/pycasacore/parameters.json

@@ -1,7 +1,7 @@
 {
     "task": "pycasacore", 
     "base": "stimela/meqtrees", 
-    "tag": "1.0.1", 
+    "tag": "1.1.3", 
     "description": "Runs a user defined script in a python environment with python-casaore", 
     "prefix": "", 
     "binary": "python", 

stimela/cargo/cab/sharpener/Dockerfile

@@ -0,0 +1,5 @@
+FROM stimela/sharpener:1.1.3
+MAINTAINER <[email protected]>
+ADD src /scratch/code
+ENV LOGFILE ${OUTPUT}/logfile.txt
+CMD /scratch/code/run.sh

stimela/cargo/cab/sharpener/parameters.json

@@ -0,0 +1,258 @@
+{
+    "task": "sharpener",
+    "base": "stimela/sharpener",
+    "tag": "1.1.3",
+    "description": "A tool to extract a spectrum from continuum sources in a cube.",
+    "prefix": "",
+    "binary": "run_sharpener",
+    "msdir": false,
+    "use_graphics": true,
+    "parameters": [
+        {
+            "info": "Name of continuum image to extract sources",
+            "dtype": "file",
+            "default": null,
+            "name": "contname",
+            "required": false,
+            "io": "input"
+        },
+        {
+            "info": "Name of cube image to extract spectrum of sources",
+            "dtype": "file",
+            "default": null,
+            "name": "cubename",
+            "required": true,
+            "io": "input"
+        },
+        {
+            "info": "Prefix in the plots title",
+            "dtype": "str",
+            "default": "SHARP",
+            "name": "label",
+            "required": false
+        },
+        {
+            "info": "Name of catalog file used to locate sources in a cube",
+            "dtype": "file",
+            "default": null,
+            "name": "catalog_file",
+            "required": false,
+            "io": "input"
+        },
+        {
+            "info": "Name of working directory",
+            "dtype": "str",
+            "default": null,
+            "required": true,
+            "name": "workdir"
+        },
+        {
+            "info": "Format of plots produced",
+            "dtype": "str",
+            "default": "pdf",
+            "required": false,
+            "name": "plot_format"
+        },
+        {
+            "info": "Merge all the plots",
+            "dtype": "bool",
+            "default": true,
+            "required": false,
+            "name": "merge_plots"
+        },
+        {
+            "info": "plot spectra from folder plot in workdir/sharpOut/plot/",
+            "dtype": "bool",
+            "default": true,
+            "required": false,
+            "name": "enable_abs_plot"
+        },
+        {
+            "info": "Detail plot showing specified number of channels per plot",
+            "dtype": "bool",
+            "default": true,
+            "required": false,
+            "name": "detailed_plot"
+        },
+        {
+            "info": "Number of channels per plot",
+            "dtype": "int",
+            "default": 50,
+            "required": false,
+            "name": "channels_per_plot"
+        },
+        {
+            "info": "Enable hanning for smoothing spectrum",
+            "dtype": "bool",
+            "default": false,
+            "required": false,
+            "name": "enable_hanning"
+        },
+        {
+            "info": "Hanning window",
+            "dtype": "int",
+            "default": 3,
+            "required": false,
+            "name": "window"
+        },
+        {
+            "info": "Subtract polynomial from spectrum to improve continuum subtraction",
+            "dtype": "bool",
+            "default": false,
+            "required": false,
+            "name": "enable_polynomial_subtraction"
+        }, 
+        {
+            "info": "Degree of polynomial",
+            "dtype": "bool",
+            "default": false,
+            "required": false,
+            "name": "degree_pol"
+        },
+        {
+            "info": "Enable sdss source matching",
+            "dtype": "bool",
+            "default": false,
+            "required": false,
+            "name": "enable_sdss_match"
+        }, 
+        {
+            "info": "Minimum frequency",
+            "dtype": "float",
+            "default": 1130.0,
+            "required": false,
+            "name": "freq_min"
+        }, 
+        {
+            "info": "Maximum frequency",
+            "dtype": "float",
+            "default": 1430.0,
+            "required": false,
+            "name": "freq_max"
+        }, 
+        {
+            "info": "Plot image",
+            "dtype": "bool",
+            "default": true,
+            "required": false,
+            "name": "plot_image"
+        }, 
+        {
+            "info": "Match catalog",
+            "dtype": "bool",
+            "default": true,
+            "required": false,
+            "name": "match_cat"
+        }, 
+        {
+            "info": "Maximum separation",
+            "dtype": "float",
+            "default": 30.0,
+            "required": false,
+            "name": "max_sep"
+        }, 
+        {
+            "info": "Minimum radio flux",
+            "dtype": "float",
+            "default": 0.0,
+            "required": false,
+            "name": "min_radio_flux"
+        }, 
+        {
+            "info": "SDSS query timeout",
+            "dtype": "int",
+            "default": 300,
+            "required": false,
+            "name": "sdss_query_timeout"
+        }, 
+        {
+            "info": "Simulate continuum from source catalog output",
+            "dtype": "bool",
+            "default": false,
+            "required": false,
+            "name": "enable_simulate_continuum"
+        }, 
+        {
+            "info": "Find sources in a f.o.v. from source_catalog",
+            "dtype": "bool",
+            "default": false,
+            "required": false,
+            "name": "enable_source_catalog"
+        }, 
+        {
+            "info": "Type of catalog",
+            "dtype": "str",
+            "default": null,
+            "required": true,
+            "name": "catalog",
+            "choices": [
+                "FIRST", 
+                "NVSS", 
+                "PYBDSF"
+            ]
+        }, 
+        {
+            "info": "Threshold to select sources (mJy)",
+            "dtype": "float",
+            "default": 50.0,
+            "required": false,
+            "name": "thresh"
+        }, 
+        {
+            "info": "Field of view width (e.g. 5.5d)",
+            "dtype": "str",
+            "default": null,
+            "required": false,
+            "name": "width"
+        }, 
+        {
+            "info": "Find sources in continuum image using miriad imsad",
+            "dtype": "bool",
+            "default": false,
+            "required": false,
+            "name": "enable_source_finder"
+        }, 
+        {
+            "info": "Source finding clip",
+            "dtype": "float",
+            "default": null,
+            "required": false,
+            "name": "clip"
+        }, 
+        {
+            "info": "Plot image",
+            "dtype": "bool",
+            "default": true,
+            "required": false,
+            "name": "plot_image"
+        }, 
+        {
+            "info": "Extract spectra from datacube",
+            "dtype": "bool",
+            "default": true,
+            "required": false,
+            "name": "enable_spec_ex"
+        }, 
+        {
+            "info": "Chromatic aberration",
+            "dtype": "bool",
+            "default": true,
+            "required": false,
+            "name": "chrom_aberration"
+        }, 
+        {
+            "info": "Number of channels to flag",
+            "dtype": "int",
+            "default": null,
+            "required": false,
+            "name": "flag_chans"
+        }, 
+        {
+            "info": "Red shift units",
+            "dtype": "str",
+            "default": null,
+            "required": false,
+            "name": "zunit"
+        }
+    ]
+}

stimela/cargo/cab/sharpener/src/run.py

@@ -0,0 +1,58 @@
+import os
+import sys
+import yaml
+import sharpener
+
+sys.path.append('/scratch/stimela')
+import utils
+
+
+CONFIG = os.environ["CONFIG"]
+INPUT = os.environ["INPUT"]
+MSDIR = os.environ["MSDIR"]
+OUTPUT = os.environ["OUTPUT"]
+
+cab = utils.readJson(CONFIG)
+args = []
+msname = None
+
+pkg_path = os.path.dirname(os.path.realpath(sharpener.__file__))
+sharpener_file = '{:s}/sharpener_default.yml'.format(pkg_path)
+
+with open(sharpener_file) as f:
+    list_doc = yaml.load(f)
+
+list_doc['general']['workdir'] = '{:s}/'.format(OUTPUT)
+
+for param in cab['parameters']:
+    name = param['name']
+    value = param['value']
+
+    if value is None:
+        continue
+
+    if name in ['contname', 'cubename']:
+        list_doc['general'][name] = value.split('/')[-1]
+        continue
+    if name in ['catalog_file']:
+        list_doc['source_catalog'][name] = value.split('/')[-1]
+        continue
+
+
+    for key, val in list_doc.items():
+        if type(val) == dict:
+            for k1, v1 in val.items():
+                if 'enable' in name:
+                    if key in name:
+                        list_doc[key]['enable'] = value
+                elif k1 == name:
+                    list_doc[key][k1] = value
+        else:
+            if key == name:
+                list_doc[key] = value
+
+edited_file = 'sharpener_default.yml'
+with open(edited_file, "w") as f:
+    yaml.dump(list_doc, f)
+
+utils.xrun('run_sharpener -c ', [edited_file])