Merge pull request #705 from ratt-ru/SpheMakh-patch-1

initial commit

.github/workflows/python-package.yml

@@ -0,0 +1,36 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Python package
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8, 3.9]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest
+        python -m pip install .
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics

examples/simulation_pipeline.py

@@ -114,4 +114,4 @@
 },
     label="Remove MS")
 
-pipeline.run()
+pipeline.run([3])

setup.py

@@ -33,7 +33,7 @@
           "cab/*/parameters.json",
           "cab/*/src/tdlconf.profiles",
       ]},
-      python_requires='>=3.5',
+      python_requires='>=3.6',
       install_requires=requirements,
       scripts=["bin/" + i for i in os.listdir("bin")] + 
                 glob.glob("stimela/cargo/cab/stimela_runscript"),

stimela/cargo/base/meqtrees/Dockerfile

@@ -1,4 +1,4 @@
-FROM stimela/base:1.6.6
-RUN docker-apt-install meqtrees
+FROM stimela/base:1.6.0
+RUN docker-apt-install meqtrees-timba kittens owlcat
 RUN pip install -U pip
-RUN pip install owlcat scabha
+RUN pip install owlcat scabha meqtrees-cattery astro-tigger-lsm

stimela/cargo/base/tigger/Dockerfile

@@ -1,7 +1,2 @@
-FROM stimela/meqtrees:1.2.0
-RUN docker-apt-install xvfb
+FROM stimela/base:1.7.1
 RUN pip install astro-tigger-lsm
-COPY xvfb.init.d /etc/init.d/xvfb
-RUN chmod 755 /etc/init.d/xvfb
-RUN chmod 777 /var/run
-ENV DISPLAY :99

stimela/cargo/cab/__init__.py

@@ -97,7 +97,7 @@ def validate(self, value):
                     types = (t,int) if t is float else (t,)      # float permits ints as well
                     if all(isinstance(x, types) for x in value): # check that all elements are of permitted type
                         return True
-            elif item is "file":
+            elif item == "file":
                 return True
             elif isinstance(value, tuple([item]+[int] if item is float else [item])):
                 return True

stimela/cargo/cab/aegean/src/run.py

@@ -8,6 +8,7 @@
 import shlex
 import subprocess
 import yaml
+import glob
 
 from astLib.astWCS import WCS
 from astropy.table import Table
@@ -130,4 +131,4 @@ def tigger_src(src, idx):
 
 model.save(tname_lsm)
 # Rename using CORPAT
-utils.xrun("tigger-convert", [tname_lsm, "--rename -f"])
+subprocess.check_call(["tigger-convert", tname_lsm, "--rename", "-f"])

stimela/cargo/cab/autoflagger/src/run.py

@@ -4,6 +4,7 @@
 import shutil
 import subprocess
 import yaml
+import glob
 
 CONFIG = os.environ["CONFIG"]
 INPUT = os.environ["INPUT"]

stimela/cargo/cab/calibrator/parameters.json

@@ -1,7 +1,7 @@
 {
     "task": "calibrator", 
     "base": "stimela/meqtrees", 
-    "tag": "1.6.3", 
+    "tag": "1.7.1", 
     "description": "Uses MeqTrees to calibrate a measurement set, given a sky model (Gaussians and point sources, and/or visibility data in MODEL_DATA column)", 
     "prefix": " ", 
     "binary": "/usr/bin/meqtree-pipeliner.py",

stimela/cargo/cab/casa_clean/src/run.py

@@ -5,6 +5,7 @@
 import os
 import sys
 import numpy
+import astropy.io.fits as pyfits
 
 args = parameters_dict
 

stimela/cargo/cab/casa_gencal/src/run.py

@@ -1,17 +1,20 @@
 # -*- coding: future_fstrings -*-
 import Crasa.Crasa as crasa
 from scabha import config, parameters_dict, prun
+from casacore.table import table
+import os
+import numpy
 
 print(f"Running CASA task '{config.binary}'")
 
 args = parameters_dict
 
-task = crasa.CasaTask(cab["binary"], **args)
+task = crasa.CasaTask(config.binary, **args)
 task.run()
 
 gtab = args["caltable"]
 if not os.path.exists(gtab):
-    raise RuntimeError(f"The gaintable was not created. Please refer to CASA {config.binary} logfile for further details"
+    raise RuntimeError(f"The gaintable was not created. Please refer to CASA {config.binary} logfile for further details")
 
 tab = table(gtab)
 field_ids = numpy.unique(tab.getcol("FIELD_ID"))

stimela/cargo/cab/casa_makemask/src/run.py

@@ -6,7 +6,7 @@
 
 makemask_args = {}
 immath_args = {}
-for name, value in parameters_dic.items():
+for name, value in parameters_dict.items():
     if value is None:
         continue
 

stimela/cargo/cab/cleanmask/src/run.py

@@ -10,6 +10,7 @@
 CONFIG = os.environ["CONFIG"]
 INPUT = os.environ["INPUT"]
 OUTPUT = os.environ["OUTPUT"]
+MSDIR = os.environ["MSDIR"]
 
 with open(CONFIG, "r") as _std:
     cab = yaml.safe_load(_std)

stimela/cargo/cab/cubical/src/run.py

@@ -59,5 +59,5 @@
 argslist = shlex.split(_runc)
 
 # run the command
-if prun(argslist) is not 0:
+if prun(argslist) != 0:
     sys.exit(1)

stimela/cargo/cab/equolver/src/run.py

@@ -11,6 +11,6 @@
             args.pop(i)
 
 # run the command
-if prun(args) is not 0:
+if prun(args) != 0:
     sys.exit(1)
 

stimela/cargo/cab/flagms/src/run.py

@@ -4,6 +4,7 @@
 import shutil
 import subprocess
 import yaml
+import glob
 
 CONFIG = os.environ["CONFIG"]
 INPUT = os.environ["INPUT"]

stimela/cargo/cab/lwimager/src/run.py

@@ -108,7 +108,7 @@ def predict_vis(msname, image, column="MODEL_DATA",
         if chanchunk < nimgchan:
             mschanstep = numchans*chanstep/nimgchan
             if numchans % nimgchan:
-                warn(
+                print(
                     "MS channels not evenly divisible into $nimgchan image channels, chunking may be incorrect")
             chunklist = []
             for chan0 in range(0, nimgchan, chanchunk):
@@ -141,7 +141,7 @@ def predict_vis(msname, image, column="MODEL_DATA",
         if len(chunklist) > 1:
             blc[0], trc[0] = imgch0, imgch1
             print("writing CASA image for slice {0} {1}".format(blc, trc))
-            casaimage1 = II("{0}.{1}.img".format(image, ichunk1))
+            casaimage1 = "{0}.{1}.img".format(image, ichunk)
             rm_fr(casaimage1)
             print("writing CASA image for slice {0} {1} to {2}".format(
                 blc, trc, casaimage1))

stimela/cargo/cab/mvftoms/src/run.py

@@ -53,7 +53,7 @@
     subprocess.check_call(shlex.split(_runc))
 finally:
     for item in junk:
-        for dest in [OUTPUT, MSDIR]: # these are the only writable volumes in the container
+        for dest in [OUTDIR, MSDIR]: # these are the only writable volumes in the container
             items = glob.glob("{dest}/{item}".format(**locals()))
             for f in items:
                 if os.path.isfile(f):

stimela/cargo/cab/owlcat_plotelev/src/run.py

@@ -8,5 +8,5 @@
                                           positional=["msname"], mandatory=["msname"])
 
 # run the command
-if prun(args) is not 0:
+if prun(args) != 0:
     sys.exit(1)

stimela/cargo/cab/pyddi/src/run.py

@@ -4,7 +4,7 @@
 import yaml
 import glob
 import shutil
-import shelx
+import shlex
 
 CONFIG = os.environ["CONFIG"]
 INPUT = os.environ["INPUT"]

stimela/cargo/cab/ragavi/src/run.py

@@ -6,6 +6,6 @@
 args = [config.binary] + parse_parameters(repeat=" ")
 
 # run the command
-if prun(args) is not 0:
+if prun(args) != 0:
     sys.exit(1)
 

stimela/cargo/cab/ragavi_vis/src/run.py

@@ -17,6 +17,6 @@
     args += list(map(str, value))
 
 # run the command
-if prun(args) is not 0:
+if prun(args) != 0:
     sys.exit(1)
 

stimela/cargo/cab/rfimasker/src/run.py

@@ -10,6 +10,6 @@
                                           repeat_dict=dict(ms=True))
 
 # run the command
-if prun(args) is not 0:
+if prun(args) != 0:
     sys.exit(1)
 

stimela/cargo/cab/rmclean3d/src/run.py

@@ -9,5 +9,5 @@
                                           positional=["dirty-pdf", "rmsf-fwhm"], mandatory=["dirty-pdf", "rmsf-fwhm"])
 
 # run the command
-if prun(args) is not 0:
+if prun(args) !=0:
     sys.exit(1)

stimela/cargo/cab/rmsynth1d/src/run.py

@@ -9,5 +9,5 @@
                                           positional=["dataFile"], mandatory=["dataFile"])
 
 # run the command
-if prun(args) is not 0:
+if prun(args) != 0:
     sys.exit(1)

stimela/cargo/cab/rmsynth3d/src/run.py

@@ -9,5 +9,5 @@
                                           positional=["fitsq", "fitsu", "freqs"], mandatory=["fitsq", "fitsu", "freqs"])
 
 # run the command
-if prun(args) is not 0:
+if prun(args) != 0:
     sys.exit(1)

stimela/cargo/cab/sharpener/src/run.py

@@ -5,6 +5,7 @@
 import glob
 import shlex
 import subprocess
+import shutil
 
 
 CONFIG = os.environ["CONFIG"]

stimela/cargo/cab/simulator/parameters.json

@@ -1,7 +1,7 @@
 {
     "task": "simulator", 
     "base": "stimela/meqtrees",
-    "tag": "1.6.3",
+    "tag": "1.7.1",
     "description": "Uses MeqTrees to simulate a sky model (Gaussians and point sources) into measurement set (MS)", 
     "prefix": " ", 
     "binary": "meqtree-pipeliner.py", 

stimela/cargo/cab/simulator/src/run.py

@@ -147,8 +147,7 @@ def compute_vis_noise(msname, sefd, spw_id=0):
         field_center = "J2000,%frad,%frad" % (ra, dec)
     tmp = "recentered_"+os.path.basename(skymodel)
 
-    utils.xrun("tigger-convert", ["--recenter",
-                                  field_center, skymodel, tmp, "-f"])
+    prun(["tigger-convert", "--recenter", field_center, skymodel, tmp, "-f"])
     options["tiggerlsm.filename"] = tmp
 
 prefix = ['-s {}'.format(saveconf) if saveconf else ''] + \
@@ -163,8 +162,7 @@ def compute_vis_noise(msname, sefd, spw_id=0):
     args.append('{0}={1}'.format(key, value))
 
 _runc = " ".join([config.binary] + prefix + args + suffix)
+
 exitcode = prun(_runc)
 if exitcode != 0:
     raise RuntimeError(f"Meqtrees failed with exit code {exitcode}. See logs for more details")
-
-

stimela/cargo/cab/sofia/src/run.py

@@ -69,75 +69,3 @@
                 elif os.path.isdir(f):
                     shutil.rmtree(f)
                 # Leave other types
-
-if not port2tigger:
-    sys.exit(0)
-
-# convert to data file to Tigger LSM
-# First make dummy tigger model
-tfile = tempfile.NamedTemporaryFile(suffix='.txt')
-tfile.flush()
-
-if image and writecat and parameterise:
-    pass
-else:
-    sys.exit(0)
-
-prefix = os.path.splitext(image)[0]
-tname_lsm = prefix + ".lsm.html"
-with open(tfile.name, "w") as stdw:
-    stdw.write("#format:name ra_d dec_d i emaj_s emin_s pa_d\n")
-
-model = Tigger.load(tfile.name)
-tfile.close()
-
-
-def tigger_src(src, idx):
-
-    name = "SRC%d" % idx
-    flux = ModelClasses.Polarization(src["f_int"], 0, 0, 0)
-    ra = numpy.deg2rad(src["ra"])
-    dec = numpy.deg2rad(src["dec"])
-    pos = ModelClasses.Position(ra, dec)
-    ex = numpy.deg2rad(src["ell_maj"])
-    ey = numpy.deg2rad(src["ell_min"])
-    pa = numpy.deg2rad(src["ell_pa"])
-
-    if ex and ey:
-        shape = ModelClasses.Gaussian(ex, ey, pa)
-    else:
-        shape = None
-    source = SkyModel.Source(name, pos, flux, shape=shape)
-    # Adding source peak flux (error) as extra flux attributes for sources,
-    # and to avoid null values for point sources I_peak = src["Total_flux"]
-    if shape:
-        source.setAttribute("I_peak", float(src["f_peak"]))
-    else:
-        source.setAttribute("I_peak", float(src["f_int"]))
-
-    return source
-
-
-with open('{0}_cat.ascii'.format(prefix)) as stdr:
-    # Header
-    stdr.readline()
-    # Column names
-    names = stdr.readline().split("#")[1].strip().split()
-    # Units
-    stdr.readline()
-    # Column numbers
-    stdr.readline()
-    sys.stdout.write(" ".join(names))
-    data = numpy.genfromtxt(stdr,
-                            names=names + ["col"])
-
-for i, src in enumerate(data):
-    model.sources.append(tigger_src(src, i))
-
-wcs = WCS(image)
-centre = wcs.getCentreWCSCoords()
-model.ra0, model.dec0 = map(numpy.deg2rad, centre)
-
-model.save(tname_lsm)
-# Rename using CORPAT
-utils.xrun("tigger-convert", [tname_lsm, "--rename -f"])

stimela/cargo/cab/tigger_tag/src/run.py

@@ -51,10 +51,10 @@
 args += ['{0}{1} {2}'.format(cab['prefix'], name, value)
          for name, value in params.iteritems()]
 
-_runc = " ".join([cab['binary']] args+[inlsm, tag])
+_runc = " ".join([cab.binary, inlsm, tag] + args)
 
 try:
-    subprocess.check_call(shlex.split(_runc))
+ subprocess.check_call(shlex.split(_runc))
 finally:
     for item in junk:
         for dest in [OUTPUT, MSDIR]: # these are the only writable volumes in the container