[WIP] Prepare 0.2.0

.github/workflows/ci.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.6", "3.7", "3.8"]
+        python-version: ["3.8", "3.9", "3.10"]
 
     steps:
       - name: Set up Python ${{ matrix.python-version }}

Jenkinsfile.sh

@@ -10,32 +10,32 @@ TEST_DATA_DIR="$WORKSPACE/../../../test-data"
 
 
 # build
-docker build -f ${WORKSPACE_ROOT}/projects/tricolour/docker/python36.docker -t tricolour.1804.py36:${BUILD_NUMBER} ${WORKSPACE_ROOT}/projects/tricolour/
 docker build -f ${WORKSPACE_ROOT}/projects/tricolour/docker/python38.docker -t tricolour.2004.py38:${BUILD_NUMBER} ${WORKSPACE_ROOT}/projects/tricolour/
+docker build -f ${WORKSPACE_ROOT}/projects/tricolour/docker/python310.docker -t tricolour.2204.py310:${BUILD_NUMBER} ${WORKSPACE_ROOT}/projects/tricolour/
 
 #run tests
 tar xvf $TEST_DATA_DIR/acceptance_test_data.tar.gz -C $TEST_OUTPUT_DIR
 TEST_MS_REL=1519747221.subset.ms
 
-# test 3.6
+# test 3.8
 docker run \
     --rm \
     -v $TEST_OUTPUT_DIR:/testdata \
     --env TRICOLOUR_TEST_MS=/testdata/$TEST_MS_REL \
     --workdir /code \
     --entrypoint /bin/sh \
-    tricolour.1804.py36:${BUILD_NUMBER} \
+    tricolour.2004.py38:${BUILD_NUMBER} \
     -c "python3 -m pytest --flake8 -s -vvv ."
 
 rm -rf $TEST_OUTPUT_DIR/TEST_MS_REL
 tar xvf $TEST_DATA_DIR/acceptance_test_data.tar.gz -C $TEST_OUTPUT_DIR
 
-# test 3.8
+# test 3.10
 docker run \
     --rm \
     -v $TEST_OUTPUT_DIR:/testdata \
     --env TRICOLOUR_TEST_MS=/testdata/$TEST_MS_REL \
     --workdir /code \
     --entrypoint /bin/sh \
-    tricolour.2004.py38:${BUILD_NUMBER} \
-    -c "python3 -m pytest --flake8 -s -vvv ."
+    tricolour.2204.py310:${BUILD_NUMBER} \
+    -c "python3 -m pytest --flake8 -s -vvv ."

docker/python310.docker

@@ -0,0 +1,22 @@
+FROM ubuntu:22.04
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBIAN_PRIORITY critical
+
+# Install base requirements
+RUN apt update && apt install -y python3-pip \
+                              build-essential \
+                              cmake \
+                              casacore-dev \
+                              python3-numpy \
+                              python3-setuptools \
+                              libboost-python-dev \
+                              libcfitsio-dev \
+                              wcslib-dev
+
+RUN pip3 install --upgrade pip setuptools
+ADD . /code
+WORKDIR /code
+
+# Install base + testing packages
+RUN pip3 install .[testing]

docker/python38.docker

@@ -16,4 +16,4 @@ ADD . /code
 WORKDIR /code
 
 # Install base + testing packages
-RUN pip3 install .[testing]
+RUN pip3 install .[testing]

setup.py

@@ -9,19 +9,19 @@
     readme = readme_file.read()
 
 requirements = [
-    'dask[array] == 2021.2.0',
-    'donfig >= 0.4.0',
-    'numpy >= 1.14.0, <= 1.19.5',  # breakage in newer numpy + numerical errors
-    'numba >= 0.43.0',
-    'scipy >= 1.2.0',
-    'threadpoolctl >= 1.0.0',
-    'dask-ms == 0.2.6',
-    'zarr >= 2.3.1'
+    'dask[array] >= 2021.2.0, <=2024.3.1',
+    'donfig >= 0.4.0, <0.8.2',
+    'numpy >= 1.14.0, <= 1.22.0',
+    'numba >= 0.43.0, <= 0.59.1',
+    'scipy >= 1.2.0, <=1.12.0',
+    'threadpoolctl >= 1.0.0, <=3.4.0',
+    'dask-ms >= 0.2.6, <=0.2.20',
+    'zarr >= 2.3.1, <=2.17.1'
 ]
 
 extras_require = {'testing': ['pytest',
                               'pytest-flake8',
-                              'flake8 >= 4.0.0, <5.0.0',
+                              "flake8 >= 4.0.0, <5.0.0",
                               'requests', 'gdown']}
 
 setup(

tricolour/apps/tricolour/app.py

@@ -37,6 +37,7 @@
 from tricolour.window_statistics import (window_stats,
                                          combine_window_stats,
                                          summarise_stats)
+from datetime import datetime
 
 ##############################################################
 # Initialize Post Mortem debugger
@@ -61,18 +62,30 @@ def create_logger():
     log = logging.getLogger("tricolour")
     cfmt = logging.Formatter(u'%(name)s - %(asctime)s '
                              '%(levelname)s - %(message)s')
-    log.setLevel(logging.DEBUG)
-    filehandler = logging.FileHandler("tricolour.log")
-    filehandler.setFormatter(cfmt)
-    log.addHandler(filehandler)
     log.setLevel(logging.INFO)
-
     console = logging.StreamHandler()
     console.setLevel(logging.INFO)
     console.setFormatter(cfmt)
-
     log.addHandler(console)
 
+    # add an optional file handler
+    logger_path = os.environ.get("TRICOLOUR_LOGPATH", os.getcwd())
+    nowT = int(np.ceil(datetime.timestamp(datetime.now())))
+    logfile = os.path.join(logger_path,
+                           f"tricolour.{nowT}.log")
+    try:
+        with open(logfile, "w") as f:
+            f.write("")
+        filehandler = logging.FileHandler(logfile)
+        filehandler.setFormatter(cfmt)
+        log.addHandler(filehandler)
+        if logger_path != os.getcwd():
+            log.info(f"A copy of this log is available at {logfile}")
+    except PermissionError:
+        log.warning(f"Failed to initialize logfile for this run. "
+                    f"Check your permissions and available space on "
+                    f"'{logger_path}'. Proceeding without writing "
+                    f"a logfile.")
     return log
 
 
@@ -386,7 +399,7 @@ def _main(args):
         # Generate unflagged defaults if we should ignore existing flags
         # otherwise take flags from the dataset
         if args.ignore_flags is True:
-            flags = da.full_like(vis, False, dtype=np.bool)
+            flags = da.full_like(vis, False, dtype=bool)
             log.critical("Completely ignoring measurement set "
                          "flags as per '-if' request. "
                          "Strategy WILL NOT or with original flags, even if "

tricolour/flagging.py

@@ -26,7 +26,7 @@
 """  # noqa
 
 
-@numba.njit(nogil=True, cache=True)
+@numba.jit(nopython=True, nogil=True, cache=True)
 def flag_nans_and_zeros(vis_windows, flag_windows):
     """
     Flag nan and zero visibilities.
@@ -190,8 +190,20 @@ def _as_min_dtype(value):
     return np.array(value, dtype)
 
 
-@numba.generated_jit(nopython=True, nogil=True, cache=True)
+@numba.jit(nopython=True, nogil=True, cache=True)
 def _asbool(data):
+    return _asbool_impl(data)
+
+
+def _asbool_impl(data):
+    if data.dtype.itemsize == 1:
+        return data.view(np.bool_)
+    else:
+        return data.astype(np.bool_)
+
+
+@numba.extending.overload(_asbool_impl, nopython=True, nogil=True, cache=True)
+def _asbool_impl_jit(data):
     """Create a boolean array with the same values as `data`.
 
     The `data` contain only 0's and 1's. If possible, a view is returned,

tricolour/mask.py

@@ -60,7 +60,7 @@ def load_mask(filename, dilate):
     # Load mask
     mask = np.load(filename)
 
-    if mask.dtype[0] != np.bool or mask.dtype[1] != np.float64:
+    if mask.dtype[0] != bool or mask.dtype[1] != np.float64:
         raise ValueError("Mask %s is not a valid static mask "
                          "with labelled channel axis "
                          "[dtype == (bool, float64)]" % filename)

tricolour/packing.py

@@ -90,7 +90,7 @@ def _create_window_dask(name, ntime, nchan, nbl, ncorr, token,
 
     graph = HighLevelGraph.from_collections(collection_name, layers, ())
     chunks = ((0,),)  # One chunk containing single zarr array object
-    return da.Array(graph, collection_name, chunks, dtype=np.object)
+    return da.Array(graph, collection_name, chunks, dtype=object)
 
 
 def create_vis_windows(ntime, nchan, nbl, ncorr, token,
@@ -343,7 +343,7 @@ def pack_data(time_inv, ubl,
                            flags, ("row", "chan", "corr"),
                            vis_win_obj, ("windim",),
                            flag_win_obj, ("windim",),
-                           dtype=np.bool)
+                           dtype=bool)
 
     # Expose visibility data at it's full resolution
     vis_windows = da.blockwise(_packed_windows, _WINDOW_SCHEMA,

tricolour/tests/test_flagging.py

@@ -5,11 +5,11 @@
 import scipy.interpolate
 from scipy.ndimage import gaussian_filter1d, gaussian_filter
 import pytest
-
 from tricolour import flagging
+import unittest
 
 
-class TestAsbool(object):
+class TestAsbool(unittest.TestCase):
     def _test(self, dtype, expect_view):
         a = np.array([0, 1, 1, 0, 1, 0, 0, 1], dtype)
         expected = a.astype(np.bool_)
@@ -33,8 +33,8 @@ def test_bool(self):
         self._test(np.bool_, True)
 
 
-class TestAverageFreq(object):
-    def setup(self):
+class TestAverageFreq(unittest.TestCase):
+    def setUp(self):
         self.small_data = np.arange(30, dtype=np.float32).reshape(1, 5, 6)
         self.small_data = self.small_data.repeat(2, axis=0)
         self.small_flags = np.zeros(self.small_data.shape, np.bool_)
@@ -151,11 +151,11 @@ def test_time_median():
     np.testing.assert_array_equal(expected_flags, out_flags)
 
 
-class TestMedianAbs(object):
+class TestMedianAbs(unittest.TestCase):
     """Tests for :func:`katsdpsigproc.rfi.flagging._median_abs` and
     :func:`katsdpsigproc.rfi.flagging._median_abs_axis0`."""
 
-    def setup(self):
+    def setUp(self):
         self.data = np.array([[-2.0, -6.0, 4.5], [1.5, 3.3, 0.5]], np.float32)
         self.flags = np.array([[0, 0, 0], [0, 1, 0]], np.uint8)
 
@@ -179,12 +179,12 @@ def test_axis0_all_flagged(self):
         np.testing.assert_array_equal(expected, out)
 
 
-class TestLinearlyInterpolateNans(object):
+class TestLinearlyInterpolateNans(unittest.TestCase):
     """
     Tests for :func:`katsdpsigproc.rfi.flagging._linearly_interpolate_nans`.
     """
 
-    def setup(self):
+    def setUp(self):
         self.y = np.array([np.nan, np.nan, 4.0, np.nan, np.nan,
                            10.0, np.nan, -2.0, np.nan, np.nan])
         self.expected = np.array([4.0, 4.0, 4.0, 6.0, 8.0,
@@ -224,7 +224,7 @@ def test_2d(self):
         np.testing.assert_allclose(expected, y)
 
 
-class TestBoxGaussianFilter(object):
+class TestBoxGaussianFilter(unittest.TestCase):
     def test_one_pass(self):
         """Test that _box_gaussian_filter1d places the box correctly"""
         a = np.array([50.0, 10.0, 60.0, -70.0, 30.0, 20.0, -15.0], np.float32)
@@ -289,8 +289,8 @@ def test_edge(self):
         np.testing.assert_allclose(fdata[:, 80:120], fcore, rtol=1e-5)
 
 
-class TestMaskedGaussianFilter(object):
-    def setup(self):
+class TestMaskedGaussianFilter(unittest.TestCase):
+    def setUp(self):
         self.rs = np.random.RandomState(seed=1)
         shape = (77, 53)
         self.data = self.rs.uniform(size=shape).astype(np.float32)
@@ -332,15 +332,15 @@ def test_nan(self):
         assert 0 < np.sum(np.isnan(expected))
 
 
-class TestGetBackground2D(object):
+class TestGetBackground2D(unittest.TestCase):
     """Tests for :func:`katsdpsigproc.rfi.flagging._get_background2d`.
 
     This is a difficult function to test, because it's not really practical to
     determine expected results by hand. The tests mainly check corner cases
     where large regions are flagged.
     """
 
-    def setup(self):
+    def setUp(self):
         self.shape = (95, 86)
         self.data = np.ones(self.shape, np.float32) * 7.5
         self.flags = np.zeros(self.shape, np.uint8)
@@ -421,8 +421,8 @@ def test_iterations(self):
         np.testing.assert_allclose(expected, background, rtol=1e-2)
 
 
-class TestSumThreshold(object):
-    def setup(self):
+class TestSumThreshold(unittest.TestCase):
+    def setUp(self):
         self.small_data = np.arange(30, dtype=np.float32).reshape(5, 6)
         self.small_flags = np.zeros(self.small_data.shape, np.bool_)
         self.small_flags[3, :] = 1
@@ -501,10 +501,10 @@ def test_sum_threshold_existing(self):
                                       out_flags[70, :4])
 
 
-class TestSumThresholdFlagger(object):
+class TestSumThresholdFlagger(unittest.TestCase):
     """Tests for :class:`katsdpsigproc.rfi.flagging.SumThresholdFlagger`."""
 
-    def setup(self):
+    def setUp(self):
         self.flagger = flagging.SumThresholdFlagger()
 
     def _make_background(self, shape, rs):

tricolour/tests/test_flagging_additional.py

@@ -131,7 +131,7 @@ def test_apply_static_mask(wsrt_ants, unique_baselines,
                                   accumulation_mode="or")
 
     # Check that first mask's flags are applied
-    chan_sel = np.zeros(chan_freqs.shape[0], dtype=np.bool)
+    chan_sel = np.zeros(chan_freqs.shape[0], dtype=bool)
     chan_sel[[2, 10]] = True
 
     assert np.all(new_flags[:, :, :, chan_sel] == 1)
@@ -144,7 +144,7 @@ def test_apply_static_mask(wsrt_ants, unique_baselines,
                                   accumulation_mode="or")
 
     # Check that both mask's flags have been applied
-    chan_sel = np.zeros(chan_freqs.shape[0], dtype=np.bool)
+    chan_sel = np.zeros(chan_freqs.shape[0], dtype=bool)
     chan_sel[[2, 10, 4, 11, 5]] = True
 
     assert np.all(new_flags[:, :, :, chan_sel] == 1)
@@ -157,7 +157,7 @@ def test_apply_static_mask(wsrt_ants, unique_baselines,
                                   accumulation_mode="override")
 
     # Check that only last mask's flags applied
-    chan_sel = np.zeros(chan_freqs.shape[0], dtype=np.bool)
+    chan_sel = np.zeros(chan_freqs.shape[0], dtype=bool)
     chan_sel[[4, 11, 5]] = True
 
     assert np.all(new_flags[:, :, :, chan_sel] == 1)
@@ -176,7 +176,7 @@ def test_apply_static_mask(wsrt_ants, unique_baselines,
                                   uvrange=uvrange)
 
     # Check that both mask's flags have been applied
-    chan_sel = np.zeros(chan_freqs.shape[0], dtype=np.bool)
+    chan_sel = np.zeros(chan_freqs.shape[0], dtype=bool)
     chan_sel[[2, 10, 4, 11, 5]] = True
 
     # Select baselines based on the uvrange