Migrate to Python 3.10 and TF 2.15 (#281)

This adds support for Python 3.10 and moves MB up to 2.15 on that version (which requires C++20 support (since 2.13).

I have maintained the stepped approach to tensorflow versions making the software backwards compatible to Python 3.6.

.ci/withoutnvcc.py3.10.docker

@@ -0,0 +1,80 @@
+FROM ubuntu:22.04
+
+WORKDIR /optsoft/tensorflow
+ENV DEBIAN_FRONTEND=noninteractive
+ENV DEBIAN_DEPENDENCIES wget \
+                        build-essential \
+                        python3-pip \
+                        git \
+                        python3-casacore \
+                        casacore-data
+
+# setup base dependencies
+RUN apt update && apt install -y ${DEBIAN_DEPENDENCIES}
+RUN pip3 install -U pip setuptools wheel
+
+#####################################################
+# install montblanc including specified tensorflow
+#####################################################
+# copy into workspace
+WORKDIR /src/montblanc
+ADD docs /src/montblanc/docs
+ADD montblanc /src/montblanc/montblanc
+ADD CHANGELOG.md /src/montblanc/CHANGELOG.log
+ADD CONTRIBUTORS.md /src/montblanc/CONTRIBUTORS.md
+ADD LICENSE.md /src/montblanc/LICENSE.md
+ADD MANIFEST.in /src/montblanc/MANIFEST.in
+ADD pyproject.toml /src/montblanc/pyproject.toml
+ADD README.rst /src/montblanc/README.rst
+ADD readthedocs.yml /src/montblanc/readthedocs.yml
+ADD requirements.readthedocs.txt /src/montblanc/requirements.readthedocs.txt
+ADD setup.cfg /src/montblanc/setup.cfg
+ADD setup.py /src/montblanc/setup.py
+
+RUN pip3 install -U setuptools wheel pip
+RUN pip3 install /src/montblanc
+
+WORKDIR /mb_testing
+RUN python3 -c "from montblanc.impl.rime.tensorflow import load_tf_lib; load_tf_lib()"
+# to run tests we need to be in dev install - pkg_requirements return the path to the source, 
+# not the install otherwise leading to library load path errors
+RUN pip3 install -e /src/montblanc
+
+# run tests
+WORKDIR /mb_testing
+ENV TESTS_TO_RUN /src/montblanc/montblanc/tests/beam_factory.py \
+                 /src/montblanc/montblanc/tests/test_antenna_uvw_decomposition.py \
+                 /src/montblanc/montblanc/tests/test_source_utils.py \
+                 /src/montblanc/montblanc/tests/test_source_utils.py \
+                 /src/montblanc/montblanc/tests/test_utils.py \
+                 /src/montblanc/montblanc/impl/rime/tensorflow/rime_ops/test_b_sqrt.py \
+                 /src/montblanc/montblanc/impl/rime/tensorflow/rime_ops/test_create_antenna_jones.py \
+                 /src/montblanc/montblanc/impl/rime/tensorflow/rime_ops/test_e_beam.py \
+                 /src/montblanc/montblanc/impl/rime/tensorflow/rime_ops/test_feed_rotation.py \
+                 /src/montblanc/montblanc/impl/rime/tensorflow/rime_ops/test_gauss_shape.py \
+                 /src/montblanc/montblanc/impl/rime/tensorflow/rime_ops/test_parallactic_angle_sin_cos.py \
+                 /src/montblanc/montblanc/impl/rime/tensorflow/rime_ops/test_phase.py \ 
+                 /src/montblanc/montblanc/impl/rime/tensorflow/rime_ops/test_post_process_visibilities.py \
+                 /src/montblanc/montblanc/impl/rime/tensorflow/rime_ops/test_radec_to_lm.py \
+                 /src/montblanc/montblanc/impl/rime/tensorflow/rime_ops/test_sersic_shape.py \
+                 /src/montblanc/montblanc/impl/rime/tensorflow/rime_ops/test_sum_coherencies.py
+
+# run tests
+# skip meqtrees acceptance test in this case as it is covered by other cases
+# and it does not yet build on newer GCC
+RUN pynose $TESTS_TO_RUN
+
+RUN pip install git+https://github.com/ratt-ru/simms.git@7984aee13e2d7bd32408ce9388b7960bfc0b2e27 # BH: todo until I can make a release!
+RUN pip install casadata
+WORKDIR /mb_testing
+RUN mkdir data
+WORKDIR /mb_testing/data
+RUN python3 -m pip install simms 'eidos>=1.1.1'
+RUN simms -dir "J2000,00h00m00.0000000s,-10d00m00.00s" -T meerkat -dt 30 -st 0.5 -nc 15 -f0 856MHz -df 57066.66667kHz -pl XX XY YX YY -n mk64.Lwide.0.5hr.30s.856mhz.ms
+RUN eidos -d 4 -r 0.015625 -f 856 1712 142 -P test_beam_ -o8
+
+#run example montblanc recipe
+WORKDIR /mb_testing
+RUN python3 /src/montblanc/montblanc/examples/MS_tf_example.py /mb_testing/data/mk64.Lwide.0.5hr.30s.856mhz.ms \
+            -b /mb_testing/data/test_beam_
+

.dockerignore

@@ -1 +1,3 @@
 .ci
+rime.so
+build

Jenkinsfile.sh

@@ -19,4 +19,6 @@ docker build -t "$IMAGENAME:$BUILD_NUMBER" --no-cache=false -f .ci/py3.8.docker
 IMAGENAME="mb_py36"
 docker build -t "$IMAGENAME:$BUILD_NUMBER" --no-cache=false -f .ci/py3.6.docker .
 IMAGENAME="mb_py38nonvss"
-docker build -t "$IMAGENAME:$BUILD_NUMBER" --no-cache=false -f .ci/withoutnvcc.py3.8.docker .
+docker build -t "$IMAGENAME:$BUILD_NUMBER" --no-cache=false -f .ci/withoutnvcc.py3.8.docker .
+IMAGENAME="mb_py310nonvss"
+docker build -t "$IMAGENAME:$BUILD_NUMBER" --no-cache=false -f .ci/withoutnvcc.py3.10.docker .

montblanc/__init__.py

@@ -23,8 +23,8 @@
 
 from montblanc.logsetup import setup_logging, setup_test_logging
 from montblanc.tests import test
-
-__version__ = "0.7.0"
+import pkg_resources
+__version__ = pkg_resources.get_distribution("montblanc").version
 
 log = setup_logging()
 

montblanc/impl/rime/tensorflow/RimeSolver.py

@@ -31,7 +31,9 @@
 import numpy as np
 import tensorflow as tf
 from tensorflow.python.client import timeline
-from attrdict import AttrDict
+
+import attridict as AttrDict
+
 import attr
 
 import montblanc
@@ -836,12 +838,10 @@ def _construct_tensorflow_feed_data(dfs, cube, iter_dims,
     nr_of_input_staging_areas):
 
     FD = AttrDict()
-    # https://github.com/bcj/AttrDict/issues/34
-    FD._setattr('_sequence_type', list)
+    FD['_sequence_type'] = list
     # Reference local staging_areas
     FD.local = local = AttrDict()
-    # https://github.com/bcj/AttrDict/issues/34
-    local._setattr('_sequence_type', list)
+    local['_sequence_type'] = list
 
     # Create placholder variables for source counts
     FD.src_ph_vars = AttrDict({

montblanc/impl/rime/tensorflow/__init__.py

@@ -34,6 +34,6 @@ def load_tf_lib():
         from montblanc import ext
         rime_lib_path = os.path.join(os.path.dirname(ext.__file__), 'rime.so')
     if not os.path.isfile(rime_lib_path):
-        raise RuntimeError("Montblanc backend not found: '{rime_lib_path}'. Have you compiled the backend?")
+        raise RuntimeError(f"Montblanc backend not found: '{rime_lib_path}'. Have you compiled the backend?")
     return tf.load_op_library(rime_lib_path)
 

montblanc/impl/rime/tensorflow/queue_wrapper.py

@@ -2,7 +2,9 @@
 import itertools
 import sys
 import six
-from attrdict import AttrDict
+
+import attridict as AttrDict
+
 import tensorflow as tf
 
 def _get_queue_types(fed_arrays, data_sources):

montblanc/impl/rime/tensorflow/rime_ops/b_sqrt_op.h

@@ -1,6 +1,8 @@
 #ifndef RIME_B_SQRT_OP_H_
 #define RIME_B_SQRT_OP_H_
 
+#include "montblanc_macros.h"
+
 namespace montblanc {
 namespace bsqrt {
 

montblanc/impl/rime/tensorflow/rime_ops/b_sqrt_op_cpu.cpp

@@ -40,7 +40,7 @@ auto bsqrt_shape_function = [](InferenceContext* c) {
     c->set_output(0, bsqrt);
     c->set_output(1, sgn_brightness);
 
-    return Status::OK();
+    return MB_STAT_OK();
 };
 
 REGISTER_OP("BSqrt")

montblanc/impl/rime/tensorflow/rime_ops/create_antenna_jones_op.h

@@ -1,6 +1,7 @@
 #ifndef RIME_CREATE_ANTENNA_JONES_OP_H
 #define RIME_CREATE_ANTENNA_JONES_OP_H
 
+#include "montblanc_macros.h"
 // montblanc namespace start and stop defines
 #define MONTBLANC_NAMESPACE_BEGIN namespace montblanc {
 #define MONTBLANC_NAMESPACE_STOP }

montblanc/impl/rime/tensorflow/rime_ops/create_antenna_jones_op_cpu.cpp

@@ -62,7 +62,7 @@ auto ekb_shape_function = [](InferenceContext* c) {
     // Set the output shape
     c->set_output(0, ant_jones);
 
-    return Status::OK();
+    return MB_STAT_OK();
 };
 
 

montblanc/impl/rime/tensorflow/rime_ops/e_beam_op.h

@@ -1,6 +1,7 @@
 #ifndef RIME_E_BEAM_OP_H_
 #define RIME_E_BEAM_OP_H_
 
+#include "montblanc_macros.h"
 #include "rime_constant_structures.h"
 
 namespace montblanc {

montblanc/impl/rime/tensorflow/rime_ops/e_beam_op_cpu.cpp

@@ -94,7 +94,7 @@ auto ebeam_shape_function = [](InferenceContext* c) {
     // Set the output shape
     c->set_output(0, ejones);
 
-    return Status::OK();
+    return MB_STAT_OK();
 };
 
 

montblanc/impl/rime/tensorflow/rime_ops/feed_rotation_op.h

@@ -1,6 +1,7 @@
 #ifndef RIME_FEED_ROTATION_OP_H
 #define RIME_FEED_ROTATION_OP_H
 
+#include "montblanc_macros.h"
 // montblanc namespace start and stop defines
 #define MONTBLANC_NAMESPACE_BEGIN namespace montblanc {
 #define MONTBLANC_NAMESPACE_STOP }

montblanc/impl/rime/tensorflow/rime_ops/feed_rotation_op_cpu.cpp

@@ -48,7 +48,7 @@ auto shape_function = [](InferenceContext* c) {
 
     // printf("output shape %s\\n", c->DebugString(out).c_str());;
 
-    return Status::OK();
+    return MB_STAT_OK();
 };
 
 // Register the FeedRotation operator.

montblanc/impl/rime/tensorflow/rime_ops/gauss_shape_op.h

@@ -1,6 +1,7 @@
 #ifndef RIME_GAUSS_SHAPE_OP_H
 #define RIME_GAUSS_SHAPE_OP_H
 
+#include "montblanc_macros.h"
 // montblanc namespace start and stop defines
 #define MONTBLANC_NAMESPACE_BEGIN namespace montblanc {
 #define MONTBLANC_NAMESPACE_STOP }

montblanc/impl/rime/tensorflow/rime_ops/gauss_shape_op_cpu.cpp

@@ -55,7 +55,7 @@ auto gauss_shape_shape_function = [](InferenceContext* c) {
     // Set the output shape
     c->set_output(0, output);
 
-    return Status::OK();
+    return MB_STAT_OK();
 };
 
 

montblanc/impl/rime/tensorflow/rime_ops/montblanc_macros.h

@@ -0,0 +1,10 @@
+#ifndef MONTBLANC_MACROS_H
+#define MONTBLANC_MACROS_H
+
+#ifdef DCOMPAT_TF2_4
+    #define MB_STAT_OK Status::OK
+#else
+    #define MB_STAT_OK Status
+#endif //DCOMPAT_TF2_4
+
+#endif //MONTBLANC_MACROS_H

montblanc/impl/rime/tensorflow/rime_ops/parallactic_angle_sin_cos_op.h

@@ -1,6 +1,7 @@
 #ifndef RIME_PARALLACTIC_ANGLE_SIN_COS_OP_H
 #define RIME_PARALLACTIC_ANGLE_SIN_COS_OP_H
 
+#include "montblanc_macros.h"
 // montblanc namespace start and stop defines
 #define MONTBLANC_NAMESPACE_BEGIN namespace montblanc {
 #define MONTBLANC_NAMESPACE_STOP }

montblanc/impl/rime/tensorflow/rime_ops/parallactic_angle_sin_cos_op_cpu.cpp

@@ -41,7 +41,7 @@ auto shape_function = [](InferenceContext* c) {
 
     // printf("output shape %s\\n", c->DebugString(out).c_str());;
 
-    return Status::OK();
+    return MB_STAT_OK();
 };
 
 // Register the ParallacticAngleSinCos operator.

montblanc/impl/rime/tensorflow/rime_ops/phase_op.h

@@ -1,6 +1,7 @@
 #ifndef RIME_PHASE_OP_H_
 #define RIME_PHASE_OP_H_
 
+#include "montblanc_macros.h"
 namespace montblanc {
 namespace phase {
 

montblanc/impl/rime/tensorflow/rime_ops/phase_op_cpu.cpp

@@ -48,7 +48,7 @@ auto phase_shape_function = [](InferenceContext* c) {
     // Set the output shape
     c->set_output(0, output);
 
-    return Status::OK();
+    return MB_STAT_OK();
 };
 
 REGISTER_OP("Phase")

montblanc/impl/rime/tensorflow/rime_ops/post_process_visibilities_op.h

@@ -1,6 +1,7 @@
 #ifndef RIME_POST_PROCESS_VISIBILITIES_OP_H
 #define RIME_POST_PROCESS_VISIBILITIES_OP_H
 
+#include "montblanc_macros.h"
 // montblanc namespace start and stop defines
 #define MONTBLANC_NAMESPACE_BEGIN namespace montblanc {
 #define MONTBLANC_NAMESPACE_STOP }

montblanc/impl/rime/tensorflow/rime_ops/post_process_visibilities_op_cpu.cpp

@@ -111,7 +111,7 @@ auto shape_function = [](InferenceContext* c) {
 
     // printf("output shape %s\\n", c->DebugString(out).c_str());;
 
-    return Status::OK();
+    return MB_STAT_OK();
 };
 
 // Register the PostProcessVisibilities operator.

montblanc/impl/rime/tensorflow/rime_ops/radec_to_lm_op.h

@@ -1,6 +1,7 @@
 #ifndef RIME_RADEC_TO_LM_OP_H
 #define RIME_RADEC_TO_LM_OP_H
 
+#include "montblanc_macros.h"
 // montblanc namespace start and stop defines
 #define MONTBLANC_NAMESPACE_BEGIN namespace montblanc {
 #define MONTBLANC_NAMESPACE_STOP }

montblanc/impl/rime/tensorflow/rime_ops/radec_to_lm_op_cpu.cpp

@@ -52,7 +52,7 @@ auto shape_function(InferenceContext* c) {
 
     // printf("output shape %s\\n", c->DebugString(out).c_str());;
 
-    return Status::OK();
+    return MB_STAT_OK();
 };
 
 // Register the RadecToLm operator.

montblanc/impl/rime/tensorflow/rime_ops/sersic_shape_op.h

@@ -1,6 +1,7 @@
 #ifndef RIME_SERSIC_SHAPE_OP_H
 #define RIME_SERSIC_SHAPE_OP_H
 
+#include "montblanc_macros.h"
 // montblanc namespace start and stop defines
 #define MONTBLANC_NAMESPACE_BEGIN namespace montblanc {
 #define MONTBLANC_NAMESPACE_STOP }

montblanc/impl/rime/tensorflow/rime_ops/sersic_shape_op_cpu.cpp

@@ -55,7 +55,7 @@ auto sersic_shape_shape_function = [](InferenceContext* c) {
     // Set the output shape
     c->set_output(0, output);
 
-    return Status::OK();
+    return MB_STAT_OK();
 };
 
 

montblanc/impl/rime/tensorflow/rime_ops/sum_coherencies_op_cpu.cpp

@@ -61,7 +61,7 @@ auto sum_coherencies_shape_function = [](InferenceContext* c) {
     // Set the output shape
     c->set_output(0, coherencies);
 
-    return Status::OK();
+    return MB_STAT_OK();
 };
 
 

montblanc/impl/rime/tensorflow/rime_ops/sum_coherencies_op_cpu.h

@@ -1,6 +1,7 @@
 #ifndef RIME_SUM_COHERENCIES_OP_CPU_H
 #define RIME_SUM_COHERENCIES_OP_CPU_H
 
+#include "montblanc_macros.h"
 #include "sum_coherencies_op.h"
 
 // Required in order for Eigen::ThreadPoolDevice to be an actual type

montblanc/impl/rime/tensorflow/staging_area_wrapper.py

@@ -1,4 +1,4 @@
-from attrdict import AttrDict
+import attridict as AttrDict
 
 import tensorflow as tf
 from tensorflow.python.ops import data_flow_ops

pyproject.toml

@@ -1,5 +1,6 @@
 [build-system]
-requires = ['tensorflow >= 2.7.0,<=2.8.4; python_version >="3.8"',
+requires = ['tensorflow >= 2.7.0,<=2.15.0; python_version >="3.10"',
+            'tensorflow >= 2.7.0,<=2.8.4; python_version >="3.8" and python_version <"3.10"',
             'tensorflow <=2.4.4; python_version <"3.8"',
             "setuptools >= 0.53.0"]
 build-backend = "setuptools.build_meta"