Update linting config and fix

Remove flake8 ignores for various rules: only ignore the rules which
must be ignored to be black-compatible.

Remove exclusions from pre-commit config.

.flake8

@@ -1,27 +1,4 @@
-[flake8]
-# TODO: remove all of these ignores other than W503,W504,B008
-# `black` will handle enforcement of styling, and we will have no opinionated
-# ignore rules
-# any cases in which we actually need to ignore a rule (e.g. E402) we will mark
-# the relevant segment with noqa comments as necessary
-#
-# D203: 1 blank line required before class docstring
-# E124: closing bracket does not match visual indentation
-# E126: continuation line over-indented for hanging indent
-# This one is bad. Sometimes ordering matters, conditional imports
-# setting env vars necessary etc.
-# E402: module level import not at top of file
-# E129: Visual indent to not match indent as next line, counter eg here:
-# https://github.com/PyCQA/pycodestyle/issues/386
-#
-# E203,W503,W504: conflict with black formatting sometimes
-# B008: a flake8-bugbear rule which fails on idiomatic typer usage (consider
-# re-enabling this once everything else is fixed and updating usage)
-ignore = D203, E124, E126, E402, E129, W605, W503, W504, E203, F401, B008
+[flake8]  # black-compatible
+ignore = W503, W504, E203, B008
 # TODO: reduce this to 88 once `black` is applied to all code
-max-line-length = 160
-exclude = parsl/executors/serialize/, test_import_fail.py
-# F632 is comparing constant literals with == instead of "is"
-per-file-ignores = funcx_sdk/funcx/sdk/client.py:F632,
-    funcx_endpoint/funcx/endpoint/auth.py:F821,
-    funcx_endpoint/funcx/serialize/base.py:F821
+max-line-length = 88

.pre-commit-config.yaml

@@ -8,9 +8,6 @@ repos:
   hooks:
     - id: check-merge-conflict
     - id: trailing-whitespace
-      # FIXME: temporary exclude to reduce conflicts, remove this
-      exclude: ^funcx_endpoint/
-      # end FIXME
 - repo: https://github.com/sirosen/check-jsonschema
   rev: 0.3.1
   hooks:
@@ -19,27 +16,18 @@ repos:
   rev: 21.5b1
   hooks:
     - id: black
-      # FIXME: temporary exclude to reduce conflicts, remove this
-      exclude: ^funcx_endpoint/
-      # end FIXME
 - repo: https://github.com/timothycrosley/isort
   rev: 5.8.0
   hooks:
     - id: isort
       # explicitly pass settings file so that isort does not try to deduce
       # which settings to use based on a file's directory
       args: ["--settings-path", ".isort.cfg"]
-      # FIXME: temporary exclude to reduce conflicts, remove this
-      exclude: ^funcx_endpoint/
-      # end FIXME
 - repo: https://github.com/asottile/pyupgrade
   rev: v2.17.0
   hooks:
     - id: pyupgrade
       args: ["--py36-plus"]
-      # FIXME: temporary exclude to reduce conflicts, remove this
-      exclude: ^funcx_endpoint/
-      # end FIXME
 - repo: https://gitlab.com/pycqa/flake8
   rev: 3.9.2
   hooks:

docs/conf.py

@@ -16,10 +16,8 @@
 import os
 import sys
 
-import requests
-
 sys.path.insert(0, os.path.abspath("../funcx_sdk/"))
-import funcx
+import funcx  # noqa:E402
 
 # -- Project information -----------------------------------------------------
 

docs/configs/bluewaters.py

@@ -10,7 +10,7 @@
 # PLEASE UPDATE user_opts BEFORE USE
 user_opts = {
     'bluewaters': {
-        'worker_init': 'module load bwpy;source anaconda3/etc/profile.d/conda.sh;conda activate funcx_testing_py3.7',
+        'worker_init': 'module load bwpy;source anaconda3/etc/profile.d/conda.sh;conda activate funcx_testing_py3.7',  # noqa: E501
         'scheduler_options': '',
     }
 }

docs/configs/polaris.py

@@ -1,4 +1,3 @@
-from parsl.addresses import address_by_hostname
 from parsl.launchers import SingleNodeLauncher
 from parsl.providers import PBSProProvider
 

docs/configs/uchicago_ai_cluster.py

@@ -1,6 +1,6 @@
 from parsl.addresses import address_by_hostname
 from parsl.launchers import SrunLauncher
-from parsl.providers import LocalProvider, SlurmProvider
+from parsl.providers import SlurmProvider
 
 from funcx_endpoint.endpoint.utils.config import Config
 from funcx_endpoint.executors import HighThroughputExecutor
@@ -28,12 +28,17 @@
                 partition='general',
 
                 # Launch 4 managers per node, each bound to 1 GPU
-                # This is a hack. We use hostname ; to terminate the srun command, and start our own
+                # This is a hack. We use hostname ; to terminate the srun command, and
+                # start our own
+                #
                 # DO NOT MODIFY unless you know what you are doing.
-                launcher=SrunLauncher(overrides=(f'hostname; srun --ntasks={TOTAL_WORKERS} '
-                                                 f'--ntasks-per-node={WORKERS_PER_NODE} '
-                                                 f'--gpus-per-task=rtx2080ti:{GPUS_PER_WORKER} '
-                                                 f'--gpu-bind=map_gpu:{GPU_MAP}')
+                launcher=SrunLauncher(
+                    overrides=(
+                        f'hostname; srun --ntasks={TOTAL_WORKERS} '
+                        f'--ntasks-per-node={WORKERS_PER_NODE} '
+                        f'--gpus-per-task=rtx2080ti:{GPUS_PER_WORKER} '
+                        f'--gpu-bind=map_gpu:{GPU_MAP}'
+                    )
                 ),
 
                 # Scale between 0-1 blocks with 2 nodes per block

funcx_endpoint/funcx_endpoint/endpoint/config.py

@@ -1,17 +1,16 @@
-import globus_sdk
-import parsl
 import os
 
-from parsl.config import Config
+import globus_sdk
+from parsl.addresses import address_by_route
 from parsl.channels import LocalChannel
-from parsl.providers import LocalProvider, KubernetesProvider
+from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
-from parsl.addresses import address_by_route
+from parsl.providers import KubernetesProvider, LocalProvider
 
 # GlobusAuth-related secrets
-SECRET_KEY = os.environ.get('secret_key')
-GLOBUS_KEY = os.environ.get('globus_key')
-GLOBUS_CLIENT = os.environ.get('globus_client')
+SECRET_KEY = os.environ.get("secret_key")
+GLOBUS_KEY = os.environ.get("globus_key")
+GLOBUS_CLIENT = os.environ.get("globus_client")
 
 FUNCX_URL = "https://funcx.org/"
 FUNCX_HUB_URL = "3.88.81.131"
@@ -31,10 +30,9 @@ def _load_auth_client():
     _prod = True
 
     if _prod:
-        app = globus_sdk.ConfidentialAppAuthClient(GLOBUS_CLIENT,
-                                                   GLOBUS_KEY)
+        app = globus_sdk.ConfidentialAppAuthClient(GLOBUS_CLIENT, GLOBUS_KEY)
     else:
-        app = globus_sdk.ConfidentialAppAuthClient('', '')
+        app = globus_sdk.ConfidentialAppAuthClient("", "")
     return app
 
 
@@ -62,7 +60,7 @@ def _get_parsl_config():
                 ),
             )
         ],
-        strategy=None
+        strategy=None,
     )
     return config
 
@@ -77,31 +75,35 @@ def _get_executor(container):
     """
 
     executor = HighThroughputExecutor(
-                   label=container['container_uuid'],
-                   cores_per_worker=1,
-                   max_workers=1,
-                   poll_period=10,
-                   # launch_cmd="ls; sleep 3600",
-                   worker_logdir_root='runinfo',
-                   # worker_debug=True,
-                   address=address_by_route(),
-                   provider=KubernetesProvider(
-                       namespace="dlhub-privileged",
-                       image=container['location'],
-                       nodes_per_block=1,
-                       init_blocks=1,
-                       max_blocks=1,
-                       parallelism=1,
-                       worker_init="""pip install git+https://github.com/Parsl/parsl;
+        label=container["container_uuid"],
+        cores_per_worker=1,
+        max_workers=1,
+        poll_period=10,
+        # launch_cmd="ls; sleep 3600",
+        worker_logdir_root="runinfo",
+        # worker_debug=True,
+        address=address_by_route(),
+        provider=KubernetesProvider(
+            namespace="dlhub-privileged",
+            image=container["location"],
+            nodes_per_block=1,
+            init_blocks=1,
+            max_blocks=1,
+            parallelism=1,
+            worker_init="""pip install git+https://github.com/Parsl/parsl;
                                    pip install git+https://github.com/funcx-faas/funcX;
                                    export PYTHONPATH=$PYTHONPATH:/home/ubuntu:/app""",
-                       # security=None,
-                       secret="ryan-kube-secret",
-                       pod_name=container['name'].replace('.', '-').replace("_", '-').replace('/', '-').lower(),
-                       # secret="minikube-aws-ecr",
-                       # user_id=32781,
-                       # group_id=10253,
-                       # run_as_non_root=True
-                   ),
-               )
+            # security=None,
+            secret="ryan-kube-secret",
+            pod_name=container["name"]
+            .replace(".", "-")
+            .replace("_", "-")
+            .replace("/", "-")
+            .lower(),
+            # secret="minikube-aws-ecr",
+            # user_id=32781,
+            # group_id=10253,
+            # run_as_non_root=True
+        ),
+    )
     return [executor]

funcx_endpoint/funcx_endpoint/endpoint/default_config.py

@@ -1,16 +1,19 @@
+from parsl.providers import LocalProvider
+
 from funcx_endpoint.endpoint.utils.config import Config
 from funcx_endpoint.executors import HighThroughputExecutor
-from parsl.providers import LocalProvider
 
 config = Config(
-    executors=[HighThroughputExecutor(
-        provider=LocalProvider(
-            init_blocks=1,
-            min_blocks=0,
-            max_blocks=1,
-        ),
-    )],
-    funcx_service_address='https://api2.funcx.org/v2'
+    executors=[
+        HighThroughputExecutor(
+            provider=LocalProvider(
+                init_blocks=1,
+                min_blocks=0,
+                max_blocks=1,
+            ),
+        )
+    ],
+    funcx_service_address="https://api2.funcx.org/v2",
 )
 
 # For now, visible_to must be a list of URNs for globus auth users or groups, e.g.:
@@ -22,5 +25,5 @@
     "organization": "",
     "department": "",
     "public": False,
-    "visible_to": []
+    "visible_to": [],
 }