From 6e627165111cd73f0c95045aa949e415c94ed3f1 Mon Sep 17 00:00:00 2001 From: Stephen Rosen Date: Thu, 28 Oct 2021 16:44:26 +0000 Subject: [PATCH] Fix CI and the way dependencies are specified test and dev are extras for installing tools, no multiple requirements files scattered across the repo. CI workflows should not be installing multiple distinct applications into a single venv and then trying to test. This muddies the waters and makes the purpose and requirements of each CI step unclear. Therefore, in addition to putting the requirement data where it really belongs, this fixes the workflows to specify a better isolated test configuration. Workflow jobs are used more thoughtfully to produce isolated VMs for distinct build and testing processes. New daily tests run Monday-through-Friday in the AM for US timezones, doing `safety check` and little else. Safety checking on dependencies is done separately for funcx-sdk and funcx-endpoint. The CI workflow for testing PRs and branches also does `safety check`, but importantly the hourly test does not. Notifications from the daily and hourly workflows are done in a separate dedicated job. This means that new jobs can be added upstream without needing distinct or rewritten notification logic. The absence of any pytest run for the funcx-sdk package is more obvious and identified properly as a problem. It is paired with the "import test" so that when enabled, the import test removal will be localized to the same area of the workflow file. Linting is a dedicated job which runs as part of the CI workflow and uses pre-commit on the full repo. No additional lint steps (e.g. one-off flake8 runs) are used or should be needed. Testing documentation is updated minimally to refer to the installation of extras, rather than requirements files. The doc site requirements for some (undocumented) reason are installing funcx-sdk test requirements. This has been removed. Any documentation requirements can continue to be specified in the doc directory, or under a new `[docs]` extra if necessary. It should not "inherit" the requirements of the testing process. The codecov integration does not deliver value in the absence of good unit testing and a test matrix of size greater than 1. Therefore, until such a time as codecov is useful and a worthwhile configuration can be identified, the integration has been removed. The funcx-endpoint setup.py file was autoformatted with `black` to simplify the formatting of the requirements data. No workflow steps are allowed to use `actions/checkout@master`. This is unnecessary and dangerous usage. `checkout@v2` is used -- as the GitHub Action's own documentation instructs users to do -- instead. --- .github/workflows/ci.yaml | 115 +++++++++++++-------------- .github/workflows/daily.yaml | 57 +++++++++++++ .github/workflows/hourly.yaml | 41 +++++----- .github/workflows/smoke_test.yaml | 8 +- CONTRIBUTING.md | 15 ++++ docs/doc-requirements.txt | 3 +- funcx_endpoint/requirements.txt | 45 ----------- funcx_endpoint/setup.py | 93 ++++++++++++++++------ funcx_endpoint/test-requirements.txt | 5 -- funcx_sdk/funcx/tests/README.rst | 6 +- funcx_sdk/requirements.txt | 18 ----- funcx_sdk/setup.py | 35 +++++++- funcx_sdk/test-requirements.txt | 9 --- 13 files changed, 254 insertions(+), 196 deletions(-) create mode 100644 .github/workflows/daily.yaml delete mode 100644 funcx_endpoint/requirements.txt delete mode 100644 funcx_endpoint/test-requirements.txt delete mode 100644 funcx_sdk/requirements.txt delete mode 100644 funcx_sdk/test-requirements.txt diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 391985466..b2327551d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,73 +9,68 @@ on: pull_request: jobs: - test: - strategy: - matrix: - python-version: [3.7] + lint: runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + - name: install pre-commit + run: | + python -m pip install -U pip setuptools wheel + python -m pip install pre-commit + - name: run pre-commit + run: pre-commit run -a + test-sdk: + runs-on: ubuntu-latest steps: - - uses: actions/checkout@master - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Get latest pip version - run: | - python -m pip install --upgrade pip setuptools wheel - - name: Lint - run: | - pip install pre-commit - pre-commit run -a - - name: Install dependencies for funcx-sdk - run: | - python -m pip install -r funcx_sdk/requirements.txt - python -m pip install -r funcx_sdk/test-requirements.txt - pip list - - name: Check for vulnerabilities in libraries - run: | - pip install safety - pip freeze | safety check - - name: Test sdk by just importing - run: | - cd funcx_sdk - pip install . - python -c "from funcx.sdk.client import FuncXClient" - cd .. -# - name: Test with pytest -# run: | -# pytest - - name: Install dependencies for funcx-endpoint - run: | - python -m pip install -r funcx_endpoint/requirements.txt - python -m pip install -r funcx_endpoint/test-requirements.txt - pip list - - name: Check for vulnerabilities in libraries - run: | - pip install safety - pip freeze | safety check - - name: Test funcx-endpoint by just importing - run: | - cd funcx_endpoint - pip install . - python -c "from funcx_endpoint.version import VERSION" - funcx-endpoint -v - cd .. - - name: Lint with Flake8 - run: | - flake8 funcx_endpoint - - name: Test with pytest - run: | - PYTHONPATH=funcx_endpoint python -m coverage run -m pytest funcx_endpoint/tests/funcx_endpoint - - name: Report coverage with Codecov - run: | - codecov --token=${{ secrets.CODECOV_TOKEN }} + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: install requirements + run: | + python -m pip install -U pip setuptools wheel + python -m pip install './funcx_sdk[test]' + pip install safety + - name: run safety check + run: safety check + + # TODO: remove this test + # This is the weakest test which does anything, checking that the client can + # be imported. As soon as pytest is running again, remove this. + - name: check importable + run: python -c "from funcx.sdk.client import FuncXClient" + # - name: run pytest + # run: | + # cd funcx_sdk + # pytest + + test-endpoint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: install requirements + run: | + python -m pip install -U pip setuptools wheel + python -m pip install './funcx_endpoint[test]' + pip install safety + - name: run safety check + run: safety check + - name: run pytest + run: | + PYTHONPATH=funcx_endpoint python -m coverage run -m pytest funcx_endpoint/tests/funcx_endpoint publish: # only trigger on pushes to the main repo (not forks, and not PRs) if: ${{ github.repository == 'funcx-faas/funcX' && github.event_name == 'push' }} - needs: test + needs: + - lint + - test-sdk + - test-endpoint runs-on: ubuntu-latest strategy: matrix: diff --git a/.github/workflows/daily.yaml b/.github/workflows/daily.yaml new file mode 100644 index 000000000..8e1a3183e --- /dev/null +++ b/.github/workflows/daily.yaml @@ -0,0 +1,57 @@ +name: daily +on: + # build every weekday at 4:00 AM UTC + schedule: + - cron: '0 4 * * 1-5' + workflow_dispatch: + +jobs: + safety-check-sdk: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: main + - uses: actions/setup-python@v1 + - name: install requirements + run: | + python -m pip install --upgrade pip setuptools wheel + python -m pip install './funcx_sdk' + python -m pip install safety + - name: run safety check + run: safety check + + safety-check-endpoint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: main + - uses: actions/setup-python@v1 + - name: install requirements + run: | + python -m pip install --upgrade pip setuptools wheel + python -m pip install './funcx_endpoint' + python -m pip install safety + - name: run safety check + run: safety check + + notify: + runs-on: ubuntu-latest + needs: + - safety-check-sdk + - safety-check-endpoint + if: failure() + steps: + # FIXME: make this send to a listhost or Slack + - name: Send mail + uses: dawidd6/action-send-mail@v3 + with: + server_address: smtp.gmail.com + server_port: 465 + username: ${{secrets.MAIL_USERNAME}} + password: ${{secrets.MAIL_PASSWORD}} + subject: ${{ github.repository }} - Daily Check ${{ job.status }} + to: ryan.chard@gmail.com,rchard@anl.gov,chard@uchicago.edu,yadudoc1729@gmail.com,josh@globus.org,bengal1@illinois.edu,benc@hawaga.org.uk,sirosen@globus.org,uriel@globus.org + from: funcX Tests # + body: The daily ${{ github.repository }} workflow failed! diff --git a/.github/workflows/hourly.yaml b/.github/workflows/hourly.yaml index 4abfb9d7c..3b127c59f 100644 --- a/.github/workflows/hourly.yaml +++ b/.github/workflows/hourly.yaml @@ -10,7 +10,7 @@ on: description: "manual test" jobs: - tutorial_test: + smoke-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -22,25 +22,26 @@ jobs: - name: Install dependencies for funcx-sdk and test requirements run: | python -m pip install --upgrade pip setuptools wheel - python -m pip install ./funcx_sdk - python -m pip install -r funcx_sdk/test-requirements.txt - - name: Check for vulnerabilities in libraries - run: | - pip install safety - safety check + python -m pip install './funcx_sdk[test]' + python -m pip install safety - name: Run smoke tests to check liveness of hosted services run: | pytest -v funcx_endpoint/tests/smoke_tests --api-client-id ${{ secrets.API_CLIENT_ID }} --api-client-secret ${{ secrets.API_CLIENT_SECRET }} - # FIXME: make this send to a listhost or Slack - - name: Send mail - if: ${{ failure() }} - uses: dawidd6/action-send-mail@v3 - with: - server_address: smtp.gmail.com - server_port: 465 - username: ${{secrets.MAIL_USERNAME}} - password: ${{secrets.MAIL_PASSWORD}} - subject: ${{ github.repository }} - Tutorial test ${{ job.status }} - to: ryan.chard@gmail.com,rchard@anl.gov,chard@uchicago.edu,yadudoc1729@gmail.com,josh@globus.org,bengal1@illinois.edu,benc@hawaga.org.uk,sirosen@globus.org,uriel@globus.org - from: funcX Tests # - body: The ${{ github.repository }} test ${{ github.workflow }} exited with status - ${{ job.status }}! + + notify: + runs-on: ubuntu-latest + needs: [smoke-test] + if: failure() + steps: + # FIXME: make this send to a listhost or Slack + - name: Send mail + uses: dawidd6/action-send-mail@v3 + with: + server_address: smtp.gmail.com + server_port: 465 + username: ${{secrets.MAIL_USERNAME}} + password: ${{secrets.MAIL_PASSWORD}} + subject: ${{ github.repository }} - Tutorial test ${{ job.status }} + to: ryan.chard@gmail.com,rchard@anl.gov,chard@uchicago.edu,yadudoc1729@gmail.com,josh@globus.org,bengal1@illinois.edu,benc@hawaga.org.uk,sirosen@globus.org,uriel@globus.org + from: funcX Tests # + body: The hourly ${{ github.repository }} workflow failed! diff --git a/.github/workflows/smoke_test.yaml b/.github/workflows/smoke_test.yaml index 6aa569e9a..1e38e2d16 100644 --- a/.github/workflows/smoke_test.yaml +++ b/.github/workflows/smoke_test.yaml @@ -21,14 +21,10 @@ jobs: - uses: actions/setup-python@v1 with: python-version: 3.7 - - name: Install dependencies for funcx-sdk and test requirements + - name: install requirements run: | python -m pip install --upgrade pip setuptools wheel - python -m pip install ./funcx_sdk - python -m pip install -r funcx_sdk/test-requirements.txt - - name: Test sdk by just importing - run: | - python -c "from funcx.sdk.client import FuncXClient" + python -m pip install './funcx_sdk[test]' - name: Run smoke tests to check liveness of hosted services run: | pytest -v funcx_endpoint/tests/smoke_tests --api-client-id ${{ secrets.API_CLIENT_ID }} --api-client-secret ${{ secrets.API_CLIENT_SECRET }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 01773fa79..c701642ee 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,3 +30,18 @@ After installing `pre-commit`, run in the repo to configure hooks. > NOTE: If necessary, you can always skip hooks with `git commit --no-verify` + +## Installing Testing Requirements + +Testing requirements for each of the two packages in this repository +(funcx-sdk and funcx-endpoint) are specified as installable extras. + +To install the funcx-sdk test requirements + + cd funcx_sdk + pip install '.[test]' + +To install the funcx-endpoint test requirements + + cd funcx_endpoint + pip install '.[test]' diff --git a/docs/doc-requirements.txt b/docs/doc-requirements.txt index ba67ae0ad..528f1a9a6 100644 --- a/docs/doc-requirements.txt +++ b/docs/doc-requirements.txt @@ -1,4 +1,3 @@ --r ../funcx_sdk/requirements.txt --r ../funcx_sdk/test-requirements.txt +../funcx_sdk nbsphinx sphinx_rtd_theme diff --git a/funcx_endpoint/requirements.txt b/funcx_endpoint/requirements.txt deleted file mode 100644 index 52f7b0bf4..000000000 --- a/funcx_endpoint/requirements.txt +++ /dev/null @@ -1,45 +0,0 @@ -requests>=2.20.0,<3 -globus_sdk<3 -funcx>=0.3.3,<0.4.0 - -# table printing used in list-endpoints -texttable>=1.6.4,<2 - -# although psutil does not declare itself to use semver, it appears to offer -# strong backwards-compatibility promises based on its changelog, usage, and -# history -# -# TODO: re-evaluate bound after we have an answer of some kind from psutil -# see: -# https://github.com/giampaolo/psutil/issues/2002 -psutil<6 - -# provides easy daemonization of the endpoint -python-daemon>=2,<3 - -# TODO: replace use of `typer` with `click` because -# 1. `typer` is a thin wrapper over `click` offering very minimal additional -# functionality -# 2. `click` follows semver and releases new major versions when known -# backwards-incompatible changes are introduced, making our application -# safer to distribute -typer==0.4.0 - - -# disallow use of 22.3.0; the whl package on some platforms causes ZMQ issues -# -# NOTE: 22.3.0 introduced a patched version of libzmq.so to the wheel packaging -# which may be the source of the problems , the problem can be fixed by -# building from source, which may mean there's an issue in the packaged library -# further investigation may be needed if the issue persists in the next pyzmq -# release -pyzmq>=22.0.0,!=22.3.0 - -# TODO: evaluate removal of the 'retry' library after the update to -# globus-sdk v3, which provides automatic retries on all API calls -retry==0.9.2 - -# 'parsl' is a core requirement of the funcx-endpoint, essential to a range -# of different features and functions -# pin exact versions because it does not use semver -parsl==1.1.0 diff --git a/funcx_endpoint/setup.py b/funcx_endpoint/setup.py index fb0c266d0..6008f513f 100644 --- a/funcx_endpoint/setup.py +++ b/funcx_endpoint/setup.py @@ -1,20 +1,70 @@ import os -from setuptools import setup, find_packages + +from setuptools import find_packages, setup + +REQUIRES = [ + "requests>=2.20.0,<3", + "globus_sdk<3", + "funcx>=0.3.3,<0.4.0", + # table printing used in list-endpoints + "texttable>=1.6.4,<2", + # although psutil does not declare itself to use semver, it appears to offer + # strong backwards-compatibility promises based on its changelog, usage, and + # history + # + # TODO: re-evaluate bound after we have an answer of some kind from psutil + # see: + # https://github.com/giampaolo/psutil/issues/2002 + "psutil<6", + # provides easy daemonization of the endpoint + "python-daemon>=2,<3", + # TODO: replace use of `typer` with `click` because + # 1. `typer` is a thin wrapper over `click` offering very minimal additional + # functionality + # 2. `click` follows semver and releases new major versions when known + # backwards-incompatible changes are introduced, making our application + # safer to distribute + "typer==0.4.0", + # disallow use of 22.3.0; the whl package on some platforms causes ZMQ issues + # + # NOTE: 22.3.0 introduced a patched version of libzmq.so to the wheel packaging + # which may be the source of the problems , the problem can be fixed by + # building from source, which may mean there's an issue in the packaged library + # further investigation may be needed if the issue persists in the next pyzmq + # release + "pyzmq>=22.0.0,!=22.3.0", + # TODO: evaluate removal of the 'retry' library after the update to + # globus-sdk v3, which provides automatic retries on all API calls + "retry==0.9.2", + # 'parsl' is a core requirement of the funcx-endpoint, essential to a range + # of different features and functions + # pin exact versions because it does not use semver + "parsl==1.1.0", +] + +TEST_REQUIRES = [ + "pytest>=5.2", + "coverage>=5.2", + "codecov==2.1.8", + "pytest-mock==3.2.0", + "flake8>=3.8", +] + version_ns = {} with open(os.path.join("funcx_endpoint", "version.py")) as f: exec(f.read(), version_ns) -version = version_ns['VERSION'] - -with open('requirements.txt') as f: - install_requires = f.readlines() +version = version_ns["VERSION"] setup( - name='funcx-endpoint', + name="funcx-endpoint", version=version, packages=find_packages(), - description='funcX: High Performance Function Serving for Science', - install_requires=install_requires, + description="funcX: High Performance Function Serving for Science", + install_requires=REQUIRES, + extras_require={ + "test": TEST_REQUIRES, + }, python_requires=">=3.6.0", classifiers=[ "Development Status :: 3 - Alpha", @@ -23,23 +73,20 @@ "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Topic :: Scientific/Engineering" - ], - keywords=[ - "funcX", - "FaaS", - "Function Serving" + "Topic :: Scientific/Engineering", ], - entry_points={'console_scripts': - ['funcx-endpoint=funcx_endpoint.endpoint.endpoint:cli_run', - 'funcx-interchange=funcx_endpoint.executors.high_throughput.interchange:cli_run', - 'funcx-manager=funcx_endpoint.executors.high_throughput.funcx_manager:cli_run', - 'funcx-worker=funcx_endpoint.executors.high_throughput.funcx_worker:cli_run', - ] + keywords=["funcX", "FaaS", "Function Serving"], + entry_points={ + "console_scripts": [ + "funcx-endpoint=funcx_endpoint.endpoint.endpoint:cli_run", + "funcx-interchange=funcx_endpoint.executors.high_throughput.interchange:cli_run", + "funcx-manager=funcx_endpoint.executors.high_throughput.funcx_manager:cli_run", + "funcx-worker=funcx_endpoint.executors.high_throughput.funcx_worker:cli_run", + ] }, include_package_data=True, - author='funcX team', - author_email='labs@globus.org', + author="funcX team", + author_email="labs@globus.org", license="Apache License, Version 2.0", - url="https://github.com/funcx-faas/funcx" + url="https://github.com/funcx-faas/funcx", ) diff --git a/funcx_endpoint/test-requirements.txt b/funcx_endpoint/test-requirements.txt deleted file mode 100644 index fcb0abfc0..000000000 --- a/funcx_endpoint/test-requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -pytest>=5.2 -coverage>=5.2 -codecov==2.1.8 -pytest-mock==3.2.0 -flake8>=3.8 diff --git a/funcx_sdk/funcx/tests/README.rst b/funcx_sdk/funcx/tests/README.rst index 2e0f34801..9579d3586 100644 --- a/funcx_sdk/funcx/tests/README.rst +++ b/funcx_sdk/funcx/tests/README.rst @@ -24,9 +24,8 @@ From Pypi conda create -y --name funcx_testing_py3.8 python=3.8 conda activate funcx_testing_py3.8 - pip install funcx==0.0.6a5 + pip install 'funcx[test]==0.0.6a5' pip install funcx-endpoint==0.0.6a5 - pip install -r ./funcx_sdk/test-requirements.txt From Source ^^^^^^^^^^^ @@ -37,9 +36,8 @@ Here's a sequence of steps that should be copy-pastable: conda create -y --name funcx_testing_py3.8 python=3.8 conda activate funcx_testing_py3.8 - pip install ./funcx_sdk/ + pip install './funcx_sdk[test] pip install ./funcx_endpoint/ - pip install -r ./funcx_sdk/test-requirements.txt Setup an endpoint ----------------- diff --git a/funcx_sdk/requirements.txt b/funcx_sdk/requirements.txt deleted file mode 100644 index 810968bcf..000000000 --- a/funcx_sdk/requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -# request sending and authorization tools -requests>=2.20.0 -globus-sdk<3 - -# 'websockets' is used for the client-side websocket listener -websockets==9.1 - -# table printing used in search result rendering -texttable>=1.6.4,<2 - -# versions >=0.2.3 requires globus-sdk v3 -# TODO: update pin to latest when globus-sdk is updated -fair_research_login==0.2.2 - -# dill is an extension of `pickle` to a wider array of native python types -# pin to the latest version, as 'dill' is not at 1.0 and does not have a clear -# versioning and compatibility policy -dill==0.3.4 diff --git a/funcx_sdk/setup.py b/funcx_sdk/setup.py index 5688332d7..f12aff36a 100644 --- a/funcx_sdk/setup.py +++ b/funcx_sdk/setup.py @@ -2,20 +2,47 @@ from setuptools import find_namespace_packages, setup +REQUIRES = [ + # request sending and authorization tools + "requests>=2.20.0", + "globus-sdk<3", + # 'websockets' is used for the client-side websocket listener + "websockets==9.1", + # table printing used in search result rendering + "texttable>=1.6.4,<2", + # versions >=0.2.3 requires globus-sdk v3 + # TODO: update pin to latest when globus-sdk is updated + "fair_research_login==0.2.2", + # dill is an extension of `pickle` to a wider array of native python types + # pin to the latest version, as 'dill' is not at 1.0 and does not have a clear + # versioning and compatibility policy + "dill==0.3.4", +] + +TEST_REQUIRES = [ + "flake8==3.8.0", + "numpy", + "pytest", +] +DEV_REQUIRES = TEST_REQUIRES + [ + "pre-commit", +] + version_ns = {} with open(os.path.join("funcx", "sdk", "version.py")) as f: exec(f.read(), version_ns) version = version_ns["VERSION"] -with open("requirements.txt") as f: - install_requires = f.readlines() - setup( name="funcx", version=version, packages=find_namespace_packages(include=["funcx", "funcx.*"]), description="funcX: High Performance Function Serving for Science", - install_requires=install_requires, + install_requires=REQUIRES, + extras_require={ + "dev": DEV_REQUIRES, + "test": TEST_REQUIRES, + }, python_requires=">=3.6.0", classifiers=[ "Development Status :: 3 - Alpha", diff --git a/funcx_sdk/test-requirements.txt b/funcx_sdk/test-requirements.txt deleted file mode 100644 index c6574756d..000000000 --- a/funcx_sdk/test-requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -flake8==3.8.0 -numpy -pytest - -# TODO: when we refactor dev requirements to be kept in extras, move this into -# `funcx_sdk[dev]` and the other requirements into `funcx_sdk[test]` -# for an example of this kind of config, see attrs: -# https://github.com/python-attrs/attrs/blob/fb154878ebd2758d2a3b4dc518d21fd4f73e12d2/setup.py#L64-L67 -pre-commit