diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 391985466..b2327551d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,73 +9,68 @@ on: pull_request: jobs: - test: - strategy: - matrix: - python-version: [3.7] + lint: runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + - name: install pre-commit + run: | + python -m pip install -U pip setuptools wheel + python -m pip install pre-commit + - name: run pre-commit + run: pre-commit run -a + test-sdk: + runs-on: ubuntu-latest steps: - - uses: actions/checkout@master - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Get latest pip version - run: | - python -m pip install --upgrade pip setuptools wheel - - name: Lint - run: | - pip install pre-commit - pre-commit run -a - - name: Install dependencies for funcx-sdk - run: | - python -m pip install -r funcx_sdk/requirements.txt - python -m pip install -r funcx_sdk/test-requirements.txt - pip list - - name: Check for vulnerabilities in libraries - run: | - pip install safety - pip freeze | safety check - - name: Test sdk by just importing - run: | - cd funcx_sdk - pip install . - python -c "from funcx.sdk.client import FuncXClient" - cd .. -# - name: Test with pytest -# run: | -# pytest - - name: Install dependencies for funcx-endpoint - run: | - python -m pip install -r funcx_endpoint/requirements.txt - python -m pip install -r funcx_endpoint/test-requirements.txt - pip list - - name: Check for vulnerabilities in libraries - run: | - pip install safety - pip freeze | safety check - - name: Test funcx-endpoint by just importing - run: | - cd funcx_endpoint - pip install . - python -c "from funcx_endpoint.version import VERSION" - funcx-endpoint -v - cd .. - - name: Lint with Flake8 - run: | - flake8 funcx_endpoint - - name: Test with pytest - run: | - PYTHONPATH=funcx_endpoint python -m coverage run -m pytest funcx_endpoint/tests/funcx_endpoint - - name: Report coverage with Codecov - run: | - codecov --token=${{ secrets.CODECOV_TOKEN }} + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: install requirements + run: | + python -m pip install -U pip setuptools wheel + python -m pip install './funcx_sdk[test]' + pip install safety + - name: run safety check + run: safety check + + # TODO: remove this test + # This is the weakest test which does anything, checking that the client can + # be imported. As soon as pytest is running again, remove this. + - name: check importable + run: python -c "from funcx.sdk.client import FuncXClient" + # - name: run pytest + # run: | + # cd funcx_sdk + # pytest + + test-endpoint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: install requirements + run: | + python -m pip install -U pip setuptools wheel + python -m pip install './funcx_endpoint[test]' + pip install safety + - name: run safety check + run: safety check + - name: run pytest + run: | + PYTHONPATH=funcx_endpoint python -m coverage run -m pytest funcx_endpoint/tests/funcx_endpoint publish: # only trigger on pushes to the main repo (not forks, and not PRs) if: ${{ github.repository == 'funcx-faas/funcX' && github.event_name == 'push' }} - needs: test + needs: + - lint + - test-sdk + - test-endpoint runs-on: ubuntu-latest strategy: matrix: diff --git a/.github/workflows/daily.yaml b/.github/workflows/daily.yaml new file mode 100644 index 000000000..8e1a3183e --- /dev/null +++ b/.github/workflows/daily.yaml @@ -0,0 +1,57 @@ +name: daily +on: + # build every weekday at 4:00 AM UTC + schedule: + - cron: '0 4 * * 1-5' + workflow_dispatch: + +jobs: + safety-check-sdk: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: main + - uses: actions/setup-python@v1 + - name: install requirements + run: | + python -m pip install --upgrade pip setuptools wheel + python -m pip install './funcx_sdk' + python -m pip install safety + - name: run safety check + run: safety check + + safety-check-endpoint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: main + - uses: actions/setup-python@v1 + - name: install requirements + run: | + python -m pip install --upgrade pip setuptools wheel + python -m pip install './funcx_endpoint' + python -m pip install safety + - name: run safety check + run: safety check + + notify: + runs-on: ubuntu-latest + needs: + - safety-check-sdk + - safety-check-endpoint + if: failure() + steps: + # FIXME: make this send to a listhost or Slack + - name: Send mail + uses: dawidd6/action-send-mail@v3 + with: + server_address: smtp.gmail.com + server_port: 465 + username: ${{secrets.MAIL_USERNAME}} + password: ${{secrets.MAIL_PASSWORD}} + subject: ${{ github.repository }} - Daily Check ${{ job.status }} + to: ryan.chard@gmail.com,rchard@anl.gov,chard@uchicago.edu,yadudoc1729@gmail.com,josh@globus.org,bengal1@illinois.edu,benc@hawaga.org.uk,sirosen@globus.org,uriel@globus.org + from: funcX Tests # + body: The daily ${{ github.repository }} workflow failed! diff --git a/.github/workflows/hourly.yaml b/.github/workflows/hourly.yaml index 4abfb9d7c..3b127c59f 100644 --- a/.github/workflows/hourly.yaml +++ b/.github/workflows/hourly.yaml @@ -10,7 +10,7 @@ on: description: "manual test" jobs: - tutorial_test: + smoke-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -22,25 +22,26 @@ jobs: - name: Install dependencies for funcx-sdk and test requirements run: | python -m pip install --upgrade pip setuptools wheel - python -m pip install ./funcx_sdk - python -m pip install -r funcx_sdk/test-requirements.txt - - name: Check for vulnerabilities in libraries - run: | - pip install safety - safety check + python -m pip install './funcx_sdk[test]' + python -m pip install safety - name: Run smoke tests to check liveness of hosted services run: | pytest -v funcx_endpoint/tests/smoke_tests --api-client-id ${{ secrets.API_CLIENT_ID }} --api-client-secret ${{ secrets.API_CLIENT_SECRET }} - # FIXME: make this send to a listhost or Slack - - name: Send mail - if: ${{ failure() }} - uses: dawidd6/action-send-mail@v3 - with: - server_address: smtp.gmail.com - server_port: 465 - username: ${{secrets.MAIL_USERNAME}} - password: ${{secrets.MAIL_PASSWORD}} - subject: ${{ github.repository }} - Tutorial test ${{ job.status }} - to: ryan.chard@gmail.com,rchard@anl.gov,chard@uchicago.edu,yadudoc1729@gmail.com,josh@globus.org,bengal1@illinois.edu,benc@hawaga.org.uk,sirosen@globus.org,uriel@globus.org - from: funcX Tests # - body: The ${{ github.repository }} test ${{ github.workflow }} exited with status - ${{ job.status }}! + + notify: + runs-on: ubuntu-latest + needs: [smoke-test] + if: failure() + steps: + # FIXME: make this send to a listhost or Slack + - name: Send mail + uses: dawidd6/action-send-mail@v3 + with: + server_address: smtp.gmail.com + server_port: 465 + username: ${{secrets.MAIL_USERNAME}} + password: ${{secrets.MAIL_PASSWORD}} + subject: ${{ github.repository }} - Tutorial test ${{ job.status }} + to: ryan.chard@gmail.com,rchard@anl.gov,chard@uchicago.edu,yadudoc1729@gmail.com,josh@globus.org,bengal1@illinois.edu,benc@hawaga.org.uk,sirosen@globus.org,uriel@globus.org + from: funcX Tests # + body: The hourly ${{ github.repository }} workflow failed! diff --git a/.github/workflows/smoke_test.yaml b/.github/workflows/smoke_test.yaml index 6aa569e9a..1e38e2d16 100644 --- a/.github/workflows/smoke_test.yaml +++ b/.github/workflows/smoke_test.yaml @@ -21,14 +21,10 @@ jobs: - uses: actions/setup-python@v1 with: python-version: 3.7 - - name: Install dependencies for funcx-sdk and test requirements + - name: install requirements run: | python -m pip install --upgrade pip setuptools wheel - python -m pip install ./funcx_sdk - python -m pip install -r funcx_sdk/test-requirements.txt - - name: Test sdk by just importing - run: | - python -c "from funcx.sdk.client import FuncXClient" + python -m pip install './funcx_sdk[test]' - name: Run smoke tests to check liveness of hosted services run: | pytest -v funcx_endpoint/tests/smoke_tests --api-client-id ${{ secrets.API_CLIENT_ID }} --api-client-secret ${{ secrets.API_CLIENT_SECRET }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 01773fa79..c701642ee 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,3 +30,18 @@ After installing `pre-commit`, run in the repo to configure hooks. > NOTE: If necessary, you can always skip hooks with `git commit --no-verify` + +## Installing Testing Requirements + +Testing requirements for each of the two packages in this repository +(funcx-sdk and funcx-endpoint) are specified as installable extras. + +To install the funcx-sdk test requirements + + cd funcx_sdk + pip install '.[test]' + +To install the funcx-endpoint test requirements + + cd funcx_endpoint + pip install '.[test]' diff --git a/docs/doc-requirements.txt b/docs/doc-requirements.txt index ba67ae0ad..528f1a9a6 100644 --- a/docs/doc-requirements.txt +++ b/docs/doc-requirements.txt @@ -1,4 +1,3 @@ --r ../funcx_sdk/requirements.txt --r ../funcx_sdk/test-requirements.txt +../funcx_sdk nbsphinx sphinx_rtd_theme diff --git a/funcx_endpoint/requirements.txt b/funcx_endpoint/requirements.txt deleted file mode 100644 index 52f7b0bf4..000000000 --- a/funcx_endpoint/requirements.txt +++ /dev/null @@ -1,45 +0,0 @@ -requests>=2.20.0,<3 -globus_sdk<3 -funcx>=0.3.3,<0.4.0 - -# table printing used in list-endpoints -texttable>=1.6.4,<2 - -# although psutil does not declare itself to use semver, it appears to offer -# strong backwards-compatibility promises based on its changelog, usage, and -# history -# -# TODO: re-evaluate bound after we have an answer of some kind from psutil -# see: -# https://github.com/giampaolo/psutil/issues/2002 -psutil<6 - -# provides easy daemonization of the endpoint -python-daemon>=2,<3 - -# TODO: replace use of `typer` with `click` because -# 1. `typer` is a thin wrapper over `click` offering very minimal additional -# functionality -# 2. `click` follows semver and releases new major versions when known -# backwards-incompatible changes are introduced, making our application -# safer to distribute -typer==0.4.0 - - -# disallow use of 22.3.0; the whl package on some platforms causes ZMQ issues -# -# NOTE: 22.3.0 introduced a patched version of libzmq.so to the wheel packaging -# which may be the source of the problems , the problem can be fixed by -# building from source, which may mean there's an issue in the packaged library -# further investigation may be needed if the issue persists in the next pyzmq -# release -pyzmq>=22.0.0,!=22.3.0 - -# TODO: evaluate removal of the 'retry' library after the update to -# globus-sdk v3, which provides automatic retries on all API calls -retry==0.9.2 - -# 'parsl' is a core requirement of the funcx-endpoint, essential to a range -# of different features and functions -# pin exact versions because it does not use semver -parsl==1.1.0 diff --git a/funcx_endpoint/setup.py b/funcx_endpoint/setup.py index fb0c266d0..6008f513f 100644 --- a/funcx_endpoint/setup.py +++ b/funcx_endpoint/setup.py @@ -1,20 +1,70 @@ import os -from setuptools import setup, find_packages + +from setuptools import find_packages, setup + +REQUIRES = [ + "requests>=2.20.0,<3", + "globus_sdk<3", + "funcx>=0.3.3,<0.4.0", + # table printing used in list-endpoints + "texttable>=1.6.4,<2", + # although psutil does not declare itself to use semver, it appears to offer + # strong backwards-compatibility promises based on its changelog, usage, and + # history + # + # TODO: re-evaluate bound after we have an answer of some kind from psutil + # see: + # https://github.com/giampaolo/psutil/issues/2002 + "psutil<6", + # provides easy daemonization of the endpoint + "python-daemon>=2,<3", + # TODO: replace use of `typer` with `click` because + # 1. `typer` is a thin wrapper over `click` offering very minimal additional + # functionality + # 2. `click` follows semver and releases new major versions when known + # backwards-incompatible changes are introduced, making our application + # safer to distribute + "typer==0.4.0", + # disallow use of 22.3.0; the whl package on some platforms causes ZMQ issues + # + # NOTE: 22.3.0 introduced a patched version of libzmq.so to the wheel packaging + # which may be the source of the problems , the problem can be fixed by + # building from source, which may mean there's an issue in the packaged library + # further investigation may be needed if the issue persists in the next pyzmq + # release + "pyzmq>=22.0.0,!=22.3.0", + # TODO: evaluate removal of the 'retry' library after the update to + # globus-sdk v3, which provides automatic retries on all API calls + "retry==0.9.2", + # 'parsl' is a core requirement of the funcx-endpoint, essential to a range + # of different features and functions + # pin exact versions because it does not use semver + "parsl==1.1.0", +] + +TEST_REQUIRES = [ + "pytest>=5.2", + "coverage>=5.2", + "codecov==2.1.8", + "pytest-mock==3.2.0", + "flake8>=3.8", +] + version_ns = {} with open(os.path.join("funcx_endpoint", "version.py")) as f: exec(f.read(), version_ns) -version = version_ns['VERSION'] - -with open('requirements.txt') as f: - install_requires = f.readlines() +version = version_ns["VERSION"] setup( - name='funcx-endpoint', + name="funcx-endpoint", version=version, packages=find_packages(), - description='funcX: High Performance Function Serving for Science', - install_requires=install_requires, + description="funcX: High Performance Function Serving for Science", + install_requires=REQUIRES, + extras_require={ + "test": TEST_REQUIRES, + }, python_requires=">=3.6.0", classifiers=[ "Development Status :: 3 - Alpha", @@ -23,23 +73,20 @@ "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Topic :: Scientific/Engineering" - ], - keywords=[ - "funcX", - "FaaS", - "Function Serving" + "Topic :: Scientific/Engineering", ], - entry_points={'console_scripts': - ['funcx-endpoint=funcx_endpoint.endpoint.endpoint:cli_run', - 'funcx-interchange=funcx_endpoint.executors.high_throughput.interchange:cli_run', - 'funcx-manager=funcx_endpoint.executors.high_throughput.funcx_manager:cli_run', - 'funcx-worker=funcx_endpoint.executors.high_throughput.funcx_worker:cli_run', - ] + keywords=["funcX", "FaaS", "Function Serving"], + entry_points={ + "console_scripts": [ + "funcx-endpoint=funcx_endpoint.endpoint.endpoint:cli_run", + "funcx-interchange=funcx_endpoint.executors.high_throughput.interchange:cli_run", + "funcx-manager=funcx_endpoint.executors.high_throughput.funcx_manager:cli_run", + "funcx-worker=funcx_endpoint.executors.high_throughput.funcx_worker:cli_run", + ] }, include_package_data=True, - author='funcX team', - author_email='labs@globus.org', + author="funcX team", + author_email="labs@globus.org", license="Apache License, Version 2.0", - url="https://github.com/funcx-faas/funcx" + url="https://github.com/funcx-faas/funcx", ) diff --git a/funcx_endpoint/test-requirements.txt b/funcx_endpoint/test-requirements.txt deleted file mode 100644 index fcb0abfc0..000000000 --- a/funcx_endpoint/test-requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -pytest>=5.2 -coverage>=5.2 -codecov==2.1.8 -pytest-mock==3.2.0 -flake8>=3.8 diff --git a/funcx_sdk/funcx/tests/README.rst b/funcx_sdk/funcx/tests/README.rst index 2e0f34801..9579d3586 100644 --- a/funcx_sdk/funcx/tests/README.rst +++ b/funcx_sdk/funcx/tests/README.rst @@ -24,9 +24,8 @@ From Pypi conda create -y --name funcx_testing_py3.8 python=3.8 conda activate funcx_testing_py3.8 - pip install funcx==0.0.6a5 + pip install 'funcx[test]==0.0.6a5' pip install funcx-endpoint==0.0.6a5 - pip install -r ./funcx_sdk/test-requirements.txt From Source ^^^^^^^^^^^ @@ -37,9 +36,8 @@ Here's a sequence of steps that should be copy-pastable: conda create -y --name funcx_testing_py3.8 python=3.8 conda activate funcx_testing_py3.8 - pip install ./funcx_sdk/ + pip install './funcx_sdk[test] pip install ./funcx_endpoint/ - pip install -r ./funcx_sdk/test-requirements.txt Setup an endpoint ----------------- diff --git a/funcx_sdk/requirements.txt b/funcx_sdk/requirements.txt deleted file mode 100644 index 810968bcf..000000000 --- a/funcx_sdk/requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -# request sending and authorization tools -requests>=2.20.0 -globus-sdk<3 - -# 'websockets' is used for the client-side websocket listener -websockets==9.1 - -# table printing used in search result rendering -texttable>=1.6.4,<2 - -# versions >=0.2.3 requires globus-sdk v3 -# TODO: update pin to latest when globus-sdk is updated -fair_research_login==0.2.2 - -# dill is an extension of `pickle` to a wider array of native python types -# pin to the latest version, as 'dill' is not at 1.0 and does not have a clear -# versioning and compatibility policy -dill==0.3.4 diff --git a/funcx_sdk/setup.py b/funcx_sdk/setup.py index 5688332d7..f12aff36a 100644 --- a/funcx_sdk/setup.py +++ b/funcx_sdk/setup.py @@ -2,20 +2,47 @@ from setuptools import find_namespace_packages, setup +REQUIRES = [ + # request sending and authorization tools + "requests>=2.20.0", + "globus-sdk<3", + # 'websockets' is used for the client-side websocket listener + "websockets==9.1", + # table printing used in search result rendering + "texttable>=1.6.4,<2", + # versions >=0.2.3 requires globus-sdk v3 + # TODO: update pin to latest when globus-sdk is updated + "fair_research_login==0.2.2", + # dill is an extension of `pickle` to a wider array of native python types + # pin to the latest version, as 'dill' is not at 1.0 and does not have a clear + # versioning and compatibility policy + "dill==0.3.4", +] + +TEST_REQUIRES = [ + "flake8==3.8.0", + "numpy", + "pytest", +] +DEV_REQUIRES = TEST_REQUIRES + [ + "pre-commit", +] + version_ns = {} with open(os.path.join("funcx", "sdk", "version.py")) as f: exec(f.read(), version_ns) version = version_ns["VERSION"] -with open("requirements.txt") as f: - install_requires = f.readlines() - setup( name="funcx", version=version, packages=find_namespace_packages(include=["funcx", "funcx.*"]), description="funcX: High Performance Function Serving for Science", - install_requires=install_requires, + install_requires=REQUIRES, + extras_require={ + "dev": DEV_REQUIRES, + "test": TEST_REQUIRES, + }, python_requires=">=3.6.0", classifiers=[ "Development Status :: 3 - Alpha", diff --git a/funcx_sdk/test-requirements.txt b/funcx_sdk/test-requirements.txt deleted file mode 100644 index c6574756d..000000000 --- a/funcx_sdk/test-requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -flake8==3.8.0 -numpy -pytest - -# TODO: when we refactor dev requirements to be kept in extras, move this into -# `funcx_sdk[dev]` and the other requirements into `funcx_sdk[test]` -# for an example of this kind of config, see attrs: -# https://github.com/python-attrs/attrs/blob/fb154878ebd2758d2a3b4dc518d21fd4f73e12d2/setup.py#L64-L67 -pre-commit