Merge pull request globus#624 from sirosen/fix-dependency-specification

Fix CI and the way dependencies are specified

.github/workflows/ci.yaml

@@ -9,73 +9,68 @@ on:
   pull_request:
 
 jobs:
-  test:
-    strategy:
-      matrix:
-        python-version: [3.7]
+  lint:
     runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v1
+      - name: install pre-commit
+        run: |
+          python -m pip install -U pip setuptools wheel
+          python -m pip install pre-commit
+      - name: run pre-commit
+        run: pre-commit run -a
 
+  test-sdk:
+    runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@master
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Get latest pip version
-      run: |
-        python -m pip install --upgrade pip setuptools wheel
-    - name: Lint
-      run: |
-        pip install pre-commit
-        pre-commit run -a
-    - name: Install dependencies for funcx-sdk
-      run: |
-        python -m pip install -r funcx_sdk/requirements.txt
-        python -m pip install -r funcx_sdk/test-requirements.txt
-        pip list
-    - name: Check for vulnerabilities in libraries
-      run: |
-        pip install safety
-        pip freeze | safety check
-    - name: Test sdk by just importing
-      run: |
-        cd funcx_sdk
-        pip install .
-        python -c "from funcx.sdk.client import FuncXClient"
-        cd ..
-#    - name: Test with pytest
-#      run: |
-#        pytest
-    - name: Install dependencies for funcx-endpoint
-      run: |
-        python -m pip install -r funcx_endpoint/requirements.txt
-        python -m pip install -r funcx_endpoint/test-requirements.txt
-        pip list
-    - name: Check for vulnerabilities in libraries
-      run: |
-        pip install safety
-        pip freeze | safety check
-    - name: Test funcx-endpoint by just importing
-      run: |
-        cd funcx_endpoint
-        pip install .
-        python -c "from funcx_endpoint.version import VERSION"
-        funcx-endpoint -v
-        cd ..
-    - name: Lint with Flake8
-      run: |
-        flake8 funcx_endpoint
-    - name: Test with pytest
-      run: |
-        PYTHONPATH=funcx_endpoint python -m coverage run -m  pytest funcx_endpoint/tests/funcx_endpoint
-    - name: Report coverage with Codecov
-      run: |
-        codecov --token=${{ secrets.CODECOV_TOKEN }}
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: install requirements
+        run: |
+          python -m pip install -U pip setuptools wheel
+          python -m pip install './funcx_sdk[test]'
+          pip install safety
+      - name: run safety check
+        run: safety check
+
+      # TODO: remove this test
+      # This is the weakest test which does anything, checking that the client can
+      # be imported. As soon as pytest is running again, remove this.
+      - name: check importable
+        run: python -c "from funcx.sdk.client import FuncXClient"
+      # - name: run pytest
+      #   run: |
+      #     cd funcx_sdk
+      #     pytest
+
+  test-endpoint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: install requirements
+        run: |
+          python -m pip install -U pip setuptools wheel
+          python -m pip install './funcx_endpoint[test]'
+          pip install safety
+      - name: run safety check
+        run: safety check
+      - name: run pytest
+        run: |
+          PYTHONPATH=funcx_endpoint python -m coverage run -m pytest funcx_endpoint/tests/funcx_endpoint
 
   publish:
     # only trigger on pushes to the main repo (not forks, and not PRs)
     if: ${{ github.repository == 'funcx-faas/funcX' && github.event_name == 'push' }}
-    needs: test
+    needs:
+      - lint
+      - test-sdk
+      - test-endpoint
     runs-on: ubuntu-latest
     strategy:
       matrix:

.github/workflows/daily.yaml

@@ -0,0 +1,57 @@
+name: daily
+on:
+  # build every weekday at 4:00 AM UTC
+  schedule:
+    - cron: '0 4 * * 1-5'
+  workflow_dispatch:
+
+jobs:
+  safety-check-sdk:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: main
+    - uses: actions/setup-python@v1
+    - name: install requirements
+      run: |
+        python -m pip install --upgrade pip setuptools wheel
+        python -m pip install './funcx_sdk'
+        python -m pip install safety
+    - name: run safety check
+      run: safety check
+
+  safety-check-endpoint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: main
+    - uses: actions/setup-python@v1
+    - name: install requirements
+      run: |
+        python -m pip install --upgrade pip setuptools wheel
+        python -m pip install './funcx_endpoint'
+        python -m pip install safety
+    - name: run safety check
+      run: safety check
+
+  notify:
+    runs-on: ubuntu-latest
+    needs:
+      - safety-check-sdk
+      - safety-check-endpoint
+    if: failure()
+    steps:
+      # FIXME: make this send to a listhost or Slack
+      - name: Send mail
+        uses: dawidd6/action-send-mail@v3
+        with:
+          server_address: smtp.gmail.com
+          server_port: 465
+          username: ${{secrets.MAIL_USERNAME}}
+          password: ${{secrets.MAIL_PASSWORD}}
+          subject: ${{ github.repository }} - Daily Check ${{ job.status }}
+          to: [email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected]
+          from: funcX Tests # <[email protected]>
+          body: The daily ${{ github.repository }} workflow failed!

.github/workflows/hourly.yaml

@@ -10,7 +10,7 @@ on:
         description: "manual test"
 
 jobs:
-  tutorial_test:
+  smoke-test:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -22,25 +22,26 @@ jobs:
     - name: Install dependencies for funcx-sdk and test requirements
       run: |
         python -m pip install --upgrade pip setuptools wheel
-        python -m pip install ./funcx_sdk
-        python -m pip install -r funcx_sdk/test-requirements.txt
-    - name: Check for vulnerabilities in libraries
-      run: |
-        pip install safety
-        safety check
+        python -m pip install './funcx_sdk[test]'
+        python -m pip install safety
     - name: Run smoke tests to check liveness of hosted services
       run: |
         pytest -v funcx_endpoint/tests/smoke_tests --api-client-id ${{ secrets.API_CLIENT_ID }} --api-client-secret ${{ secrets.API_CLIENT_SECRET }}
-     # FIXME: make this send to a listhost or Slack
-    - name: Send mail
-      if: ${{ failure() }}
-      uses: dawidd6/action-send-mail@v3
-      with:
-        server_address: smtp.gmail.com
-        server_port: 465
-        username: ${{secrets.MAIL_USERNAME}}
-        password: ${{secrets.MAIL_PASSWORD}}
-        subject: ${{ github.repository }} - Tutorial test ${{ job.status }}
-        to: [email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected]
-        from: funcX Tests # <[email protected]>
-        body: The ${{ github.repository }} test ${{ github.workflow }} exited with status - ${{ job.status }}!
+
+  notify:
+    runs-on: ubuntu-latest
+    needs: [smoke-test]
+    if: failure()
+    steps:
+      # FIXME: make this send to a listhost or Slack
+      - name: Send mail
+        uses: dawidd6/action-send-mail@v3
+        with:
+          server_address: smtp.gmail.com
+          server_port: 465
+          username: ${{secrets.MAIL_USERNAME}}
+          password: ${{secrets.MAIL_PASSWORD}}
+          subject: ${{ github.repository }} - Tutorial test ${{ job.status }}
+          to: [email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected]
+          from: funcX Tests # <[email protected]>
+          body: The hourly ${{ github.repository }} workflow failed!

.github/workflows/smoke_test.yaml

@@ -21,14 +21,10 @@ jobs:
     - uses: actions/setup-python@v1
       with:
         python-version: 3.7
-    - name: Install dependencies for funcx-sdk and test requirements
+    - name: install requirements
       run: |
         python -m pip install --upgrade pip setuptools wheel
-        python -m pip install ./funcx_sdk
-        python -m pip install -r funcx_sdk/test-requirements.txt
-    - name: Test sdk by just importing
-      run: |
-        python -c "from funcx.sdk.client import FuncXClient"
+        python -m pip install './funcx_sdk[test]'
     - name: Run smoke tests to check liveness of hosted services
       run: |
         pytest -v funcx_endpoint/tests/smoke_tests --api-client-id ${{ secrets.API_CLIENT_ID }} --api-client-secret ${{ secrets.API_CLIENT_SECRET }}

CONTRIBUTING.md

@@ -30,3 +30,18 @@ After installing `pre-commit`, run
 in the repo to configure hooks.
 
 > NOTE: If necessary, you can always skip hooks with `git commit --no-verify`
+
+## Installing Testing Requirements
+
+Testing requirements for each of the two packages in this repository
+(funcx-sdk and funcx-endpoint) are specified as installable extras.
+
+To install the funcx-sdk test requirements
+
+    cd funcx_sdk
+    pip install '.[test]'
+
+To install the funcx-endpoint test requirements
+
+    cd funcx_endpoint
+    pip install '.[test]'

docs/doc-requirements.txt

@@ -1,4 +1,3 @@
--r ../funcx_sdk/requirements.txt
--r ../funcx_sdk/test-requirements.txt
+../funcx_sdk
 nbsphinx
 sphinx_rtd_theme