Data Cleaning #21627
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Data Cleaning | |
| on: | |
| push: | |
| paths: | |
| - 'cleansing/full.py' | |
| schedule: | |
| - cron: '0 7/12 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| whotriggered: | |
| description: 'Manual Trigger' | |
| default: 'Why' | |
| required: false | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout current repo | |
| uses: actions/checkout@v2 | |
| with: | |
| path: covid19-eu-data-cleansing | |
| - name: Checkout raw data repo | |
| uses: actions/checkout@v2 | |
| with: | |
| token: ${{ secrets.ACCESS_TOKEN }} | |
| repository: covid19-eu-zh/covid19-eu-data | |
| path: covid19-eu-data | |
| - name: List contents in current folder | |
| run: | | |
| pwd | |
| ls | |
| - name: Copy raw dataset to current repo | |
| run: | | |
| echo "create dataset folder in current repo" | |
| mkdir -p covid19-eu-data-cleansing/dataset | |
| echo "Copy raw dataset to current repo" | |
| cp -r covid19-eu-data/dataset/* covid19-eu-data-cleansing/dataset/ | |
| - uses: actions/setup-python@v1 | |
| with: | |
| python-version: '3.x' | |
| - name: Install Python Requirements | |
| working-directory: ./covid19-eu-data-cleansing/cleansing | |
| run: pip install -r requirements.txt | |
| - name: Clean up dataset | |
| working-directory: ./covid19-eu-data-cleansing | |
| run: | | |
| DB_FOLDER=db | |
| echo "Creating folder $DB_FOLDER" | |
| mkdir -p $DB_FOLDER | |
| echo "Cleaning up data and save as json in $DB_FOLDER" | |
| python cleansing/full.py | |
| echo "All files in $DB_FOLDER" | |
| ls -alht $DB_FOLDER | |
| echo "git add commit" | |
| git config --local user.email "[email protected]" | |
| git config --local user.name "GitHub Action" | |
| git status | |
| git add . | |
| git commit -m "Update Datasets" || echo "Nothing to update" | |
| git status | |
| - name: Push changes | |
| uses: ad-m/github-push-action@master | |
| with: | |
| directory: covid19-eu-data-cleansing | |
| branch: master | |
| github_token: ${{ secrets.GITHUB_TOKEN }} |