upload-open-data-assets #65
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: upload-open-data-assets | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| socrata_asset: | |
| type: choice | |
| description: Target Socrata asset | |
| options: | |
| - Appeals | |
| - Assessed Values | |
| - Parcel Addresses | |
| - Parcel Proximity | |
| - Parcel Sales | |
| - Parcel Status | |
| - Permits | |
| - Property Tax-Exempt Parcels | |
| - Parcel Universe (Historic) | |
| - Parcel Universe (Current Year) | |
| - Residential Condominium Unit Characteristics | |
| - Single and Multi-Family Improvement Characteristics | |
| default: Parcel Universe (Current Year) | |
| required: true | |
| overwrite: | |
| # True for overwrite, False for update | |
| type: boolean | |
| description: Overwrite socrata asset | |
| required: true | |
| years: | |
| # Comma separated list of years | |
| type: string | |
| description: Years to update or overwrite | |
| default: 'all' | |
| required: false | |
| by_township: | |
| type: boolean | |
| description: Chunk Socrata upload by township | |
| required: false | |
| env: | |
| UV_SYSTEM_PYTHON: 1 | |
| jobs: | |
| upload-open-data-assets: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| # contents:read and id-token:write permissions are needed to interact | |
| # with GitHub's OIDC Token endpoint so that we can authenticate with AWS | |
| contents: read | |
| id-token: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| socrata/pyproject.toml | |
| socrata/uv.lock | |
| - name: Install Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version-file: socrata/.python-version | |
| - name: Setup dbt | |
| uses: ./.github/actions/setup_dbt | |
| with: | |
| role-to-assume: ${{ secrets.AWS_IAM_ROLE_TO_ASSUME_ARN }} | |
| # Set these env vars as secrets so they get masked in the GitHub | |
| # Actions logs | |
| - name: Install Python dependencies | |
| working-directory: socrata | |
| shell: bash | |
| run: uv pip install . | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: us-east-1 | |
| role-to-assume: ${{ secrets.AWS_IAM_ROLE_TO_ASSUME_ARN }} | |
| - name: Upload to Socrata | |
| env: | |
| AWS_IAM_ROLE_TO_ASSUME_ARN: ${{ secrets.AWS_IAM_ROLE_TO_ASSUME_ARN }} | |
| AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} | |
| AWS_ATHENA_S3_STAGING_DIR: ${{ vars.AWS_ATHENA_S3_STAGING_DIR }} | |
| SOCRATA_APP_TOKEN: ${{ secrets.SOCRATA_APP_TOKEN }} | |
| SOCRATA_USERNAME: ${{ secrets.SOCRATA_USERNAME }} | |
| SOCRATA_PASSWORD: ${{ secrets.SOCRATA_PASSWORD }} | |
| SOCRATA_ASSET: ${{ inputs.socrata_asset }} | |
| OVERWRITE: ${{ inputs.overwrite }} | |
| YEARS: ${{ inputs.years }} | |
| BY_TOWNSHIP: ${{ inputs.by_township }} | |
| run: python ./socrata/socrata_upload.py |