From e3fc15406c61b2877c4fc0600a8d822fe2bd5a44 Mon Sep 17 00:00:00 2001 From: reubensamuel Date: Tue, 14 Feb 2023 16:08:32 +0000 Subject: [PATCH 01/10] GAT-1887: Added Production pipeline --- .github/workflows/prod_deployment.yaml | 55 ++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/prod_deployment.yaml diff --git a/.github/workflows/prod_deployment.yaml b/.github/workflows/prod_deployment.yaml new file mode 100644 index 00000000..686e4e07 --- /dev/null +++ b/.github/workflows/prod_deployment.yaml @@ -0,0 +1,55 @@ +name: Deploy to Cloud Run + +on: + pull_request: + types: + - closed + branches: + - 'master' + +env: + PROJECT_ID: '${{ secrets.PROJECT_ID }}' # TODO: update Google Cloud project id. + GAR_LOCATION: '${{ secrets.GAR_LOCATION }}' # TODO: update Artifact Registry location + +jobs: # Deployment please don't modify anything here as the infrastructure is controlled by terraform any changes here please agree with chris and reuben + deploy: + if: github.event.pull_request.merged == true + permissions: + contents: write + id-token: write + + runs-on: ubuntu-latest + environment: legacy-prod + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: master + + - name: Google Auth + id: auth + uses: 'google-github-actions/auth@v0' + with: + token_format: 'access_token' + workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' + service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' + + - name: Read VERSION file + id: getversion + run: echo "version=$(cat Chart.yaml)" >> $GITHUB_OUTPUT + + - name: Deploy to Cloud Run + uses: actions-hub/gcloud@master + id: deploy + env: + PROJECT_ID: ${{ secrets.PROJECT_ID }} + DEV_PROJECT_ID: ${{ secrets.DEV_PROJECT_ID }} + GAR_LOCATION: ${{ secrets.GAR_LOCATION }} + GAR_NAME: ${{ secrets.GAR_NAME_API }} + SERVICE_NAME: '${{ secrets.SERVICE_NAME_API }}' + SERVICE_REGION: '${{ secrets.SERVICE_REGION_API }}' + + with: + # args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' + args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:latest --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' + # If required, use the Cloud Run url output in later steps \ No newline at end of file From 7d158aae92fe365d51aeba21d688c8d4e47232e3 Mon Sep 17 00:00:00 2001 From: reubensamuel <110553227+reubensamuel@users.noreply.github.com> Date: Tue, 14 Feb 2023 16:13:22 +0000 Subject: [PATCH 02/10] Revert "GAT-1887: Testing New Release from preprod to prod gateway-api" --- .github/workflows/dev_deployment.yaml | 138 ---------------------- .github/workflows/preprod_deployment.yaml | 55 --------- .github/workflows/uat_deployment.yaml | 63 ---------- Chart.yaml | 1 - src/config/server.js | 2 - src/resources/auth/auth.route.js | 2 +- src/resources/auth/utils.js | 2 +- src/resources/dataset/dataset.entity.js | 2 +- src/resources/metadata/metadata.route.js | 22 ---- src/utils/datasetonboarding.util.js | 2 +- 10 files changed, 4 insertions(+), 285 deletions(-) delete mode 100644 .github/workflows/dev_deployment.yaml delete mode 100644 .github/workflows/preprod_deployment.yaml delete mode 100644 .github/workflows/uat_deployment.yaml delete mode 100644 Chart.yaml delete mode 100644 src/resources/metadata/metadata.route.js diff --git a/.github/workflows/dev_deployment.yaml b/.github/workflows/dev_deployment.yaml deleted file mode 100644 index 3e3242a9..00000000 --- a/.github/workflows/dev_deployment.yaml +++ /dev/null @@ -1,138 +0,0 @@ -name: Build and Deploy to Cloud Run - -on: - push: - paths: - - 'Chart.yaml' - -env: - PROJECT_ID: '${{ secrets.PROJECT_ID }}' # TODO: update Google Cloud project id. - GAR_LOCATION: '${{ secrets.GAR_LOCATION }}' # TODO: update Artifact Registry location - -jobs: - build: - # needs: analyze - # Add 'id-token' with the intended permissions for workload identity federation - permissions: - contents: write - id-token: write - - runs-on: ubuntu-latest - environment: legacy-dev - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: dev - - - name: Read VERSION file - id: getversion - run: echo "version=$(cat Chart.yaml)" >> $GITHUB_OUTPUT - # Used for production - # - uses: "marvinpinto/action-automatic-releases@latest" - # with: - # repo_token: "${{ secrets.GITHUB_TOKEN }}" - # automatic_release_tag: ${{ steps.getversion.outputs.version }} - # prerelease: false - - - name: Google Auth - id: auth - uses: 'google-github-actions/auth@v0' - with: - token_format: 'access_token' - workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' - service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' - - - name: Login to GAR - uses: docker/login-action@v2 - with: - registry: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.PROJECT_ID }} - username: oauth2accesstoken - password: ${{ steps.auth.outputs.access_token }} - - - name: Build and Push Container - shell: bash - env: - GAR_LOCATION: ${{ secrets.GAR_LOCATION }} - PROJECT_ID: ${{ secrets.PROJECT_ID }} - GAR_NAME: ${{ secrets.GAR_NAME_API }} - - run: |- - docker build -t '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} -t '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:latest ./ - docker push --all-tags '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }} - # END - Docker auth and build - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Deployment please don't modify anything here as the infrastructure is controlled by terraform any changes here please agree with chris and reuben - deploy: - needs: build - permissions: - contents: write - id-token: write - - runs-on: ubuntu-latest - environment: legacy-dev - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: dev - - - name: Google Auth - id: auth - uses: 'google-github-actions/auth@v0' - with: - token_format: 'access_token' - workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' - service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' - - - name: Read VERSION file - id: getversion - run: echo "version=$(cat Chart.yaml)" >> $GITHUB_OUTPUT - - - name: Deploy to Cloud Run - uses: actions-hub/gcloud@master - id: deploy - env: - PROJECT_ID: ${{ secrets.PROJECT_ID }} - GAR_LOCATION: ${{ secrets.GAR_LOCATION }} - GAR_NAME: ${{ secrets.GAR_NAME_API }} - SERVICE_NAME: '${{ secrets.SERVICE_NAME_API }}' - SERVICE_REGION: '${{ secrets.SERVICE_REGION_API }}' - - with: - args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' - # If required, use the Cloud Run url output in later steps \ No newline at end of file diff --git a/.github/workflows/preprod_deployment.yaml b/.github/workflows/preprod_deployment.yaml deleted file mode 100644 index b5e08ed1..00000000 --- a/.github/workflows/preprod_deployment.yaml +++ /dev/null @@ -1,55 +0,0 @@ -name: Deploy to Cloud Run - -on: - pull_request: - types: - - closed - branches: - - 'preprod' - -env: - PROJECT_ID: '${{ secrets.PROJECT_ID }}' # TODO: update Google Cloud project id. - GAR_LOCATION: '${{ secrets.GAR_LOCATION }}' # TODO: update Artifact Registry location - -jobs: # Deployment please don't modify anything here as the infrastructure is controlled by terraform any changes here please agree with chris and reuben - deploy: - if: github.event.pull_request.merged == true - permissions: - contents: write - id-token: write - - runs-on: ubuntu-latest - environment: legacy-preprod - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: preprod - - - name: Google Auth - id: auth - uses: 'google-github-actions/auth@v0' - with: - token_format: 'access_token' - workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' - service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' - - - name: Read VERSION file - id: getversion - run: echo "version=$(cat Chart.yaml)" >> $GITHUB_OUTPUT - - - name: Deploy to Cloud Run - uses: actions-hub/gcloud@master - id: deploy - env: - PROJECT_ID: ${{ secrets.PROJECT_ID }} - DEV_PROJECT_ID: ${{ secrets.DEV_PROJECT_ID }} - GAR_LOCATION: ${{ secrets.GAR_LOCATION }} - GAR_NAME: ${{ secrets.GAR_NAME_API }} - SERVICE_NAME: '${{ secrets.SERVICE_NAME_API }}' - SERVICE_REGION: '${{ secrets.SERVICE_REGION_API }}' - - with: - # args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' - args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:latest --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' - # If required, use the Cloud Run url output in later steps \ No newline at end of file diff --git a/.github/workflows/uat_deployment.yaml b/.github/workflows/uat_deployment.yaml deleted file mode 100644 index c4a1b1cb..00000000 --- a/.github/workflows/uat_deployment.yaml +++ /dev/null @@ -1,63 +0,0 @@ -name: Deploy to Cloud Run - -on: - pull_request: - types: - - closed - branches: - - 'release' - - -env: - PROJECT_ID: '${{ secrets.PROJECT_ID }}' # TODO: update Google Cloud project id. - GAR_LOCATION: '${{ secrets.GAR_LOCATION }}' # TODO: update Artifact Registry location - -jobs: # Deployment please don't modify anything here as the infrastructure is controlled by terraform any changes here please agree with chris and reuben. - # catchsha: - # uses: HDRUK/gateway-api/.github/workflows/dev_deployment.yaml@dev - deploy: - # if: github.event.pull_request.merged == true - permissions: - contents: write - id-token: write - - runs-on: ubuntu-latest - environment: legacy-uat - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: release - - - name: Google Auth - id: auth - uses: 'google-github-actions/auth@v0' - with: - token_format: 'access_token' - workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' - service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' - - - name: Read VERSION file - id: getversion - # run: echo "::set-output name=version::$(cat Chart.yaml)" - run: echo "version=$(cat Chart.yaml)" >> $GITHUB_OUTPUT - - # - name: Get SHA - # id: getsha - # run: echo ${{ needs.catchsha.outputs.GITHUB_SHA }} - - - name: Deploy to Cloud Run - uses: actions-hub/gcloud@master - id: deploy - env: - PROJECT_ID: ${{ secrets.PROJECT_ID }} - DEV_PROJECT_ID: ${{ secrets.DEV_PROJECT_ID }} - GAR_LOCATION: ${{ secrets.GAR_LOCATION }} - GAR_NAME: ${{ secrets.GAR_NAME_API }} - SERVICE_NAME: '${{ secrets.SERVICE_NAME_API }}' - SERVICE_REGION: '${{ secrets.SERVICE_REGION_API }}' - - with: - # args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:{{ steps.catchsha.outputs.GITHUB_SHA}} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' - # Functionality not supported by Github Actions one to ccheck back agin in the future - args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:latest --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' \ No newline at end of file diff --git a/Chart.yaml b/Chart.yaml deleted file mode 100644 index 5eb312c6..00000000 --- a/Chart.yaml +++ /dev/null @@ -1 +0,0 @@ -v0.0.0 \ No newline at end of file diff --git a/src/config/server.js b/src/config/server.js index 57dcc6aa..60b5c46f 100644 --- a/src/config/server.js +++ b/src/config/server.js @@ -236,8 +236,6 @@ app.use('/api/v2/questionbank', require('../resources/questionbank/questionbank. app.use('/api/v2/data-use-registers', require('../resources/dataUseRegister/dataUseRegister.route')); app.use('/api/v1/locations', require('../resources/spatialfilter/SpatialRouter')); -app.use('/api/v1/metadata', require('../resources/metadata/metadata.route')); - initialiseAuthentication(app); // launch our backend into a port diff --git a/src/resources/auth/auth.route.js b/src/resources/auth/auth.route.js index 687959c4..fdc342f3 100644 --- a/src/resources/auth/auth.route.js +++ b/src/resources/auth/auth.route.js @@ -25,7 +25,7 @@ router.get('/status', function (req, res, next) { if (err || !user) { return res.json({ success: true, - data: [{ role: 'Reader', id: null, name: null, loggedIn: false, tempProp: true }], + data: [{ role: 'Reader', id: null, name: null, loggedIn: false }], }); } else { // 1. Reformat teams array for frontend diff --git a/src/resources/auth/utils.js b/src/resources/auth/utils.js index 231e6543..380d9243 100644 --- a/src/resources/auth/utils.js +++ b/src/resources/auth/utils.js @@ -129,7 +129,7 @@ const getTeams = async () => { const catchLoginErrorAndRedirect = (req, res, next) => { if (req.auth.err || !req.auth.user) { - if (req.auth.err === 'loginError' || req.auth.user === undefined) { + if (req.auth.err === 'loginError') { return res.status(200).redirect(process.env.homeURL + '/loginerror'); } diff --git a/src/resources/dataset/dataset.entity.js b/src/resources/dataset/dataset.entity.js index 2e00900d..2edc0815 100644 --- a/src/resources/dataset/dataset.entity.js +++ b/src/resources/dataset/dataset.entity.js @@ -50,7 +50,7 @@ export default class DatasetClass extends Entity { transformedObject.dataset['@schema'] = { type: `Dataset`, version: `2.0.0`, - url: `https://raw.githubusercontent.com/HDRUK/schemata/master/schema/dataset/2.1.0/dataset.schema.json`, + url: `https://raw.githubusercontent.com/HDRUK/schemata/master/schema/dataset/latest/dataset.schema.json`, } // Return v2 object diff --git a/src/resources/metadata/metadata.route.js b/src/resources/metadata/metadata.route.js deleted file mode 100644 index cc9e4596..00000000 --- a/src/resources/metadata/metadata.route.js +++ /dev/null @@ -1,22 +0,0 @@ -import express from 'express'; -import passport from 'passport'; - -import { utils } from '../auth'; -import { ROLES } from '../user/user.roles'; - -import datasetonboardingUtil from '../../utils/datasetonboarding.util'; - -const router = express.Router({ mergeParams: true }); - -router.post('/scoring', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin), async (req, res) => { - const { dataset } = req.body; - - if (!dataset) { - res.json({ success: false, error: 'Dataset object must be supplied and contain all required data', status: 400 }); - } - - const verdict = await datasetonboardingUtil.buildMetadataQuality(dataset, dataset.datasetv2, dataset.pid); - res.json({ success: true, data: verdict, status: 200 }); -}); - -module.exports = router; \ No newline at end of file diff --git a/src/utils/datasetonboarding.util.js b/src/utils/datasetonboarding.util.js index b151df00..139e79c7 100644 --- a/src/utils/datasetonboarding.util.js +++ b/src/utils/datasetonboarding.util.js @@ -758,7 +758,7 @@ const buildMetadataQuality = async (dataset, v2Object, pid) => { let rawdata = fs.readFileSync(__dirname + '/schema.json'); schema = JSON.parse(rawdata); - const ajv = new Ajv({ strict: false, allErrors: false }); + const ajv = new Ajv({ strict: false, allErrors: true }); addFormats(ajv); const validate = ajv.compile(schema); validate(cleanV2Object); From 22de1b89da2a2c8c30808882e313013da87deb02 Mon Sep 17 00:00:00 2001 From: reubensamuel <110553227+reubensamuel@users.noreply.github.com> Date: Tue, 14 Feb 2023 16:19:00 +0000 Subject: [PATCH 03/10] Revert "Revert "GAT-1887: Testing New Release from preprod to prod gateway-api"" --- .github/workflows/dev_deployment.yaml | 138 ++++++++++++++++++++++ .github/workflows/preprod_deployment.yaml | 55 +++++++++ .github/workflows/uat_deployment.yaml | 63 ++++++++++ Chart.yaml | 1 + src/config/server.js | 2 + src/resources/auth/auth.route.js | 2 +- src/resources/auth/utils.js | 2 +- src/resources/dataset/dataset.entity.js | 2 +- src/resources/metadata/metadata.route.js | 22 ++++ src/utils/datasetonboarding.util.js | 2 +- 10 files changed, 285 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/dev_deployment.yaml create mode 100644 .github/workflows/preprod_deployment.yaml create mode 100644 .github/workflows/uat_deployment.yaml create mode 100644 Chart.yaml create mode 100644 src/resources/metadata/metadata.route.js diff --git a/.github/workflows/dev_deployment.yaml b/.github/workflows/dev_deployment.yaml new file mode 100644 index 00000000..3e3242a9 --- /dev/null +++ b/.github/workflows/dev_deployment.yaml @@ -0,0 +1,138 @@ +name: Build and Deploy to Cloud Run + +on: + push: + paths: + - 'Chart.yaml' + +env: + PROJECT_ID: '${{ secrets.PROJECT_ID }}' # TODO: update Google Cloud project id. + GAR_LOCATION: '${{ secrets.GAR_LOCATION }}' # TODO: update Artifact Registry location + +jobs: + build: + # needs: analyze + # Add 'id-token' with the intended permissions for workload identity federation + permissions: + contents: write + id-token: write + + runs-on: ubuntu-latest + environment: legacy-dev + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: dev + + - name: Read VERSION file + id: getversion + run: echo "version=$(cat Chart.yaml)" >> $GITHUB_OUTPUT + # Used for production + # - uses: "marvinpinto/action-automatic-releases@latest" + # with: + # repo_token: "${{ secrets.GITHUB_TOKEN }}" + # automatic_release_tag: ${{ steps.getversion.outputs.version }} + # prerelease: false + + - name: Google Auth + id: auth + uses: 'google-github-actions/auth@v0' + with: + token_format: 'access_token' + workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' + service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' + + - name: Login to GAR + uses: docker/login-action@v2 + with: + registry: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.PROJECT_ID }} + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} + + - name: Build and Push Container + shell: bash + env: + GAR_LOCATION: ${{ secrets.GAR_LOCATION }} + PROJECT_ID: ${{ secrets.PROJECT_ID }} + GAR_NAME: ${{ secrets.GAR_NAME_API }} + + run: |- + docker build -t '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} -t '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:latest ./ + docker push --all-tags '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }} + # END - Docker auth and build + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + # Deployment please don't modify anything here as the infrastructure is controlled by terraform any changes here please agree with chris and reuben + deploy: + needs: build + permissions: + contents: write + id-token: write + + runs-on: ubuntu-latest + environment: legacy-dev + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: dev + + - name: Google Auth + id: auth + uses: 'google-github-actions/auth@v0' + with: + token_format: 'access_token' + workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' + service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' + + - name: Read VERSION file + id: getversion + run: echo "version=$(cat Chart.yaml)" >> $GITHUB_OUTPUT + + - name: Deploy to Cloud Run + uses: actions-hub/gcloud@master + id: deploy + env: + PROJECT_ID: ${{ secrets.PROJECT_ID }} + GAR_LOCATION: ${{ secrets.GAR_LOCATION }} + GAR_NAME: ${{ secrets.GAR_NAME_API }} + SERVICE_NAME: '${{ secrets.SERVICE_NAME_API }}' + SERVICE_REGION: '${{ secrets.SERVICE_REGION_API }}' + + with: + args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' + # If required, use the Cloud Run url output in later steps \ No newline at end of file diff --git a/.github/workflows/preprod_deployment.yaml b/.github/workflows/preprod_deployment.yaml new file mode 100644 index 00000000..b5e08ed1 --- /dev/null +++ b/.github/workflows/preprod_deployment.yaml @@ -0,0 +1,55 @@ +name: Deploy to Cloud Run + +on: + pull_request: + types: + - closed + branches: + - 'preprod' + +env: + PROJECT_ID: '${{ secrets.PROJECT_ID }}' # TODO: update Google Cloud project id. + GAR_LOCATION: '${{ secrets.GAR_LOCATION }}' # TODO: update Artifact Registry location + +jobs: # Deployment please don't modify anything here as the infrastructure is controlled by terraform any changes here please agree with chris and reuben + deploy: + if: github.event.pull_request.merged == true + permissions: + contents: write + id-token: write + + runs-on: ubuntu-latest + environment: legacy-preprod + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: preprod + + - name: Google Auth + id: auth + uses: 'google-github-actions/auth@v0' + with: + token_format: 'access_token' + workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' + service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' + + - name: Read VERSION file + id: getversion + run: echo "version=$(cat Chart.yaml)" >> $GITHUB_OUTPUT + + - name: Deploy to Cloud Run + uses: actions-hub/gcloud@master + id: deploy + env: + PROJECT_ID: ${{ secrets.PROJECT_ID }} + DEV_PROJECT_ID: ${{ secrets.DEV_PROJECT_ID }} + GAR_LOCATION: ${{ secrets.GAR_LOCATION }} + GAR_NAME: ${{ secrets.GAR_NAME_API }} + SERVICE_NAME: '${{ secrets.SERVICE_NAME_API }}' + SERVICE_REGION: '${{ secrets.SERVICE_REGION_API }}' + + with: + # args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' + args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:latest --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' + # If required, use the Cloud Run url output in later steps \ No newline at end of file diff --git a/.github/workflows/uat_deployment.yaml b/.github/workflows/uat_deployment.yaml new file mode 100644 index 00000000..c4a1b1cb --- /dev/null +++ b/.github/workflows/uat_deployment.yaml @@ -0,0 +1,63 @@ +name: Deploy to Cloud Run + +on: + pull_request: + types: + - closed + branches: + - 'release' + + +env: + PROJECT_ID: '${{ secrets.PROJECT_ID }}' # TODO: update Google Cloud project id. + GAR_LOCATION: '${{ secrets.GAR_LOCATION }}' # TODO: update Artifact Registry location + +jobs: # Deployment please don't modify anything here as the infrastructure is controlled by terraform any changes here please agree with chris and reuben. + # catchsha: + # uses: HDRUK/gateway-api/.github/workflows/dev_deployment.yaml@dev + deploy: + # if: github.event.pull_request.merged == true + permissions: + contents: write + id-token: write + + runs-on: ubuntu-latest + environment: legacy-uat + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: release + + - name: Google Auth + id: auth + uses: 'google-github-actions/auth@v0' + with: + token_format: 'access_token' + workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' + service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' + + - name: Read VERSION file + id: getversion + # run: echo "::set-output name=version::$(cat Chart.yaml)" + run: echo "version=$(cat Chart.yaml)" >> $GITHUB_OUTPUT + + # - name: Get SHA + # id: getsha + # run: echo ${{ needs.catchsha.outputs.GITHUB_SHA }} + + - name: Deploy to Cloud Run + uses: actions-hub/gcloud@master + id: deploy + env: + PROJECT_ID: ${{ secrets.PROJECT_ID }} + DEV_PROJECT_ID: ${{ secrets.DEV_PROJECT_ID }} + GAR_LOCATION: ${{ secrets.GAR_LOCATION }} + GAR_NAME: ${{ secrets.GAR_NAME_API }} + SERVICE_NAME: '${{ secrets.SERVICE_NAME_API }}' + SERVICE_REGION: '${{ secrets.SERVICE_REGION_API }}' + + with: + # args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:{{ steps.catchsha.outputs.GITHUB_SHA}} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' + # Functionality not supported by Github Actions one to ccheck back agin in the future + args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:latest --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' \ No newline at end of file diff --git a/Chart.yaml b/Chart.yaml new file mode 100644 index 00000000..5eb312c6 --- /dev/null +++ b/Chart.yaml @@ -0,0 +1 @@ +v0.0.0 \ No newline at end of file diff --git a/src/config/server.js b/src/config/server.js index 60b5c46f..57dcc6aa 100644 --- a/src/config/server.js +++ b/src/config/server.js @@ -236,6 +236,8 @@ app.use('/api/v2/questionbank', require('../resources/questionbank/questionbank. app.use('/api/v2/data-use-registers', require('../resources/dataUseRegister/dataUseRegister.route')); app.use('/api/v1/locations', require('../resources/spatialfilter/SpatialRouter')); +app.use('/api/v1/metadata', require('../resources/metadata/metadata.route')); + initialiseAuthentication(app); // launch our backend into a port diff --git a/src/resources/auth/auth.route.js b/src/resources/auth/auth.route.js index fdc342f3..687959c4 100644 --- a/src/resources/auth/auth.route.js +++ b/src/resources/auth/auth.route.js @@ -25,7 +25,7 @@ router.get('/status', function (req, res, next) { if (err || !user) { return res.json({ success: true, - data: [{ role: 'Reader', id: null, name: null, loggedIn: false }], + data: [{ role: 'Reader', id: null, name: null, loggedIn: false, tempProp: true }], }); } else { // 1. Reformat teams array for frontend diff --git a/src/resources/auth/utils.js b/src/resources/auth/utils.js index 380d9243..231e6543 100644 --- a/src/resources/auth/utils.js +++ b/src/resources/auth/utils.js @@ -129,7 +129,7 @@ const getTeams = async () => { const catchLoginErrorAndRedirect = (req, res, next) => { if (req.auth.err || !req.auth.user) { - if (req.auth.err === 'loginError') { + if (req.auth.err === 'loginError' || req.auth.user === undefined) { return res.status(200).redirect(process.env.homeURL + '/loginerror'); } diff --git a/src/resources/dataset/dataset.entity.js b/src/resources/dataset/dataset.entity.js index 2edc0815..2e00900d 100644 --- a/src/resources/dataset/dataset.entity.js +++ b/src/resources/dataset/dataset.entity.js @@ -50,7 +50,7 @@ export default class DatasetClass extends Entity { transformedObject.dataset['@schema'] = { type: `Dataset`, version: `2.0.0`, - url: `https://raw.githubusercontent.com/HDRUK/schemata/master/schema/dataset/latest/dataset.schema.json`, + url: `https://raw.githubusercontent.com/HDRUK/schemata/master/schema/dataset/2.1.0/dataset.schema.json`, } // Return v2 object diff --git a/src/resources/metadata/metadata.route.js b/src/resources/metadata/metadata.route.js new file mode 100644 index 00000000..cc9e4596 --- /dev/null +++ b/src/resources/metadata/metadata.route.js @@ -0,0 +1,22 @@ +import express from 'express'; +import passport from 'passport'; + +import { utils } from '../auth'; +import { ROLES } from '../user/user.roles'; + +import datasetonboardingUtil from '../../utils/datasetonboarding.util'; + +const router = express.Router({ mergeParams: true }); + +router.post('/scoring', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin), async (req, res) => { + const { dataset } = req.body; + + if (!dataset) { + res.json({ success: false, error: 'Dataset object must be supplied and contain all required data', status: 400 }); + } + + const verdict = await datasetonboardingUtil.buildMetadataQuality(dataset, dataset.datasetv2, dataset.pid); + res.json({ success: true, data: verdict, status: 200 }); +}); + +module.exports = router; \ No newline at end of file diff --git a/src/utils/datasetonboarding.util.js b/src/utils/datasetonboarding.util.js index 139e79c7..b151df00 100644 --- a/src/utils/datasetonboarding.util.js +++ b/src/utils/datasetonboarding.util.js @@ -758,7 +758,7 @@ const buildMetadataQuality = async (dataset, v2Object, pid) => { let rawdata = fs.readFileSync(__dirname + '/schema.json'); schema = JSON.parse(rawdata); - const ajv = new Ajv({ strict: false, allErrors: true }); + const ajv = new Ajv({ strict: false, allErrors: false }); addFormats(ajv); const validate = ajv.compile(schema); validate(cleanV2Object); From 28236704bb0167028acbc9d0869f546c22995b61 Mon Sep 17 00:00:00 2001 From: reubensamuel Date: Wed, 15 Feb 2023 14:26:03 +0000 Subject: [PATCH 04/10] GAT-1887: Added Automated release and tag creation --- .github/workflows/prod_deployment.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/prod_deployment.yaml b/.github/workflows/prod_deployment.yaml index 686e4e07..5152f649 100644 --- a/.github/workflows/prod_deployment.yaml +++ b/.github/workflows/prod_deployment.yaml @@ -37,7 +37,14 @@ jobs: # Deployment please don't modify anything here as the infrastructure - name: Read VERSION file id: getversion run: echo "version=$(cat Chart.yaml)" >> $GITHUB_OUTPUT - + + - name: Create the release + uses: "marvinpinto/action-automatic-releases@latest" + with: + repo_token: "${{ secrets.GITHUB_TOKEN }}" + automatic_release_tag: 'GAT Live Release - '${{ steps.getversion.outputs.version }} + prerelease: false + - name: Deploy to Cloud Run uses: actions-hub/gcloud@master id: deploy From 35be6a95de4cbb5f043a49e7aa23753612bb80ca Mon Sep 17 00:00:00 2001 From: reubensamuel <110553227+reubensamuel@users.noreply.github.com> Date: Tue, 21 Feb 2023 17:00:45 +0000 Subject: [PATCH 05/10] GAT-1887: Slack Channel notifications --- .github/workflows/prod_deployment.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/prod_deployment.yaml b/.github/workflows/prod_deployment.yaml index 5152f649..f0c9a1e2 100644 --- a/.github/workflows/prod_deployment.yaml +++ b/.github/workflows/prod_deployment.yaml @@ -10,6 +10,8 @@ on: env: PROJECT_ID: '${{ secrets.PROJECT_ID }}' # TODO: update Google Cloud project id. GAR_LOCATION: '${{ secrets.GAR_LOCATION }}' # TODO: update Artifact Registry location + SLACK_WEBHOOK_URL: '${{ secrets.SLACK_WEBHOOK_URL}}' + SLACK_CHANNEL: '${{ secrets.GITHUBACTIONS_SLACK_CHANNEL }}' jobs: # Deployment please don't modify anything here as the infrastructure is controlled by terraform any changes here please agree with chris and reuben deploy: @@ -42,7 +44,7 @@ jobs: # Deployment please don't modify anything here as the infrastructure uses: "marvinpinto/action-automatic-releases@latest" with: repo_token: "${{ secrets.GITHUB_TOKEN }}" - automatic_release_tag: 'GAT Live Release - '${{ steps.getversion.outputs.version }} + automatic_release_tag: 'GAT Live Release - ''${{ steps.getversion.outputs.version }}' prerelease: false - name: Deploy to Cloud Run @@ -59,4 +61,14 @@ jobs: # Deployment please don't modify anything here as the infrastructure with: # args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:latest --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' - # If required, use the Cloud Run url output in later steps \ No newline at end of file + # If required, use the Cloud Run url output in later steps + + - name: Deploy Notification + id: deploynotificationsent + uses: act10ns/slack@v1 + with: + status: ${{ job.status }} + steps: ${{ toJson(steps) }} + channel: ${{ env.SLACK_CHANNEL }} + message: Deploying {{ env.GITHUB_REF_NAME }} branch + if: always() From dd723b5fa4094287bb7994b468ced5a3ba28b08e Mon Sep 17 00:00:00 2001 From: reubensamuel <110553227+reubensamuel@users.noreply.github.com> Date: Tue, 21 Feb 2023 17:04:25 +0000 Subject: [PATCH 06/10] GAT-1887: Slack Channel notifications --- .github/workflows/prod_deployment.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/prod_deployment.yaml b/.github/workflows/prod_deployment.yaml index 5152f649..18e2f377 100644 --- a/.github/workflows/prod_deployment.yaml +++ b/.github/workflows/prod_deployment.yaml @@ -10,6 +10,8 @@ on: env: PROJECT_ID: '${{ secrets.PROJECT_ID }}' # TODO: update Google Cloud project id. GAR_LOCATION: '${{ secrets.GAR_LOCATION }}' # TODO: update Artifact Registry location + SLACK_WEBHOOK_URL: '${{ secrets.SLACK_WEBHOOK_URL}}' + SLACK_CHANNEL: '${{ secrets.GITHUBACTIONS_SLACK_CHANNEL }}' jobs: # Deployment please don't modify anything here as the infrastructure is controlled by terraform any changes here please agree with chris and reuben deploy: @@ -42,7 +44,7 @@ jobs: # Deployment please don't modify anything here as the infrastructure uses: "marvinpinto/action-automatic-releases@latest" with: repo_token: "${{ secrets.GITHUB_TOKEN }}" - automatic_release_tag: 'GAT Live Release - '${{ steps.getversion.outputs.version }} + automatic_release_tag: 'GAT Live Release -''${{ steps.getversion.outputs.version }}' prerelease: false - name: Deploy to Cloud Run @@ -59,4 +61,14 @@ jobs: # Deployment please don't modify anything here as the infrastructure with: # args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:latest --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' - # If required, use the Cloud Run url output in later steps \ No newline at end of file + # If required, use the Cloud Run url output in later steps + + - name: Deploy Notification + id: deploynotificationsent + uses: act10ns/slack@v1 + with: + status: ${{ job.status }} + steps: ${{ toJson(steps) }} + channel: ${{ env.SLACK_CHANNEL }} + message: Deploying {{ env.GITHUB_REF_NAME }} branch + if: always() From 5940fa1151d7b01afe4924b1bdbc9361c962f023 Mon Sep 17 00:00:00 2001 From: reubensamuel <110553227+reubensamuel@users.noreply.github.com> Date: Tue, 28 Feb 2023 13:57:58 +0000 Subject: [PATCH 07/10] GAT-2050: Update Release bug --- .github/workflows/prod_deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prod_deployment.yaml b/.github/workflows/prod_deployment.yaml index f0c9a1e2..50cd4717 100644 --- a/.github/workflows/prod_deployment.yaml +++ b/.github/workflows/prod_deployment.yaml @@ -44,7 +44,7 @@ jobs: # Deployment please don't modify anything here as the infrastructure uses: "marvinpinto/action-automatic-releases@latest" with: repo_token: "${{ secrets.GITHUB_TOKEN }}" - automatic_release_tag: 'GAT Live Release - ''${{ steps.getversion.outputs.version }}' + automatic_release_tag: 'GAT Live Release - v4.0.0' prerelease: false - name: Deploy to Cloud Run From 0e0cbf4ae89bed4e440cd1f34b44073b52c6414c Mon Sep 17 00:00:00 2001 From: reubensamuel <110553227+reubensamuel@users.noreply.github.com> Date: Tue, 28 Feb 2023 14:01:31 +0000 Subject: [PATCH 08/10] GAT-2050: Update release tag --- .github/workflows/prod_deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prod_deployment.yaml b/.github/workflows/prod_deployment.yaml index 50cd4717..bc2607a0 100644 --- a/.github/workflows/prod_deployment.yaml +++ b/.github/workflows/prod_deployment.yaml @@ -44,7 +44,7 @@ jobs: # Deployment please don't modify anything here as the infrastructure uses: "marvinpinto/action-automatic-releases@latest" with: repo_token: "${{ secrets.GITHUB_TOKEN }}" - automatic_release_tag: 'GAT Live Release - v4.0.0' + automatic_release_tag: v4.0.0 prerelease: false - name: Deploy to Cloud Run From 8db46f602b958bc8a6034c47eba4948e313273db Mon Sep 17 00:00:00 2001 From: reubensamuel <110553227+reubensamuel@users.noreply.github.com> Date: Tue, 28 Feb 2023 14:44:20 +0000 Subject: [PATCH 09/10] GAT-1887:Fix Tag bug --- .github/workflows/prod_deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prod_deployment.yaml b/.github/workflows/prod_deployment.yaml index bc2607a0..3c62899f 100644 --- a/.github/workflows/prod_deployment.yaml +++ b/.github/workflows/prod_deployment.yaml @@ -44,7 +44,7 @@ jobs: # Deployment please don't modify anything here as the infrastructure uses: "marvinpinto/action-automatic-releases@latest" with: repo_token: "${{ secrets.GITHUB_TOKEN }}" - automatic_release_tag: v4.0.0 + automatic_release_tag: '${{ steps.getversion.outputs.version }}' prerelease: false - name: Deploy to Cloud Run From 448ef42379c3f8815a1c99570f5dcabf647b6e82 Mon Sep 17 00:00:00 2001 From: reubensamuel Date: Wed, 8 Mar 2023 10:18:36 +0000 Subject: [PATCH 10/10] GAT-1887: Disable 0/1 testing --- .github/workflows/dev_deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev_deployment.yaml b/.github/workflows/dev_deployment.yaml index a4aaf2b5..eaa86c42 100644 --- a/.github/workflows/dev_deployment.yaml +++ b/.github/workflows/dev_deployment.yaml @@ -4,8 +4,8 @@ on: push: branches: - 'dev' - paths: - - 'Chart.yaml' + # paths: + # - 'Chart.yaml' env: PROJECT_ID: '${{ secrets.PROJECT_ID }}' # TODO: update Google Cloud project id.