diff --git a/package.json b/package.json index 1ec7466b..96ce04a2 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,18 @@ { "name": "hdruk-rdt-api", - "version": "0.1.0", + "config": { + "mongodbMemoryServer": { + "version": "latest" + } + }, + "version": "0.1.1", "private": true, "dependencies": { "@google-cloud/monitoring": "^2.1.0", "@google-cloud/storage": "^5.3.0", "@sendgrid/mail": "^7.1.0", - "@sentry/node": "^5.29.0", + "@sentry/node": "^6.3.5", + "@sentry/tracing": "^6.3.5", "ajv": "^8.1.0", "ajv-formats": "^2.0.2", "async": "^3.2.0", @@ -36,6 +42,7 @@ "jsonwebtoken": "^8.5.1", "keygrip": "^1.1.0", "lodash": "^4.17.19", + "mailchimp-api-v3": "^1.15.0", "moment": "^2.27.0", "mongoose": "^5.9.12", "morgan": "^1.10.0", @@ -63,7 +70,7 @@ "babel-jest": "^26.6.3", "eslint": "^7.20.0", "jest": "^26.6.3", - "mongodb-memory-server": "^6.9.2", + "mongodb-memory-server": "6.9.2", "nodemon": "^2.0.3", "supertest": "^4.0.2" }, diff --git a/src/config/account.js b/src/config/account.js index 6b942409..402b3b42 100755 --- a/src/config/account.js +++ b/src/config/account.js @@ -1,4 +1,5 @@ import { getUserByUserId } from '../resources/user/user.repository'; +import ga4ghUtils from '../resources/utilities/ga4gh.utils'; import { to } from 'await-to-js'; import { isNil } from 'lodash'; @@ -40,6 +41,9 @@ class Account { if (claimsToSend.includes('rquestroles')) { claim.rquestroles = user.advancedSearchRoles; } + if (claimsToSend.includes('ga4gh_passport_v1')) { + claim.ga4gh_passport_v1 = await ga4ghUtils.buildGa4ghVisas(user); + } } return claim; diff --git a/src/config/configuration.js b/src/config/configuration.js index d7b1ebeb..a11b3cf0 100755 --- a/src/config/configuration.js +++ b/src/config/configuration.js @@ -39,6 +39,16 @@ export const clients = [ id_token_signed_response_alg: 'HS256', post_logout_redirect_uris: ['https://web.uatbeta.healthdatagateway.org/search?search=&logout=true'], }, + { + //GA4GH passports + client_id: process.env.GA4GHClientID, + client_secret: process.env.GA4GHClientSecret, + grant_types: ['authorization_code', 'implicit'], + response_types: ['code id_token'], + redirect_uris: process.env.GA4GHRedirectURI.split(',') || [''], + id_token_signed_response_alg: 'HS256', + post_logout_redirect_uris: ['https://web.uatbeta.healthdatagateway.org/search?search=&logout=true'], + }, ]; export const interactions = { @@ -58,6 +68,7 @@ export const claims = { email: ['email'], profile: ['firstname', 'lastname'], rquestroles: ['rquestroles'], + ga4gh_passport_v1: ['ga4gh_passport_v1'], }; export const features = { diff --git a/src/config/server.js b/src/config/server.js index de88bc0c..9b30b29c 100644 --- a/src/config/server.js +++ b/src/config/server.js @@ -13,23 +13,39 @@ import cookieParser from 'cookie-parser'; import { connectToDatabase } from './db'; import { initialiseAuthentication } from '../resources/auth'; import * as Sentry from '@sentry/node'; +import * as Tracing from '@sentry/tracing'; import helper from '../resources/utilities/helper.util'; require('dotenv').config(); -if (helper.getEnvironment() !== 'local') { - Sentry.init({ - dsn: 'https://b6ea46f0fbe048c9974718d2c72e261b@o444579.ingest.sentry.io/5653683', - environment: helper.getEnvironment(), - }); -} +var app = express(); + +Sentry.init({ + dsn: 'https://b6ea46f0fbe048c9974718d2c72e261b@o444579.ingest.sentry.io/5653683', + environment: helper.getEnvironment(), + integrations: [ + // enable HTTP calls tracing + new Sentry.Integrations.Http({ tracing: true }), + // enable Express.js middleware tracing + new Tracing.Integrations.Express({ + // trace all requests to the default router + app, + }), + ], + tracesSampleRate: 1.0, +}); +// RequestHandler creates a separate execution context using domains, so that every +// transaction/span/breadcrumb is attached to its own Hub instance +app.use(Sentry.Handlers.requestHandler()); +// TracingHandler creates a trace for every incoming request +app.use(Sentry.Handlers.tracingHandler()); +app.use(Sentry.Handlers.errorHandler()); const Account = require('./account'); const configuration = require('./configuration'); const API_PORT = process.env.PORT || 3001; const session = require('express-session'); -var app = express(); app.disable('x-powered-by'); configuration.findAccount = Account.findAccount; @@ -213,6 +229,7 @@ app.use('/api/v2/papers', require('../resources/paper/v2/paper.route')); app.use('/api/v1/counter', require('../resources/tool/counter.route')); app.use('/api/v1/coursecounter', require('../resources/course/coursecounter.route')); +app.use('/api/v1/collectioncounter', require('../resources/collections/collectioncounter.route')); app.use('/api/v1/discourse', require('../resources/discourse/discourse.route')); @@ -231,6 +248,8 @@ app.use('/api/v1/help', require('../resources/help/help.router')); app.use('/api/v2/filters', require('../resources/filters/filters.route')); +app.use('/api/v1/mailchimp', require('../services/mailchimp/mailchimp.route')); + initialiseAuthentication(app); // launch our backend into a port diff --git a/src/resources/account/account.route.js b/src/resources/account/account.route.js index 75275f62..fb148a46 100644 --- a/src/resources/account/account.route.js +++ b/src/resources/account/account.route.js @@ -227,6 +227,6 @@ async function sendEmailNotifications(tool, activeflag) { if (err) { return new Error({ success: false, error: err }); } - emailGenerator.sendEmail(emailRecipients, `${hdrukEmail}`, subject, html); + emailGenerator.sendEmail(emailRecipients, `${hdrukEmail}`, subject, html, false); }); } diff --git a/src/resources/auth/auth.route.js b/src/resources/auth/auth.route.js index 2c60bf82..39b8a69c 100644 --- a/src/resources/auth/auth.route.js +++ b/src/resources/auth/auth.route.js @@ -96,6 +96,7 @@ router.get('/status', function (req, res, next) { id: req.user.id, name: req.user.firstname + ' ' + req.user.lastname, loggedIn: true, + email: req.user.email, teams: [...adminArray, ...teamArray], provider: req.user.provider, advancedSearchRoles: req.user.advancedSearchRoles, diff --git a/src/resources/auth/strategies/oidc.js b/src/resources/auth/strategies/oidc.js index e8bd242f..665d7be6 100644 --- a/src/resources/auth/strategies/oidc.js +++ b/src/resources/auth/strategies/oidc.js @@ -11,10 +11,11 @@ import { ROLES } from '../../user/user.roles'; import queryString from 'query-string'; import Url from 'url'; import { discourseLogin } from '../sso/sso.discourse.service'; - +import { isNil } from 'lodash'; const OidcStrategy = passportOidc.Strategy; const baseAuthUrl = process.env.AUTH_PROVIDER_URI; const eventLogController = require('../../eventlog/eventlog.controller'); +import { UserModel } from '../../user/user.model'; const strategy = app => { const strategyOptions = { @@ -34,6 +35,9 @@ const strategy = app => { let [err, user] = await to(getUserByProviderId(profile._json.eduPersonTargetedID)); if (err || user) { + if (user && !user.affiliation) { + UserModel.findOneAndUpdate({ id: user.id }, { $set: { affiliation: profile._json.eduPersonScopedAffilation } }); + } return done(err, user); } @@ -41,6 +45,7 @@ const strategy = app => { createUser({ provider: 'oidc', providerId: profile._json.eduPersonTargetedID, + affiliation: !isNil(profile._json.eduPersonScopedAffilation) ? profile._json.eduPersonScopedAffilation : 'no.organization', firstname: '', lastname: '', email: '', diff --git a/src/resources/collections/collectioncounter.route.js b/src/resources/collections/collectioncounter.route.js new file mode 100644 index 00000000..b0e4bedd --- /dev/null +++ b/src/resources/collections/collectioncounter.route.js @@ -0,0 +1,21 @@ +import express from 'express'; +import { Collections } from './collections.model'; +const rateLimit = require('express-rate-limit'); + +const router = express.Router(); + +const datasetLimiter = rateLimit({ + windowMs: 60 * 1000, // 1 minute window + max: 50, // start blocking after 50 requests + message: 'Too many calls have been made to this api from this IP, please try again after an hour', +}); + +router.post('/update', datasetLimiter, async (req, res) => { + const { id, counter } = req.body; + Collections.findOneAndUpdate({ id: id }, { counter: counter }, err => { + if (err) return res.json({ success: false, error: err }); + return res.json({ success: true }); + }); +}); + +module.exports = router; \ No newline at end of file diff --git a/src/resources/collections/collections.model.js b/src/resources/collections/collections.model.js index 380204db..bba81132 100644 --- a/src/resources/collections/collections.model.js +++ b/src/resources/collections/collections.model.js @@ -6,6 +6,7 @@ const CollectionSchema = new Schema( id: Number, name: String, description: String, + updatedon: Date, imageLink: String, authors: [Number], // emailNotifications: Boolean, diff --git a/src/resources/collections/collections.route.js b/src/resources/collections/collections.route.js index 7a45a69a..bbff007c 100644 --- a/src/resources/collections/collections.route.js +++ b/src/resources/collections/collections.route.js @@ -115,13 +115,15 @@ router.get('/entityid/:entityID', async (req, res) => { }); router.put('/edit', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, ROLES.Creator), async (req, res) => { - const collectionCreator = req.body.collectionCreator; - let { id, name, description, imageLink, authors, relatedObjects, publicflag, keywords, previousPublicFlag } = req.body; + let { id, name, description, imageLink, authors, relatedObjects, publicflag, keywords, previousPublicFlag, collectionCreator } = req.body; imageLink = urlValidator.validateURL(imageLink); + let updatedon = Date.now(); + + let collectionId = parseInt(id); await Collections.findOneAndUpdate( - { id: id }, - { + { id: collectionId }, + { //lgtm [js/sql-injection] name: inputSanitizer.removeNonBreakingSpaces(name), description: inputSanitizer.removeNonBreakingSpaces(description), imageLink: imageLink, @@ -129,6 +131,7 @@ router.put('/edit', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admi relatedObjects: relatedObjects, publicflag: publicflag, keywords: keywords, + updatedon: updatedon }, err => { if (err) { @@ -139,7 +142,7 @@ router.put('/edit', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admi return res.json({ success: true }); }); - await Collections.find({ id: id }, { publicflag: 1, id: 1, activeflag: 1, authors: 1, name: 1 }).then(async res => { + await Collections.find({ id: collectionId }, { publicflag: 1, id: 1, activeflag: 1, authors: 1, name: 1 }).then(async res => { if (previousPublicFlag === false && publicflag === true) { await sendEmailNotifications(res[0], res[0].activeflag, collectionCreator, true); @@ -170,6 +173,7 @@ router.post('/add', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admi collections.activeflag = 'active'; collections.publicflag = publicflag; collections.keywords = keywords; + collections.updatedon = Date.now(); if (collections.authors) { collections.authors.forEach(async authorId => { @@ -191,10 +195,11 @@ router.post('/add', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admi }); router.put('/status', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, ROLES.Creator), async (req, res) => { - var { id, activeflag } = req.body; - var isAuthorAdmin = false; + let { id, activeflag } = req.body; + let isAuthorAdmin = false; + let collectionId = parseInt(id); - var q = Collections.aggregate([{ $match: { $and: [{ id: parseInt(req.body.id) }, { authors: req.user.id }] } }]); + let q = Collections.aggregate([{ $match: { $and: [{ id: parseInt(req.body.id) }, { authors: req.user.id }] } }]); q.exec((err, data) => { if (data.length === 1) { isAuthorAdmin = true; @@ -206,8 +211,8 @@ router.put('/status', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Ad if (isAuthorAdmin) { Collections.findOneAndUpdate( - { id: id }, - { + { id: collectionId }, + { //lgtm [js/sql-injection] activeflag: activeflag, }, err => { diff --git a/src/resources/course/course.repository.js b/src/resources/course/course.repository.js index 21eb50aa..acb6cc1c 100644 --- a/src/resources/course/course.repository.js +++ b/src/resources/course/course.repository.js @@ -105,7 +105,7 @@ const editCourse = async (req, res) => { Course.findOneAndUpdate( { id: id }, - { + { //lgtm [js/sql-injection] title: inputSanitizer.removeNonBreakingSpaces(req.body.title), link: urlValidator.validateURL(inputSanitizer.removeNonBreakingSpaces(req.body.link)), provider: inputSanitizer.removeNonBreakingSpaces(req.body.provider), @@ -398,7 +398,7 @@ async function sendEmailNotifications(tool, activeflag, rejectionReason) { if (err) { return new Error({ success: false, error: err }); } - emailGenerator.sendEmail(emailRecipients, `${hdrukEmail}`, subject, html); + emailGenerator.sendEmail(emailRecipients, `${hdrukEmail}`, subject, html, false); }); } else { // 3. Find the creator of the course if they have opted in to email updates @@ -418,7 +418,7 @@ async function sendEmailNotifications(tool, activeflag, rejectionReason) { if (err) { return new Error({ success: false, error: err }); } - emailGenerator.sendEmail(emailRecipients, `${hdrukEmail}`, subject, html); + emailGenerator.sendEmail(emailRecipients, `${hdrukEmail}`, subject, html, false); }); // 5. Find all admins regardless of email opt-in preference diff --git a/src/resources/datarequest/datarequest.controller.js b/src/resources/datarequest/datarequest.controller.js index c0814f5e..f2bb1cad 100644 --- a/src/resources/datarequest/datarequest.controller.js +++ b/src/resources/datarequest/datarequest.controller.js @@ -2011,7 +2011,6 @@ module.exports = { ); options = { - userType: '', userEmail: appEmail, publisher, datasetTitles, @@ -2109,6 +2108,7 @@ module.exports = { ) { // Retrieve all custodian user Ids to generate notifications custodianManagers = teamController.getTeamMembersByRole(accessRecord.datasets[0].publisher.team, constants.roleTypes.MANAGER); + // check if publisher.team has email notifications custodianUserIds = custodianManagers.map(user => user.id); await notificationBuilder.triggerNotificationMessage( custodianUserIds, diff --git a/src/resources/datarequest/datarequest.model.js b/src/resources/datarequest/datarequest.model.js index 9c541022..49350a35 100644 --- a/src/resources/datarequest/datarequest.model.js +++ b/src/resources/datarequest/datarequest.model.js @@ -80,6 +80,7 @@ const DataRequestSchema = new Schema( questionAnswers: { type: Object, default: {} }, }, ], + originId: { type: Schema.Types.ObjectId, ref: 'data_request' } }, { timestamps: true, diff --git a/src/resources/datarequest/datarequest.route.js b/src/resources/datarequest/datarequest.route.js index 460f7b2d..20f6aaf5 100644 --- a/src/resources/datarequest/datarequest.route.js +++ b/src/resources/datarequest/datarequest.route.js @@ -3,6 +3,7 @@ import passport from 'passport'; import _ from 'lodash'; import multer from 'multer'; import { param } from 'express-validator'; +import { logger } from '../utilities/logger'; const amendmentController = require('./amendment/amendment.controller'); const datarequestController = require('./datarequest.controller'); const fs = require('fs'); @@ -16,18 +17,29 @@ const storage = multer.diskStorage({ }, }); const multerMid = multer({ storage: storage }); +const logCategory = 'Data Access Request'; const router = express.Router(); // @route GET api/v1/data-access-request // @desc GET Access requests for user // @access Private - Applicant (Gateway User) and Custodian Manager/Reviewer -router.get('/', passport.authenticate('jwt'), datarequestController.getAccessRequestsByUser); +router.get( + '/', + passport.authenticate('jwt'), + logger.logRequestMiddleware({ logCategory, action: 'Viewed personal Data Access Request dashboard' }), + datarequestController.getAccessRequestsByUser +); // @route GET api/v1/data-access-request/:requestId // @desc GET a single data access request by Id // @access Private - Applicant (Gateway User) and Custodian Manager/Reviewer -router.get('/:requestId', passport.authenticate('jwt'), datarequestController.getAccessRequestById); +router.get( + '/:requestId', + passport.authenticate('jwt'), + logger.logRequestMiddleware({ logCategory, action: 'Opened a Data Access Request application' }), + datarequestController.getAccessRequestById +); // @route GET api/v1/data-access-request/dataset/:datasetId // @desc GET Access request for user diff --git a/src/resources/datarequest/utils/datarequest.util.js b/src/resources/datarequest/utils/datarequest.util.js index af0d143a..98d7f0a0 100644 --- a/src/resources/datarequest/utils/datarequest.util.js +++ b/src/resources/datarequest/utils/datarequest.util.js @@ -210,7 +210,7 @@ const matchCurrentUser = (user, auditField) => { const cloneIntoExistingApplication = (appToClone, appToUpdate) => { // 1. Extract values required to clone into existing application - const { questionAnswers } = appToClone; + const { questionAnswers, _id } = appToClone; const { jsonSchema: schemaToUpdate } = appToUpdate; // 2. Extract and append any user repeated sections from the original form @@ -220,13 +220,13 @@ const cloneIntoExistingApplication = (appToClone, appToUpdate) => { } // 3. Return updated application - return { ...appToUpdate, questionAnswers }; + return { ...appToUpdate, questionAnswers, originId: _id }; }; const cloneIntoNewApplication = async (appToClone, context) => { // 1. Extract values required to clone existing application const { userId, datasetIds, datasetTitles, publisher } = context; - const { questionAnswers } = appToClone; + const { questionAnswers, _id } = appToClone; // 2. Get latest publisher schema const { jsonSchema, version, _id: schemaId, isCloneable = false, formType } = await getLatestPublisherSchema(publisher); @@ -246,6 +246,7 @@ const cloneIntoNewApplication = async (appToClone, context) => { aboutApplication: {}, amendmentIterations: [], applicationStatus: constants.applicationStatuses.INPROGRESS, + originId: _id }; // 4. Extract and append any user repeated sections from the original form diff --git a/src/resources/dataset/dataset.util.js b/src/resources/dataset/dataset.util.js new file mode 100644 index 00000000..63edfc73 --- /dev/null +++ b/src/resources/dataset/dataset.util.js @@ -0,0 +1,24 @@ +// Federated Metadata Catalogue +export const metadataCatalogues = Object.keys(process.env) + .filter(key => key.match(/^MDC_Config_/g)) + .reduce((obj, key) => { + const match = key.match(/^MDC_Config_(.*?)_(.*?)$/); + const catalogue = match[1]; + const catalogueParam = match[2]; + const catalogueValue = process.env[key]; + + if (Object.keys(obj).some(key => key === catalogue)) { + obj = { ...obj, [`${catalogue}`]: { ...obj[`${catalogue}`], [`${catalogueParam}`]: catalogueValue } }; + } else { + obj = { ...obj, [`${catalogue}`]: { [`${catalogueParam}`]: catalogueValue } }; + } + return obj; + }, {}); + +export const validateCatalogueParams = (params) => { + const { metadataUrl, username, password, source, instanceType } = params; + if(!metadataUrl || !username || !password || !source || !instanceType ) { + return false; + } + return true; +}; diff --git a/src/resources/dataset/v1/dataset.route.js b/src/resources/dataset/v1/dataset.route.js index 566ebda3..16fd6086 100644 --- a/src/resources/dataset/v1/dataset.route.js +++ b/src/resources/dataset/v1/dataset.route.js @@ -1,6 +1,6 @@ import express from 'express'; import { Data } from '../../tool/data.model'; -import { loadDatasets, updateExternalDatasetServices } from './dataset.service'; +import { saveUptime, importCatalogues, updateExternalDatasetServices } from './dataset.service'; import { getAllTools } from '../../tool/data.repository'; import { isEmpty, isNil } from 'lodash'; import escape from 'escape-html'; @@ -18,26 +18,29 @@ const datasetLimiter = rateLimit({ router.post('/', async (req, res) => { try { - //Check to see if header is in json format + // Check to see if header is in json format let parsedBody = {}; if (req.header('content-type') === 'application/json') { parsedBody = req.body; } else { parsedBody = JSON.parse(req.body); } - //Check for key + // Check for key if (parsedBody.key !== process.env.cachingkey) { return res.status(400).json({ success: false, error: 'Caching could not be started' }); } - + // Throw error if parsing failed if (parsedBody.error === true) { throw new Error('cache error test'); } - - loadDatasets(parsedBody.override || false).then(() => { + // Deconstruct body params + const { catalogues = [], override = false, limit } = parsedBody; + // Run catalogue importer + importCatalogues(catalogues, override, limit).then(() => { filtersService.optimiseFilters('dataset'); + saveUptime(); }); - + // Return response indicating job has started (do not await async import) return res.status(200).json({ success: true, message: 'Caching started' }); } catch (err) { Sentry.captureException(err); diff --git a/src/resources/dataset/v1/dataset.service.js b/src/resources/dataset/v1/dataset.service.js index 95545f03..c4a120a3 100644 --- a/src/resources/dataset/v1/dataset.service.js +++ b/src/resources/dataset/v1/dataset.service.js @@ -4,6 +4,16 @@ import axios from 'axios'; import * as Sentry from '@sentry/node'; import { v4 as uuidv4 } from 'uuid'; import { PublisherModel } from '../../publisher/publisher.model'; +import { metadataCatalogues, validateCatalogueParams } from '../dataset.util'; +import { has } from 'lodash'; + +let metadataQualityList = [], + phenotypesList = [], + dataUtilityList = [], + onboardedCustodians = [], + datasetsMDCList = [], + datasetsMDCIDs = [], + counter = 0; export async function updateExternalDatasetServices(services) { for (let service of services) { @@ -52,93 +62,441 @@ export async function updateExternalDatasetServices(services) { } } -export async function loadDatasets(override) { - console.log('Starting run at ' + Date()); - let metadataCatalogueLink = process.env.metadataURL || 'https://metadata-catalogue.org/hdruk'; +/** + * Import Metadata Catalogues + * + * @desc Performs the import of a given array of catalogues + * @param {Array} cataloguesToImport The recognised names of each catalogue to import with this request + * @param {Boolean} override Overriding forces the import of each catalogue requested regardless of differential in datasets + * @param {Number} limit The maximum number of datasets to import from each catalogue requested + */ +export async function importCatalogues(cataloguesToImport, override = false, limit) { + onboardedCustodians = await getCustodians(); + for (const catalogue in metadataCatalogues) { + if (!cataloguesToImport.includes(catalogue)) { + continue; + } + const isValid = validateCatalogueParams(metadataCatalogues[catalogue]); + if (!isValid) { + console.error('Catalogue failed to run due to incorrect or incomplete parameters'); + continue; + } + const { metadataUrl, dataModelExportRoute, username, password, source, instanceType } = metadataCatalogues[catalogue]; + const options = { + instanceType, + credentials: { + username, + password, + }, + override, + limit, + }; + initialiseImporter(); + await importMetadataFromCatalogue(metadataUrl, dataModelExportRoute, source, options); + } +} - let datasetsMDCCount = await new Promise(function (resolve, reject) { - axios - .post( - metadataCatalogueLink + - '/api/profiles/uk.ac.hdrukgateway/HdrUkProfilePluginService/customSearch?searchTerm=&domainType=DataModel&limit=1' - ) - .then(function (response) { - resolve(response.data.count); - }) - .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'The caching run has failed because it was unable to get a count from the MDC', - level: Sentry.Severity.Fatal, - }); - Sentry.captureException(err); - reject(err); - }); - }).catch(() => { - return 'Update failed'; +export async function saveUptime() { + const monitoring = require('@google-cloud/monitoring'); + const projectId = 'hdruk-gateway'; + const client = new monitoring.MetricServiceClient(); + + var selectedMonthStart = new Date(); + selectedMonthStart.setMonth(selectedMonthStart.getMonth() - 1); + selectedMonthStart.setDate(1); + selectedMonthStart.setHours(0, 0, 0, 0); + + var selectedMonthEnd = new Date(); + selectedMonthEnd.setDate(0); + selectedMonthEnd.setHours(23, 59, 59, 999); + + const request = { + name: client.projectPath(projectId), + filter: + 'metric.type="monitoring.googleapis.com/uptime_check/check_passed" AND resource.type="uptime_url" AND metric.label."check_id"="check-production-web-app-qsxe8fXRrBo" AND metric.label."checker_location"="eur-belgium"', + + interval: { + startTime: { + seconds: selectedMonthStart.getTime() / 1000, + }, + endTime: { + seconds: selectedMonthEnd.getTime() / 1000, + }, + }, + aggregation: { + alignmentPeriod: { + seconds: '86400s', + }, + crossSeriesReducer: 'REDUCE_NONE', + groupByFields: ['metric.label."checker_location"', 'resource.label."instance_id"'], + perSeriesAligner: 'ALIGN_FRACTION_TRUE', + }, + }; + + // Writes time series data + const [timeSeries] = await client.listTimeSeries(request); + var dailyUptime = []; + var averageUptime; + + timeSeries.forEach(data => { + data.points.forEach(data => { + dailyUptime.push(data.value.doubleValue); + }); + + averageUptime = (dailyUptime.reduce((a, b) => a + b, 0) / dailyUptime.length) * 100; }); - if (datasetsMDCCount === 'Update failed') return; + var metricsData = new MetricsData(); + metricsData.uptime = averageUptime; + await metricsData.save(); +} - // Compare counts from HDR and MDC, if greater drop of 10%+ then stop process and email support queue - var datasetsHDRCount = await Data.countDocuments({ type: 'dataset', activeflag: 'active' }); +/** + * Initialise Importer Instance + * + * @desc Resets the importer module scoped variables to original values for next run + */ +function initialiseImporter() { + metadataQualityList = []; + phenotypesList = []; + dataUtilityList = []; + datasetsMDCList = []; + datasetsMDCIDs = []; + counter = 0; +} - // Get active custodians on HDR Gateway - const publishers = await PublisherModel.find().select('name').lean(); - const onboardedCustodians = publishers.map(publisher => publisher.name); +async function importMetadataFromCatalogue(baseUri, dataModelExportRoute, source, { instanceType, credentials, override = false, limit }) { + const startCacheTime = Date.now(); + console.log( + `Starting metadata import for ${source} on ${instanceType} at ${Date()} with base URI ${baseUri}, override:${override}, limit:${ + limit || 'all' + }` + ); + datasetsMDCList = await getDataModels(baseUri); + if (datasetsMDCList === 'Update failed') return; + + const isDifferentialValid = await checkDifferentialValid(datasetsMDCList.count, source, override); + if (!isDifferentialValid) return; - if ((datasetsMDCCount / datasetsHDRCount) * 100 < 90 && !override) { + metadataQualityList = await getMetadataQualityExport(); + phenotypesList = await getPhenotypesExport(); + dataUtilityList = await getDataUtilityExport(); + + await logoutCatalogue(baseUri); + await loginCatalogue(baseUri, credentials); + await loadDatasets(baseUri, dataModelExportRoute, datasetsMDCList.items, datasetsMDCList.count, source, limit).catch(err => { Sentry.addBreadcrumb({ category: 'Caching', - message: `The caching run has failed because the counts from the MDC (${datasetsMDCCount}) where ${ - 100 - (datasetsMDCCount / datasetsHDRCount) * 100 - }% lower than the number stored in the DB (${datasetsHDRCount})`, - level: Sentry.Severity.Fatal, + message: `Unable to complete the metadata import for ${source} ${err.message}`, + level: Sentry.Severity.Error, }); - Sentry.captureException(); - return; + Sentry.captureException(err); + console.error(`Unable to complete the metadata import for ${source} ${err.message}`); + }); + await logoutCatalogue(baseUri); + await archiveMissingDatasets(source); + + const totalCacheTime = ((Date.now() - startCacheTime) / 1000).toFixed(3); + console.log(`Run Completed for ${source} at ${Date()} - Run took ${totalCacheTime}s`); +} + +async function loadDatasets(baseUri, dataModelExportRoute, datasetsToImport, datasetsToImportCount, source, limit) { + if (limit) { + datasetsToImport = [...datasetsToImport.slice(0, limit)]; + datasetsToImportCount = datasetsToImport.length; } + for (const datasetMDC of datasetsToImport) { + counter++; + console.log(`Starting ${counter} of ${datasetsToImportCount} datasets (${datasetMDC.id})`); - //datasetsMDCCount = 1; //For testing to limit the number brought down + let datasetHDR = await Data.findOne({ datasetid: datasetMDC.id }); + datasetsMDCIDs.push({ datasetid: datasetMDC.id }); - var datasetsMDCList = await new Promise(function (resolve, reject) { - axios - .post( - metadataCatalogueLink + - '/api/profiles/uk.ac.hdrukgateway/HdrUkProfilePluginService/customSearch?searchTerm=&domainType=DataModel&limit=' + - datasetsMDCCount - ) - .then(function (response) { - resolve(response.data); + const metadataQuality = metadataQualityList.data.find(x => x.id === datasetMDC.id); + const dataUtility = dataUtilityList.data.find(x => x.id === datasetMDC.id); + const phenotypes = phenotypesList.data[datasetMDC.id] || []; + + const startImportTime = Date.now(); + + const exportUri = `${baseUri}${dataModelExportRoute}`.replace('@datasetid@', datasetMDC.id); + const datasetMDCJSON = await axios + .get(exportUri, { + timeout: 60000, }) .catch(err => { Sentry.addBreadcrumb({ category: 'Caching', - message: 'The caching run has failed because it was unable to pull the datasets from the MDC', - level: Sentry.Severity.Fatal, + message: 'Unable to get dataset JSON ' + err.message, + level: Sentry.Severity.Error, }); Sentry.captureException(err); - reject(err); + console.error('Unable to get metadata JSON ' + err.message); }); - }).catch(() => { - return 'Update failed'; - }); - if (datasetsMDCList === 'Update failed') return; + const elapsedTime = ((Date.now() - startImportTime) / 1000).toFixed(3); + console.log(`Time taken to import JSON ${elapsedTime} (${datasetMDC.id})`); - const metadataQualityList = await axios - .get('https://raw.githubusercontent.com/HDRUK/datasets/master/reports/metadata_quality.json', { timeout: 10000 }) + const metadataSchemaCall = axios //Paul - Remove and populate gateway side + .get(`${baseUri}/api/profiles/uk.ac.hdrukgateway/HdrUkProfilePluginService/schema.org/${datasetMDC.id}`, { + timeout: 10000, + }) + .catch(err => { + Sentry.addBreadcrumb({ + category: 'Caching', + message: 'Unable to get metadata schema ' + err.message, + level: Sentry.Severity.Error, + }); + Sentry.captureException(err); + console.error('Unable to get metadata schema ' + err.message); + }); + + const versionLinksCall = axios.get(`${baseUri}/api/catalogueItems/${datasetMDC.id}/semanticLinks`, { timeout: 10000 }).catch(err => { + Sentry.addBreadcrumb({ + category: 'Caching', + message: 'Unable to get version links ' + err.message, + level: Sentry.Severity.Error, + }); + Sentry.captureException(err); + console.error('Unable to get version links ' + err.message); + }); + + const [metadataSchema, versionLinks] = await axios.all([metadataSchemaCall, versionLinksCall]); + + let datasetv1Object = {}, + datasetv2Object = {}; + if (has(datasetMDCJSON, 'data.dataModel.metadata')) { + datasetv1Object = populateV1datasetObject(datasetMDCJSON.data.dataModel.metadata); + datasetv2Object = populateV2datasetObject(datasetMDCJSON.data.dataModel.metadata); + } + + // Get technical details data classes + let technicaldetails = []; + if (has(datasetMDCJSON, 'data.dataModel.childDataClasses')) { + for (const dataClassMDC of datasetMDCJSON.data.dataModel.childDataClasses) { + if (dataClassMDC.childDataElements) { + // Map out data class elements to attach to class + const dataClassElementArray = dataClassMDC.childDataElements.map(element => { + return { + domainType: element.domainType, + label: element.label, + description: element.description, + dataType: { + domainType: element.dataType.domainType, + label: element.dataType.label, + }, + }; + }); + + // Create class object + const technicalDetailClass = { + domainType: dataClassMDC.domainType, + label: dataClassMDC.label, + description: dataClassMDC.description, + elements: dataClassElementArray, + }; + + technicaldetails = [...technicaldetails, technicalDetailClass]; + } + } + } + + // Detect if dataset uses 5 Safes form for access + const is5Safes = onboardedCustodians.includes(datasetMDC.publisher); + const hasTechnicalDetails = technicaldetails.length > 0; + + if (datasetHDR) { + //Edit + if (!datasetHDR.pid) { + let uuid = uuidv4(); + let listOfVersions = []; + datasetHDR.pid = uuid; + datasetHDR.datasetVersion = '0.0.1'; + + if (versionLinks && versionLinks.data && versionLinks.data.items && versionLinks.data.items.length > 0) { + versionLinks.data.items.forEach(item => { + if (!listOfVersions.find(x => x.id === item.source.id)) { + listOfVersions.push({ id: item.source.id, version: item.source.documentationVersion }); + } + if (!listOfVersions.find(x => x.id === item.target.id)) { + listOfVersions.push({ id: item.target.id, version: item.target.documentationVersion }); + } + }); + + listOfVersions.forEach(async item => { + if (item.id !== datasetMDC.id) { + await Data.findOneAndUpdate({ datasetid: item.id }, { pid: uuid, datasetVersion: item.version }); + } else { + datasetHDR.pid = uuid; + datasetHDR.datasetVersion = item.version; + } + }); + } + } + + let keywordArray = splitString(datasetv1Object.keywords); + let physicalSampleAvailabilityArray = splitString(datasetv1Object.physicalSampleAvailability); + let geographicCoverageArray = splitString(datasetv1Object.geographicCoverage); + + await Data.findOneAndUpdate( + { datasetid: datasetMDC.id }, + { + pid: datasetHDR.pid, + datasetVersion: datasetHDR.datasetVersion, + name: datasetMDC.label, + description: datasetMDC.description, + source, + is5Safes: is5Safes, + hasTechnicalDetails, + activeflag: 'active', + license: datasetv1Object.license, + tags: { + features: keywordArray, + }, + datasetfields: { + publisher: datasetv1Object.publisher, + geographicCoverage: geographicCoverageArray, + physicalSampleAvailability: physicalSampleAvailabilityArray, + abstract: datasetv1Object.abstract, + releaseDate: datasetv1Object.releaseDate, + accessRequestDuration: datasetv1Object.accessRequestDuration, + conformsTo: datasetv1Object.conformsTo, + accessRights: datasetv1Object.accessRights, + jurisdiction: datasetv1Object.jurisdiction, + datasetStartDate: datasetv1Object.datasetStartDate, + datasetEndDate: datasetv1Object.datasetEndDate, + statisticalPopulation: datasetv1Object.statisticalPopulation, + ageBand: datasetv1Object.ageBand, + contactPoint: datasetv1Object.contactPoint, + periodicity: datasetv1Object.periodicity, + + metadataquality: metadataQuality ? metadataQuality : {}, + datautility: dataUtility ? dataUtility : {}, + metadataschema: metadataSchema && metadataSchema.data ? metadataSchema.data : {}, + technicaldetails: technicaldetails, + versionLinks: versionLinks && versionLinks.data && versionLinks.data.items ? versionLinks.data.items : [], + phenotypes, + }, + datasetv2: datasetv2Object, + } + ); + console.log(`Dataset Editted (${datasetMDC.id})`); + } else { + //Add + let uuid = uuidv4(); + let listOfVersions = []; + let pid = uuid; + let datasetVersion = '0.0.1'; + + if (versionLinks && versionLinks.data && versionLinks.data.items && versionLinks.data.items.length > 0) { + versionLinks.data.items.forEach(item => { + if (!listOfVersions.find(x => x.id === item.source.id)) { + listOfVersions.push({ id: item.source.id, version: item.source.documentationVersion }); + } + if (!listOfVersions.find(x => x.id === item.target.id)) { + listOfVersions.push({ id: item.target.id, version: item.target.documentationVersion }); + } + }); + + for (const item of listOfVersions) { + if (item.id !== datasetMDC.id) { + var existingDataset = await Data.findOne({ datasetid: item.id }); + if (existingDataset && existingDataset.pid) pid = existingDataset.pid; + else { + await Data.findOneAndUpdate({ datasetid: item.id }, { pid: uuid, datasetVersion: item.version }); + } + } else { + datasetVersion = item.version; + } + } + } + + let uniqueID = ''; + while (uniqueID === '') { + uniqueID = parseInt(Math.random().toString().replace('0.', '')); + if ((await Data.find({ id: uniqueID }).length) === 0) { + uniqueID = ''; + } + } + + let { keywords = '', physicalSampleAvailability = '', geographicCoverage = '' } = datasetv1Object; + let keywordArray = splitString(keywords); + let physicalSampleAvailabilityArray = splitString(physicalSampleAvailability); + let geographicCoverageArray = splitString(geographicCoverage); + + let data = new Data(); + data.pid = pid; + data.datasetVersion = datasetVersion; + data.id = uniqueID; + data.datasetid = datasetMDC.id; + data.type = 'dataset'; + data.activeflag = 'active'; + data.source = source; + data.is5Safes = is5Safes; + data.hasTechnicalDetails = hasTechnicalDetails; + + data.name = datasetMDC.label; + data.description = datasetMDC.description; + data.license = datasetv1Object.license; + data.tags.features = keywordArray; + data.datasetfields.publisher = datasetv1Object.publisher; + data.datasetfields.geographicCoverage = geographicCoverageArray; + data.datasetfields.physicalSampleAvailability = physicalSampleAvailabilityArray; + data.datasetfields.abstract = datasetv1Object.abstract; + data.datasetfields.releaseDate = datasetv1Object.releaseDate; + data.datasetfields.accessRequestDuration = datasetv1Object.accessRequestDuration; + data.datasetfields.conformsTo = datasetv1Object.conformsTo; + data.datasetfields.accessRights = datasetv1Object.accessRights; + data.datasetfields.jurisdiction = datasetv1Object.jurisdiction; + data.datasetfields.datasetStartDate = datasetv1Object.datasetStartDate; + data.datasetfields.datasetEndDate = datasetv1Object.datasetEndDate; + data.datasetfields.statisticalPopulation = datasetv1Object.statisticalPopulation; + data.datasetfields.ageBand = datasetv1Object.ageBand; + data.datasetfields.contactPoint = datasetv1Object.contactPoint; + data.datasetfields.periodicity = datasetv1Object.periodicity; + + data.datasetfields.metadataquality = metadataQuality ? metadataQuality : {}; + data.datasetfields.datautility = dataUtility ? dataUtility : {}; + data.datasetfields.metadataschema = metadataSchema && metadataSchema.data ? metadataSchema.data : {}; + data.datasetfields.technicaldetails = technicaldetails; + data.datasetfields.versionLinks = versionLinks && versionLinks.data && versionLinks.data.items ? versionLinks.data.items : []; + data.datasetfields.phenotypes = phenotypes; + data.datasetv2 = datasetv2Object; + await data.save(); + console.log(`Dataset Added (${datasetMDC.id})`); + } + + console.log(`Finished ${counter} of ${datasetsToImportCount} datasets (${datasetMDC.id})`); + } +} + +/** + * Get Data Utility Export + * + * @desc Gets a JSON extract from GitHub containing all HDRUK Data Utility data + * @returns {Array} JSON response from HDRUK GitHub + */ +async function getDataUtilityExport() { + return await axios + .get('https://raw.githubusercontent.com/HDRUK/datasets/master/reports/data_utility.json', { timeout: 10000 }) .catch(err => { Sentry.addBreadcrumb({ category: 'Caching', - message: 'Unable to get metadata quality value ' + err.message, + message: 'Unable to get data utility ' + err.message, level: Sentry.Severity.Error, }); Sentry.captureException(err); - console.error('Unable to get metadata quality value ' + err.message); + console.error('Unable to get data utility ' + err.message); }); +} - const phenotypesList = await axios +/** + * Get Phenotypes Export + * + * @desc Gets a JSON extract from GitHub containing all HDRUK recognised Phenotypes + * @returns {Array} Json response from HDRUK GitHub + */ +async function getPhenotypesExport() { + return await axios .get('https://raw.githubusercontent.com/spiros/hdr-caliber-phenome-portal/master/_data/dataset2phenotypes.json', { timeout: 10000 }) .catch(err => { Sentry.addBreadcrumb({ @@ -149,319 +507,92 @@ export async function loadDatasets(override) { Sentry.captureException(err); console.error('Unable to get metadata quality value ' + err.message); }); +} - const dataUtilityList = await axios - .get('https://raw.githubusercontent.com/HDRUK/datasets/master/reports/data_utility.json', { timeout: 10000 }) +/** + * Get Metadata Quality Export + * + * @desc Gets a JSON extract from GitHub containing all HDRUK dataset Metadata Quality + * @returns {Array} Json response from HDRUK GitHub + */ +async function getMetadataQualityExport() { + return await axios + .get('https://raw.githubusercontent.com/HDRUK/datasets/master/reports/metadata_quality.json', { timeout: 10000 }) .catch(err => { Sentry.addBreadcrumb({ category: 'Caching', - message: 'Unable to get data utility ' + err.message, + message: 'Unable to get metadata quality value ' + err.message, level: Sentry.Severity.Error, }); Sentry.captureException(err); - console.error('Unable to get data utility ' + err.message); + console.error('Unable to get metadata quality value ' + err.message); }); +} - var datasetsMDCIDs = []; - var counter = 0; - - await datasetsMDCList.results.reduce( - (p, datasetMDC) => - p.then( - () => - new Promise(resolve => { - setTimeout(async function () { - try { - counter++; - var datasetHDR = await Data.findOne({ datasetid: datasetMDC.id }); - datasetsMDCIDs.push({ datasetid: datasetMDC.id }); - - const metadataQuality = metadataQualityList.data.find(x => x.id === datasetMDC.id); - const dataUtility = dataUtilityList.data.find(x => x.id === datasetMDC.id); - const phenotypes = phenotypesList.data[datasetMDC.id] || []; - - const metadataSchemaCall = axios - .get(metadataCatalogueLink + '/api/profiles/uk.ac.hdrukgateway/HdrUkProfilePluginService/schema.org/' + datasetMDC.id, { - timeout: 10000, - }) - .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'Unable to get metadata schema ' + err.message, - level: Sentry.Severity.Error, - }); - Sentry.captureException(err); - console.error('Unable to get metadata schema ' + err.message); - }); - - const dataClassCall = axios - .get(metadataCatalogueLink + '/api/dataModels/' + datasetMDC.id + '/dataClasses?max=300', { timeout: 10000 }) - .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'Unable to get dataclass ' + err.message, - level: Sentry.Severity.Error, - }); - Sentry.captureException(err); - console.error('Unable to get dataclass ' + err.message); - }); - - const versionLinksCall = axios - .get(metadataCatalogueLink + '/api/catalogueItems/' + datasetMDC.id + '/semanticLinks', { timeout: 10000 }) - .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'Unable to get version links ' + err.message, - level: Sentry.Severity.Error, - }); - Sentry.captureException(err); - console.error('Unable to get version links ' + err.message); - }); - - const datasetV2Call = axios - .get(metadataCatalogueLink + '/api/facets/' + datasetMDC.id + '/metadata?all=true', { timeout: 5000 }) - .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'Unable to get dataset version 2 ' + err.message, - level: Sentry.Severity.Error, - }); - Sentry.captureException(err); - console.error('Unable to get dataset version 2 ' + err.message); - }); - - const [metadataSchema, dataClass, versionLinks, datasetV2] = await axios.all([ - metadataSchemaCall, - dataClassCall, - versionLinksCall, - datasetV2Call, - ]); - - // Safely destructure data class items to protect against undefined and HTTP failures - const { data: { items: dataClassItems = [] } = {} } = dataClass || []; - - // Get technical details data classes - let technicaldetails = []; - - for (const dataClassMDC of dataClassItems) { - // Get data elements for each class - const { data: { items: dataClassElements = [] } = {} } = await axios - .get(`${metadataCatalogueLink}/api/dataModels/${datasetMDC.id}/dataClasses/${dataClassMDC.id}/dataElements?max=300`, { - timeout: 10000, - }) - .catch(err => { - console.error('Unable to get dataclass element ' + err.message); - }); - - // Map out data class elements to attach to class - const dataClassElementArray = dataClassElements.map(element => { - return { - id: element.id, - domainType: element.domainType, - label: element.label, - description: element.description, - dataType: { - id: element.dataType.id, - domainType: element.dataType.domainType, - label: element.dataType.label, - }, - }; - }); - - // Create class object - const technicalDetailClass = { - id: dataClassMDC.id, - domainType: dataClassMDC.domainType, - label: dataClassMDC.label, - description: dataClassMDC.description, - elements: dataClassElementArray, - }; - - technicaldetails = [...technicaldetails, technicalDetailClass]; - } - - let datasetv2Object = populateV2datasetObject(datasetV2.data.items); - - // Detect if dataset uses 5 Safes form for access - const is5Safes = onboardedCustodians.includes(datasetMDC.publisher); - const hasTechnicalDetails = technicaldetails.length > 0; - - if (datasetHDR) { - //Edit - if (!datasetHDR.pid) { - let uuid = uuidv4(); - let listOfVersions = []; - datasetHDR.pid = uuid; - datasetHDR.datasetVersion = '0.0.1'; - - if (versionLinks && versionLinks.data && versionLinks.data.items && versionLinks.data.items.length > 0) { - versionLinks.data.items.forEach(item => { - if (!listOfVersions.find(x => x.id === item.source.id)) { - listOfVersions.push({ id: item.source.id, version: item.source.documentationVersion }); - } - if (!listOfVersions.find(x => x.id === item.target.id)) { - listOfVersions.push({ id: item.target.id, version: item.target.documentationVersion }); - } - }); - - listOfVersions.forEach(async item => { - if (item.id !== datasetMDC.id) { - await Data.findOneAndUpdate({ datasetid: item.id }, { pid: uuid, datasetVersion: item.version }); - } else { - datasetHDR.pid = uuid; - datasetHDR.datasetVersion = item.version; - } - }); - } - } - - let keywordArray = splitString(datasetMDC.keywords); - let physicalSampleAvailabilityArray = splitString(datasetMDC.physicalSampleAvailability); - let geographicCoverageArray = splitString(datasetMDC.geographicCoverage); - // Update dataset - await Data.findOneAndUpdate( - { datasetid: datasetMDC.id }, - { - pid: datasetHDR.pid, - datasetVersion: datasetHDR.datasetVersion, - name: datasetMDC.title, - description: datasetMDC.description, - source: 'HDRUK MDC', - is5Safes, - hasTechnicalDetails, - activeflag: 'active', - license: datasetMDC.license, - tags: { - features: keywordArray, - }, - datasetfields: { - publisher: datasetMDC.publisher, - geographicCoverage: geographicCoverageArray, - physicalSampleAvailability: physicalSampleAvailabilityArray, - abstract: datasetMDC.abstract, - releaseDate: datasetMDC.releaseDate, - accessRequestDuration: datasetMDC.accessRequestDuration, - conformsTo: datasetMDC.conformsTo, - accessRights: datasetMDC.accessRights, - jurisdiction: datasetMDC.jurisdiction, - datasetStartDate: datasetMDC.datasetStartDate, - datasetEndDate: datasetMDC.datasetEndDate, - statisticalPopulation: datasetMDC.statisticalPopulation, - ageBand: datasetMDC.ageBand, - contactPoint: datasetMDC.contactPoint, - periodicity: datasetMDC.periodicity, - - metadataquality: metadataQuality ? metadataQuality : {}, - datautility: dataUtility ? dataUtility : {}, - metadataschema: metadataSchema && metadataSchema.data ? metadataSchema.data : {}, - technicaldetails: technicaldetails, - versionLinks: versionLinks && versionLinks.data && versionLinks.data.items ? versionLinks.data.items : [], - phenotypes, - }, - datasetv2: datasetv2Object, - } - ); - } else { - //Add - let uuid = uuidv4(); - let listOfVersions = []; - let pid = uuid; - let datasetVersion = '0.0.1'; - - if (versionLinks && versionLinks.data && versionLinks.data.items && versionLinks.data.items.length > 0) { - versionLinks.data.items.forEach(item => { - if (!listOfVersions.find(x => x.id === item.source.id)) { - listOfVersions.push({ id: item.source.id, version: item.source.documentationVersion }); - } - if (!listOfVersions.find(x => x.id === item.target.id)) { - listOfVersions.push({ id: item.target.id, version: item.target.documentationVersion }); - } - }); - - for (const item of listOfVersions) { - if (item.id !== datasetMDC.id) { - var existingDataset = await Data.findOne({ datasetid: item.id }); - if (existingDataset && existingDataset.pid) pid = existingDataset.pid; - else { - await Data.findOneAndUpdate({ datasetid: item.id }, { pid: uuid, datasetVersion: item.version }); - } - } else { - datasetVersion = item.version; - } - } - } - - var uniqueID = ''; - while (uniqueID === '') { - uniqueID = parseInt(Math.random().toString().replace('0.', '')); - if ((await Data.find({ id: uniqueID }).length) === 0) { - uniqueID = ''; - } - } - - var keywordArray = splitString(datasetMDC.keywords); - var physicalSampleAvailabilityArray = splitString(datasetMDC.physicalSampleAvailability); - var geographicCoverageArray = splitString(datasetMDC.geographicCoverage); - - var data = new Data(); - data.pid = pid; - data.datasetVersion = datasetVersion; - data.id = uniqueID; - data.datasetid = datasetMDC.id; - data.type = 'dataset'; - data.activeflag = 'active'; - data.source = 'HDRUK MDC'; - data.is5Safes = is5Safes; - data.hasTechnicalDetails = hasTechnicalDetails; - - data.name = datasetMDC.title; - data.description = datasetMDC.description; - data.license = datasetMDC.license; - data.tags.features = keywordArray; - data.datasetfields.publisher = datasetMDC.publisher; - data.datasetfields.geographicCoverage = geographicCoverageArray; - data.datasetfields.physicalSampleAvailability = physicalSampleAvailabilityArray; - data.datasetfields.abstract = datasetMDC.abstract; - data.datasetfields.releaseDate = datasetMDC.releaseDate; - data.datasetfields.accessRequestDuration = datasetMDC.accessRequestDuration; - data.datasetfields.conformsTo = datasetMDC.conformsTo; - data.datasetfields.accessRights = datasetMDC.accessRights; - data.datasetfields.jurisdiction = datasetMDC.jurisdiction; - data.datasetfields.datasetStartDate = datasetMDC.datasetStartDate; - data.datasetfields.datasetEndDate = datasetMDC.datasetEndDate; - data.datasetfields.statisticalPopulation = datasetMDC.statisticalPopulation; - data.datasetfields.ageBand = datasetMDC.ageBand; - data.datasetfields.contactPoint = datasetMDC.contactPoint; - data.datasetfields.periodicity = datasetMDC.periodicity; - - data.datasetfields.metadataquality = metadataQuality ? metadataQuality : {}; - data.datasetfields.datautility = dataUtility ? dataUtility : {}; - data.datasetfields.metadataschema = metadataSchema && metadataSchema.data ? metadataSchema.data : {}; - data.datasetfields.technicaldetails = technicaldetails; - data.datasetfields.versionLinks = - versionLinks && versionLinks.data && versionLinks.data.items ? versionLinks.data.items : []; - data.datasetfields.phenotypes = phenotypes; - data.datasetv2 = datasetv2Object; - await data.save(); - } - console.log(`Finished ${counter} of ${datasetsMDCCount} datasets (${datasetMDC.id})`); - resolve(null); - } catch (err) { - Sentry.addBreadcrumb({ - category: 'Caching', - message: `Failed to add ${datasetMDC.id} to the DB with the error of ${err.message}`, - level: Sentry.Severity.Fatal, - }); - Sentry.captureException(err); - console.error(`Failed to add ${datasetMDC.id} to the DB with the error of ${err.message}`); - } - }, 500); - }) - ), - Promise.resolve(null) - ); +async function getDataModels(baseUri) { + return await new Promise(function (resolve, reject) { + axios + .get(baseUri + '/api/dataModels') + .then(function (response) { + resolve(response.data); + }) + .catch(err => { + Sentry.addBreadcrumb({ + category: 'Caching', + message: 'The caching run has failed because it was unable to get a count from the MDC', + level: Sentry.Severity.Fatal, + }); + Sentry.captureException(err); + reject(err); + }); + }).catch(() => { + return 'Update failed'; + }); +} + +async function checkDifferentialValid(incomingMetadataCount, source, override) { + // Compare counts from HDR and MDC, if greater drop of 10%+ then stop process and email support queue + const datasetsHDRCount = await Data.countDocuments({ type: 'dataset', activeflag: 'active', source }); - var datasetsHDRIDs = await Data.aggregate([{ $match: { type: 'dataset' } }, { $project: { _id: 0, datasetid: 1 } }]); + if ((incomingMetadataCount / datasetsHDRCount) * 100 < 90 && !override) { + Sentry.addBreadcrumb({ + category: 'Caching', + message: `The caching run has failed because the counts from the MDC (${incomingMetadataCount}) where ${ + 100 - (incomingMetadataCount / datasetsHDRCount) * 100 + }% lower than the number stored in the DB (${datasetsHDRCount})`, + level: Sentry.Severity.Fatal, + }); + Sentry.captureException(); + return false; + } + return true; +} + +async function getCustodians() { + const publishers = await PublisherModel.find().select('name').lean(); + return publishers.map(publisher => publisher.name); +} + +async function logoutCatalogue(baseUri) { + await axios.post(`${baseUri}/api/authentication/logout`, { withCredentials: true, timeout: 10000 }).catch(err => { + console.error(`Error when trying to logout of the MDC - ${err.message}`); + }); +} + +async function loginCatalogue(baseUri, credentials) { + let response = await axios.post(`${baseUri}/api/authentication/login`, credentials, { + withCredentials: true, + timeout: 10000, + }); + + axios.defaults.headers.Cookie = response.headers['set-cookie'][0]; +} + +async function archiveMissingDatasets(source) { + let datasetsHDRIDs = await Data.aggregate([ + { $match: { type: 'dataset', activeflag: 'active', source } }, + { $project: { _id: 0, datasetid: 1 } }, + ]); let datasetsNotFound = datasetsHDRIDs.filter(o1 => !datasetsMDCIDs.some(o2 => o1.datasetid === o2.datasetid)); @@ -476,10 +607,44 @@ export async function loadDatasets(override) { ); }) ); +} - saveUptime(); - console.log('Update Completed at ' + Date()); - return; +function populateV1datasetObject(v1Data) { + let datasetV1List = v1Data.filter(item => item.namespace === 'uk.ac.hdrukgateway'); + let datasetv1Object = {}; + if (datasetV1List.length > 0) { + datasetv1Object = { + keywords: datasetV1List.find(x => x.key === 'keywords') ? datasetV1List.find(x => x.key === 'keywords').value : '', + license: datasetV1List.find(x => x.key === 'license') ? datasetV1List.find(x => x.key === 'license').value : '', + publisher: datasetV1List.find(x => x.key === 'publisher') ? datasetV1List.find(x => x.key === 'publisher').value : '', + geographicCoverage: datasetV1List.find(x => x.key === 'geographicCoverage') + ? datasetV1List.find(x => x.key === 'geographicCoverage').value + : '', + physicalSampleAvailability: datasetV1List.find(x => x.key === 'physicalSampleAvailability') + ? datasetV1List.find(x => x.key === 'physicalSampleAvailability').value + : '', + abstract: datasetV1List.find(x => x.key === 'abstract') ? datasetV1List.find(x => x.key === 'abstract').value : '', + releaseDate: datasetV1List.find(x => x.key === 'releaseDate') ? datasetV1List.find(x => x.key === 'releaseDate').value : '', + accessRequestDuration: datasetV1List.find(x => x.key === 'accessRequestDuration') + ? datasetV1List.find(x => x.key === 'accessRequestDuration').value + : '', + conformsTo: datasetV1List.find(x => x.key === 'conformsTo') ? datasetV1List.find(x => x.key === 'conformsTo').value : '', + accessRights: datasetV1List.find(x => x.key === 'accessRights') ? datasetV1List.find(x => x.key === 'accessRights').value : '', + jurisdiction: datasetV1List.find(x => x.key === 'jurisdiction') ? datasetV1List.find(x => x.key === 'jurisdiction').value : '', + datasetStartDate: datasetV1List.find(x => x.key === 'datasetStartDate') + ? datasetV1List.find(x => x.key === 'datasetStartDate').value + : '', + datasetEndDate: datasetV1List.find(x => x.key === 'datasetEndDate') ? datasetV1List.find(x => x.key === 'datasetEndDate').value : '', + statisticalPopulation: datasetV1List.find(x => x.key === 'statisticalPopulation') + ? datasetV1List.find(x => x.key === 'statisticalPopulation').value + : '', + ageBand: datasetV1List.find(x => x.key === 'ageBand') ? datasetV1List.find(x => x.key === 'ageBand').value : '', + contactPoint: datasetV1List.find(x => x.key === 'contactPoint') ? datasetV1List.find(x => x.key === 'contactPoint').value : '', + periodicity: datasetV1List.find(x => x.key === 'periodicity') ? datasetV1List.find(x => x.key === 'periodicity').value : '', + }; + } + + return datasetv1Object; } function populateV2datasetObject(v2Data) { @@ -760,58 +925,3 @@ function splitString(array) { } return returnArray; } - -async function saveUptime() { - const monitoring = require('@google-cloud/monitoring'); - const projectId = 'hdruk-gateway'; - const client = new monitoring.MetricServiceClient(); - - var selectedMonthStart = new Date(); - selectedMonthStart.setMonth(selectedMonthStart.getMonth() - 1); - selectedMonthStart.setDate(1); - selectedMonthStart.setHours(0, 0, 0, 0); - - var selectedMonthEnd = new Date(); - selectedMonthEnd.setDate(0); - selectedMonthEnd.setHours(23, 59, 59, 999); - - const request = { - name: client.projectPath(projectId), - filter: - 'metric.type="monitoring.googleapis.com/uptime_check/check_passed" AND resource.type="uptime_url" AND metric.label."check_id"="check-production-web-app-qsxe8fXRrBo" AND metric.label."checker_location"="eur-belgium"', - - interval: { - startTime: { - seconds: selectedMonthStart.getTime() / 1000, - }, - endTime: { - seconds: selectedMonthEnd.getTime() / 1000, - }, - }, - aggregation: { - alignmentPeriod: { - seconds: '86400s', - }, - crossSeriesReducer: 'REDUCE_NONE', - groupByFields: ['metric.label."checker_location"', 'resource.label."instance_id"'], - perSeriesAligner: 'ALIGN_FRACTION_TRUE', - }, - }; - - // Writes time series data - const [timeSeries] = await client.listTimeSeries(request); - var dailyUptime = []; - var averageUptime; - - timeSeries.forEach(data => { - data.points.forEach(data => { - dailyUptime.push(data.value.doubleValue); - }); - - averageUptime = (dailyUptime.reduce((a, b) => a + b, 0) / dailyUptime.length) * 100; - }); - - var metricsData = new MetricsData(); - metricsData.uptime = averageUptime; - await metricsData.save(); -} diff --git a/src/resources/linkchecker/linkchecker.router.js b/src/resources/linkchecker/linkchecker.router.js index 072bb895..467d43d9 100644 --- a/src/resources/linkchecker/linkchecker.router.js +++ b/src/resources/linkchecker/linkchecker.router.js @@ -1,4 +1,5 @@ import express from 'express'; +import * as Sentry from '@sentry/node'; import { getObjectResult } from './linkchecker.repository'; import { getUserByUserId } from '../user/user.repository'; import { Data } from '../tool/data.model'; @@ -109,7 +110,16 @@ router.post('/', async (req, res) => {

${footer}`, }; - await sgMail.send(msg); + await sgMail.send(msg, false, err => { + if (err) { + Sentry.addBreadcrumb({ + category: 'SendGrid', + message: 'Sending email failed', + level: Sentry.Severity.Warning, + }); + Sentry.captureException(err); + } + }); } } }; diff --git a/src/resources/message/message.controller.js b/src/resources/message/message.controller.js index 8a4d4c8f..e5c07c01 100644 --- a/src/resources/message/message.controller.js +++ b/src/resources/message/message.controller.js @@ -4,183 +4,225 @@ import { TopicModel } from '../topic/topic.model'; import mongoose from 'mongoose'; import { UserModel } from '../user/user.model'; import emailGenerator from '../utilities/emailGenerator.util'; +import teamController from '../team/team.controller'; + import { Data as ToolModel } from '../tool/data.model'; import constants from '../utilities/constants.util'; const topicController = require('../topic/topic.controller'); module.exports = { - // POST /api/v1/messages - createMessage: async (req, res) => { - try { - const { _id: createdBy, firstname, lastname } = req.user - let { messageType = 'message', topic = '', messageDescription, relatedObjectIds } = req.body; - let topicObj = {}; - // 1. If the message type is 'message' and topic id is empty - if(messageType === 'message') { - if(_.isEmpty(topic)) { - // 2. Create new topic - topicObj = await topicController.buildTopic({createdBy, relatedObjectIds }); - // 3. If topic was not successfully created, throw error response - if(!topicObj) - return res.status(500).json({ success: false, message: 'Could not save topic to database.' }); - // 4. Pass new topic Id - topic = topicObj._id; - } else { - // 2. Find the existing topic - topicObj = await topicController.findTopic(topic, createdBy); - // 3. Return not found if it was not found - if(!topicObj) { - return res.status(404).json({ success: false, message: 'The topic specified could not be found' }); - } - // 4. Find the related object(s) in MongoDb and include team data to update topic recipients in case teams have changed - const tools = await ToolModel.find().where('_id').in(relatedObjectIds).populate({ path: 'publisher', populate: { path: 'team' }}); - // 5. Return undefined if no object(s) exists - if(_.isEmpty(tools)) - return undefined; - - // 6. Get recipients for new message - let { publisher = '' } = tools[0]; - if(_.isEmpty(publisher)) { - console.error(`No publisher associated to this dataset`); - return res.status(500).json({ success: false, message: 'No publisher associated to this dataset' }); - } - let { team = [] } = publisher; - if(_.isEmpty(team)) { - console.error(`No team associated to publisher, cannot message`); - return res.status(500).json({ success: false, message: 'No team associated to publisher, cannot message' }); - } - topicObj.recipients = await topicController.buildRecipients(team, topicObj.createdBy); - await topicObj.save(); - } - } - // 5. Create new message - const message = await MessagesModel.create({ - messageID: parseInt(Math.random().toString().replace('0.', '')), - messageObjectID: parseInt(Math.random().toString().replace('0.', '')), - messageDescription, - topic, - createdBy, - messageType, - readBy: [new mongoose.Types.ObjectId(createdBy)] - }); - // 6. Return 500 error if message was not successfully created - if(!message) - return res.status(500).json({ success: false, message: 'Could not save message to database.' }); - // 7. Prepare to send email if a new message has been created - if(messageType === 'message') { - // 8. Find recipients who have opted in to email updates and exclude the requesting user - let messageRecipients = await UserModel.find({ _id: { $in: topicObj.recipients } }).populate('additionalInfo'); - let optedInEmailRecipients = [...messageRecipients].filter(function(user) { - let { additionalInfo: { emailNotifications }, _id} = user; - return emailNotifications === true && _id.toString() !== createdBy.toString(); - }); - // 9. Send email - emailGenerator.sendEmail( - optedInEmailRecipients, - constants.hdrukEmail, - `You have received a new message on the HDR UK Innovation Gateway`, - `You have received a new message on the HDR UK Innovation Gateway.
Log in to view your messages here : HDR UK Innovation Gateway` - ); - } - // 10. Return successful response with message data - message.createdByName = { firstname, lastname }; - return res.status(201).json({ success: true, message }); - } catch (err) { - console.error(err.message); - return res.status(500).json(err.message); - } - }, - // DELETE /api/v1/messages/:id - deleteMessage: async(req, res) => { - try { - const { id } = req.params; - const { _id: userId } = req.user; - // 1. Return not found error if id not passed - if(!id) - return res.status(404).json({ success: false, message: 'Message Id not found.' }); - // 2. Get message by Id from MongoDb - const message = await MessagesModel.findOne({ _id: id }); - // 3. Return not found if message not found - if(!message) { - return res.status(404).json({ success: false, message: 'Message not found.' }); - } - // 4. Check that the message was created by requesting user otherwise return unathorised - if(message.createdBy.toString() !== userId.toString()) { - return res.status(401).json({ success: false, message: 'Not authorised to delete this message' }); - } - // 5. Delete message by id - await MessagesModel.remove({ _id: id }); - // 6. Check attached topic for other messages to avoid orphaning topic documents - const messagesCount = await MessagesModel.count({ topic:message.topic }); - // 7. If no other messages remain then delete the topic - if(!messagesCount) { - await TopicModel.remove({ _id: new mongoose.Types.ObjectId(message.topic) }); - } - // 8. Return successful response - return res.status(204).json({ success: true }); - } catch (err) { - console.error(err.message); - return res.status(500).json(err.message); - } - }, - // PUT /api/v1/messages - updateMessage: async(req, res) => { - try { - let { _id: userId } = req.user; - let { messageId, isRead, messageDescription = '', messageType = '' } = req.body; - // 1. Return not found error if id not passed - if(!messageId) - return res.status(404).json({ success: false, message: 'Message Id not found.' }); - // 2. Get message by object id - const message = await MessagesModel.findOne( - { _id: messageId } - ); - // 3. Return not found if message not found - if(!message) { - return res.status(404).json({ success: false, message: 'Message not found.' }); - } - // 4. Update message params - readBy is an array of users who have read the message - if(isRead && !message.readBy.includes(userId.toString())) { - message.readBy.push(userId); - } - if(isRead) { message.isRead = isRead; } - if(!_.isEmpty(messageDescription)) { message.messageDescription = messageDescription; } - if(!_.isEmpty(messageType)) {message.messageType = messageType; } - // 5. Save message to Mongo - await message.save(); - // 6. Return success no content - return res.status(204).json({ success:true }); - } catch(err) { - console.error(err.message); - return res.status(500).json(err.message); - } - }, - // GET api/v1/messages/unread/count - getUnreadMessageCount: async(req, res) => { - try { - let {_id: userId } = req.user; - let unreadMessageCount = 0; + // POST /api/v1/messages + createMessage: async (req, res) => { + try { + const { _id: createdBy, firstname, lastname } = req.user; + let { messageType = 'message', topic = '', messageDescription, relatedObjectIds } = req.body; + let topicObj = {}; + let team; + // 1. If the message type is 'message' and topic id is empty + if (messageType === 'message') { + // 2. Find the related object(s) in MongoDb and include team data to update topic recipients in case teams have changed + const tools = await ToolModel.find() + .where('_id') + .in(relatedObjectIds) + .populate({ path: 'publisher', populate: { path: 'team' } }); + // 3. Return undefined if no object(s) exists + if (_.isEmpty(tools)) return undefined; + + // 4. Get recipients for new message + let { publisher = '' } = tools[0]; + + if (_.isEmpty(publisher)) { + console.error(`No publisher associated to this dataset`); + return res.status(500).json({ success: false, message: 'No publisher associated to this dataset' }); + } + // 5. get team + ({ team = [] } = publisher); + if (_.isEmpty(team)) { + console.error(`No team associated to publisher, cannot message`); + return res.status(500).json({ success: false, message: 'No team associated to publisher, cannot message' }); + } + + if (_.isEmpty(topic)) { + // 6. Create new topic + topicObj = await topicController.buildTopic({ createdBy, relatedObjectIds }); + // 7. If topic was not successfully created, throw error response + if (!topicObj) return res.status(500).json({ success: false, message: 'Could not save topic to database.' }); + // 8. Pass new topic Id + topic = topicObj._id; + } else { + // 9. Find the existing topic + topicObj = await topicController.findTopic(topic, createdBy); + // 10. Return not found if it was not found + if (!topicObj) { + return res.status(404).json({ success: false, message: 'The topic specified could not be found' }); + } + topicObj.recipients = await topicController.buildRecipients(team, topicObj.createdBy); + await topicObj.save(); + } + } + // 11. Create new message + const message = await MessagesModel.create({ + messageID: parseInt(Math.random().toString().replace('0.', '')), + messageObjectID: parseInt(Math.random().toString().replace('0.', '')), + messageDescription, + topic, + createdBy, + messageType, + readBy: [new mongoose.Types.ObjectId(createdBy)], + }); + // 12. Return 500 error if message was not successfully created + if (!message) return res.status(500).json({ success: false, message: 'Could not save message to database.' }); + + // 13. Prepare to send email if a new message has been created + if (messageType === 'message') { + let optIn, subscribedEmails; + // 14. Find recipients who have opted in to email updates and exclude the requesting user + let messageRecipients = await UserModel.find({ _id: { $in: topicObj.recipients } }).populate('additionalInfo'); + let optedInEmailRecipients = [...messageRecipients].filter(function (user) { + let { + additionalInfo: { emailNotifications }, + _id, + } = user; + return emailNotifications === true && _id.toString() !== createdBy.toString(); + }); + + if (!_.isEmpty(team) || !_.isNil(team)) { + // 15. team all users for notificationType + generic email + // Retrieve notifications for the team based on type return {notificationType, subscribedEmails, optIn} + let teamNotifications = teamController.getTeamNotificationByType(team, constants.teamNotificationTypes.DATAACCESSREQUEST); + // only deconstruct if team notifications object returns - safeguard code + if (!_.isEmpty(teamNotifications)) { + // Get teamNotification emails if optIn true + ({ optIn = false, subscribedEmails = [] } = teamNotifications); + // check subscribedEmails and optIn send back emails or blank [] + let teamNotificationEmails = teamController.getTeamNotificationEmails(optIn, subscribedEmails); + // get users from team.members with notification type and optedIn only + const subscribedMembersByType = teamController.filterMembersByNoticationTypesOptIn( + [...team.members], + [constants.teamNotificationTypes.DATAACCESSREQUEST] + ); + if (!_.isEmpty(subscribedMembersByType)) { + // build cleaner array of memberIds from subscribedMembersByType + const memberIds = [...subscribedMembersByType].map(m => m.memberid); + // returns array of objects [{email: 'email@email.com '}] for members in subscribed emails users is list of full user object + const { memberEmails } = teamController.getMemberDetails([...memberIds], [...messageRecipients]); + optedInEmailRecipients = [...teamNotificationEmails, ...memberEmails]; + } else { + // only if not membersByType but has a team email setup + optedInEmailRecipients = [...optedInEmailRecipients, ...teamNotificationEmails]; + } + } + } + + // 16. Send email + emailGenerator.sendEmail( + optedInEmailRecipients, + constants.hdrukEmail, + `You have received a new message on the HDR UK Innovation Gateway`, + `You have received a new message on the HDR UK Innovation Gateway.
Log in to view your messages here : HDR UK Innovation Gateway`, + false + ); + } + // 17. Return successful response with message data + message.createdByName = { firstname, lastname }; + return res.status(201).json({ success: true, message }); + } catch (err) { + console.error(err.message); + return res.status(500).json(err.message); + } + }, + // DELETE /api/v1/messages/:id + deleteMessage: async (req, res) => { + try { + const { id } = req.params; + const { _id: userId } = req.user; + // 1. Return not found error if id not passed + if (!id) return res.status(404).json({ success: false, message: 'Message Id not found.' }); + // 2. Get message by Id from MongoDb + const message = await MessagesModel.findOne({ _id: id }); + // 3. Return not found if message not found + if (!message) { + return res.status(404).json({ success: false, message: 'Message not found.' }); + } + // 4. Check that the message was created by requesting user otherwise return unathorised + if (message.createdBy.toString() !== userId.toString()) { + return res.status(401).json({ success: false, message: 'Not authorised to delete this message' }); + } + // 5. Delete message by id + await MessagesModel.remove({ _id: id }); + // 6. Check attached topic for other messages to avoid orphaning topic documents + const messagesCount = await MessagesModel.count({ topic: message.topic }); + // 7. If no other messages remain then delete the topic + if (!messagesCount) { + await TopicModel.remove({ _id: new mongoose.Types.ObjectId(message.topic) }); + } + // 8. Return successful response + return res.status(204).json({ success: true }); + } catch (err) { + console.error(err.message); + return res.status(500).json(err.message); + } + }, + // PUT /api/v1/messages + updateMessage: async (req, res) => { + try { + let { _id: userId } = req.user; + let { messageId, isRead, messageDescription = '', messageType = '' } = req.body; + // 1. Return not found error if id not passed + if (!messageId) return res.status(404).json({ success: false, message: 'Message Id not found.' }); + // 2. Get message by object id + const message = await MessagesModel.findOne({ _id: messageId }); + // 3. Return not found if message not found + if (!message) { + return res.status(404).json({ success: false, message: 'Message not found.' }); + } + // 4. Update message params - readBy is an array of users who have read the message + if (isRead && !message.readBy.includes(userId.toString())) { + message.readBy.push(userId); + } + if (isRead) { + message.isRead = isRead; + } + if (!_.isEmpty(messageDescription)) { + message.messageDescription = messageDescription; + } + if (!_.isEmpty(messageType)) { + message.messageType = messageType; + } + // 5. Save message to Mongo + await message.save(); + // 6. Return success no content + return res.status(204).json({ success: true }); + } catch (err) { + console.error(err.message); + return res.status(500).json(err.message); + } + }, + // GET api/v1/messages/unread/count + getUnreadMessageCount: async (req, res) => { + try { + let { _id: userId } = req.user; + let unreadMessageCount = 0; - // 1. Find all active topics the user is a member of - const topics = await TopicModel.find({ - recipients: { $elemMatch : { $eq: userId }}, - status: 'active' - }); - // 2. Iterate through each topic and aggregate unread messages - topics.forEach(topic => { - topic.topicMessages.forEach(message => { - if(!message.readBy.includes(userId)) { - unreadMessageCount ++; - } - }) - }); - // 3. Return the number of unread messages - return res.status(200).json({ success: true, count: unreadMessageCount }); - } - catch (err) { - console.error(err.message); - return res.status(500).json(err.message); - } - } -} + // 1. Find all active topics the user is a member of + const topics = await TopicModel.find({ + recipients: { $elemMatch: { $eq: userId } }, + status: 'active', + }); + // 2. Iterate through each topic and aggregate unread messages + topics.forEach(topic => { + topic.topicMessages.forEach(message => { + if (!message.readBy.includes(userId)) { + unreadMessageCount++; + } + }); + }); + // 3. Return the number of unread messages + return res.status(200).json({ success: true, count: unreadMessageCount }); + } catch (err) { + console.error(err.message); + return res.status(500).json(err.message); + } + }, +}; diff --git a/src/resources/person/person.route.js b/src/resources/person/person.route.js index 7e0991e9..9e7a75e8 100644 --- a/src/resources/person/person.route.js +++ b/src/resources/person/person.route.js @@ -5,37 +5,15 @@ import passport from 'passport'; import { ROLES } from '../user/user.roles'; import { getAllTools } from '../tool/data.repository'; import { UserModel } from '../user/user.model'; +import mailchimpConnector from '../../services/mailchimp/mailchimp'; +import constants from '../utilities/constants.util'; import helper from '../utilities/helper.util'; -import _ from 'lodash'; +import { isEmpty, isNil } from 'lodash'; const urlValidator = require('../utilities/urlValidator'); const inputSanitizer = require('../utilities/inputSanitizer'); const router = express.Router(); -router.post('/', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, ROLES.Creator), async (req, res) => { - const { firstname, lastname, bio, emailNotifications, terms, sector, organisation, showOrganisation, tags } = req.body; - let link = urlValidator.validateURL(inputSanitizer.removeNonBreakingSpaces(req.body.link)); - let orcid = req.body.orcid !== '' ? urlValidator.validateOrcidURL(inputSanitizer.removeNonBreakingSpaces(req.body.orcid)) : ''; - let data = Data(); - data.id = parseInt(Math.random().toString().replace('0.', '')); - (data.firstname = inputSanitizer.removeNonBreakingSpaces(firstname)), - (data.lastname = inputSanitizer.removeNonBreakingSpaces(lastname)), - (data.type = 'person'); - data.bio = inputSanitizer.removeNonBreakingSpaces(bio); - data.link = link; - data.orcid = orcid; - data.emailNotifications = emailNotifications; - data.terms = terms; - data.sector = inputSanitizer.removeNonBreakingSpaces(sector); - data.organisation = inputSanitizer.removeNonBreakingSpaces(organisation); - data.showOrganisation = showOrganisation; - data.tags = inputSanitizer.removeNonBreakingSpaces(tags); - let newPersonObj = await data.save(); - if (!newPersonObj) return res.json({ success: false, error: "Can't persist data to DB" }); - - return res.json({ success: true, data: newPersonObj }); -}); - router.put('/', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, ROLES.Creator), async (req, res) => { let { id, @@ -46,7 +24,8 @@ router.put('/', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, R showBio, showLink, showOrcid, - emailNotifications, + feedback, + news, terms, sector, showSector, @@ -66,8 +45,13 @@ router.put('/', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, R organisation = inputSanitizer.removeNonBreakingSpaces(organisation); tags.topics = inputSanitizer.removeNonBreakingSpaces(tags.topics); + const userId = parseInt(id); + const { news: newsOriginalValue, feedback: feedbackOriginalValue } = await UserModel.findOne({ id: userId }, 'news feedback').lean(); + const newsDirty = newsOriginalValue !== news && !isNil(news); + const feedbackDirty = feedbackOriginalValue !== feedback && !isNil(feedback); + await Data.findOneAndUpdate( - { id: id }, + { id: userId }, { firstname, lastname, @@ -78,7 +62,6 @@ router.put('/', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, R showLink, orcid, showOrcid, - emailNotifications, terms, sector, showSector, @@ -87,10 +70,23 @@ router.put('/', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, R tags, showDomain, profileComplete, - }, - { new: true } - ); - await UserModel.findOneAndUpdate({ id: id }, { $set: { firstname: firstname, lastname: lastname, email: email } }) + } + ); + + if (newsDirty) { + const newsSubscriptionId = process.env.MAILCHIMP_NEWS_AUDIENCE_ID; + const newsStatus = news ? constants.mailchimpSubscriptionStatuses.SUBSCRIBED : constants.mailchimpSubscriptionStatuses.UNSUBSCRIBED; + await mailchimpConnector.updateSubscriptionUsers(newsSubscriptionId, [req.user], newsStatus); + } + if (feedbackDirty) { + const feedbackSubscriptionId = process.env.MAILCHIMP_FEEDBACK_AUDIENCE_ID; + const feedbackStatus = feedback + ? constants.mailchimpSubscriptionStatuses.SUBSCRIBED + : constants.mailchimpSubscriptionStatuses.UNSUBSCRIBED; + await mailchimpConnector.updateSubscriptionUsers(feedbackSubscriptionId, [req.user], feedbackStatus); + } + + await UserModel.findOneAndUpdate({ id: userId }, { $set: { firstname, lastname, email, feedback, news } }) .then(person => { return res.json({ success: true, data: person }); }) @@ -102,28 +98,42 @@ router.put('/', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, R // @router GET /api/v1/person/unsubscribe/:userObjectId // @desc Unsubscribe a single user from email notifications without challenging authentication // @access Public -router.put('/unsubscribe/:userObjectId', async (req, res) => { - const userId = req.params.userObjectId; - // 1. Use _id param issued by MongoDb as unique reference to find user entry - await UserModel.findOne({ _id: userId }) - .then(async user => { - // 2. Find person entry using numeric id and update email notifications to false - await Data.findOneAndUpdate( - { id: user.id }, - { - emailNotifications: false, - } - ).then(() => { - // 3a. Return success message - return res.json({ - success: true, - msg: "You've been successfully unsubscribed from all emails. You can change this setting via your account.", - }); - }); +// router.put('/unsubscribe/:userObjectId', async (req, res) => { +// const userId = req.params.userObjectId; +// // 1. Use _id param issued by MongoDb as unique reference to find user entry +// await UserModel.findOne({ _id: userId }) +// .then(async user => { +// // 2. Find person entry using numeric id and update email notifications to false +// await Data.findOneAndUpdate( +// { id: user.id }, +// { +// emailNotifications: false, +// } +// ).then(() => { +// // 3a. Return success message +// return res.json({ +// success: true, +// msg: "You've been successfully unsubscribed from all emails. You can change this setting via your account.", +// }); +// }); +// }) +// .catch(() => { +// // 3b. Return generic failure message in all cases without disclosing reason or data structure +// return res.status(500).send({ success: false, msg: 'A problem occurred unsubscribing from email notifications.' }); +// }); +// }); + +// @router PATCH /api/v1/person/profileComplete/:id +// @desc Set profileComplete to true +// @access Private +router.patch('/profileComplete/:id', passport.authenticate('jwt'), async (req, res) => { + const id = req.params.id; + await Data.findOneAndUpdate({ id }, { profileComplete: true }) + .then(response => { + return res.json({ success: true, response }); }) - .catch(() => { - // 3b. Return generic failure message in all cases without disclosing reason or data structure - return res.status(500).send({ success: false, msg: 'A problem occurred unsubscribing from email notifications.' }); + .catch(err => { + return res.json({ success: false, error: err.message }); }); }); @@ -139,7 +149,7 @@ router.get('/:id', async (req, res) => { return res.json({ success: false, error: err }); }); - if (_.isEmpty(person)) { + if (isEmpty(person)) { return res.status(404).send(`Person not found for Id: ${escape(req.params.id)}`); } else { person = helper.hidePrivateProfileDetails([person])[0]; @@ -150,24 +160,27 @@ router.get('/:id', async (req, res) => { // @router GET /api/v1/person/profile/:id // @desc Get person info for their account router.get('/profile/:id', async (req, res) => { - let profileData = Data.aggregate([ - { $match: { $and: [{ id: parseInt(req.params.id) }] } }, - { $lookup: { from: 'tools', localField: 'id', foreignField: 'authors', as: 'tools' } }, - { $lookup: { from: 'reviews', localField: 'id', foreignField: 'reviewerID', as: 'reviews' } }, - ]); - profileData.exec((err, data) => { - if (err) return res.json({ success: false, error: err }); + try { + let person = await Data.findOne({ id: parseInt(req.params.id) }) + .populate([{ path: 'tools' }, { path: 'reviews' }, { path: 'user', select: 'feedback news' }]) + .lean(); + const { feedback, news } = person.user; + person = { ...person, feedback, news }; + let data = [person]; return res.json({ success: true, data: data }); - }); + } catch (err) { + console.error(err.message); + return res.json({ success: false, error: err.message }); + } }); -// @router GET /api/v1/person +// @router GET /api/v1/person // @desc Get paper for an author // @access Private router.get('/', async (req, res) => { let personArray = []; - req.params.type = 'person'; - await getAllTools(req) + req.params.type = 'person'; + await getAllTools(req) .then(data => { data.map(personObj => { personArray.push({ diff --git a/src/resources/search/search.repository.js b/src/resources/search/search.repository.js index 60765598..103bbd6f 100644 --- a/src/resources/search/search.repository.js +++ b/src/resources/search/search.repository.js @@ -8,7 +8,6 @@ import moment from 'moment'; import helperUtil from '../utilities/helper.util'; export async function getObjectResult(type, searchAll, searchQuery, startIndex, maxResults, sort) { - let collection = Data; if (type === 'course') { collection = Course; @@ -58,7 +57,158 @@ export async function getObjectResult(type, searchAll, searchQuery, startIndex, }, ]; } else if (type === 'collection') { - queryObject = [{ $match: newSearchQuery }, { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }]; + queryObject = [ + { $match: newSearchQuery }, + { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }, + { + $project: { + _id: 0, + id: 1, + name: 1, + description: 1, + imageLink: 1, + relatedObjects: 1, + + 'persons.id': 1, + 'persons.firstname': 1, + 'persons.lastname': 1, + + activeflag: 1, + counter: 1, + latestUpdate: { + $cond: { + if: { $gte: ['$createdAt', '$updatedon'] }, + then: '$createdAt', + else: '$updatedon', + }, + }, + relatedresources: { $cond: { if: { $isArray: '$relatedObjects' }, then: { $size: '$relatedObjects' }, else: 0 } }, + }, + }, + ]; + } else if (type === 'dataset') { + queryObject = [ + { $match: newSearchQuery }, + { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }, + { + $lookup: { + from: 'tools', + let: { + pid: '$pid', + }, + pipeline: [ + { $unwind: '$relatedObjects' }, + { + $match: { + $expr: { + $and: [ + { + $eq: ['$relatedObjects.pid', '$$pid'], + }, + { + $eq: ['$activeflag', 'active'], + }, + ], + }, + }, + }, + { $group: { _id: null, count: { $sum: 1 } } }, + ], + as: 'relatedResourcesTools', + }, + }, + { + $lookup: { + from: 'course', + let: { + pid: '$pid', + }, + pipeline: [ + { $unwind: '$relatedObjects' }, + { + $match: { + $expr: { + $and: [ + { + $eq: ['$relatedObjects.pid', '$$pid'], + }, + { + $eq: ['$activeflag', 'active'], + }, + ], + }, + }, + }, + { $group: { _id: null, count: { $sum: 1 } } }, + ], + as: 'relatedResourcesCourses', + }, + }, + { + $project: { + _id: 0, + id: 1, + name: 1, + type: 1, + description: 1, + bio: { + $cond: { + if: { $eq: [false, '$showBio'] }, + then: '$$REMOVE', + else: '$bio', + }, + }, + 'categories.category': 1, + 'categories.programmingLanguage': 1, + 'programmingLanguage.programmingLanguage': 1, + 'programmingLanguage.version': 1, + license: 1, + 'tags.features': 1, + 'tags.topics': 1, + firstname: 1, + lastname: 1, + datasetid: 1, + pid: 1, + 'datasetfields.publisher': 1, + 'datasetfields.geographicCoverage': 1, + 'datasetfields.physicalSampleAvailability': 1, + 'datasetfields.abstract': 1, + 'datasetfields.ageBand': 1, + 'datasetfields.phenotypes': 1, + 'datasetv2.summary.publisher.name': 1, + 'datasetv2.summary.publisher.logo': 1, + 'datasetv2.summary.publisher.memberOf': 1, + + 'persons.id': 1, + 'persons.firstname': 1, + 'persons.lastname': 1, + + activeflag: 1, + counter: 1, + 'datasetfields.metadataquality.quality_score': 1, + + latestUpdate: '$timestamps.updated', + relatedresources: { + $add: [ + { + $cond: { + if: { $eq: [{ $size: '$relatedResourcesTools' }, 0] }, + then: 0, + else: { $first: '$relatedResourcesTools.count' }, + }, + }, + { + $cond: { + if: { $eq: [{ $size: '$relatedResourcesCourses' }, 0] }, + then: 0, + else: { $first: '$relatedResourcesCourses.count' }, + }, + }, + ], + }, + }, + }, + ]; } else { queryObject = [ { $match: newSearchQuery }, @@ -105,12 +255,28 @@ export async function getObjectResult(type, searchAll, searchQuery, startIndex, activeflag: 1, counter: 1, 'datasetfields.metadataquality.quality_score': 1, + latestUpdate: { + $cond: { + if: { $gte: ['$createdAt', '$updatedon'] }, + then: '$createdAt', + else: '$updatedon', + }, + }, + relatedresources: { $cond: { if: { $isArray: '$relatedObjects' }, then: { $size: '$relatedObjects' }, else: 0 } }, }, }, ]; } - if (sort === '' || sort === 'relevance') { + if (sort === '') { + if (type === 'dataset') { + if (searchAll) queryObject.push({ $sort: { 'datasetfields.metadataquality.quality_score': -1, name: 1 } }); + else queryObject.push({ $sort: { score: { $meta: 'textScore' } } }); + } else { + if (searchAll) queryObject.push({ $sort: { latestUpdate: -1 } }); + else queryObject.push({ $sort: { score: { $meta: 'textScore' } } }); + } + } else if (sort === 'relevance') { if (type === 'person') { if (searchAll) queryObject.push({ $sort: { lastname: 1 } }); else queryObject.push({ $sort: { score: { $meta: 'textScore' } } }); @@ -132,7 +298,14 @@ export async function getObjectResult(type, searchAll, searchQuery, startIndex, } else if (sort === 'startdate') { if (searchAll) queryObject.push({ $sort: { 'courseOptions.startDate': 1 } }); else queryObject.push({ $sort: { 'courseOptions.startDate': 1, score: { $meta: 'textScore' } } }); + } else if (sort === 'latest') { + if (searchAll) queryObject.push({ $sort: { latestUpdate: -1 } }); + else queryObject.push({ $sort: { latestUpdate: -1, score: { $meta: 'textScore' } } }); + } else if (sort === 'resources') { + if (searchAll) queryObject.push({ $sort: { relatedresources: -1 } }); + else queryObject.push({ $sort: { relatedresources: -1, score: { $meta: 'textScore' } } }); } + // Get paged results based on query params const searchResults = await collection.aggregate(queryObject).skip(parseInt(startIndex)).limit(parseInt(maxResults)); // Return data diff --git a/src/resources/stats/stats.router.js b/src/resources/stats/stats.router.js index 2992105e..fa2430f9 100644 --- a/src/resources/stats/stats.router.js +++ b/src/resources/stats/stats.router.js @@ -387,10 +387,11 @@ router.get('', async (req, res) => { activeflag: 1, datasetv2: 1, datasetfields: 1, - updatedAt: 1, + description: 1, + 'timestamps.updated': 1, } ) - .sort({ updatedAt: -1, name: 1 }) + .sort({ 'timestamps.updated': -1, name: 1 }) .limit(10); } else if (req.query.type && req.query.type !== 'course' && req.query.type !== 'dataset') { recentlyUpdated = Data.find( diff --git a/src/resources/team/team.controller.js b/src/resources/team/team.controller.js index 4691e6a7..48e33a95 100644 --- a/src/resources/team/team.controller.js +++ b/src/resources/team/team.controller.js @@ -18,9 +18,7 @@ const getTeamById = async (req, res) => { let { members } = team; let authorised = false; if (members) { - authorised = members.some( - (el) => el.memberid.toString() === _id.toString() - ); + authorised = members.some(el => el.memberid.toString() === _id.toString()); } // 3. If not return unauthorised if (!authorised) { @@ -64,32 +62,30 @@ const getTeamMembers = async (req, res) => { } }; -const formatTeamUsers = (team) => { +const formatTeamUsers = team => { let { users = [] } = team; - users = users.map((user) => { - let { - firstname, - lastname, - id, - _id, - email, - additionalInfo: { organisation, bio, showOrganisation, showBio }, - } = user; - let userMember = team.members.find( - (el) => el.memberid.toString() === user._id.toString() - ); - let { roles = [] } = userMember; - return { - firstname, - lastname, - id, - _id, - email, - roles, - organisation: showOrganisation ? organisation : '', - bio: showBio ? bio : '', - }; - }); + users = users.map(user => { + let { + firstname, + lastname, + id, + _id, + email, + additionalInfo: { organisation, bio, showOrganisation, showBio }, + } = user; + let userMember = team.members.find(el => el.memberid.toString() === user._id.toString()); + let { roles = [] } = userMember; + return { + firstname, + lastname, + id, + _id, + email, + roles, + organisation: showOrganisation ? organisation : '', + bio: showBio ? bio : '', + }; + }); return users; }; @@ -110,10 +106,7 @@ const addTeamMembers = async (req, res) => { }); } // 2. Find team by Id passed - const team = await TeamModel.findOne({ _id: id }).populate([ - { path: 'users' }, - { path: 'publisher', select: 'name' }, - ]); + const team = await TeamModel.findOne({ _id: id }).populate([{ path: 'users' }, { path: 'publisher', select: 'name' }]); // 3. Return 404 if no team found matching Id if (!team) { return res.status(404).json({ @@ -121,28 +114,19 @@ const addTeamMembers = async (req, res) => { }); } // 4. Ensure the user has permissions to perform this operation - let authorised = checkTeamPermissions( - 'manager', - team.toObject(), - req.user._id - ); + let authorised = checkTeamPermissions('manager', team.toObject(), req.user._id); // 5. If not return unauthorised if (!authorised) { return res.status(401).json({ success: false }); } // 6. Filter out any existing members to avoid duplication let teamObj = team.toObject(); - newMembers = [...newMembers].filter( - (newMem) => - !teamObj.members.some( - (mem) => newMem.memberid.toString() === mem.memberid.toString() - ) - ); - + newMembers = [...newMembers].filter(newMem => !teamObj.members.some(mem => newMem.memberid.toString() === mem.memberid.toString())); + // 8. Add members to MongoDb collection using model validation team.members = [...team.members, ...newMembers]; // 9. Save members handling error callback if validation fails - team.save(async (err) => { + team.save(async err => { if (err) { console.error(err.message); return res.status(400).json({ @@ -151,14 +135,9 @@ const addTeamMembers = async (req, res) => { }); } else { // 10. Issue notification to added members - let newMemberIds = newMembers.map((mem) => mem.memberid); + let newMemberIds = newMembers.map(mem => mem.memberid); let newUsers = await UserModel.find({ _id: newMemberIds }); - createNotifications( - constants.notificationTypes.MEMBERADDED, - { newUsers }, - team, - req.user - ); + createNotifications(constants.notificationTypes.MEMBERADDED, { newUsers }, team, req.user); // 11. Get updated team users including bio data const updatedTeam = await TeamModel.findOne({ _id: req.params.id }).populate({ path: 'users', @@ -191,6 +170,293 @@ const addTeamMembers = async (req, res) => { */ const updateTeamMember = async (req, res) => {}; +/** + * GET api/v1/teams/:id/notifications + * + * @desc Get team notifications by :id + */ +const getTeamNotifications = async (req, res) => { + try { + const team = await TeamModel.findOne({ _id: req.params.id }); + if (!team) { + return res.status(404).json({ success: false }); + } + // 2. Check the current user is a member of the team + const { + user: { _id }, + } = req; + + let { members } = team; + let authorised = false; + // 3. check if member is inside the team of members + if (members) { + authorised = members.some(el => el.memberid.toString() === _id.toString()); + } + // 4. If not return unauthorised + if (!authorised) return res.status(401).json({ success: false }); + + // 5. get member details + let member = [...members].find(el => el.memberid.toString() === _id.toString()); + + // 6. format teamNotifications for FE + const teamNotifications = formatTeamNotifications(team); + // 7. return optimal payload needed for FE containing memberNotifications and teamNotifications + let notifications = { + memberNotifications: member.notifications ? member.notifications : [], + teamNotifications, + }; + // 8. return 200 success + return res.status(200).json(notifications); + } catch (err) { + console.error(err.message); + return res.status(500).json({ + success: false, + message: 'An error occurred retrieving team notifications', + }); + } +}; + +/** + * PUT api/v1/team/:id/notifications + * + * @desc Update Team notification preferences + * + */ +const updateNotifications = async (req, res) => { + try { + // 1. Get the team from the database include user documents for each member + const team = await TeamModel.findOne({ _id: req.params.id }).populate([{ path: 'users' }, { path: 'publisher', select: 'name' }]); + + if (!team) { + return res.status(404).json({ success: false }); + } + // 2. Check the current user is a member of the team + const { + user: { _id }, + body: data, + } = req; + + let { members, users, notifications } = team; + let authorised = false; + + if (members) { + authorised = [...members].some(el => el.memberid.toString() === _id.toString()); + } + // 3. If not return unauthorised + if (!authorised) return res.status(401).json({ success: false }); + // 4. get member details + let member = [...members].find(el => el.memberid.toString() === _id.toString()); + // 5. get member roles and notifications + let { roles = [] } = member; + + // 6. get user role + let isManager = roles.includes('manager'); + + // 7. req data from FE + let { memberNotifications = [], teamNotifications = [] } = data; + + // 8. commonality = can only turn off personal notification for each type if team has subscribed emails for desired type **As of M2 DAR** + let missingOptIns = {}; + + // 9. if member has notifications - backend check to ensure optIn is true if team notifications opted out for member + if (!_.isEmpty(memberNotifications) && !_.isEmpty(teamNotifications)) { + missingOptIns = findMissingOptIns(memberNotifications, teamNotifications); + } + + // 10. return missingOptIns to FE and do not update + if (!_.isEmpty(missingOptIns)) return res.status(400).json({ success: false, message: missingOptIns }); + + // 11. if manager updates team notifications, check if we have any team notifications optedOut + if (isManager) { + // 1. filter team.notification types that are opted out ie { optIn: false, ... } + const optedOutTeamNotifications = [...teamNotifications].filter(notification => !notification.optIn) || []; + // 2. if there are opted out team notifications find members who have these notifications turned off and turn on if any + if (!_.isEmpty(optedOutTeamNotifications)) { + // loop over each notification type that has optOut + optedOutTeamNotifications.forEach(teamNotification => { + // get notification type + let { notificationType } = teamNotification; + // loop members + members.forEach(member => { + // get member notifications + let { notifications = [] } = member; + // if notifications exist + if (!_.isEmpty(notifications)) { + // find the notification by notificationType + let notificationIndex = notifications.findIndex(n => n.notificationType.toUpperCase() === notificationType.toUpperCase()); + // if notificationType exists update optIn and notificationMessage + if (!notifications[notificationIndex].optIn) { + notifications[notificationIndex].optIn = true; + notifications[notificationIndex].message = constants.teamNotificationMessages[notificationType.toUpperCase()]; + } + } + // update member notifications + member.notifications = notifications; + }); + }); + } + + // compare db / payload notifications for each type and send email ** when more types update email logic only to capture multiple types as it only outs one currently as per design *** + // check if team has team.notificaitons loop over db notifications array - process emails missing / added for the type if manager has saved + if (!_.isEmpty(notifications)) { + // get manager who has submitted the request + let manager = [...users].find(user => user._id.toString() === member.memberid.toString()); + + [...notifications].forEach(dbNotification => { + // extract notification type from team.notifications ie dataAccessRequest + let { notificationType } = dbNotification; + // find the notificationType in the teamNotifications incoming from FE + const notificationPayload = + [...teamNotifications].find(n => n.notificationType.toUpperCase() === notificationType.toUpperCase()) || {}; + // if found process next phase + if (!_.isEmpty(notificationPayload)) { + // get db subscribedEmails and rename to dbSubscribedEmails + let { subscribedEmails: dbSubscribedEmails, optIn: dbOptIn } = dbNotification; + // get incoming subscribedEmails and rename to payLoadSubscribedEmails + let { subscribedEmails: payLoadSubscribedEmails, optIn: payLoadOptIn } = notificationPayload; + // compare team.notifications by notificationType subscribed emails against the incoming payload to get emails that have been removed + const removedEmails = _.difference([...dbSubscribedEmails], [...payLoadSubscribedEmails]) || []; + // compare incoming payload notificationTypes subscribed emails to get emails that have been added against db + const addedEmails = _.difference([...payLoadSubscribedEmails], [...dbSubscribedEmails]) || []; + // get all members who have notifications by the type + const subscribedMembersByType = filterMembersByNoticationTypes([...members], [notificationType]); + if (!_.isEmpty(subscribedMembersByType)) { + // build cleaner array of memberIds from subscribedMembersByType + const memberIds = [...subscribedMembersByType].map(m => m.memberid); + // returns array of objects [{email: 'email@email.com '}] for members in subscribed emails users is list of full user object in team + const { memberEmails, userIds } = getMemberDetails([...memberIds], [...users]); + // email options and html template + let html = ''; + let options = { + managerName: `${manager.firstname} ${manager.lastname}`, + notificationRemoved: false, + disabled: false, + header: '', + emailAddresses: [], + }; + // check if removed emails and send email subscribedEmails or if the emails are turned off + if (!_.isEmpty(removedEmails) || (dbOptIn && !payLoadOptIn)) { + // update the options + options = { + ...options, + notificationRemoved: true, + disabled: !payLoadOptIn ? true : false, + header: `A manager for ${team.publisher ? team.publisher.name : 'a team'} has ${ + dbOptIn && !payLoadOptIn ? 'disabled all' : 'removed a' + } generic team email address(es)`, + emailAddresses: dbOptIn && !payLoadOptIn ? payLoadSubscribedEmails : removedEmails, + publisherId: team.publisher._id.toString(), + }; + // get html template + html = emailGenerator.generateTeamNotificationEmail(options); + // send email + emailGenerator.sendEmail( + memberEmails, + constants.hdrukEmail, + `A manager for ${team.publisher ? team.publisher.name : 'a team'} has ${ + dbOptIn && !payLoadOptIn ? 'disabled all' : 'removed a' + } generic team email address(es)`, + html, + true + ); + + notificationBuilder.triggerNotificationMessage( + [...userIds], + `A manager for ${team.publisher ? team.publisher.name : 'a team'} has ${ + dbOptIn && !payLoadOptIn ? 'disabled all' : 'removed a' + } generic team email address(es)`, + 'team', + team.publisher ? team.publisher.name : 'Undefined' + ); + } + // check if added emails and send email to subscribedEmails or if the dbOpt is false but the manager is turning back on team notifications + if (!_.isEmpty(addedEmails) || (!dbOptIn && payLoadOptIn)) { + // update options + options = { + ...options, + notificationRemoved: false, + header: `A manager for ${team.publisher ? team.publisher.name : 'a team'} has ${ + !dbOptIn && payLoadOptIn ? 'enabled all' : 'added a' + } generic team email address(es)`, + emailAddresses: payLoadSubscribedEmails, + publisherId: team.publisher._id.toString(), + }; + // get html template + html = emailGenerator.generateTeamNotificationEmail(options); + // send email + emailGenerator.sendEmail( + memberEmails, + constants.hdrukEmail, + `A manager for ${team.publisher ? team.publisher.name : 'a team'} has ${ + !dbOptIn && payLoadOptIn ? 'enabled all' : 'added a' + } generic team email address(es)`, + html, + true + ); + + notificationBuilder.triggerNotificationMessage( + [...userIds], + `A manager for ${team.publisher ? team.publisher.name : 'a team'} has ${ + !dbOptIn && payLoadOptIn ? 'enabled all' : 'added a' + } generic team email address(es)`, + 'team', + team.publisher ? team.publisher.name : 'Undefined' + ); + } + } + } + }); + } + // update team notifications + team.notifications = teamNotifications; + } + // 11. update member notifications + member.notifications = memberNotifications; + // 12. save changes to team + await team.save(); + // 13. return 201 with new team + return res.status(201).json(team); + } catch (err) { + console.error(err.message); + return res.status(500).json({ + success: false, + message: 'An error occurred updating team notifications', + }); + } +}; + +/** + * PUT api/v1/team/:id/notification-messages + * + * @desc Update Individal User messages against their own notifications + * + */ +const updateNotificationMessages = async (req, res) => { + try { + const { + user: { _id }, + } = req; + await TeamModel.update( + { _id: req.params.id }, + { $set: { 'members.$[m].notifications.$[].message': '' } }, + { arrayFilters: [{ 'm.memberid': _id }], multi: true } + ) + .then(resp => { + return res.status(201).json(); + }) + .catch(err => { + console.log(err); + res.status(500).json({ success: false, message: err.message }); + }); + } catch (err) { + console.error(err.message); + return res.status(500).json({ + success: false, + message: 'An error occurred updating notification messages', + }); + } +}; + /** * Deletes a team member from a team * @@ -207,10 +473,7 @@ const deleteTeamMember = async (req, res) => { }); } // 2. Find team by Id passed - const team = await TeamModel.findOne({ _id: id }).populate([ - { path: 'users' }, - { path: 'publisher', select: 'name' }, - ]); + const team = await TeamModel.findOne({ _id: id }).populate([{ path: 'users' }, { path: 'publisher', select: 'name' }]); // 3. Return 404 if no team found matching Id if (!team) { return res.status(404).json({ @@ -218,22 +481,14 @@ const deleteTeamMember = async (req, res) => { }); } // 4. Ensure the user has permissions to perform this operation - let authorised = checkTeamPermissions( - 'manager', - team.toObject(), - req.user._id - ); + let authorised = checkTeamPermissions('manager', team.toObject(), req.user._id); // 5. If not return unauthorised if (!authorised) { return res.status(401).json({ success: false }); } // 6. Ensure at least one manager will remain if this member is deleted let { members = [], users = [] } = team; - let managerCount = members.filter( - (mem) => - mem.roles.includes('manager') && - mem.memberid.toString() !== req.user._id.toString() - ).length; + let managerCount = members.filter(mem => mem.roles.includes('manager') && mem.memberid.toString() !== req.user._id.toString()).length; if (managerCount === 0) { return res.status(400).json({ success: false, @@ -241,9 +496,7 @@ const deleteTeamMember = async (req, res) => { }); } // 7. Filter out removed member - let updatedMembers = [...members].filter( - (mem) => mem.memberid.toString() !== memberid.toString() - ); + let updatedMembers = [...members].filter(mem => mem.memberid.toString() !== memberid.toString()); if (members.length === updatedMembers.length) { return res.status(400).json({ success: false, @@ -261,15 +514,8 @@ const deleteTeamMember = async (req, res) => { }); } else { // 9. Issue notification to removed member - let removedUser = users.find( - (user) => user._id.toString() === memberid.toString() - ); - createNotifications( - constants.notificationTypes.MEMBERREMOVED, - { removedUser }, - team, - req.user - ); + let removedUser = users.find(user => user._id.toString() === memberid.toString()); + createNotifications(constants.notificationTypes.MEMBERREMOVED, { removedUser }, team, req.user); // 10. Return success response return res.status(204).json({ success: true, @@ -278,10 +524,7 @@ const deleteTeamMember = async (req, res) => { }); } catch (err) { console.error(err.message); - return res.status(500).json({ - success: false, - message: 'An error occurred deleting the team member', - }); + res.status(500).json({ status: 'error', message: err.message }); } }; @@ -298,17 +541,11 @@ const checkTeamPermissions = (role, team, userId) => { // 2. Extract team members let { members } = team; // 3. Find the current user - let userMember = members.find( - (el) => el.memberid.toString() === userId.toString() - ); + let userMember = members.find(el => el.memberid.toString() === userId.toString()); // 4. If the user was found check they hold the minimum required role if (userMember) { let { roles = [] } = userMember; - if ( - roles.includes(role) || - roles.includes(constants.roleTypes.MANAGER) || - role === '' - ) { + if (roles.includes(role) || roles.includes(constants.roleTypes.MANAGER) || role === '') { return true; } } @@ -320,11 +557,9 @@ const getTeamMembersByRole = (team, role) => { // Destructure members array and populated users array (populate 'users' must be included in the original Mongo query) let { members = [], users = [] } = team; // Get all userIds for role within team - let userIds = members - .filter((mem) => mem.roles.includes(role)) - .map((mem) => mem.memberid.toString()); + let userIds = members.filter(mem => mem.roles.includes(role)).map(mem => mem.memberid.toString()); // return all user records for role - return users.filter((user) => userIds.includes(user._id.toString())); + return users.filter(user => userIds.includes(user._id.toString())); }; /** @@ -332,7 +567,7 @@ const getTeamMembersByRole = (team, role) => { * * @param {object} team The team object containing its name or linked object containing name e.g. publisher */ -const getTeamName = (team) => { +const getTeamName = team => { let teamObj = team.toObject(); if (_.has(teamObj, 'publisher') && !_.isNull(teamObj.publisher)) { let { @@ -344,6 +579,145 @@ const getTeamName = (team) => { } }; +/** + * [Get teams notification by type ] + * + * @param {Object} team [team object] + * @param {String} notificationType [notificationType dataAccessRequest] + * @return {Object} [return team notification object {notificaitonType, optIn, subscribedEmails }] + */ +const getTeamNotificationByType = (team = {}, notificationType = '') => { + let teamObj = team.toObject(); + if (_.has(teamObj, 'notifications') && !_.isNull(teamObj.notifications) && !_.isEmpty(notificationType)) { + let { notifications } = teamObj; + let notification = [...notifications].find(n => n.notificationType.toUpperCase() === notificationType.toUpperCase()); + if (typeof notification !== 'undefined') return notification; + else return {}; + } else { + return {}; + } +}; + +const findTeamMemberById = (members = [], custodianManager = {}) => { + if (!_.isEmpty(members) && !_.isEmpty(custodianManager)) + return [...members].find(member => member.memberid.toString() === custodianManager._id.toString()) || {}; + + return {}; +}; + +const findByNotificationType = (notificaitons = [], notificationType = '') => { + if (!_.isEmpty(notificaitons) && !_.isEmpty(notificationType)) { + return [...notificaitons].find(notification => notification.notificationType === notificationType) || {}; + } + return {}; +}; + +/** + * filterMembersByNoticationTypes *nifty* + * + * @param {Array} members [members] + * @param {Array} notificationTypes [notificationTypes] + * @return {Array} [return all members with notification types] + */ +const filterMembersByNoticationTypes = (members, notificationTypes) => { + return _.filter(members, member => { + return _.some(member.notifications, notification => { + return _.includes(notificationTypes, notification.notificationType); + }); + }); +}; + +/** + * filterMembersByNoticationTypesOptIn *nifty* + * + * @param {Array} members [members] + * @param {Array} notificationTypes [notificationTypes] + * @return {Array} [return all members with notification types] + */ +const filterMembersByNoticationTypesOptIn = (members, notificationTypes) => { + return _.filter(members, member => { + return _.some(member.notifications, notification => { + return _.includes(notificationTypes, notification.notificationType) && notification.optIn; + }); + }); +}; + +/** + * getMemberDetails + * + * @param {Array} memberIds [memberIds from team.members] + * @param {Array} users [array of user objects that are in the team] + * @return {Array} [return all emails for memberIds from user aray] + */ +const getMemberDetails = (memberIds = [], users = []) => { + if (!_.isEmpty(memberIds) && !_.isEmpty(users)) { + return [...users].reduce( + (arr, user) => { + const member = [...memberIds].find(m => m.toString() === user._id.toString()) || {}; + if (!_.isEmpty(member)) { + let { email, id } = user; + return { + memberEmails: [...arr['memberEmails'], { email }], + userIds: [...arr['userIds'], id], + }; + } + return arr; + }, + { memberEmails: [], userIds: [] } + ); + } + return []; +}; + +const buildOptedInEmailList = (custodianManagers = [], team = {}, notificationType = '') => { + let { members = [] } = team; + if (!_.isEmpty(custodianManagers)) { + // loop over custodianManagers + return [...custodianManagers].reduce((acc, custodianManager) => { + let custodianNotificationObj, member, notifications, optIn; + // if memebers exist only do the following + if (!_.isEmpty(members)) { + // find member in team.members array + member = findTeamMemberById(members, custodianManager); + if (!_.isEmpty(member)) { + // deconstruct members + ({ notifications = [] } = member); + // if the custodian has notifications + if (!_.isEmpty(notifications)) { + // find the notification type in the notifications array + custodianNotificationObj = findByNotificationType(notifications, notificationType); + if (!_.isEmpty(custodianNotificationObj)) { + ({ optIn } = custodianNotificationObj); + if (optIn) return [...acc, { email: custodianManager.email }]; + else return acc; + } + } else { + // if no notifications found optIn by default (safeguard) + return [...acc, { email: custodianManager.email }]; + } + } + } + }, []); + } else { + return []; + } +}; + +/** + * [Get subscribedEmails from optIn status ] + * + * @param {Boolean} optIn [optIn Status ] + * @param {Array} subscribedEmails [the list of subscribed emails for notification type] + * @return {Array} [formatted array of [{email: email}]] + */ +const getTeamNotificationEmails = (optIn = false, subscribedEmails) => { + if (optIn && !_.isEmpty(subscribedEmails)) { + return [...subscribedEmails].map(email => ({ email })); + } + + return []; +}; + const createNotifications = async (type, context, team, user) => { const teamName = getTeamName(team); let options = {}; @@ -365,18 +739,12 @@ const createNotifications = async (type, context, team, user) => { teamName, }; html = emailGenerator.generateRemovedFromTeam(options); - emailGenerator.sendEmail( - [removedUser], - constants.hdrukEmail, - `You have been removed from the team ${teamName}`, - html, - false - ); + emailGenerator.sendEmail([removedUser], constants.hdrukEmail, `You have been removed from the team ${teamName}`, html, false); break; case constants.notificationTypes.MEMBERADDED: // 1. Get users added const { newUsers } = context; - const newUserIds = newUsers.map((user) => user.id); + const newUserIds = newUsers.map(user => user.id); // 2. Create user notifications notificationBuilder.triggerNotificationMessage( newUserIds, @@ -416,13 +784,72 @@ const createNotifications = async (type, context, team, user) => { } }; +const formatTeamNotifications = team => { + let { notifications = [] } = team; + if (!_.isEmpty(notifications)) { + // 1. reduce for mapping over team notifications + return [...notifications].reduce((arr, notification) => { + let teamNotificationEmails = []; + let { notificationType = '', optIn = false, subscribedEmails = [] } = notification; + // 2. check subscribedEmails has length + if (!_.isEmpty(subscribedEmails)) teamNotificationEmails = [...subscribedEmails].map(email => ({ value: email, error: '' })); + else teamNotificationEmails = [{ value: '', error: '' }]; + + // 3. return optimal payload for formated notification + let formattedNotification = { + notificationType, + optIn, + subscribedEmails: teamNotificationEmails, + }; + + arr = [...arr, formattedNotification]; + + return arr; + }, []); + } else { + return []; + } +}; + +const findMissingOptIns = (memberNotifications, teamNotifications) => { + return [...memberNotifications].reduce((neededOptIns, memberNotification) => { + let { notificationType: memberNotificationType, optIn: memberOptIn } = memberNotification; + // find the matching notification type within the teams notification + let teamNotification = + [...teamNotifications].find(teamNotification => teamNotification.notificationType === memberNotificationType) || {}; + // if the team has the same notification type test + if (!_.isEmpty(teamNotification)) { + let { notificationType, optIn: teamOptIn, subscribedEmails } = teamNotification; + // if both are turned off build and return new error + if ((!teamOptIn && !memberOptIn) || (!memberOptIn && subscribedEmails.length <= 0)) { + neededOptIns = { + ...neededOptIns, + [`${notificationType}`]: `Notifications must be enabled for ${constants.teamNotificationTypesHuman[notificationType]}`, + }; + } + } + return neededOptIns; + }, {}); +}; + export default { getTeamById: getTeamById, + getTeamNotificationByType: getTeamNotificationByType, + getTeamNotificationEmails: getTeamNotificationEmails, + findTeamMemberById: findTeamMemberById, + findByNotificationType: findByNotificationType, + filterMembersByNoticationTypes: filterMembersByNoticationTypes, + filterMembersByNoticationTypesOptIn: filterMembersByNoticationTypesOptIn, + buildOptedInEmailList: buildOptedInEmailList, getTeamMembers: getTeamMembers, + getMemberDetails: getMemberDetails, + getTeamNotifications: getTeamNotifications, addTeamMembers: addTeamMembers, updateTeamMember: updateTeamMember, + updateNotifications: updateNotifications, + updateNotificationMessages: updateNotificationMessages, deleteTeamMember: deleteTeamMember, checkTeamPermissions: checkTeamPermissions, getTeamMembersByRole: getTeamMembersByRole, - createNotifications: createNotifications + createNotifications: createNotifications, }; diff --git a/src/resources/team/team.model.js b/src/resources/team/team.model.js index ed5df37a..3797aba7 100644 --- a/src/resources/team/team.model.js +++ b/src/resources/team/team.model.js @@ -1,4 +1,5 @@ import { model, Schema } from 'mongoose'; +import constants from '../utilities/constants.util'; const TeamSchema = new Schema( { @@ -14,8 +15,12 @@ const TeamSchema = new Schema( dateUpdated: Date, notifications: [ { - type: String, // metadataonbarding || dataaccessrequest + notificationType:{ + type: String, + enum: Object.values(constants.teamNotificationTypes) + } , // metadataonbarding || dataaccessrequest optIn: { type: Boolean, default: true }, + message: String }, ], }, @@ -27,7 +32,11 @@ const TeamSchema = new Schema( }, notifications: [ { - type: String, // metadataonbarding || dataaccessrequest + notificationType: { + type: String, // metadataonbarding || dataaccessrequest + default: constants.teamNotificationTypes.DATAACCESSREQUEST, + enum: Object.values(constants.teamNotificationTypes), + }, optIn: { type: Boolean, default: false }, subscribedEmails: [String], }, diff --git a/src/resources/team/team.route.js b/src/resources/team/team.route.js index 1983654e..798a230d 100644 --- a/src/resources/team/team.route.js +++ b/src/resources/team/team.route.js @@ -20,14 +20,30 @@ router.get('/:id/members', passport.authenticate('jwt'), teamController.getTeamM // @access Private router.post('/:id/members', passport.authenticate('jwt'), teamController.addTeamMembers); -// @route PUT api/teams/:id/members +// @route PUT api/v1/teams/:id/members // @desc Edit a team member // @access Private router.put('/:id/members/:memberid', passport.authenticate('jwt'), teamController.updateTeamMember); + // @route DELETE api/teams/:id/members // @desc Delete a team member // @access Private router.delete('/:id/members/:memberid', passport.authenticate('jwt'), teamController.deleteTeamMember); +// @route GET api/v1/teams/:id/notifications +// @desc Get team notifications +// @access Private +router.get('/:id/notifications', passport.authenticate('jwt'), teamController.getTeamNotifications); + +// @route PUT api/v1/teams/:id/notifications +// @desc Update notifications +// @access Private +router.put('/:id/notifications', passport.authenticate('jwt'), teamController.updateNotifications); + +// @route PUT api/v1/teams/:id/notification-messages +// @desc Update notifications +// @access Private +router.put('/:id/notification-messages', passport.authenticate('jwt'), teamController.updateNotificationMessages); + module.exports = router; diff --git a/src/resources/tool/data.model.js b/src/resources/tool/data.model.js index 90814971..f8aa24d4 100644 --- a/src/resources/tool/data.model.js +++ b/src/resources/tool/data.model.js @@ -63,7 +63,7 @@ const DataSchema = new Schema( showBio: Boolean, orcid: String, showOrcid: Boolean, - emailNotifications: Boolean, + emailNotifications: { type: Boolean, default: true }, terms: Boolean, sector: String, showSector: Boolean, @@ -157,4 +157,11 @@ DataSchema.virtual('persons', { localField: 'authors', }); +DataSchema.virtual('user', { + ref: 'User', + foreignField: 'id', + localField: 'id', + justOne: true +}); + export const Data = model('Data', DataSchema); diff --git a/src/resources/tool/data.repository.js b/src/resources/tool/data.repository.js index e7ee31ae..756cb575 100644 --- a/src/resources/tool/data.repository.js +++ b/src/resources/tool/data.repository.js @@ -460,7 +460,7 @@ async function sendEmailNotifications(tool, activeflag, rejectionReason) { if (err) { return new Error({ success: false, error: err }); } - emailGenerator.sendEmail(emailRecipients, `${hdrukEmail}`, subject, html); + emailGenerator.sendEmail(emailRecipients, `${hdrukEmail}`, subject, html, false); }); } @@ -489,7 +489,8 @@ async function sendEmailNotificationToAuthors(tool, toolOwner) { emailRecipients, `${hdrukEmail}`, `${toolOwner.name} added you as an author of the tool ${tool.name}`, - `${toolOwner.name} added you as an author of the tool ${tool.name}

${toolLink}` + `${toolOwner.name} added you as an author of the tool ${tool.name}

${toolLink}`, + false ); }); } diff --git a/src/resources/tool/v1/tool.route.js b/src/resources/tool/v1/tool.route.js index b5541317..7f5af54b 100644 --- a/src/resources/tool/v1/tool.route.js +++ b/src/resources/tool/v1/tool.route.js @@ -12,6 +12,7 @@ import inputSanitizer from '../../utilities/inputSanitizer'; import _ from 'lodash'; import helper from '../../utilities/helper.util'; import escape from 'escape-html'; +import helperUtil from '../../utilities/helper.util'; const hdrukEmail = `enquiry@healthdatagateway.org`; const router = express.Router(); @@ -43,15 +44,15 @@ router.put('/:id', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin }); // @router GET /api/v1/get/admin -// @desc Returns List of Tool objects +// @desc Returns List of Tool objects // @access Private router.get('/getList', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, ROLES.Creator), async (req, res) => { req.params.type = 'tool'; let role = req.user.role; if (role === ROLES.Admin) { - await getToolsAdmin(req) - .then(data => { + await getToolsAdmin(req) + .then(data => { return res.json({ success: true, data }); }) .catch(err => { @@ -68,7 +69,7 @@ router.get('/getList', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.A } }); -// @router GET /api/v1/ +// @router GET /api/v1/ // @desc Returns List of Tool Objects No auth // This unauthenticated route was created specifically for API-docs // @access Public @@ -94,7 +95,7 @@ router.patch('/:id', passport.authenticate('jwt'), async (req, res) => { .catch(err => { return res.json({ success: false, error: err.message }); }); -}); +}); /** * {get} /tool/:id Tool @@ -348,13 +349,15 @@ router.delete('/review/delete', passport.authenticate('jwt'), utils.checkIsInRol // } // ); -// @router GET /api/v1/project/tag/name +// @router GET /api/v1/project/tag/ // @desc Get tools by tag search // @access Private -router.get('/:type/tag/:name', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, ROLES.Creator), async (req, res) => { +router.get('/:type/tag', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, ROLES.Creator), async (req, res) => { try { // 1. Destructure tag name parameter passed - let { type, name } = req.params; + let { type } = req.params; + let { name } = req.query; + // 2. Check if parameters are empty if (_.isEmpty(name) || _.isEmpty(type)) { return res.status(400).json({ @@ -362,10 +365,21 @@ router.get('/:type/tag/:name', passport.authenticate('jwt'), utils.checkIsInRole message: 'Entity type and tag are required', }); } + // 3. Find matching projects in MongoDb selecting name and id - let entities = await Data.find({ - $and: [{ type }, { $or: [{ 'tags.topics': name }, { 'tags.features': name }] }], - }).select('id name'); + let filterValues = name.split(','); + let searchQuery = { $and: [{ type }] }; + searchQuery['$and'].push({ + $or: filterValues.map(value => { + return { + $or: [ + { 'tags.topics': { $regex: helperUtil.escapeRegexChars(value), $options: 'i' } }, + { 'tags.features': { $regex: helperUtil.escapeRegexChars(value), $options: 'i' } }, + ], + }; + }), + }); + let entities = await Data.find(searchQuery).select('id name'); // 4. Return projects return res.status(200).json({ success: true, entities }); } catch (err) { @@ -455,7 +469,8 @@ async function sendEmailNotifications(review) { emailRecipients, `${hdrukEmail}`, `Someone reviewed your tool`, - `${reviewer.firstname} ${reviewer.lastname} gave a ${review.rating}-star review to your tool ${tool.name}

${toolLink}` + `${reviewer.firstname} ${reviewer.lastname} gave a ${review.rating}-star review to your tool ${tool.name}

${toolLink}`, + false ); }); } diff --git a/src/resources/user/user.model.js b/src/resources/user/user.model.js index 9ec59574..49f5ce3e 100644 --- a/src/resources/user/user.model.js +++ b/src/resources/user/user.model.js @@ -7,6 +7,8 @@ const UserSchema = new Schema( unique: true, }, email: String, + feedback: { type: Boolean, default: false }, //email subscription + news: { type: Boolean, default: false }, //email subscription password: String, businessName: String, firstname: String, @@ -14,6 +16,7 @@ const UserSchema = new Schema( displayname: String, providerId: { type: String, required: true }, provider: String, + affiliation: String, role: String, redirectURL: String, discourseUsername: String, diff --git a/src/resources/user/user.register.route.js b/src/resources/user/user.register.route.js index e67ad06f..b3a8fac2 100644 --- a/src/resources/user/user.register.route.js +++ b/src/resources/user/user.register.route.js @@ -5,6 +5,7 @@ import { updateUser } from '../user/user.service'; import { createPerson } from '../person/person.service'; import { getUserByUserId } from '../user/user.repository'; import { registerDiscourseUser } from '../discourse/discourse.service'; +import mailchimpConnector from '../../services/mailchimp/mailchimp'; const urlValidator = require('../utilities/urlValidator'); const eventLogController = require('../eventlog/eventlog.controller'); const router = express.Router(); @@ -32,11 +33,12 @@ router.post('/', async (req, res) => { showBio, showLink, showOrcid, - redirectURL, + redirectURL: redirectURLis = '', sector, showSector, organisation, - emailNotifications, + feedback, + news, terms, tags, showDomain, @@ -45,7 +47,6 @@ router.post('/', async (req, res) => { } = req.body; let link = urlValidator.validateURL(req.body.link); let orcid = urlValidator.validateOrcidURL(req.body.orcid); - let username = `${firstname.toLowerCase()}.${lastname.toLowerCase()}`; let discourseUsername, discourseKey = ''; @@ -62,11 +63,13 @@ router.post('/', async (req, res) => { email, discourseKey, discourseUsername, + feedback, + news, }) ); // 2. Create person entry in tools - let [personErr, person] = await to( + await to( createPerson({ id, firstname, @@ -77,7 +80,6 @@ router.post('/', async (req, res) => { showLink, orcid, showOrcid, - emailNotifications, terms, sector, showSector, @@ -97,6 +99,16 @@ router.post('/', async (req, res) => { email, }); + // 4. Create subscriptions in MailChimp for news and feedback if opted in + if (news) { + const newsSubscriptionId = process.env.MAILCHIMP_NEWS_AUDIENCE_ID; + await mailchimpConnector.addSubscriptionMember(newsSubscriptionId, user); + } + if (feedback) { + const feedbackSubscriptionId = process.env.MAILCHIMP_FEEDBACK_AUDIENCE_ID; + await mailchimpConnector.addSubscriptionMember(feedbackSubscriptionId, user); + } + const [loginErr, token] = await to(login(req, user)); if (loginErr) { @@ -104,12 +116,6 @@ router.post('/', async (req, res) => { return res.status(500).json({ success: false, data: 'Authentication error!' }); } - var redirectURLis = redirectURL; - - if (redirectURLis === null || redirectURLis === '') { - redirectURLis = ''; - } - //Build event object for user registered and log it to DB let eventObj = { userId: req.user.id, diff --git a/src/resources/user/user.route.js b/src/resources/user/user.route.js index 3a4cb5e7..526a3dff 100644 --- a/src/resources/user/user.route.js +++ b/src/resources/user/user.route.js @@ -7,7 +7,7 @@ import { utils } from '../auth'; import { UserModel } from './user.model'; import { Data } from '../tool/data.model'; import helper from '../utilities/helper.util'; -import { createServiceAccount } from './user.repository'; +//import { createServiceAccount } from './user.repository'; const router = express.Router(); diff --git a/src/resources/user/user.service.js b/src/resources/user/user.service.js index 0e4de56d..27787cf0 100644 --- a/src/resources/user/user.service.js +++ b/src/resources/user/user.service.js @@ -18,7 +18,7 @@ export async function createUser({ firstname, lastname, email, providerId, provi }); } -export async function updateUser({ id, firstname, lastname, email, discourseKey, discourseUsername }) { +export async function updateUser({ id, firstname, lastname, email, discourseKey, discourseUsername, feedback, news }) { return new Promise(async (resolve, reject) => { return resolve( await UserModel.findOneAndUpdate( @@ -29,6 +29,8 @@ export async function updateUser({ id, firstname, lastname, email, discourseKey, email, discourseKey, discourseUsername, + feedback, + news } ) ); diff --git a/src/resources/utilities/constants.util.js b/src/resources/utilities/constants.util.js index d134c87d..84c61fc1 100644 --- a/src/resources/utilities/constants.util.js +++ b/src/resources/utilities/constants.util.js @@ -9,6 +9,31 @@ const _formTypes = Object.freeze({ Extended5Safe: '5 safe', }); +const _teamNotificationTypes = Object.freeze({ + DATAACCESSREQUEST: 'dataAccessRequest', + METADATAONBOARDING: 'metaDataOnboarding', +}); + +const _teamNotificationMessages = { + DATAACCESSREQUEST: 'A team manager removed team email addresses. Your email notifications are now being sent to your gateway email', +}; + +const _teamNotificationEmailContentTypes = { + TEAMEMAILHEADERADD: 'A team manager has added a new team email address', + TEAMEMAILHEADEREMOVE: 'A team manager has removed a team email address', + TEAMEMAILSUBHEADERADD: + 'has added a new team email address. All emails relating to pre-submission messages from researchers will be sent to the following email addresses:', + TEAMEMAILSUBHEADEREMOVE: + 'has removed a team email address. All emails relating to pre-submission messages from researchers will no longer be sent to the following email addresses:', + TEAMEMAILFOOTERREMOVE: + 'If you had stopped emails being sent to your gateway log in email address and no team email address is now active, your emails will have reverted back to your gateway log in email.', +}; + +const _teamNotificationTypesHuman = Object.freeze({ + dataAccessRequest: 'Data access request', + metaDataOnboarding: 'Meta data on-boarding', +}); + const _enquiryFormId = '5f0c4af5d138d3e486270031'; const _userQuestionActions = { @@ -367,10 +392,26 @@ const _datatsetStatuses = { const _hdrukEmail = 'enquiry@healthdatagateway.org'; +const _mailchimpSubscriptionStatuses = { + SUBSCRIBED: 'subscribed', + UNSUBSCRIBED: 'unsubscribed', + CLEANED: 'cleaned', + PENDING: 'pending', +}; + +const _logTypes = { + SYSTEM: 'System', + USER: 'User' +} + export default { userTypes: _userTypes, enquiryFormId: _enquiryFormId, formTypes: _formTypes, + teamNotificationTypes: _teamNotificationTypes, + teamNotificationMessages: _teamNotificationMessages, + teamNotificationTypesHuman: _teamNotificationTypesHuman, + teamNotificationEmailContentTypes: _teamNotificationEmailContentTypes, userQuestionActions: _userQuestionActions, navigationFlags: _navigationFlags, amendmentStatuses: _amendmentStatuses, @@ -383,5 +424,7 @@ export default { darPanelMapper: _darPanelMapper, submissionEmailRecipientTypes: _submissionEmailRecipientTypes, hdrukEmail: _hdrukEmail, + mailchimpSubscriptionStatuses: _mailchimpSubscriptionStatuses, datatsetStatuses: _datatsetStatuses, + logTypes: _logTypes }; diff --git a/src/resources/utilities/emailGenerator.util.js b/src/resources/utilities/emailGenerator.util.js index ca12a2d0..a9b37f10 100644 --- a/src/resources/utilities/emailGenerator.util.js +++ b/src/resources/utilities/emailGenerator.util.js @@ -2,8 +2,8 @@ import _, { isNil, isEmpty, capitalize, groupBy, forEach } from 'lodash'; import moment from 'moment'; import { UserModel } from '../user/user.model'; import helper from '../utilities/helper.util'; -import teamController from '../team/team.controller'; import constants from '../utilities/constants.util'; +import * as Sentry from '@sentry/node'; const sgMail = require('@sendgrid/mail'); let parent, qsId; @@ -1579,6 +1579,80 @@ const _generateRemovedFromTeam = options => { return body; }; +const _displayViewEmailNotifications = publisherId => { + let link = `${process.env.homeURL}/account?tab=teamManagement&innertab=notifications&team=${publisherId}`; + return ` + + + +
+ View email notifications +
`; +}; + +const _formatEmails = emails => { + return [...emails].map((email, i) => ` ${email}`); +}; + +const _generateTeamNotificationEmail = options => { + let { managerName, notificationRemoved, emailAddresses, header, disabled, publisherId } = options; + let formattedEmails = _formatEmails(emailAddresses); + + let body = `
+ + + + + + + + + + + + + + +
+ ${header} +
+ ${ + notificationRemoved + ? `${managerName} ${constants.teamNotificationEmailContentTypes.TEAMEMAILSUBHEADEREMOVE}` + : `${managerName} ${constants.teamNotificationEmailContentTypes.TEAMEMAILSUBHEADERADD}` + } +
+ + + + + +
Team email address + ${formattedEmails} +
+ ${disabled ? _generateTeamEmailRevert(notificationRemoved) : ''} + ${_displayViewEmailNotifications(publisherId)} +
+
`; + return body; +}; + +const _generateTeamEmailRevert = notificationRemoved => { + return ` + + + +
+ If you had stopped emails being sent to your gateway log in email address and no team email address is now active, your emails will have reverted back to your gateway log in email. +
`; +}; + const _generateAddedToTeam = options => { let { teamName, role } = options; let header = `You've been added to the ${teamName} team as a ${role} on the HDR Innovation Gateway`; @@ -1783,7 +1857,16 @@ const _sendEmail = async (to, from, subject, html, allowUnsubscribe = true, atta }; // 4. Send email using SendGrid - await sgMail.send(msg); + await sgMail.send(msg, false, err => { + if (err) { + Sentry.addBreadcrumb({ + category: 'SendGrid', + message: 'Sending email failed', + level: Sentry.Severity.Warning, + }); + Sentry.captureException(err); + } + }); } }; @@ -1858,6 +1941,7 @@ export default { generateReviewDeadlineWarning: _generateReviewDeadlineWarning, generateReviewDeadlinePassed: _generateReviewDeadlinePassed, generateFinalDecisionRequiredEmail: _generateFinalDecisionRequiredEmail, + generateTeamNotificationEmail: _generateTeamNotificationEmail, generateRemovedFromTeam: _generateRemovedFromTeam, generateAddedToTeam: _generateAddedToTeam, //Workflows diff --git a/src/resources/utilities/ga4gh.utils.js b/src/resources/utilities/ga4gh.utils.js new file mode 100644 index 00000000..739c5bea --- /dev/null +++ b/src/resources/utilities/ga4gh.utils.js @@ -0,0 +1,127 @@ +import { signToken } from '../auth/utils'; +import { DataRequestModel } from '../datarequest/datarequest.model'; +import { Data } from '../tool/data.model'; + +const _buildGa4ghVisas = async user => { + let passportDecoded = [], + passportEncoded = []; + + //AffiliationAndRole + if (user.provider === 'oidc') { + passportDecoded.push({ + iss: 'https://www.healthdatagateway.org', + sub: user.id, + ga4gh_visa_v1: { + type: 'AffiliationAndRole', + asserted: user.createdAt.getTime(), + value: user.affiliation || 'no.organization', //open athens EDUPersonRole + source: 'https://www.healthdatagateway.org', //TODO: update when value confirmed + }, + }); + } + + //AcceptTermsAndPolicies + passportDecoded.push({ + iss: 'https://www.healthdatagateway.org', + sub: user.id, + ga4gh_visa_v1: { + type: 'AcceptTermsAndPolicies', + asserted: user.createdAt.getTime(), + value: 'https://www.hdruk.ac.uk/infrastructure/gateway/terms-and-conditions/', + source: 'https://www.healthdatagateway.org', + by: 'self', + }, + }); + + if (user.acceptedAdvancedSearchTerms) { + passportDecoded.push({ + iss: 'https://www.healthdatagateway.org', + sub: user.id, + ga4gh_visa_v1: { + type: 'AcceptTermsAndPolicies', + asserted: user.createdAt.getTime(), + value: 'https://www.healthdatagateway.org/advanced-search-terms/', + source: 'https://www.healthdatagateway.org', + by: 'self', + }, + }); + } + + //ResearcherStatus + passportDecoded.push({ + iss: 'https://www.healthdatagateway.org', + sub: user.id, + ga4gh_visa_v1: { + type: 'ResearcherStatus', + asserted: user.createdAt.getTime(), + value: getResearchStatus(user), + source: 'https://www.healthdatagateway.org', + }, + }); + + //ControlledAccessGrants + let approvedDARApplications = await getApprovedDARApplications(user); + approvedDARApplications.forEach(dar => { + passportDecoded.push({ + iss: 'https://www.healthdatagateway.org', + sub: user.id, + ga4gh_visa_v1: { + type: 'ControlledAccessGrants', + asserted: dar.dateFinalStatus.getTime(), //date DAR was approved + value: dar.pids.map(pid => { + return 'https://web.www.healthdatagateway.org/dataset/' + pid; + }), //URL to each dataset that they have been approved for + source: 'https://www.healthdatagateway.org', + by: 'dac', + }, + }); + }); + + passportDecoded.forEach(visa => { + const expires_in = 900; + const jwt = signToken(visa, expires_in); + passportEncoded.push(jwt); + }); + + return passportEncoded; +}; + +const getApprovedDARApplications = async user => { + let approvedApplications = await DataRequestModel.find( + { $and: [{ userId: user.id }, { applicationStatus: { $in: ['approved', 'approved with conditions'] } }] }, + { datasetIds: 1, dateFinalStatus: 1 } + ).lean(); + + let approvedApplicationsWithPIDs = Promise.all( + approvedApplications.map(async dar => { + let pids = await Promise.all( + dar.datasetIds.map(async datasetId => { + let result = await Data.findOne({ datasetid: datasetId }, { pid: 1 }).lean(); + return result.pid; + }) + ); + return { pids, dateFinalStatus: dar.dateFinalStatus }; + }) + ); + + return approvedApplicationsWithPIDs; +}; + +const getResearchStatus = user => { + const statuses = { + UNKNOWN: 'unknown', + BONAFIDE: 'bona fide', + ACCREDITED: 'accredited', + APPROVED: 'approved', + }; + if (user.provider != 'oidc') { + return statuses.UNKNOWN; + } else { + return statuses.BONAFIDE; + } + // TODO: Integrate with ONS API when it becomes available +}; + +export default { + buildGa4ghVisas: _buildGa4ghVisas, +}; diff --git a/src/resources/utilities/logger.js b/src/resources/utilities/logger.js new file mode 100644 index 00000000..ad6a579f --- /dev/null +++ b/src/resources/utilities/logger.js @@ -0,0 +1,51 @@ +import * as Sentry from '@sentry/node'; +import constants from './constants.util'; + +const logRequestMiddleware = options => { + return (req, res, next) => { + const { logCategory, action } = options; + logger.logUserActivity(req.user, logCategory, constants.logTypes.USER, { action }); + next(); + }; +}; + +const logSystemActivity = options => { + const { category = 'Action not categorised', action = 'Action not described' } = options; + Sentry.addBreadcrumb({ + category, + message: action, + level: Sentry.Severity.Info, + }); + // Save to database +}; + +const logUserActivity = (user, category, type, context) => { + const { action } = context; + Sentry.addBreadcrumb({ + category, + message: action, + level: Sentry.Severity.Info, + }); + console.log(`${action}`); + // Log date/time + // Log action + // Log if user was logged in + // Log userId and _id + // Save to database +}; + +const logError = (err, category) => { + Sentry.captureException(err, { + tags: { + area: category, + }, + }); + console.error(`The following error occurred: ${err.message}`); +}; + +export const logger = { + logRequestMiddleware, + logSystemActivity, + logUserActivity, + logError, +}; diff --git a/src/services/mailchimp/mailchimp.js b/src/services/mailchimp/mailchimp.js new file mode 100644 index 00000000..3eb3b4dd --- /dev/null +++ b/src/services/mailchimp/mailchimp.js @@ -0,0 +1,254 @@ +import Mailchimp from 'mailchimp-api-v3'; +import * as Sentry from '@sentry/node'; +import Crypto from 'crypto'; +import constants from '../../resources/utilities/constants.util'; +import { UserModel } from '../../resources/user/user.model'; + +// See MailChimp API documentation for supporting info https://mailchimp.com/developer/marketing/api/ + +// Default service params +const apiKey = process.env.MAILCHIMP_API_KEY; +let mailchimp; +if (apiKey) mailchimp = new Mailchimp(apiKey); +const tags = ['Gateway User']; +const defaultSubscriptionStatus = constants.mailchimpSubscriptionStatuses.SUBSCRIBED; + +/** + * Create MailChimp Subscription Subscriber + * + * @desc Adds a subscriber to a subscription with the status provided + * @param {String} subscriptionId Unique identifier for a subscription to update + * @param {String} user User object containing bio details + * @param {String} status New status to assign to an email address for a subscription - subscribed or pending + */ +const addSubscriptionMember = async (subscriptionId, user, status) => { + if (apiKey) { + // 1. Assemble payload POST body for MailChimp Marketing API + let { id, email, firstname, lastname } = user; + const body = { + email_address: email, + status: status || defaultSubscriptionStatus, + status_if_new: status || defaultSubscriptionStatus, + tags, + merge_fields: { + FNAME: firstname, + LNAME: lastname, + }, + }; + // 2. Track attempted update in Sentry using log + Sentry.addBreadcrumb({ + category: 'MailChimp', + message: `Adding subscription for user: ${id} to subscription: ${subscriptionId}`, + level: Sentry.Severity.Log, + }); + // 3. POST to MailChimp Marketing API to add the Gateway user to the MailChimp subscription members + const md5email = Crypto.createHash('md5').update(email).digest('hex'); + await mailchimp.put(`lists/${subscriptionId}/members/${md5email}`, body).catch(err => { + Sentry.captureException(err); + console.error(`Message: ${err.message} Errors: ${JSON.stringify(err.errors)}`); + }); + } +}; + +/** + * Update MailChimp Subscription Membership For Gateway Users + * + * @desc Updates memberships to a MailChimp subscription for a list of Gateway Users + * @param {String} subscriptionId Unique identifier for a subscription to update + * @param {Array} users List of Gateway Users (max 500 per operation) + * @param {String} status New status to assign to each membership for a subscription - subscribed, unsubscribed, cleaned or pending + */ +const updateSubscriptionUsers = async (subscriptionId, users = [], status) => { + if (apiKey) { + // 1. Build members array providing required metadata for MailChimp + const members = users.map(user => { + return { + userId: user.id, + email_address: user.email, + status, + tags, + merge_fields: { + FNAME: user.firstName, + LNAME: user.lastName, + }, + }; + }); + // 2. Update subscription members in MailChimp + await updateSubscriptionMembers(subscriptionId, members); + } +}; + +/** + * Update MailChimp Subscription Subscribers + * + * @desc Updates a subscription with a new status for each provided email address, new email addresses will be added + * @param {String} subscriptionId Unique identifier for a subscription to update + * @param {Array} members List of email addresses to update (max 500) + */ +const updateSubscriptionMembers = async (subscriptionId, members) => { + if (apiKey) { + // 1. Assemble payload POST body for MailChimp Marketing API + const body = { + members, + skip_merge_validation: true, + skip_duplicate_check: true, + update_existing: true, + }; + // 2. Track attempted updates in Sentry using log + Sentry.addBreadcrumb({ + category: 'MailChimp', + message: `Updating subscribed for members: ${members.map( + member => `${member.userId} to ${member.status}` + )} against subscription: ${subscriptionId}`, + level: Sentry.Severity.Log, + }); + // 3. POST to MailChimp Marketing API to update member statuses + await mailchimp.post(`lists/${subscriptionId}`, body).catch(err => { + Sentry.captureException(err); + console.error(`Message: ${err.message} Errors: ${JSON.stringify(err.errors)}`); + }); + } +}; + +/** + * Sync Gateway User With MailChimp Members Subscription Statuses + * + * @desc Synchronises all subscription statuses between Gateway users based on account preferences with any changes reflected in MailChimp + * via external subscribe or unsubscribe methods e.g. mail link or sign up form sent via campaign + * @param {String} subscriptionId Unique identifier for a subscription to update + */ +const syncSubscriptionMembers = async subscriptionId => { + if (apiKey) { + // 1. Track attempted sync in Sentry using log + Sentry.addBreadcrumb({ + category: 'MailChimp', + message: `Syncing users for subscription: ${subscriptionId}`, + level: Sentry.Severity.Log, + }); + // 2. Get total member count to anticipate chunking required to process all contacts + const { + stats: { member_count: subscribedCount, unsubscribe_count: unsubscribedCount }, + } = await mailchimp.get(`lists/${subscriptionId}?fields=stats.member_count,stats.unsubscribe_count`).catch(err => { + Sentry.captureException(err); + console.error(`Message: ${err.message} Errors: ${JSON.stringify(err.errors)}`); + }); + const memberCount = subscribedCount + unsubscribedCount; + // 3. Batch update database to sync MailChimp to reflect users unsubscribed/subscribed externally + await batchImportFromMailChimp(subscriptionId, memberCount); + // 4. Push any unsynchronised new contacts from Gateway to MailChimp + await batchExportToMailChimp(subscriptionId); + } +}; + +/** + * Trigger MailChimp Status Import For Subscription + * + * @desc Triggers an import of all member statuses for a subscription and updates the database with all changes found in MailChimp + * @param {String} subscriptionId Unique identifier for a subscription to update + * @param {Number} memberCount The number of members currently registered against a subscription in MailChimp + * @param {String} subscriptionBoolKey The name of the boolean key that tracks the subscription status in the Gateway database + */ +const batchImportFromMailChimp = async (subscriptionId, memberCount) => { + if (apiKey) { + // 1. Get corresponding Gateway subscription variable e.g. feedback, news + const subscriptionBoolKey = getSubscriptionBoolKey(subscriptionId); + let processedCount = 0; + // 2. Iterate bulk update process until all contacts have been processed from MailChimp + while (processedCount < memberCount) { + const { members = [] } = await mailchimp.get(`lists/${subscriptionId}/members?count=100&offset=${processedCount}`); + let ops = []; + // 3. For each member returned by MailChimp, create a bulk update operation to update the corresponding Gateway user if they exist + members.forEach(member => { + const subscribedBoolValue = member.status === 'subscribed' ? true : false; + const { email_address: email } = member; + ops.push({ + updateMany: { + filter: { email }, + update: { + [subscriptionBoolKey]: subscribedBoolValue, + }, + upsert: false, + }, + }); + }); + // 4. Run bulk update + await UserModel.bulkWrite(ops); + // 5. Increment counter to move onto next chunk of members + processedCount = processedCount + members.length; + } + } +}; + +/** + * Sync MailChimp Subscription Members With Users In Gateway + * + * @desc Updates Subscription In MailChimp + * @param {String} subscriptionId Unique identifier for a subscription to find the database key for + */ +const batchExportToMailChimp = async subscriptionId => { + if (apiKey) { + const subscriptionBoolKey = getSubscriptionBoolKey(subscriptionId); + // 1. Get all users from db + const users = await UserModel.find().select('id email firstname lastname news feedback').lean(); + // 2. Build members array providing required metadata for MailChimp + const members = users.reduce((arr, user) => { + // Get subscription status from user profile + const status = user[subscriptionBoolKey] + ? constants.mailchimpSubscriptionStatuses.SUBSCRIBED + : constants.mailchimpSubscriptionStatuses.UNSUBSCRIBED; + // Check if the same email address has already been processed (email address can be attached to multiple user accounts) + const memberIdx = arr.findIndex(member => member.email_address === user.email); + if (memberIdx === -1) { + // If email address has not be processed, return updated membership object + return [ + ...arr, + { + userId: user.id, + email_address: user.email, + status, + tags, + merge_fields: { + FNAME: user.firstname, + LNAME: user.lastname, + }, + }, + ]; + } else { + // If email address has been processed, and the current status is unsubscribed, override membership status + if (status === constants.mailchimpSubscriptionStatuses.UNSUBSCRIBED) { + arr[memberIdx].status = constants.mailchimpSubscriptionStatuses.UNSUBSCRIBED; + } + return arr; + } + }, []); + // 3. Update subscription members in MailChimp + await updateSubscriptionMembers(subscriptionId, members); + } +}; + +/** + * Determine Database Key For Subscription Status + * + * @desc Returns the database key used to track the opt-in status per user for a subscription + * @param {String} subscriptionId Unique identifier for a subscription to find the database key for + */ +const getSubscriptionBoolKey = subscriptionId => { + // 1. Extract variables from environment settings + const newsSubscriptionId = process.env.MAILCHIMP_NEWS_AUDIENCE_ID; + const feedbackSubscriptionId = process.env.MAILCHIMP_FEEDBACK_AUDIENCE_ID; + // 2. Assemble lookup table + const lookup = { + [newsSubscriptionId]: 'news', + [feedbackSubscriptionId]: 'feedback', + }; + // 3. Return relevant key for subscription + return lookup[subscriptionId]; +}; + +const mailchimpConnector = { + addSubscriptionMember, + updateSubscriptionUsers, + syncSubscriptionMembers, +}; + +export default mailchimpConnector; diff --git a/src/services/mailchimp/mailchimp.route.js b/src/services/mailchimp/mailchimp.route.js new file mode 100644 index 00000000..cd023d3a --- /dev/null +++ b/src/services/mailchimp/mailchimp.route.js @@ -0,0 +1,40 @@ +import express from 'express'; +import * as Sentry from '@sentry/node'; +import mailchimpConnector from './mailchimp'; +const router = express.Router(); + +// @router GET /api/v1/mailchimp/:subscriptionId/sync +// @desc Performs a two-way sync of opt in preferences between MailChimp and the Gateway database +// @access Public (key required) +router.post('/sync', async (req, res) => { + try { + // Check to see if header is in json format + let parsedBody = {}; + if (req.header('content-type') === 'application/json') { + parsedBody = req.body; + } else { + parsedBody = JSON.parse(req.body); + } + // Check for key + if (parsedBody.key !== process.env.MAILCHIMP_SYNC_KEY) { + return res.status(400).json({ success: false, error: 'Sync could not be started' }); + } + // Throw error if parsing failed + if (parsedBody.error === true) { + throw new Error('Sync parsing error'); + } + let { subscriptionIds = [] } = parsedBody; + // Run sync job + for(const subscriptionId of subscriptionIds) { + mailchimpConnector.syncSubscriptionMembers(subscriptionId); + } + // Return response indicating job has started (do not await async import) + return res.status(200).json({ success: true, message: 'Sync started' }); + } catch (err) { + Sentry.captureException(err); + console.error(err.message); + return res.status(500).json({ success: false, message: 'Sync failed' }); + } +}); + +module.exports = router;