diff --git a/plugins/logger/api/api.js b/plugins/logger/api/api.js index e268dd77a09..5a16b9c82dd 100644 --- a/plugins/logger/api/api.js +++ b/plugins/logger/api/api.js @@ -4,7 +4,8 @@ var exported = {}, automaticStateManager = require('./helpers/automaticStateManager'), log = require('../../../api/utils/log.js')('logger:api'), { validateRead } = require('../../../api/utils/rights.js'); - +const JOB = require('../../../api/parts/jobs'); +const MAX_NUMBER_OF_LOG_ENTRIES = 1000; const FEATURE_NAME = 'logger'; var RequestLoggerStateEnum = { @@ -20,6 +21,13 @@ plugins.setConfigs("logger", { }); (function() { + plugins.register("/master", function() { + setTimeout(() => { + JOB.job('logger:clear', { max: MAX_NUMBER_OF_LOG_ENTRIES }) + .replace() + .schedule("every 5 minutes"); + }, 10000); + }); plugins.register("/permissions/features", function(ob) { ob.features.push(FEATURE_NAME); @@ -345,29 +353,19 @@ plugins.setConfigs("logger", { return true; } if (params.qstring.method === 'collection_info') { - validateRead(params, FEATURE_NAME, async function(parameters) { - try { - var stats = await common.db.collection('logs' + parameters.app_id).aggregate([ { $collStats: { storageStats: { } } } ]).toArray(); - common.returnOutput(parameters, {capped: stats?.[0]?.storageStats?.capped, max: stats?.[0]?.storageStats?.max}); - } - catch (ex) { - console.log("Failed fetching logs collection info: ", ex); - common.returnMessage(parameters, 400, 'Error fetching collection info'); - } + validateRead(params, FEATURE_NAME, function(parameters) { + common.db.collection('logs' + parameters.app_id).stats(function(err, stats) { + if (err) { + console.log("Failed fetching logs collection info", err); + return common.returnMessage(parameters, 400, 'Error fetching collection info'); + } + common.returnOutput(parameters, stats && {size: stats.size, count: stats.count, max: MAX_NUMBER_OF_LOG_ENTRIES} || {}); + }); }); return true; } }); - plugins.register("/i/apps/create", function(ob) { - var appId = ob.appId; - common.db.command({"convertToCapped": 'logs' + appId, size: 10000000, max: 1000}, function(err) { - if (err) { - common.db.createCollection('logs' + appId, {capped: true, size: 10000000, max: 1000}, function() {}); - } - }); - }); - plugins.register("/i/apps/delete", function(ob) { var appId = ob.appId; common.db.collection('logs' + appId).drop(function() {}); @@ -375,16 +373,12 @@ plugins.setConfigs("logger", { plugins.register("/i/apps/reset", function(ob) { var appId = ob.appId; - common.db.collection('logs' + appId).drop(function() { - common.db.createCollection('logs' + appId, {capped: true, size: 10000000, max: 1000}, function() {}); - }); + common.db.collection('logs' + appId).drop(function() {}); }); plugins.register("/i/apps/clear_all", function(ob) { var appId = ob.appId; - common.db.collection('logs' + appId).drop(function() { - common.db.createCollection('logs' + appId, {capped: true, size: 10000000, max: 1000}, function() {}); - }); + common.db.collection('logs' + appId).drop(function() {}); }); }(exported)); diff --git a/plugins/logger/api/jobs/clear.js b/plugins/logger/api/jobs/clear.js new file mode 100644 index 00000000000..ddf9376c217 --- /dev/null +++ b/plugins/logger/api/jobs/clear.js @@ -0,0 +1,48 @@ +/** + * @typedef {import("mongodb").Db} Database + * @typedef {import("mongodb").ObjectId} ObjectId + */ +const JOB = require("../../../../api/parts/jobs/job.js"); +const log = require("../../../../api/utils/log.js")("job:logger:clear"); + +const DEFAULT_MAX_ENTRIES = 1000; + +/** + * clears logs + */ +class ClearJob extends JOB.Job { + /** + * Cleans up the logs{APPID} collection + * @param {Database} db mongodb database instance + */ + async run(db) { + let max = this.data.max; + if (typeof max !== "number") { + log.e( + "Maximum number of log entries required. Falling back to default value:", + DEFAULT_MAX_ENTRIES + ); + max = DEFAULT_MAX_ENTRIES; + } + + log.d("Started: cleaning logs before the last", max); + const appIds = await db.collection("apps").find().project({ _id: 1 }).toArray(); + const loggerCollections = appIds.map(({_id}) => "logs" + _id.toString()); + + for (const colName of loggerCollections) { + const col = db.collection(colName); + // find the first entry we want to keep + const firstEntry = await col.find({}).project({ _id: 1 }).sort({ _id: -1 }) + .skip(max - 1).limit(1).toArray(); + if (!firstEntry[0]) { + continue; + } + // delete all previous entries + const result = await col.deleteMany({ _id: { $lt: firstEntry[0]._id } }); + log.d("Cleaned up", result.deletedCount, "entries from", colName); + } + log.d("Finished cleaning logs"); + } +} + +module.exports = ClearJob; \ No newline at end of file diff --git a/plugins/logger/install.js b/plugins/logger/install.js index ad9c9d8f5ca..e69de29bb2d 100644 --- a/plugins/logger/install.js +++ b/plugins/logger/install.js @@ -1,53 +0,0 @@ -var async = require('async'), - pluginManager = require('../pluginManager.js'); - -console.log("Installing logger plugin"); -pluginManager.dbConnection().then((countlyDb) => { - countlyDb.collection('apps').find({}).toArray(function(err, apps) { - - if (!apps || err) { - countlyDb.close(); - return; - } - function upgrade(app, done) { - console.log("Creating logs collection for " + app.name); - function cb() { - done(); - } - - countlyDb.command({ "listCollections": 1, "filter": { "name": "logs" + app._id } }, function(err, res) { - if (err) { - console.log(err); - cb(); - } - else { - //check if collection capped - if (res && res.cursor && res.cursor.firstBatch && res.cursor.firstBatch.length > 0) { - //collection exists - if (!res.cursor.firstBatch[0].options.capped) { - console.log("converting to the capped"); - countlyDb.command({ "convertToCapped": 'logs' + app._id, size: 10000000, max: 1000 }, - function(err) { - if (err) { - console.log(err); - } - cb(); - }); - } - else { - cb(); - } - } - else { //collection does not exist - console.log("collection does not exist"); - countlyDb.createCollection('logs' + app._id, { capped: true, size: 10000000, max: 1000 }, cb); - } - } - }); - } - async.forEach(apps, upgrade, function() { - console.log("Logger plugin installation finished"); - countlyDb.close(); - }); - }); -}); \ No newline at end of file