From c902705b0b2129588d9a706b066bdc6dbcdcded2 Mon Sep 17 00:00:00 2001 From: skadefro Date: Mon, 20 Nov 2023 19:26:46 +0100 Subject: [PATCH 01/32] bump --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c36a9572..6f47f670 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@openiap/openflow", - "version": "1.5.2", + "version": "1.5.3", "description": "Simple wrapper around NodeRed, RabbitMQ and MongoDB to support a more scaleable NodeRed implementation.\r Also the \"backend\" for [OpenRPA](https://github.com/skadefro/OpenRPA)", "main": "index.js", "scripts": { From f95c072d1fe2b303964c0fac211073419ecb1462 Mon Sep 17 00:00:00 2001 From: skadefro Date: Tue, 21 Nov 2023 20:38:55 +0100 Subject: [PATCH 02/32] up --- OpenFlow/src/DatabaseConnection.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/OpenFlow/src/DatabaseConnection.ts b/OpenFlow/src/DatabaseConnection.ts index a1f546d3..4394e4f1 100644 --- a/OpenFlow/src/DatabaseConnection.ts +++ b/OpenFlow/src/DatabaseConnection.ts @@ -3812,7 +3812,9 @@ export class DatabaseConnection extends events.EventEmitter { newObj[key] = value; } } catch (error) { - Logger.instanse.error(error, null); + // Logger.instanse.error(error, null); + var errormessage = error.message ? error.message : error; + Logger.instanse.debug(errormessage, null); newObj[key] = value; } return newObj; @@ -3830,7 +3832,7 @@ export class DatabaseConnection extends events.EventEmitter { _decryptentity(item, newObj, key) { const value: any = item[key]; try { - if (this._shouldEncryptValue(item._encrypt, key, value) && value != null) { + if (this._shouldEncryptValue(item._encrypt, key, value) && value != null && value != "") { let newvalue = Crypt.decrypt(value); if (newvalue.indexOf("{") === 0 || newvalue.indexOf("[") === 0) { try { @@ -3843,7 +3845,9 @@ export class DatabaseConnection extends events.EventEmitter { newObj[key] = value; } } catch (error) { - Logger.instanse.error(error, null); + // Logger.instanse.error(error, null); + var errormessage = error.message ? error.message : error; + Logger.instanse.debug(errormessage, null); newObj[key] = value; } return newObj; From 36c3ed92972e052f712de4b6d01e9680e941445f Mon Sep 17 00:00:00 2001 From: skadefro Date: Tue, 21 Nov 2023 20:39:51 +0100 Subject: [PATCH 03/32] up --- OpenFlow/src/Config.ts | 4 ++-- OpenFlow/src/OAuthProvider.ts | 2 +- OpenFlow/src/WebServer.ts | 15 +++++++++++++++ OpenFlow/src/index.ts | 1 + 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/OpenFlow/src/Config.ts b/OpenFlow/src/Config.ts index c8804527..4f6d7870 100644 --- a/OpenFlow/src/Config.ts +++ b/OpenFlow/src/Config.ts @@ -57,7 +57,7 @@ export class dbConfig extends Base { if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval", "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url", "nodered_domain_schema", - "domain" ].indexOf(key) > -1 ) { + "domain", "enable_openaiauth2" ].indexOf(key) > -1 ) { if(os.hostname().toLowerCase() == "nixos") { continue; } @@ -110,7 +110,7 @@ export class dbConfig extends Base { if(["db", "name", "version", "needsupdate", "updatedat"].indexOf(key) > -1 ) continue; if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval", "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url", "nodered_domain_schema", - "domain" ].indexOf(key) > -1 ) { + "domain", "enable_openaiauth2" ].indexOf(key) > -1 ) { if(os.hostname().toLowerCase() == "nixos") { continue; } diff --git a/OpenFlow/src/OAuthProvider.ts b/OpenFlow/src/OAuthProvider.ts index c5be3a4a..968f9bc4 100644 --- a/OpenFlow/src/OAuthProvider.ts +++ b/OpenFlow/src/OAuthProvider.ts @@ -168,7 +168,7 @@ export class OAuthProvider { defaultrole : "Viewer", rolemappings : { "admins": "Admin" }, clientId: "openai",client_id: "openai", - client_secret: "secret", + client_secret: "openai", token_endpoint_auth_method: "client_secret_post", response_types: ['code', 'id_token', 'code id_token'], grant_types: ['implicit', 'authorization_code'], diff --git a/OpenFlow/src/WebServer.ts b/OpenFlow/src/WebServer.ts index 71258da0..8ebb1fd4 100644 --- a/OpenFlow/src/WebServer.ts +++ b/OpenFlow/src/WebServer.ts @@ -385,6 +385,21 @@ export class WebServer { }); }); } + + public static async ProcessMessage(req: any, tuser: TokenUser, jwt: string): Promise { + const client:any = {user: tuser, jwt: jwt}; + const msg = new Message(); + const urlPath = req.path; + msg.command = urlPath.replace("/rest/v1/", "").toLowerCase(); + msg.id = NoderedUtil.GetUniqueIdentifier(); + msg.jwt = jwt; + msg.data = req.body; + msg.tuser = tuser; + msg.clientagent = req.headers["user-agent"]; + msg.clientversion = "0.0.1"; + var result = await msg.Process(client); + return result + } public static async onMessage(client: flowclient, message: any) { let command, msg, reply; try { diff --git a/OpenFlow/src/index.ts b/OpenFlow/src/index.ts index f4446d13..82d6a7f4 100644 --- a/OpenFlow/src/index.ts +++ b/OpenFlow/src/index.ts @@ -422,6 +422,7 @@ let OpenAIProxy: any = null; try { OpenAIProxy = require("./ee/OpenAIProxy"); } catch (error) { + console.error(error); } From e4a68aafc446f6aa710ea7c4c31381c2f6d8563d Mon Sep 17 00:00:00 2001 From: skadefro Date: Tue, 21 Nov 2023 20:50:44 +0100 Subject: [PATCH 04/32] rename enable_openaiauth change openai secret --- OpenFlow/src/Config.ts | 8 ++++---- OpenFlow/src/OAuthProvider.ts | 2 +- OpenFlow/src/public/Controllers.ts | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/OpenFlow/src/Config.ts b/OpenFlow/src/Config.ts index c8804527..59afb560 100644 --- a/OpenFlow/src/Config.ts +++ b/OpenFlow/src/Config.ts @@ -57,7 +57,7 @@ export class dbConfig extends Base { if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval", "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url", "nodered_domain_schema", - "domain" ].indexOf(key) > -1 ) { + "domain", "enable_openapiauth" ].indexOf(key) > -1 ) { if(os.hostname().toLowerCase() == "nixos") { continue; } @@ -110,7 +110,7 @@ export class dbConfig extends Base { if(["db", "name", "version", "needsupdate", "updatedat"].indexOf(key) > -1 ) continue; if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval", "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url", "nodered_domain_schema", - "domain" ].indexOf(key) > -1 ) { + "domain", "enable_openapiauth" ].indexOf(key) > -1 ) { if(os.hostname().toLowerCase() == "nixos") { continue; } @@ -184,7 +184,7 @@ export class Config { public static default_config: dbConfig = { enable_openai: false, enable_openapi: true, - enable_openaiauth: true, + enable_openapiauth: true, log_with_colors: true, cache_store_type: "memory", cache_store_max: 1000, @@ -442,7 +442,7 @@ export class Config { public static license_key: string = Config.getEnv("license_key", ""); public static enable_openai: boolean = Config.parseBoolean(Config.getEnv("enable_openai", "false")); public static enable_openapi: boolean = Config.parseBoolean(Config.getEnv("enable_openapi", "true")); - public static enable_openaiauth: boolean = Config.parseBoolean(Config.getEnv("enable_openaiauth", "true")); + public static enable_openapiauth: boolean = Config.parseBoolean(Config.getEnv("enable_openapiauth", "true")); public static openai_token: string = Config.getEnv("openai_token", ""); public static version: string = Config.getversion(); public static log_with_colors: boolean = Config.parseBoolean(Config.getEnv("log_with_colors", "true")); diff --git a/OpenFlow/src/OAuthProvider.ts b/OpenFlow/src/OAuthProvider.ts index c5be3a4a..968f9bc4 100644 --- a/OpenFlow/src/OAuthProvider.ts +++ b/OpenFlow/src/OAuthProvider.ts @@ -168,7 +168,7 @@ export class OAuthProvider { defaultrole : "Viewer", rolemappings : { "admins": "Admin" }, clientId: "openai",client_id: "openai", - client_secret: "secret", + client_secret: "openai", token_endpoint_auth_method: "client_secret_post", response_types: ['code', 'id_token', 'code id_token'], grant_types: ['implicit', 'authorization_code'], diff --git a/OpenFlow/src/public/Controllers.ts b/OpenFlow/src/public/Controllers.ts index 5c5ade78..2233a8c6 100644 --- a/OpenFlow/src/public/Controllers.ts +++ b/OpenFlow/src/public/Controllers.ts @@ -7410,7 +7410,7 @@ export class ConfigCtrl extends entityCtrl { {"name": "license_key", "type": "string", "default": ""}, {"name": "enable_openai", "type": "boolean", "default": "false"}, {"name": "enable_openapi", "type": "boolean", "default": "true"}, - {"name": "enable_openaiauth", "type": "boolean", "default": "true"}, + {"name": "enable_openapiauth", "type": "boolean", "default": "true"}, {"name": "openai_token", "type": "string", "default": ""}, {"name": "log_with_colors", "type": "boolean", "default": "true"}, {"name": "cache_store_type", "type": "string", "default": "memory"}, From 9c90d4f0e7aeb01e6b4809740b102f2520cb1962 Mon Sep 17 00:00:00 2001 From: skadefro Date: Wed, 22 Nov 2023 16:15:28 +0100 Subject: [PATCH 05/32] improve param parsing for createIndex --- OpenFlow/src/DatabaseConnection.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/OpenFlow/src/DatabaseConnection.ts b/OpenFlow/src/DatabaseConnection.ts index 4394e4f1..2533022e 100644 --- a/OpenFlow/src/DatabaseConnection.ts +++ b/OpenFlow/src/DatabaseConnection.ts @@ -4384,8 +4384,10 @@ export class DatabaseConnection extends events.EventEmitter { return new Promise((resolve, reject) => { try { Logger.instanse.info("Adding index " + name + " to " + collectionname, span, { collection: collectionname }); - if (NoderedUtil.IsNullUndefinded(options)) options = {}; - options["name"] = name; + if(typeof keypath === "string") keypath = JSON.parse(keypath); + if (NoderedUtil.IsNullEmpty(options)) options = {}; + if(typeof options === "string") options = JSON.parse(options); + if (!NoderedUtil.IsNullEmpty(name)) options["name"] = name; this.db.collection(collectionname).createIndex(keypath, options, (err, name) => { if (err) { Logger.instanse.error(err, span); From 2b29f0d65482981e17a0519004da678221e1baf4 Mon Sep 17 00:00:00 2001 From: skadefro Date: Wed, 22 Nov 2023 16:16:11 +0100 Subject: [PATCH 06/32] More OpenAPI improvements --- OpenFlow/src/Config.ts | 4 +- OpenFlow/src/Messages/Message.ts | 171 +++++++++++++++++++++++++- OpenFlow/src/WebServer.ts | 14 +++ OpenFlow/src/WebSocketServerClient.ts | 2 +- package.json | 3 +- 5 files changed, 189 insertions(+), 5 deletions(-) diff --git a/OpenFlow/src/Config.ts b/OpenFlow/src/Config.ts index 5db2f710..f86ab971 100644 --- a/OpenFlow/src/Config.ts +++ b/OpenFlow/src/Config.ts @@ -57,7 +57,7 @@ export class dbConfig extends Base { if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval", "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url", "nodered_domain_schema", - "domain", "enable_openapiauth", "enable_openaiauth2" ].indexOf(key) > -1 ) { + "domain", "enable_openapi", "enable_openapiauth" ].indexOf(key) > -1 ) { if(os.hostname().toLowerCase() == "nixos") { continue; } @@ -110,7 +110,7 @@ export class dbConfig extends Base { if(["db", "name", "version", "needsupdate", "updatedat"].indexOf(key) > -1 ) continue; if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval", "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url", "nodered_domain_schema", - "domain", "enable_openapiauth", "enable_openaiauth2" ].indexOf(key) > -1 ) { + "domain", "enable_openapi", "enable_openapiauth" ].indexOf(key) > -1 ) { if(os.hostname().toLowerCase() == "nixos") { continue; } diff --git a/OpenFlow/src/Messages/Message.ts b/OpenFlow/src/Messages/Message.ts index 3aff3203..6fa4d089 100644 --- a/OpenFlow/src/Messages/Message.ts +++ b/OpenFlow/src/Messages/Message.ts @@ -590,6 +590,36 @@ export class Message { case "deleteworkitem": await this.DeleteWorkitem(span); break; + case "startagent": + await this.ControlAgent(span); + break; + case "stopagent": + await this.ControlAgent(span); + break; + case "deleteagentpod": + await this.ControlAgent(span); + break; + case "getagentlog": + await this.ControlAgent(span); + break; + case "getagentpods": + await this.ControlAgent(span); + break; + case "deleteagent": + await this.ControlAgent(span); + break; + case "createindex": + await this.CreateIndex(span); + break; + case "deletepackage": + await this.DeletePackage(span); + break; + case "issuelicense": + await this.IssueLicense(cli, span); + break; + case "invokeopenra": + await this.InvokeOpenRPA(cli, span); + break; case "customcommand": await this.CustomCommand(cli, span); break; @@ -628,6 +658,145 @@ export class Message { } }); } + async ControlAgent(parent: Span) { + this.Reply(); + let msg: any = this.data + if( typeof this.data == "string") { + msg = JSON.stringify(this.data) + } + msg =JSON.parse(JSON.stringify(msg)); + try { + console.log(this.data); + if (Logger.agentdriver == null) throw new Error("No agentdriver is loaded") + var agent = null; + if((msg.agentid == null || msg.agentid == "")) { + if(this.command != "getagentpods") throw new Error("No agentid is specified"); + } else { + agent = await Config.db.GetOne({ query: { _id: msg.agentid }, collectionname: "agents", jwt:this.jwt }, parent); + if(agent == null) throw new Error("Access denied"); + if (!DatabaseConnection.hasAuthorization(this.tuser, agent, Rights.invoke)) { + throw new Error(`[${this.tuser.name}] Access denied, missing invoke permission on ${agent.name}`); + } + if(agent.image == null || agent.image == "") return; + } + + + if(this.command == "startagent") { + await Logger.agentdriver.EnsureInstance(this.tuser, this.jwt, agent, parent); + } else if(this.command == "stopagent") { + await Logger.agentdriver.RemoveInstance(this.tuser, this.jwt, agent, false, parent); + } else if(this.command == "getagentlog") { + msg.result = await Logger.agentdriver.GetInstanceLog(this.tuser, this.jwt, agent, msg.podname, parent); + } else if(this.command == "getagentpods") { + var getstats = false; + if(!NoderedUtil.IsNullEmpty(msg.stats)) getstats = msg.stats; + msg.results = await Logger.agentdriver.GetInstancePods(this.tuser, this.jwt, agent, msg.podname, parent); + // msg.results = JSON.stringify(await Logger.agentdriver.GetInstancePods(this.tuser, this.jwt, agent, msg.podname, parent)); + var b = true; + } else if(this.command == "deleteagentpod") { + await Logger.agentdriver.RemoveInstancePod(this.tuser, this.jwt, agent, msg.podname, parent); + } else if(this.command == "deleteagent") { + if (!DatabaseConnection.hasAuthorization(this.tuser, agent, Rights.delete)) { + throw new Error(`[${this.tuser.name}] Access denied, missing delete permission on ${agent.name}`); + } + await Logger.agentdriver.RemoveInstance(this.tuser, this.jwt, agent, true, parent); + Config.db.DeleteOne(agent._id, "agents", false, this.jwt, parent); + } + } finally { + this.data = JSON.stringify(msg); + } + } + async CreateIndex(parent: Span) { + this.Reply(); + let msg: any = this.data + if( typeof this.data == "string") { + msg = JSON.stringify(this.data) + } + msg = JSON.parse(JSON.stringify(msg)); + try { + if (!this.tuser.HasRoleId(WellknownIds.admins)) throw new Error("Access denied"); + msg.result = await Config.db.createIndex(msg.collectionname, msg.name, msg.index, msg.options, parent); + } finally { + this.data = JSON.stringify(msg); + } + } + async DeletePackage(parent: Span) { + this.Reply(); + let msg: any = this.data + if( typeof this.data == "string") { + msg = JSON.stringify(this.data) + } + try { + var pack = await Config.db.GetOne({ query: { _id: msg.id, "_type": "package" }, collectionname: "agents", jwt: this.jwt }, parent); + if(pack == null) throw new Error("Access denied or package not found"); + if (!DatabaseConnection.hasAuthorization(this.tuser, pack, Rights.delete)) { + throw new Error(`[${this.tuser.name}] Access denied, missing delete permission on ${pack.name}`); + } + if(pack.fileid != null && pack.fileid != "") { + const rootjwt = Crypt.rootToken(); + let query = { _id: pack.fileid }; + const item = await Config.db.GetOne({ query, collectionname: "fs.files", jwt: rootjwt }, parent); + if(item != null) { + await Config.db.DeleteOne(pack.fileid, "files", false, this.jwt, parent); + } + } + await Config.db.DeleteOne(pack._id, "agents", false, this.jwt, parent); + } finally { + this.data = JSON.stringify(msg); + } + } + async IssueLicense(cli: WebSocketServerClient, parent: Span) { + this.Reply(); + let msg: any = this.data + if( typeof this.data == "string") { + msg = JSON.stringify(this.data) + } + try { + let _lic_require: any = null; + try { + _lic_require = require("../ee/license-file"); + } catch (error) { + } + if (_lic_require == null) { + throw new Error("License module not found"); + } + Logger.License = new _lic_require.LicenseFile(); + // @ts-ignore + var data = msg.data; + try { + data = JSON.parse(data); + } catch (error) { + } + if(data == null || data == "") throw new Error("No data found"); + var domain = data.domain; + if (!this.tuser.HasRoleId(WellknownIds.admins)) { + delete data.months; + } + var exists = await Config.db.GetOne({ query: { domains: domain, "_type": "resourceusage"}, collectionname: "config", jwt:this.jwt }, parent); + if (!this.tuser.HasRoleId(WellknownIds.admins)) { + if(exists == null) throw new Error("Access denied"); + } + if(data.months == null || data.months == "") { + if(exists != null && exists.issuemonths != null) data.months = parseInt(exists.issuemonths); + } + // throw new Error("Access denied"); + msg.result = await Logger.License.generate2(data, cli?.remoteip, this.tuser, parent); + } finally { + this.data = JSON.stringify(msg); + } + } + async InvokeOpenRPA(cli: WebSocketServerClient, parent: Span) { + this.Reply(); + let msg: any = this.data + if( typeof this.data == "string") { + msg = JSON.stringify(this.data) + } + try { + throw new Error("Not implemented, only available using OpenAPI"); + } finally { + this.data = JSON.stringify(msg); + } + } async RegisterExchange(cli: WebSocketServerClient, parent: Span) { this.Reply(); let msg: RegisterExchangeMessage; @@ -977,8 +1146,8 @@ export class Message { return; } this.data = "{\"message\": \"Unknown command " + this.command + "\"}"; - this.Reply("error"); Logger.instanse.error(`UnknownCommand ${this.command}`, null); + this.Reply("error"); } private static collectionCache: any = {}; private static collectionCachetime: Date = new Date(); diff --git a/OpenFlow/src/WebServer.ts b/OpenFlow/src/WebServer.ts index 8ebb1fd4..72e60312 100644 --- a/OpenFlow/src/WebServer.ts +++ b/OpenFlow/src/WebServer.ts @@ -391,6 +391,20 @@ export class WebServer { const msg = new Message(); const urlPath = req.path; msg.command = urlPath.replace("/rest/v1/", "").toLowerCase(); + + if(msg.command == "updatedocument") { + msg.command = "updatemany" // new command to new + } + if(msg.command == "unregisterqueue") { + msg.command = "closequeue" // new command to new + } + if(msg.command == "pushworkitem") { + msg.command = "addworkitem" // new command to new + } + if(msg.command == "pushworkitems") { + msg.command = "addworkitems" // new command to new + } + msg.id = NoderedUtil.GetUniqueIdentifier(); msg.jwt = jwt; msg.data = req.body; diff --git a/OpenFlow/src/WebSocketServerClient.ts b/OpenFlow/src/WebSocketServerClient.ts index 51a75790..7a91d7fd 100644 --- a/OpenFlow/src/WebSocketServerClient.ts +++ b/OpenFlow/src/WebSocketServerClient.ts @@ -550,7 +550,7 @@ export class WebSocketServerClient { result.priority = first.priority; if (result.command != "ping" && result.command != "pong") { result.Process(this).then(msg=> { - this.Send(msg); + if(msg != null) this.Send(msg); }) .catch((error) => { Logger.instanse.error(error, span, Logger.parsecli(this)); }); diff --git a/package.json b/package.json index 6f47f670..b7f13f06 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "latest": "gulp latest", "rebuild": "docker-compose up registry -d && docker-compose build && docker-compose push --ignore-push-failures && docker-compose up -d", "build": "docker-compose up registry -d && docker-compose push --ignore-push-failures && docker-compose up -d", + "updateapilocal": "npm uninstall @openiap/nodeapi && npm i ../nodeapi", "updateapidev": "npm uninstall @openiap/nodeapi && npm i openiap/nodeapi", "updateapi": "npm uninstall @openiap/nodeapi && npm i @openiap/nodeapi" }, @@ -33,7 +34,7 @@ "@fortawesome/fontawesome-free": "^5.15.3", "@kubernetes/client-node": "^0.19.0", "@node-saml/passport-saml": "^4.0.1", - "@openiap/nodeapi": "^0.0.27", + "@openiap/nodeapi": "^0.0.30", "@openiap/openflow-api": "^2.1.11", "@opentelemetry/exporter-metrics-otlp-grpc": "^0.43.0", "@opentelemetry/sdk-node": "^0.43.0", From 8b63552cfea5005a26018803a4659878f2ddc175 Mon Sep 17 00:00:00 2001 From: skadefro Date: Thu, 23 Nov 2023 13:08:11 +0100 Subject: [PATCH 07/32] Add "pods" option on agents --- OpenFlow/src/public/Agents.html | 3 +++ OpenFlow/src/public/CommonControllers.ts | 2 +- OpenFlow/src/public/Controllers.ts | 10 +++++++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/OpenFlow/src/public/Agents.html b/OpenFlow/src/public/Agents.html index c4670c71..f184f4ff 100644 --- a/OpenFlow/src/public/Agents.html +++ b/OpenFlow/src/public/Agents.html @@ -10,6 +10,9 @@

agents

reload + + + diff --git a/OpenFlow/src/public/CommonControllers.ts b/OpenFlow/src/public/CommonControllers.ts index 9e660c2a..66751b3f 100644 --- a/OpenFlow/src/public/CommonControllers.ts +++ b/OpenFlow/src/public/CommonControllers.ts @@ -655,7 +655,7 @@ export class entitiesCtrl { this.errormessage = ""; this.loading = true; if (this.preloadData != null) { - this.preloadData(); + await this.preloadData(); } let query: object = Object.assign({}, this.basequery); let exactquery: object = null; diff --git a/OpenFlow/src/public/Controllers.ts b/OpenFlow/src/public/Controllers.ts index 2233a8c6..5319e581 100644 --- a/OpenFlow/src/public/Controllers.ts +++ b/OpenFlow/src/public/Controllers.ts @@ -7838,11 +7838,15 @@ export class AgentsCtrl extends entitiesCtrl { // } return image; } - preLoad() { - if (this.show != "all") { + async preLoad() { + if (this.show == "pods") { + this.knownpods = await NoderedUtil.CustomCommand({ command: "getagentpods" }) + const slugs = this.knownpods.map(x => x.metadata.labels.slug).filter(x => x != null); + this.basequery = { _type: "agent", slug: { $in: slugs } }; + } else if (this.show != "all" ) { this.basequery = { _type: "agent" }; this.basequery[this.show] = true; - }else { + } else { this.basequery = { _type: "agent" }; } } From 299f087af5657edd50978a0cb433a7b08e59b0c7 Mon Sep 17 00:00:00 2001 From: skadefro Date: Thu, 23 Nov 2023 13:39:26 +0100 Subject: [PATCH 08/32] fix checkbox text link --- OpenFlow/src/public/Agents.html | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/OpenFlow/src/public/Agents.html b/OpenFlow/src/public/Agents.html index f184f4ff..e898c3ab 100644 --- a/OpenFlow/src/public/Agents.html +++ b/OpenFlow/src/public/Agents.html @@ -13,17 +13,17 @@

agents

- - + + - - + + - - + + - - + + packages addagent From 3b94020af2e4b03f2687306c301a4f90bda90256 Mon Sep 17 00:00:00 2001 From: skadefro Date: Thu, 23 Nov 2023 13:39:55 +0100 Subject: [PATCH 09/32] cleanup config --- OpenFlow/src/Config.ts | 537 +++++++++--------- OpenFlow/src/Logger.ts | 2 +- OpenFlow/src/LoginProvider.ts | 10 - OpenFlow/src/Messages/Message.ts | 7 - OpenFlow/src/WebServer.ts | 2 +- OpenFlow/src/WebSocketServer.ts | 2 +- OpenFlow/src/public/CommonControllers.ts | 1 - OpenFlow/src/public/Controllers.ts | 12 +- OpenFlow/src/public/WebSocketClientService.ts | 2 - docker-compose.yml | 3 - openflow.env | 1 - test/docker-compose.yml | 2 - 12 files changed, 264 insertions(+), 317 deletions(-) diff --git a/OpenFlow/src/Config.ts b/OpenFlow/src/Config.ts index f86ab971..fc344b03 100644 --- a/OpenFlow/src/Config.ts +++ b/OpenFlow/src/Config.ts @@ -56,7 +56,7 @@ export class dbConfig extends Base { if(["db", "name", "version", "needsupdate", "updatedat"].indexOf(key) > -1 ) continue; if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval", - "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url", "nodered_domain_schema", + "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url", "domain", "enable_openapi", "enable_openapiauth" ].indexOf(key) > -1 ) { if(os.hostname().toLowerCase() == "nixos") { continue; @@ -109,7 +109,7 @@ export class dbConfig extends Base { if(key.startsWith("_")) continue; if(["db", "name", "version", "needsupdate", "updatedat"].indexOf(key) > -1 ) continue; if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval", - "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url", "nodered_domain_schema", + "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url", "domain", "enable_openapi", "enable_openapiauth" ].indexOf(key) > -1 ) { if(os.hostname().toLowerCase() == "nixos") { continue; @@ -308,7 +308,7 @@ export class Config { protocol: "http", port: 80, domain: "localhost.openiap.io", - cookie_secret: "", + cookie_secret: "NLgUIsozJaxO38ze0WuHthfj2eb1eIEu", max_ace_count: 128, amqp_reply_expiration: 60 * 1000, // 1 min @@ -328,10 +328,10 @@ export class Config { amqp_requeue_time: 1000, // 1 seconds amqp_dlx: "openflow-dlx", // Dead letter exchange, used to pickup dead or timeout messages - // mongodb_url: "mongodb://localhost:27017", - // mongodb_db: "openflow", - // mongodb_minpoolsize: 25, - // mongodb_maxpoolsize: 25, + mongodb_url: "mongodb://localhost:27017", + mongodb_db: "openflow", + mongodb_minpoolsize: 25, + mongodb_maxpoolsize: 25, skip_history_collections: "audit,openrpa_instances,workflow_instances", history_delta_count: 1000, @@ -361,22 +361,14 @@ export class Config { saml_federation_metadata: "", api_ws_url: "", - nodered_ws_url: "", - nodered_saml_entrypoint: "", agent_docker_entrypoints: "web", agent_docker_use_project: false, agent_docker_certresolver: "", namespace: "", - nodered_domain_schema: "", - nodered_initial_liveness_delay: 60, - nodered_allow_nodeselector: false, - nodered_liveness_failurethreshold: 5, - nodered_liveness_timeoutseconds: 5, - noderedcatalogues: "", + agent_allow_nodeselector: false, otel_measure_nodeid: false, otel_measure_queued_messages: false, otel_measure__mongodb_watch: false, - otel_measure_onlineuser: false, enable_analytics: true, enable_detailed_analytic: false, otel_debug_log: false, @@ -439,266 +431,255 @@ export class Config { } public static unittesting: boolean = false; public static db: DatabaseConnection = null; - public static license_key: string = Config.getEnv("license_key", ""); - public static enable_openai: boolean = Config.parseBoolean(Config.getEnv("enable_openai", "false")); - public static enable_openapi: boolean = Config.parseBoolean(Config.getEnv("enable_openapi", "true")); - public static enable_openapiauth: boolean = Config.parseBoolean(Config.getEnv("enable_openapiauth", "true")); - public static openai_token: string = Config.getEnv("openai_token", ""); + public static license_key: string = Config.getEnv("license_key"); + public static enable_openai: boolean = Config.parseBoolean(Config.getEnv("enable_openai")); + public static enable_openapi: boolean = Config.parseBoolean(Config.getEnv("enable_openapi")); + public static enable_openapiauth: boolean = Config.parseBoolean(Config.getEnv("enable_openapiauth")); + public static openai_token: string = Config.getEnv("openai_token"); public static version: string = Config.getversion(); - public static log_with_colors: boolean = Config.parseBoolean(Config.getEnv("log_with_colors", "true")); - - public static cache_store_type: string = Config.getEnv("cache_store_type", "memory"); - public static cache_store_max: number = parseInt(Config.getEnv("cache_store_max", "1000")); - public static cache_store_ttl_seconds: number = parseInt(Config.getEnv("cache_store_ttl_seconds", "300")); - public static cache_store_redis_host: string = Config.getEnv("cache_store_redis_host", ""); - public static cache_store_redis_port: number = parseInt(Config.getEnv("cache_store_redis_port", "6379")); - public static cache_store_redis_password: string = Config.getEnv("cache_store_redis_password", ""); - public static cache_workitem_queues: boolean = Config.parseBoolean(Config.getEnv("cache_workitem_queues", "false")); - - public static log_cache: boolean = Config.parseBoolean(Config.getEnv("log_cache", "false")); - public static log_amqp: boolean = Config.parseBoolean(Config.getEnv("log_amqp", "false")); - public static log_openapi: boolean = Config.parseBoolean(Config.getEnv("log_openapi", "false")); - public static log_login_provider: boolean = Config.parseBoolean(Config.getEnv("log_login_provider", "false")); - public static log_with_trace: boolean = Config.parseBoolean(Config.getEnv("log_with_trace", "false")); - public static log_websocket: boolean = Config.parseBoolean(Config.getEnv("log_websocket", "false")); - public static log_oauth: boolean = Config.parseBoolean(Config.getEnv("log_oauth", "false")); - public static log_webserver: boolean = Config.parseBoolean(Config.getEnv("log_webserver", "false")); - public static log_database: boolean = Config.parseBoolean(Config.getEnv("log_database", "false")); - public static log_database_queries: boolean = Config.parseBoolean(Config.getEnv("log_database_queries", "false")); - public static log_database_queries_ms: number = parseInt(Config.getEnv("log_database_queries_ms", "0")); - - public static log_grafana: boolean = Config.parseBoolean(Config.getEnv("log_grafana", "false")); - public static log_housekeeping: boolean = Config.parseBoolean(Config.getEnv("log_housekeeping", "false")); - public static log_otel: boolean = Config.parseBoolean(Config.getEnv("log_otel", "false")); - public static log_blocked_ips: boolean = Config.parseBoolean(Config.getEnv("log_blocked_ips", "true")); - public static log_information: boolean = Config.parseBoolean(Config.getEnv("log_information", "true")); - public static log_debug: boolean = Config.parseBoolean(Config.getEnv("log_debug", "false")); - public static log_verbose: boolean = Config.parseBoolean(Config.getEnv("log_verbose", "false")); - public static log_silly: boolean = Config.parseBoolean(Config.getEnv("log_silly", "false")); - public static log_to_exchange: boolean = Config.parseBoolean(Config.getEnv("log_to_exchange", "false")); - - public static heapdump_onstop: boolean = Config.parseBoolean(Config.getEnv("heapdump_onstop", "false")); - - public static amqp_allow_replyto_empty_queuename: boolean = Config.parseBoolean(Config.getEnv("amqp_allow_replyto_empty_queuename", "false")); - - public static openflow_uniqueid: string = Config.getEnv("openflow_uniqueid", ""); - public static enable_openflow_amqp: boolean = Config.parseBoolean(Config.getEnv("enable_openflow_amqp", "false")); - public static openflow_amqp_expiration: number = parseInt(Config.getEnv("openflow_amqp_expiration", (60 * 1000 * 25).toString())); // 25 min - public static amqp_prefetch: number = parseInt(Config.getEnv("amqp_prefetch", "25")); - public static enable_entity_restriction: boolean = Config.parseBoolean(Config.getEnv("enable_entity_restriction", "false")); - public static enable_web_tours: boolean = Config.parseBoolean(Config.getEnv("enable_web_tours", "true")); - public static enable_nodered_tours: boolean = Config.parseBoolean(Config.getEnv("enable_nodered_tours", "true")); - public static grafana_url:string = Config.getEnv("grafana_url", ""); - public static auto_hourly_housekeeping: boolean = Config.parseBoolean(Config.getEnv("auto_hourly_housekeeping", "true")); - public static housekeeping_skip_collections: string = Config.getEnv("housekeeping_skip_collections", ""); - public static workitem_queue_monitoring_enabled: boolean = Config.parseBoolean(Config.getEnv("workitem_queue_monitoring_enabled", "true")); - public static workitem_queue_monitoring_interval: number = parseInt(Config.getEnv("workitem_queue_monitoring_interval", (10 * 1000).toString())); // 10 sec - - public static upload_max_filesize_mb: number = parseInt(Config.getEnv("upload_max_filesize_mb", "25")); - - public static getting_started_url: string = Config.getEnv("getting_started_url", ""); - - public static NODE_ENV: string = Config.getEnv("NODE_ENV", "development"); - public static HTTP_PROXY: string = Config.getEnv("HTTP_PROXY", ""); - public static HTTPS_PROXY: string = Config.getEnv("HTTPS_PROXY", ""); - public static NO_PROXY: string = Config.getEnv("NO_PROXY", ""); - public static agent_HTTP_PROXY: string = Config.getEnv("agent_HTTP_PROXY", ""); - public static agent_HTTPS_PROXY: string = Config.getEnv("agent_HTTPS_PROXY", ""); - public static agent_NO_PROXY: string = Config.getEnv("agent_NO_PROXY", ""); - - public static stripe_api_key: string = Config.getEnv("stripe_api_key", ""); - public static stripe_api_secret: string = Config.getEnv("stripe_api_secret", ""); - public static stripe_force_vat: boolean = Config.parseBoolean(Config.getEnv("stripe_force_vat", "false")); - public static stripe_force_checkout: boolean = Config.parseBoolean(Config.getEnv("stripe_force_checkout", "false")); - public static stripe_allow_promotion_codes: boolean = Config.parseBoolean(Config.getEnv("stripe_allow_promotion_codes", "true")); - - public static supports_watch: boolean = Config.parseBoolean(Config.getEnv("supports_watch", "false")); - public static ensure_indexes: boolean = Config.parseBoolean(Config.getEnv("ensure_indexes", "true")); - public static text_index_name_fields: string[] = Config.parseArray(Config.getEnv("text_index_name_fields", "name,_names")); - - public static auto_create_users: boolean = Config.parseBoolean(Config.getEnv("auto_create_users", "false")); - public static auto_create_user_from_jwt: boolean = Config.parseBoolean(Config.getEnv("auto_create_user_from_jwt", "false")); - public static auto_create_domains: string[] = Config.parseArray(Config.getEnv("auto_create_domains", "")); - public static persist_user_impersonation: boolean = Config.parseBoolean(Config.getEnv("persist_user_impersonation", "false")); - public static ping_clients_interval: number = parseInt(Config.getEnv("ping_clients_interval", (10000).toString())); // 10 seconds - - public static use_ingress_beta1_syntax: boolean = Config.parseBoolean(Config.getEnv("use_ingress_beta1_syntax", "false")); - public static use_openshift_routes: boolean = Config.parseBoolean(Config.getEnv("use_openshift_routes", "false")); - public static agent_image_pull_secrets: string[] = Config.parseArray(Config.getEnv("agent_image_pull_secrets", "")); - public static auto_create_personal_nodered_group: boolean = Config.parseBoolean(Config.getEnv("auto_create_personal_nodered_group", "false")); - public static auto_create_personal_noderedapi_group: boolean = Config.parseBoolean(Config.getEnv("auto_create_personal_noderedapi_group", "false")); - public static force_add_admins: boolean = Config.parseBoolean(Config.getEnv("force_add_admins", "true")); - public static validate_emails: boolean = Config.parseBoolean(Config.getEnv("validate_emails", "false")); - public static forgot_pass_emails: boolean = Config.parseBoolean(Config.getEnv("forgot_pass_emails", "false")); - public static smtp_service: string = Config.getEnv("smtp_service", ""); - public static smtp_from: string = Config.getEnv("smtp_from", ""); - public static smtp_user: string = Config.getEnv("smtp_user", ""); - public static smtp_pass: string = Config.getEnv("smtp_pass", ""); - public static smtp_url: string = Config.getEnv("smtp_url", ""); - public static debounce_lookup: boolean = Config.parseBoolean(Config.getEnv("debounce_lookup", "false")); - public static validate_emails_disposable: boolean = Config.parseBoolean(Config.getEnv("validate_emails_disposable", "false")); - - public static tls_crt: string = Config.getEnv("tls_crt", ""); - public static tls_key: string = Config.getEnv("tls_key", ""); - public static tls_ca: string = Config.getEnv("tls_ca", ""); - public static tls_passphrase: string = Config.getEnv("tls_passphrase", ""); - - public static oidc_access_token_ttl: number = parseInt(Config.getEnv("oidc_access_token_ttl", "480")); // 8 hours - public static oidc_authorization_code_ttl: number = parseInt(Config.getEnv("oidc_authorization_code_ttl", "480")); // 8 hours - public static oidc_client_credentials_ttl: number = parseInt(Config.getEnv("oidc_client_credentials_ttl", "480")); // 8 hours - public static oidc_refresh_token_ttl: number = parseInt(Config.getEnv("oidc_refresh_token_ttl", "20160")); // 14 days in seconds - public static oidc_session_ttl: number = parseInt(Config.getEnv("oidc_session_ttl", "20160")); // 14 days in seconds - - public static oidc_cookie_key: string = Config.getEnv("oidc_cookie_key", "Y6SPiXCxDhAJbN7cbydMw5eX1wIrdy8PiWApqEcguss="); - public static api_rate_limit: boolean = Config.parseBoolean(Config.getEnv("api_rate_limit", "true")); - public static api_rate_limit_points: number = parseInt(Config.getEnv("api_rate_limit_points", "20")); - public static api_rate_limit_duration: number = parseInt(Config.getEnv("api_rate_limit_duration", "1")); - public static socket_rate_limit: boolean = Config.parseBoolean(Config.getEnv("socket_rate_limit", "true")); - public static socket_rate_limit_points: number = parseInt(Config.getEnv("socket_rate_limit_points", "30")); - public static socket_rate_limit_points_disconnect: number = parseInt(Config.getEnv("socket_rate_limit_points_disconnect", "100")); - public static socket_rate_limit_duration: number = parseInt(Config.getEnv("socket_rate_limit_duration", "1")); - public static socket_error_rate_limit_points: number = parseInt(Config.getEnv("socket_error_rate_limit_points", "30")); - public static socket_error_rate_limit_duration: number = parseInt(Config.getEnv("socket_error_rate_limit_duration", "1")); - - public static client_heartbeat_timeout: number = parseInt(Config.getEnv("client_heartbeat_timeout", "60")); - public static client_signin_timeout: number = parseInt(Config.getEnv("client_signin_timeout", "120")); - public static client_disconnect_signin_error: boolean = Config.parseBoolean(Config.getEnv("client_disconnect_signin_error", "false")); - - public static expected_max_roles: number = parseInt(Config.getEnv("expected_max_roles", "20000")); - public static decorate_roles_fetching_all_roles = Config.parseBoolean(Config.getEnv("decorate_roles_fetching_all_roles", "true")); - public static max_recursive_group_depth: number = parseInt(Config.getEnv("max_recursive_group_depth", "2")); - public static update_acl_based_on_groups: boolean = Config.parseBoolean(Config.getEnv("update_acl_based_on_groups", "true")); - public static allow_merge_acl: boolean = Config.parseBoolean(Config.getEnv("allow_merge_acl", "false")); - - public static multi_tenant: boolean = Config.parseBoolean(Config.getEnv("multi_tenant", "false")); - public static cleanup_on_delete_customer: boolean = Config.parseBoolean(Config.getEnv("cleanup_on_delete_customer", "false")); - public static cleanup_on_delete_user: boolean = Config.parseBoolean(Config.getEnv("cleanup_on_delete_user", "false")); - public static api_bypass_perm_check: boolean = Config.parseBoolean(Config.getEnv("api_bypass_perm_check", "false")); - public static ignore_expiration: boolean = Config.parseBoolean(Config.getEnv("ignore_expiration", "false")); - public static force_audit_ts: boolean = Config.parseBoolean(Config.getEnv("force_audit_ts", "false")); - public static force_dbusage_ts: boolean = Config.parseBoolean(Config.getEnv("force_dbusage_ts", "false")); - public static migrate_audit_to_ts: boolean = Config.parseBoolean(Config.getEnv("migrate_audit_to_ts", "true")); - - public static websocket_package_size: number = parseInt(Config.getEnv("websocket_package_size", "25000"), 10); - public static websocket_max_package_count: number = parseInt(Config.getEnv("websocket_max_package_count", "25000"), 10); - public static websocket_message_callback_timeout: number = parseInt(Config.getEnv("websocket_message_callback_timeout", "3600"), 10); - public static websocket_disconnect_out_of_sync: boolean = Config.parseBoolean(Config.getEnv("websocket_disconnect_out_of_sync", "false")); - public static protocol: string = Config.getEnv("protocol", "http"); // used by personal nodered and baseurl() - public static port: number = parseInt(Config.getEnv("port", "80")); - public static domain: string = Config.getEnv("domain", "localhost"); // sent to website and used in baseurl() - public static cookie_secret: string = Config.getEnv("cookie_secret", "NLgUIsozJaxO38ze0WuHthfj2eb1eIEu"); // Used to protect cookies - public static max_ace_count: number = parseInt(Config.getEnv("max_ace_count", "128"), 10); - - public static amqp_reply_expiration: number = parseInt(Config.getEnv("amqp_reply_expiration", (60 * 1000).toString())); // 1 min - public static amqp_force_queue_prefix: boolean = Config.parseBoolean(Config.getEnv("amqp_force_queue_prefix", "false")); - public static amqp_force_exchange_prefix: boolean = Config.parseBoolean(Config.getEnv("amqp_force_exchange_prefix", "false")); - public static amqp_force_sender_has_read: boolean = Config.parseBoolean(Config.getEnv("amqp_force_sender_has_read", "true")); - public static amqp_force_sender_has_invoke: boolean = Config.parseBoolean(Config.getEnv("amqp_force_sender_has_invoke", "false")); - public static amqp_force_consumer_has_update: boolean = Config.parseBoolean(Config.getEnv("amqp_force_consumer_has_update", "false")); - public static amqp_enabled_exchange: boolean = Config.parseBoolean(Config.getEnv("amqp_enabled_exchange", "false")); - public static amqp_url: string = Config.getEnv("amqp_url", "amqp://localhost"); // used to register queues and by personal nodered - public static amqp_username: string = Config.getEnv("amqp_username", "guest"); // used to talk wth rabbitmq api - public static amqp_password: string = Config.getEnv("amqp_password", "guest"); // used to talk wth rabbitmq api - - public static amqp_check_for_consumer: boolean = Config.parseBoolean(Config.getEnv("amqp_check_for_consumer", "true")); - public static amqp_check_for_consumer_count: boolean = Config.parseBoolean(Config.getEnv("amqp_check_for_consumer_count", "false")); - public static amqp_default_expiration: number = parseInt(Config.getEnv("amqp_default_expiration", (60 * 1000).toString())); // 1 min - public static amqp_requeue_time: number = parseInt(Config.getEnv("amqp_requeue_time", "1000")); // 1 seconds - public static amqp_dlx: string = Config.getEnv("amqp_dlx", "openflow-dlx"); // Dead letter exchange, used to pickup dead or timeout messages - - public static mongodb_url: string = Config.getEnv("mongodb_url", "mongodb://localhost:27017"); - public static mongodb_db: string = Config.getEnv("mongodb_db", "openflow"); - public static mongodb_minpoolsize: number = parseInt(Config.getEnv("mongodb_minpoolsize", "25")); - public static mongodb_maxpoolsize: number = parseInt(Config.getEnv("mongodb_maxpoolsize", "25")); - - public static skip_history_collections: string = Config.getEnv("skip_history_collections", "audit,openrpa_instances,workflow_instances"); - public static history_delta_count: number = parseInt(Config.getEnv("history_delta_count", "1000")); - public static allow_skiphistory: boolean = Config.parseBoolean(Config.getEnv("allow_skiphistory", "false")); - public static max_memory_restart_mb: number = parseInt(Config.getEnv("max_memory_restart_mb", "0")); - - public static saml_issuer: string = Config.getEnv("saml_issuer", "the-issuer"); // define uri of STS, also sent to personal nodereds - public static aes_secret: string = Config.getEnv("aes_secret", ""); - public static signing_crt: string = Config.getEnv("signing_crt", ""); - public static singing_key: string = Config.getEnv("singing_key", ""); - public static wapid_mail: string = Config.getEnv("wapid_mail", ""); - public static wapid_pub: string = Config.getEnv("wapid_pub", ""); - public static wapid_key: string = Config.getEnv("wapid_key", ""); - - public static shorttoken_expires_in: string = Config.getEnv("shorttoken_expires_in", "5m"); - public static longtoken_expires_in: string = Config.getEnv("longtoken_expires_in", "365d"); - public static downloadtoken_expires_in: string = Config.getEnv("downloadtoken_expires_in", "15m"); - public static personalnoderedtoken_expires_in: string = Config.getEnv("personalnoderedtoken_expires_in", "365d"); - - public static agent_images: NoderedImage[] = JSON.parse(Config.getEnv("agent_images", - JSON.stringify([{"name":"Agent", "image":"openiap/nodeagent", "languages": ["nodejs", "python"]}, {"name":"Agent+Chromium", "image":"openiap/nodechromiumagent", "chromium": true, "languages": ["nodejs", "python"]}, {"name":"NodeRED", "image":"openiap/noderedagent", "port": 3000}, {"name":"DotNet 6", "image":"openiap/dotnetagent", "languages": ["dotnet"]} , {"name":"PowerShell 7.3", "image":"openiap/nodeagent:pwsh", "languages": ["powershell"]} ]) - )); - public static agent_domain_schema: string = Config.getEnv("agent_domain_schema", ""); - public static agent_node_selector:string = Config.getEnv("agent_node_selector", ""); - - public static agent_grpc_apihost: string = Config.getEnv("agent_grpc_apihost", ""); - public static agent_ws_apihost: string = Config.getEnv("agent_ws_apihost", ""); - public static agent_oidc_config: string = Config.getEnv("agent_oidc_config", ""); - public static agent_oidc_client_id: string = Config.getEnv("agent_oidc_client_id", ""); - public static agent_oidc_client_secret: string = Config.getEnv("agent_oidc_client_secret", ""); - public static agent_oidc_userinfo_endpoint: string = Config.getEnv("agent_oidc_userinfo_endpoint", ""); - public static agent_oidc_issuer: string = Config.getEnv("agent_oidc_issuer", ""); - public static agent_oidc_authorization_endpoint: string = Config.getEnv("agent_oidc_authorization_endpoint", ""); - public static agent_oidc_token_endpoint: string = Config.getEnv("agent_oidc_token_endpoint", ""); - - public static saml_federation_metadata: string = Config.getEnv("saml_federation_metadata", ""); - public static api_ws_url: string = Config.getEnv("api_ws_url", ""); - public static nodered_ws_url: string = Config.getEnv("nodered_ws_url", ""); - public static nodered_saml_entrypoint: string = Config.getEnv("nodered_saml_entrypoint", ""); - - public static agent_docker_entrypoints: string = Config.getEnv("agent_docker_entrypoints", "web"); - public static agent_docker_use_project: boolean = Config.parseBoolean(Config.getEnv("agent_docker_use_project", "false")); - public static agent_docker_certresolver: string = Config.getEnv("agent_docker_certresolver", ""); - - public static namespace: string = Config.getEnv("namespace", ""); // also sent to website - public static nodered_domain_schema: string = Config.getEnv("nodered_domain_schema", ""); // also sent to website - public static nodered_initial_liveness_delay: number = parseInt(Config.getEnv("nodered_initial_liveness_delay", "60")); - public static nodered_allow_nodeselector: boolean = Config.parseBoolean(Config.getEnv("nodered_allow_nodeselector", "false")); - public static nodered_liveness_failurethreshold: number = parseInt(Config.getEnv("nodered_liveness_failurethreshold", "5")); - public static nodered_liveness_timeoutseconds: number = parseInt(Config.getEnv("nodered_liveness_timeoutseconds", "5")); - public static noderedcatalogues: string = Config.getEnv("noderedcatalogues", ""); - - public static otel_measure_nodeid: boolean = Config.parseBoolean(Config.getEnv("otel_measure_nodeid", "false")); - public static otel_measure_queued_messages: boolean = Config.parseBoolean(Config.getEnv("otel_measure_queued_messages", "false")); - public static otel_measure__mongodb_watch: boolean = Config.parseBoolean(Config.getEnv("otel_measure__mongodb_watch", "false")); - public static otel_measure_onlineuser: boolean = Config.parseBoolean(Config.getEnv("otel_measure_onlineuser", "false")); - public static enable_analytics: boolean = Config.parseBoolean(Config.getEnv("enable_analytics", "true")); - public static enable_detailed_analytic: boolean = Config.parseBoolean(Config.getEnv("enable_detailed_analytic", "false")); - public static otel_debug_log: boolean = Config.parseBoolean(Config.getEnv("otel_debug_log", "false")); - public static otel_warn_log: boolean = Config.parseBoolean(Config.getEnv("otel_warn_log", "false")); - public static otel_err_log: boolean = Config.parseBoolean(Config.getEnv("otel_err_log", "false")); - public static otel_trace_url: string = Config.getEnv("otel_trace_url", ""); - public static otel_metric_url: string = Config.getEnv("otel_metric_url", ""); - public static otel_trace_interval: number = parseInt(Config.getEnv("otel_trace_interval", "5000")); - public static otel_metric_interval: number = parseInt(Config.getEnv("otel_metric_interval", "5000")); - public static otel_trace_pingclients: boolean = Config.parseBoolean(Config.getEnv("otel_trace_pingclients", "false")); - public static otel_trace_dashboardauth: boolean = Config.parseBoolean(Config.getEnv("otel_trace_dashboardauth", "false")); - public static otel_trace_include_query: boolean = Config.parseBoolean(Config.getEnv("otel_trace_include_query", "false")); - public static otel_trace_connection_ips: boolean = Config.parseBoolean(Config.getEnv("otel_trace_connection_ips", "false")); - public static otel_trace_mongodb_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_per_users", "false")); - public static otel_trace_mongodb_query_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_query_per_users", "false")); - public static otel_trace_mongodb_count_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_count_per_users", "false")); - public static otel_trace_mongodb_aggregate_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_aggregate_per_users", "false")); - public static otel_trace_mongodb_insert_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_insert_per_users", "false")); - public static otel_trace_mongodb_update_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_update_per_users", "false")); - public static otel_trace_mongodb_delete_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_delete_per_users", "false")); - - public static grpc_keepalive_time_ms: number = parseInt(Config.getEnv("grpc_keepalive_time_ms", "-1")); - public static grpc_keepalive_timeout_ms: number = parseInt(Config.getEnv("grpc_keepalive_timeout_ms", "-1")); - public static grpc_http2_min_ping_interval_without_data_ms: number = parseInt(Config.getEnv("grpc_http2_min_ping_interval_without_data_ms", "-1")); - public static grpc_max_connection_idle_ms: number = parseInt(Config.getEnv("grpc_max_connection_idle_ms", "-1")); - public static grpc_max_connection_age_ms: number = parseInt(Config.getEnv("grpc_max_connection_age_ms", "-1")); - public static grpc_max_connection_age_grace_ms: number = parseInt(Config.getEnv("grpc_max_connection_age_grace_ms", "-1")); - public static grpc_http2_max_pings_without_data: number = parseInt(Config.getEnv("grpc_http2_max_pings_without_data", "-1")); - public static grpc_keepalive_permit_without_calls: number = parseInt(Config.getEnv("grpc_keepalive_permit_without_calls", "-1")); - public static grpc_max_receive_message_length: number = parseInt(Config.getEnv("grpc_max_receive_message_length", "-1")); - public static grpc_max_send_message_length: number = parseInt(Config.getEnv("grpc_max_send_message_length", "-1")); - - public static validate_user_form: string = Config.getEnv("validate_user_form", ""); + public static log_with_colors: boolean = Config.parseBoolean(Config.getEnv("log_with_colors")); + + public static cache_store_type: string = Config.getEnv("cache_store_type"); + public static cache_store_max: number = parseInt(Config.getEnv("cache_store_max")); + public static cache_store_ttl_seconds: number = parseInt(Config.getEnv("cache_store_ttl_seconds")); + public static cache_store_redis_host: string = Config.getEnv("cache_store_redis_host"); + public static cache_store_redis_port: number = parseInt(Config.getEnv("cache_store_redis_port")); + public static cache_store_redis_password: string = Config.getEnv("cache_store_redis_password"); + public static cache_workitem_queues: boolean = Config.parseBoolean(Config.getEnv("cache_workitem_queues")); + + public static log_cache: boolean = Config.parseBoolean(Config.getEnv("log_cache")); + public static log_amqp: boolean = Config.parseBoolean(Config.getEnv("log_amqp")); + public static log_openapi: boolean = Config.parseBoolean(Config.getEnv("log_openapi")); + public static log_login_provider: boolean = Config.parseBoolean(Config.getEnv("log_login_provider")); + public static log_with_trace: boolean = Config.parseBoolean(Config.getEnv("log_with_trace")); + public static log_websocket: boolean = Config.parseBoolean(Config.getEnv("log_websocket")); + public static log_oauth: boolean = Config.parseBoolean(Config.getEnv("log_oauth")); + public static log_webserver: boolean = Config.parseBoolean(Config.getEnv("log_webserver")); + public static log_database: boolean = Config.parseBoolean(Config.getEnv("log_database")); + public static log_database_queries: boolean = Config.parseBoolean(Config.getEnv("log_database_queries")); + public static log_database_queries_ms: number = parseInt(Config.getEnv("log_database_queries_ms")); + + public static log_grafana: boolean = Config.parseBoolean(Config.getEnv("log_grafana")); + public static log_housekeeping: boolean = Config.parseBoolean(Config.getEnv("log_housekeeping")); + public static log_otel: boolean = Config.parseBoolean(Config.getEnv("log_otel")); + public static log_blocked_ips: boolean = Config.parseBoolean(Config.getEnv("log_blocked_ips")); + public static log_information: boolean = Config.parseBoolean(Config.getEnv("log_information")); + public static log_debug: boolean = Config.parseBoolean(Config.getEnv("log_debug")); + public static log_verbose: boolean = Config.parseBoolean(Config.getEnv("log_verbose")); + public static log_silly: boolean = Config.parseBoolean(Config.getEnv("log_silly")); + public static log_to_exchange: boolean = Config.parseBoolean(Config.getEnv("log_to_exchange")); + + public static heapdump_onstop: boolean = Config.parseBoolean(Config.getEnv("heapdump_onstop")); + + public static amqp_allow_replyto_empty_queuename: boolean = Config.parseBoolean(Config.getEnv("amqp_allow_replyto_empty_queuename")); + + public static openflow_uniqueid: string = Config.getEnv("openflow_uniqueid"); + public static enable_openflow_amqp: boolean = Config.parseBoolean(Config.getEnv("enable_openflow_amqp")); + public static openflow_amqp_expiration: number = parseInt(Config.getEnv("openflow_amqp_expiration")); + public static amqp_prefetch: number = parseInt(Config.getEnv("amqp_prefetch")); + public static enable_entity_restriction: boolean = Config.parseBoolean(Config.getEnv("enable_entity_restriction")); + public static enable_web_tours: boolean = Config.parseBoolean(Config.getEnv("enable_web_tours")); + public static enable_nodered_tours: boolean = Config.parseBoolean(Config.getEnv("enable_nodered_tours")); + public static grafana_url:string = Config.getEnv("grafana_url"); + public static auto_hourly_housekeeping: boolean = Config.parseBoolean(Config.getEnv("auto_hourly_housekeeping")); + public static housekeeping_skip_collections: string = Config.getEnv("housekeeping_skip_collections"); + public static workitem_queue_monitoring_enabled: boolean = Config.parseBoolean(Config.getEnv("workitem_queue_monitoring_enabled")); + public static workitem_queue_monitoring_interval: number = parseInt(Config.getEnv("workitem_queue_monitoring_interval")); + + public static upload_max_filesize_mb: number = parseInt(Config.getEnv("upload_max_filesize_mb")); + + public static getting_started_url: string = Config.getEnv("getting_started_url"); + + public static NODE_ENV: string = Config.getEnv("NODE_ENV"); + public static HTTP_PROXY: string = Config.getEnv("HTTP_PROXY"); + public static HTTPS_PROXY: string = Config.getEnv("HTTPS_PROXY"); + public static NO_PROXY: string = Config.getEnv("NO_PROXY"); + public static agent_HTTP_PROXY: string = Config.getEnv("agent_HTTP_PROXY"); + public static agent_HTTPS_PROXY: string = Config.getEnv("agent_HTTPS_PROXY"); + public static agent_NO_PROXY: string = Config.getEnv("agent_NO_PROXY"); + + public static stripe_api_key: string = Config.getEnv("stripe_api_key"); + public static stripe_api_secret: string = Config.getEnv("stripe_api_secret"); + public static stripe_force_vat: boolean = Config.parseBoolean(Config.getEnv("stripe_force_vat")); + public static stripe_force_checkout: boolean = Config.parseBoolean(Config.getEnv("stripe_force_checkout")); + public static stripe_allow_promotion_codes: boolean = Config.parseBoolean(Config.getEnv("stripe_allow_promotion_codes")); + + public static supports_watch: boolean = Config.parseBoolean(Config.getEnv("supports_watch")); + public static ensure_indexes: boolean = Config.parseBoolean(Config.getEnv("ensure_indexes")); + public static text_index_name_fields: string[] = Config.parseArray(Config.getEnv("text_index_name_fields")); + + public static auto_create_users: boolean = Config.parseBoolean(Config.getEnv("auto_create_users")); + public static auto_create_user_from_jwt: boolean = Config.parseBoolean(Config.getEnv("auto_create_user_from_jwt")); + public static auto_create_domains: string[] = Config.parseArray(Config.getEnv("auto_create_domains")); + public static persist_user_impersonation: boolean = Config.parseBoolean(Config.getEnv("persist_user_impersonation")); + public static ping_clients_interval: number = parseInt(Config.getEnv("ping_clients_interval")); // 10 seconds + + public static use_ingress_beta1_syntax: boolean = Config.parseBoolean(Config.getEnv("use_ingress_beta1_syntax")); + public static use_openshift_routes: boolean = Config.parseBoolean(Config.getEnv("use_openshift_routes")); + public static agent_image_pull_secrets: string[] = Config.parseArray(Config.getEnv("agent_image_pull_secrets")); + public static auto_create_personal_nodered_group: boolean = Config.parseBoolean(Config.getEnv("auto_create_personal_nodered_group")); + public static auto_create_personal_noderedapi_group: boolean = Config.parseBoolean(Config.getEnv("auto_create_personal_noderedapi_group")); + public static force_add_admins: boolean = Config.parseBoolean(Config.getEnv("force_add_admins")); + public static validate_emails: boolean = Config.parseBoolean(Config.getEnv("validate_emails")); + public static forgot_pass_emails: boolean = Config.parseBoolean(Config.getEnv("forgot_pass_emails")); + public static smtp_service: string = Config.getEnv("smtp_service"); + public static smtp_from: string = Config.getEnv("smtp_from"); + public static smtp_user: string = Config.getEnv("smtp_user"); + public static smtp_pass: string = Config.getEnv("smtp_pass"); + public static smtp_url: string = Config.getEnv("smtp_url"); + public static debounce_lookup: boolean = Config.parseBoolean(Config.getEnv("debounce_lookup")); + public static validate_emails_disposable: boolean = Config.parseBoolean(Config.getEnv("validate_emails_disposable")); + + public static tls_crt: string = Config.getEnv("tls_crt"); + public static tls_key: string = Config.getEnv("tls_key"); + public static tls_ca: string = Config.getEnv("tls_ca"); + public static tls_passphrase: string = Config.getEnv("tls_passphrase"); + + public static oidc_access_token_ttl: number = parseInt(Config.getEnv("oidc_access_token_ttl")); + public static oidc_authorization_code_ttl: number = parseInt(Config.getEnv("oidc_authorization_code_ttl")); + public static oidc_client_credentials_ttl: number = parseInt(Config.getEnv("oidc_client_credentials_ttl")); + public static oidc_refresh_token_ttl: number = parseInt(Config.getEnv("oidc_refresh_token_ttl")); + public static oidc_session_ttl: number = parseInt(Config.getEnv("oidc_session_ttl")); + public static oidc_cookie_key: string = Config.getEnv("oidc_cookie_key"); + + public static api_rate_limit: boolean = Config.parseBoolean(Config.getEnv("api_rate_limit")); + public static api_rate_limit_points: number = parseInt(Config.getEnv("api_rate_limit_points")); + public static api_rate_limit_duration: number = parseInt(Config.getEnv("api_rate_limit_duration")); + public static socket_rate_limit: boolean = Config.parseBoolean(Config.getEnv("socket_rate_limit")); + public static socket_rate_limit_points: number = parseInt(Config.getEnv("socket_rate_limit_points")); + public static socket_rate_limit_points_disconnect: number = parseInt(Config.getEnv("socket_rate_limit_points_disconnect")); + public static socket_rate_limit_duration: number = parseInt(Config.getEnv("socket_rate_limit_duration")); + public static socket_error_rate_limit_points: number = parseInt(Config.getEnv("socket_error_rate_limit_points")); + public static socket_error_rate_limit_duration: number = parseInt(Config.getEnv("socket_error_rate_limit_duration")); + + public static client_heartbeat_timeout: number = parseInt(Config.getEnv("client_heartbeat_timeout")); + public static client_signin_timeout: number = parseInt(Config.getEnv("client_signin_timeout")); + public static client_disconnect_signin_error: boolean = Config.parseBoolean(Config.getEnv("client_disconnect_signin_error")); + + public static expected_max_roles: number = parseInt(Config.getEnv("expected_max_roles")); + public static decorate_roles_fetching_all_roles = Config.parseBoolean(Config.getEnv("decorate_roles_fetching_all_roles")); + public static max_recursive_group_depth: number = parseInt(Config.getEnv("max_recursive_group_depth")); + public static update_acl_based_on_groups: boolean = Config.parseBoolean(Config.getEnv("update_acl_based_on_groups")); + public static allow_merge_acl: boolean = Config.parseBoolean(Config.getEnv("allow_merge_acl")); + + public static multi_tenant: boolean = Config.parseBoolean(Config.getEnv("multi_tenant")); + public static cleanup_on_delete_customer: boolean = Config.parseBoolean(Config.getEnv("cleanup_on_delete_customer")); + public static cleanup_on_delete_user: boolean = Config.parseBoolean(Config.getEnv("cleanup_on_delete_user")); + public static api_bypass_perm_check: boolean = Config.parseBoolean(Config.getEnv("api_bypass_perm_check")); + public static ignore_expiration: boolean = Config.parseBoolean(Config.getEnv("ignore_expiration")); + public static force_audit_ts: boolean = Config.parseBoolean(Config.getEnv("force_audit_ts")); + public static force_dbusage_ts: boolean = Config.parseBoolean(Config.getEnv("force_dbusage_ts")); + public static migrate_audit_to_ts: boolean = Config.parseBoolean(Config.getEnv("migrate_audit_to_ts")); + + public static websocket_package_size: number = parseInt(Config.getEnv("websocket_package_size"), 10); + public static websocket_max_package_count: number = parseInt(Config.getEnv("websocket_max_package_count"), 10); + public static websocket_message_callback_timeout: number = parseInt(Config.getEnv("websocket_message_callback_timeout"), 10); + public static websocket_disconnect_out_of_sync: boolean = Config.parseBoolean(Config.getEnv("websocket_disconnect_out_of_sync")); + public static protocol: string = Config.getEnv("protocol"); + public static port: number = parseInt(Config.getEnv("port")); + public static domain: string = Config.getEnv("domain"); + public static cookie_secret: string = Config.getEnv("cookie_secret"); + public static max_ace_count: number = parseInt(Config.getEnv("max_ace_count"), 10); + + public static amqp_reply_expiration: number = parseInt(Config.getEnv("amqp_reply_expiration")); + public static amqp_force_queue_prefix: boolean = Config.parseBoolean(Config.getEnv("amqp_force_queue_prefix")); + public static amqp_force_exchange_prefix: boolean = Config.parseBoolean(Config.getEnv("amqp_force_exchange_prefix")); + public static amqp_force_sender_has_read: boolean = Config.parseBoolean(Config.getEnv("amqp_force_sender_has_read")); + public static amqp_force_sender_has_invoke: boolean = Config.parseBoolean(Config.getEnv("amqp_force_sender_has_invoke")); + public static amqp_force_consumer_has_update: boolean = Config.parseBoolean(Config.getEnv("amqp_force_consumer_has_update")); + public static amqp_enabled_exchange: boolean = Config.parseBoolean(Config.getEnv("amqp_enabled_exchange")); + public static amqp_url: string = Config.getEnv("amqp_url"); + public static amqp_username: string = Config.getEnv("amqp_username"); + public static amqp_password: string = Config.getEnv("amqp_password"); + + public static amqp_check_for_consumer: boolean = Config.parseBoolean(Config.getEnv("amqp_check_for_consumer")); + public static amqp_check_for_consumer_count: boolean = Config.parseBoolean(Config.getEnv("amqp_check_for_consumer_count")); + public static amqp_default_expiration: number = parseInt(Config.getEnv("amqp_default_expiration")); + public static amqp_requeue_time: number = parseInt(Config.getEnv("amqp_requeue_time")); + public static amqp_dlx: string = Config.getEnv("amqp_dlx"); + + public static mongodb_url: string = Config.getEnv("mongodb_url"); + public static mongodb_db: string = Config.getEnv("mongodb_db"); + public static mongodb_minpoolsize: number = parseInt(Config.getEnv("mongodb_minpoolsize")); + public static mongodb_maxpoolsize: number = parseInt(Config.getEnv("mongodb_maxpoolsize")); + + public static skip_history_collections: string = Config.getEnv("skip_history_collections"); + public static history_delta_count: number = parseInt(Config.getEnv("history_delta_count")); + public static allow_skiphistory: boolean = Config.parseBoolean(Config.getEnv("allow_skiphistory")); + public static max_memory_restart_mb: number = parseInt(Config.getEnv("max_memory_restart_mb")); + + public static saml_issuer: string = Config.getEnv("saml_issuer"); + public static aes_secret: string = Config.getEnv("aes_secret"); + public static signing_crt: string = Config.getEnv("signing_crt"); + public static singing_key: string = Config.getEnv("singing_key"); + public static wapid_mail: string = Config.getEnv("wapid_mail"); + public static wapid_pub: string = Config.getEnv("wapid_pub"); + public static wapid_key: string = Config.getEnv("wapid_key"); + + public static shorttoken_expires_in: string = Config.getEnv("shorttoken_expires_in"); + public static longtoken_expires_in: string = Config.getEnv("longtoken_expires_in"); + public static downloadtoken_expires_in: string = Config.getEnv("downloadtoken_expires_in"); + public static personalnoderedtoken_expires_in: string = Config.getEnv("personalnoderedtoken_expires_in"); + + public static agent_images: NoderedImage[] = JSON.parse(Config.getEnv("agent_images")); + public static agent_domain_schema: string = Config.getEnv("agent_domain_schema"); + public static agent_node_selector:string = Config.getEnv("agent_node_selector"); + + public static agent_grpc_apihost: string = Config.getEnv("agent_grpc_apihost"); + public static agent_ws_apihost: string = Config.getEnv("agent_ws_apihost"); + public static agent_oidc_config: string = Config.getEnv("agent_oidc_config"); + public static agent_oidc_client_id: string = Config.getEnv("agent_oidc_client_id"); + public static agent_oidc_client_secret: string = Config.getEnv("agent_oidc_client_secret"); + public static agent_oidc_userinfo_endpoint: string = Config.getEnv("agent_oidc_userinfo_endpoint"); + public static agent_oidc_issuer: string = Config.getEnv("agent_oidc_issuer"); + public static agent_oidc_authorization_endpoint: string = Config.getEnv("agent_oidc_authorization_endpoint"); + public static agent_oidc_token_endpoint: string = Config.getEnv("agent_oidc_token_endpoint"); + + public static saml_federation_metadata: string = Config.getEnv("saml_federation_metadata"); + public static api_ws_url: string = Config.getEnv("api_ws_url"); + + public static agent_docker_entrypoints: string = Config.getEnv("agent_docker_entrypoints"); + public static agent_docker_use_project: boolean = Config.parseBoolean(Config.getEnv("agent_docker_use_project")); + public static agent_docker_certresolver: string = Config.getEnv("agent_docker_certresolver"); + + public static namespace: string = Config.getEnv("namespace"); + public static agent_allow_nodeselector: boolean = Config.parseBoolean(Config.getEnv("agent_allow_nodeselector")); + + public static otel_measure_queued_messages: boolean = Config.parseBoolean(Config.getEnv("otel_measure_queued_messages")); + public static otel_measure__mongodb_watch: boolean = Config.parseBoolean(Config.getEnv("otel_measure__mongodb_watch")); + public static enable_analytics: boolean = Config.parseBoolean(Config.getEnv("enable_analytics")); + public static enable_detailed_analytic: boolean = Config.parseBoolean(Config.getEnv("enable_detailed_analytic")); + public static otel_debug_log: boolean = Config.parseBoolean(Config.getEnv("otel_debug_log")); + public static otel_warn_log: boolean = Config.parseBoolean(Config.getEnv("otel_warn_log")); + public static otel_err_log: boolean = Config.parseBoolean(Config.getEnv("otel_err_log")); + public static otel_trace_url: string = Config.getEnv("otel_trace_url"); + public static otel_metric_url: string = Config.getEnv("otel_metric_url"); + public static otel_trace_interval: number = parseInt(Config.getEnv("otel_trace_interval")); + public static otel_metric_interval: number = parseInt(Config.getEnv("otel_metric_interval")); + public static otel_trace_pingclients: boolean = Config.parseBoolean(Config.getEnv("otel_trace_pingclients")); + public static otel_trace_dashboardauth: boolean = Config.parseBoolean(Config.getEnv("otel_trace_dashboardauth")); + public static otel_trace_include_query: boolean = Config.parseBoolean(Config.getEnv("otel_trace_include_query")); + public static otel_trace_connection_ips: boolean = Config.parseBoolean(Config.getEnv("otel_trace_connection_ips")); + public static otel_trace_mongodb_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_per_users")); + public static otel_trace_mongodb_query_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_query_per_users")); + public static otel_trace_mongodb_count_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_count_per_users")); + public static otel_trace_mongodb_aggregate_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_aggregate_per_users")); + public static otel_trace_mongodb_insert_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_insert_per_users")); + public static otel_trace_mongodb_update_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_update_per_users")); + public static otel_trace_mongodb_delete_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_delete_per_users")); + + public static grpc_keepalive_time_ms: number = parseInt(Config.getEnv("grpc_keepalive_time_ms")); + public static grpc_keepalive_timeout_ms: number = parseInt(Config.getEnv("grpc_keepalive_timeout_ms")); + public static grpc_http2_min_ping_interval_without_data_ms: number = parseInt(Config.getEnv("grpc_http2_min_ping_interval_without_data_ms")); + public static grpc_max_connection_idle_ms: number = parseInt(Config.getEnv("grpc_max_connection_idle_ms")); + public static grpc_max_connection_age_ms: number = parseInt(Config.getEnv("grpc_max_connection_age_ms")); + public static grpc_max_connection_age_grace_ms: number = parseInt(Config.getEnv("grpc_max_connection_age_grace_ms")); + public static grpc_http2_max_pings_without_data: number = parseInt(Config.getEnv("grpc_http2_max_pings_without_data")); + public static grpc_keepalive_permit_without_calls: number = parseInt(Config.getEnv("grpc_keepalive_permit_without_calls")); + public static grpc_max_receive_message_length: number = parseInt(Config.getEnv("grpc_max_receive_message_length")); + public static grpc_max_send_message_length: number = parseInt(Config.getEnv("grpc_max_send_message_length")); + + public static validate_user_form: string = Config.getEnv("validate_user_form"); public static externalbaseurl(): string { @@ -732,9 +713,11 @@ export class Config { } else { result = result + "/"; } return result; } - public static getEnv(name: string, defaultvalue: string): string { + public static getEnv(name: string): string { let value: any = process.env[name]; - if (!value || value === "") { value = defaultvalue; } + if (!value || value === "") { + value = this.default_config[name] + } return value; } public static get(url: string): Promise { diff --git a/OpenFlow/src/Logger.ts b/OpenFlow/src/Logger.ts index 2f7c85f5..670e7ea8 100644 --- a/OpenFlow/src/Logger.ts +++ b/OpenFlow/src/Logger.ts @@ -152,7 +152,7 @@ export class Logger { return JSON.stringify(plainObject, filter, space); }; if (Config.log_to_exchange && !Config.unittesting) { - if (NoderedUtil.IsNullEmpty(Logger._hostname)) Logger._hostname = (Config.getEnv("HOSTNAME", undefined) || os.hostname()) || "unknown"; + if (NoderedUtil.IsNullEmpty(Logger._hostname)) Logger._hostname = (process.env.HOSTNAME || os.hostname()) || "unknown"; if (amqpwrapper.Instance() && amqpwrapper.Instance().connected && amqpwrapper.Instance().of_logger_ready) { if (typeof obj.message == "object") obj.message = JSON.parse(stringifyError(obj.message, null, 2)); amqpwrapper.Instance().send("openflow_logs", "", { ...obj, host: Logger._hostname }, 500, null, "", span, 1); diff --git a/OpenFlow/src/LoginProvider.ts b/OpenFlow/src/LoginProvider.ts index cc7c77d0..c6d0081c 100644 --- a/OpenFlow/src/LoginProvider.ts +++ b/OpenFlow/src/LoginProvider.ts @@ -1126,10 +1126,6 @@ export class LoginProvider { let _url = Config.basewsurl(); if (!NoderedUtil.IsNullEmpty(Config.api_ws_url)) _url = Config.api_ws_url; if (!_url.endsWith("/")) _url += "/"; - let nodered_domain_schema = Config.nodered_domain_schema; - if (NoderedUtil.IsNullEmpty(nodered_domain_schema)) { - nodered_domain_schema = "$nodered_id$." + Config.domain; - } let agent_domain_schema = Config.agent_domain_schema; if (NoderedUtil.IsNullEmpty(agent_domain_schema)) { agent_domain_schema = "$slug$." + Config.domain; @@ -1151,7 +1147,6 @@ export class LoginProvider { auto_create_personal_nodered_group: Config.auto_create_personal_nodered_group, auto_create_personal_noderedapi_group: Config.auto_create_personal_noderedapi_group, namespace: Config.namespace, - nodered_domain_schema: nodered_domain_schema, agent_domain_schema: agent_domain_schema, websocket_package_size: Config.websocket_package_size, version: Config.version, @@ -1477,11 +1472,6 @@ export class LoginProvider { } } - Config.smtp_service = Config.getEnv("smtp_service", ""); - Config.smtp_from = Config.getEnv("smtp_from", ""); - Config.smtp_user = Config.getEnv("smtp_user", ""); - Config.smtp_pass = Config.getEnv("smtp_service", ""); - if (Config.validate_emails) { let email: string = tuser.username; if (tuser.email && tuser.email.indexOf("@") > -1) email = tuser.email; diff --git a/OpenFlow/src/Messages/Message.ts b/OpenFlow/src/Messages/Message.ts index 6fa4d089..5f3cc152 100644 --- a/OpenFlow/src/Messages/Message.ts +++ b/OpenFlow/src/Messages/Message.ts @@ -4179,13 +4179,6 @@ export class Message { cursor = Config.db.db.collection("users").find({ "_type": "user", lastseen: { "$gte": fivedaysago } }); } else { cursor = Config.db.db.collection("users").find({ "_type": "user", lastseen: { "$gte": fivedaysago } }); - // if (Config.nodered_domain_schema == "$nodered_id$.app.openiap.io") { - // // cursor = Config.db.db.collection("users").find({ "_type": "user", "dbusage": { "$gte": 15815993 } }) - // cursor = Config.db.db.collection("users").find({ "_type": "user", "dblocked": true }) - // } else { - // cursor = Config.db.db.collection("users").find({ "_type": "user" }) - // } - // While debugging, also update users who has not been online the last 24 hours } for await (const u of cursor) { if (u.dbusage == null) u.dbusage = 0; diff --git a/OpenFlow/src/WebServer.ts b/OpenFlow/src/WebServer.ts index 72e60312..421ea377 100644 --- a/OpenFlow/src/WebServer.ts +++ b/OpenFlow/src/WebServer.ts @@ -777,7 +777,7 @@ export class WebServer { let span = Logger.otel.startSpanExpress("get_livenessprobe", req) try { const [traceId, spanId] = Logger.otel.GetTraceSpanId(span); - if (NoderedUtil.IsNullEmpty(_hostname)) _hostname = (Config.getEnv("HOSTNAME", undefined) || os.hostname()) || "unknown"; + if (NoderedUtil.IsNullEmpty(_hostname)) _hostname = (process.env.HOSTNAME || os.hostname()) || "unknown"; res.end(JSON.stringify({ "success": "true", "hostname": _hostname, dt: new Date(), traceId, spanId })); res.end(); // @ts-ignore diff --git a/OpenFlow/src/WebSocketServer.ts b/OpenFlow/src/WebSocketServer.ts index 5d1e7696..fe9e397f 100644 --- a/OpenFlow/src/WebSocketServer.ts +++ b/OpenFlow/src/WebSocketServer.ts @@ -219,7 +219,7 @@ export class WebSocketServer { try { WebSocketServer._remoteclients = []; - const hostname = (Config.getEnv("HOSTNAME", undefined) || os.hostname()) || "unknown"; + const hostname = (process.env.HOSTNAME || os.hostname()) || "unknown"; const clients: Base[] = []; for (let i = WebSocketServer._clients.length - 1; i >= 0; i--) { const cli: WebSocketServerClient = WebSocketServer._clients[i]; diff --git a/OpenFlow/src/public/CommonControllers.ts b/OpenFlow/src/public/CommonControllers.ts index 66751b3f..57a6295e 100644 --- a/OpenFlow/src/public/CommonControllers.ts +++ b/OpenFlow/src/public/CommonControllers.ts @@ -941,7 +941,6 @@ export class entityCtrl { } // @ts-ignore this.weburl = "//" + this.WebSocketClientService.agent_domain_schema.replace("$slug$", this.model.slug) - // this.weburl = "//" + this.WebSocketClientService.nodered_domain_schema.replace("$nodered_id$", this.model.slug) } catch (error) { this.loading = false; this.errormessage = JSON.stringify(error); diff --git a/OpenFlow/src/public/Controllers.ts b/OpenFlow/src/public/Controllers.ts index 5319e581..0a39c5b7 100644 --- a/OpenFlow/src/public/Controllers.ts +++ b/OpenFlow/src/public/Controllers.ts @@ -4554,7 +4554,6 @@ export class ClientsCtrl extends entitiesCtrl { let name = model.username; name = name.toLowerCase(); name = name.replace(/([^a-z0-9]+){1,63}/gi, ""); - // const noderedurl = "//" + this.WebSocketClientService.nodered_domain_schema.replace("$nodered_id$", name); const noderedurl = "//" + this.WebSocketClientService.agent_domain_schema.replace("$slug$", name); window.open(noderedurl); } @@ -7572,23 +7571,15 @@ export class ConfigCtrl extends entityCtrl { {"name": "agent_oidc_token_endpoint", "type": "string", "default": ""}, {"name": "saml_federation_metadata", "type": "string", "default": ""}, {"name": "api_ws_url", "type": "string", "default": ""}, - {"name": "nodered_ws_url", "type": "string", "default": ""}, - {"name": "nodered_saml_entrypoint", "type": "string", "default": ""}, {"name": "agent_docker_entrypoints", "type": "string", "default": "web"}, {"name": "agent_docker_use_project", "type": "boolean", "default": "false"}, {"name": "agent_docker_certresolver", "type": "string", "default": ""}, {"name": "namespace", "type": "string", "default": ""}, - {"name": "nodered_domain_schema", "type": "string", "default": ""}, - {"name": "nodered_initial_liveness_delay", "type": "number", "default": "60"}, - {"name": "nodered_allow_nodeselector", "type": "boolean", "default": "false"}, - {"name": "nodered_liveness_failurethreshold", "type": "number", "default": "5"}, - {"name": "nodered_liveness_timeoutseconds", "type": "number", "default": "5"}, - {"name": "noderedcatalogues", "type": "string", "default": ""}, + {"name": "agent_allow_nodeselector", "type": "boolean", "default": "false"}, {"name": "otel_measure_nodeid", "type": "boolean", "default": "false"}, {"name": "otel_measure_queued_messages", "type": "boolean", "default": "false"}, {"name": "otel_measure__mongodb_watch", "type": "boolean", "default": "false"}, - {"name": "otel_measure_onlineuser", "type": "boolean", "default": "false"}, {"name": "enable_analytics", "type": "boolean", "default": "true"}, {"name": "enable_detailed_analytic", "type": "boolean", "default": "false"}, {"name": "otel_debug_log", "type": "boolean", "default": "false"}, @@ -7884,7 +7875,6 @@ export class AgentsCtrl extends entitiesCtrl { if (!this.$scope.$$phase) { this.$scope.$apply(); } } weburl(model) { - // return "//" + this.WebSocketClientService.nodered_domain_schema.replace("$nodered_id$", model.slug) return "//" + this.WebSocketClientService.agent_domain_schema.replace("$slug$", model.slug) } async DeleteAgent(model: any): Promise { diff --git a/OpenFlow/src/public/WebSocketClientService.ts b/OpenFlow/src/public/WebSocketClientService.ts index c1f702ba..cb77ebc7 100644 --- a/OpenFlow/src/public/WebSocketClientService.ts +++ b/OpenFlow/src/public/WebSocketClientService.ts @@ -28,7 +28,6 @@ export class WebSocketClientService { this.auto_create_personal_nodered_group = data.auto_create_personal_nodered_group; this.namespace = data.namespace; - this.nodered_domain_schema = data.nodered_domain_schema; this.agent_domain_schema = data.agent_domain_schema; this.websocket_package_size = data.websocket_package_size; @@ -180,7 +179,6 @@ export class WebSocketClientService { public allow_user_registration: boolean = false; public auto_create_personal_nodered_group: boolean = false; public namespace: string = ""; - public nodered_domain_schema: string = ""; public agent_domain_schema: string = ""; public websocket_package_size: number = 25000; public stripe_api_key: string = ""; diff --git a/docker-compose.yml b/docker-compose.yml index ee68de8e..3d47ab75 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -67,11 +67,8 @@ services: # - enable_nodered_tours=false # - enable_openflow_amqp=true - # - nodered_ws_url=ws://api:3000 - # - domain=localhost.openiap.io # - saml_federation_metadata=http://api:3000/issue/FederationMetadata/2007-06/FederationMetadata.xml - # - nodered_saml_entrypoint=http://localhost.openiap.io/issue # - saml_issuer=uri:localhost.openiap.io # - aes_secret=7TXsxf7cn9EkUqm5h4MEWGjzkxkNCk2K diff --git a/openflow.env b/openflow.env index bc18a7bf..a12a284f 100644 --- a/openflow.env +++ b/openflow.env @@ -1,7 +1,6 @@ mongodb_url=mongodb://localhost:27017 mongodb_db=openflow namespace=openflow -# nodered_domain_schema=$nodered_id$.localhost.openiap.io port=80 signing_crt=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURqRENDQW5TZ0F3SUJBZ0lKQUp5N0tIMTE1dkQ4TUEwR0NTcUdTSWIzRFFFQkN3VUFNRnN4Q3pBSkJnTlYKQkFZVEFrUkxNUk13RVFZRFZRUUlEQXBUYjIxbExWTjBZWFJsTVNFd0h3WURWUVFLREJoSmJuUmxjbTVsZENCWAphV1JuYVhSeklGQjBlU0JNZEdReEZEQVNCZ05WQkFNTUMzTnBaMjVwYm1kalpYSjBNQjRYRFRFNU1EUXdOekUwCk5ETXpORm9YRFRJNU1EUXdOREUwTkRNek5Gb3dXekVMTUFrR0ExVUVCaE1DUkVzeEV6QVJCZ05WQkFnTUNsTnYKYldVdFUzUmhkR1V4SVRBZkJnTlZCQW9NR0VsdWRHVnlibVYwSUZkcFpHZHBkSE1nVUhSNUlFeDBaREVVTUJJRwpBMVVFQXd3TGMybG5ibWx1WjJObGNuUXdnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCRHdBd2dnRUtBb0lCCkFRQy9JSkdEaGxLTU9SWkoycXQwSWpjSDZOWUFtZDVxQzQ4dkNJRE54QWZCbmQxQnN4WlVjWkl5dkFlT28yNDcKM3I0eTYwNDgxRHVUS2JaMTBTNjRqRU05aW1XTXB1TFlJRnVyQ3BWNzVEWWhxMS85Q0FJVHJqNjlmVDluSkptcwpjM2lxTnJ1Tlg1bDlISXdadWtQM1ZNRkJRNWZVd3N1ZnE0YW1NbnVnZmtyUEVzSngxK3VJb0NYU3pyblZvcnZpClZ0ZFh4a3M4N0l1S0ZnMDJIZ1RQSzdwc0FXYTBRY3g2ck04bkV5TUhwNUdlR1Rvb1NNbkcyZ1RGNWZOSVFNdTMKVEVoc2p3SWRTYmRwck1Gb1VZV05Bc2FueTJOQk0wREhZRUdjQlZhZ0xWNUhFUW5ySUM3NEhtNjYxdG9HaU5VSAoveW04U3VndTgwWVFiVGxPcTFWNnNkaVRBZ01CQUFHalV6QlJNQjBHQTFVZERnUVdCQlMrYWc1NGtITllKZ29pCm9yRnlia293THR5R3ZqQWZCZ05WSFNNRUdEQVdnQlMrYWc1NGtITllKZ29pb3JGeWJrb3dMdHlHdmpBUEJnTlYKSFJNQkFmOEVCVEFEQVFIL01BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQTZpZHBYdzdSR2pzcUpNanI2c1YvaQpXeXFsL2xkSy9sa1NCdnNBSENieDdQYi9rVUd2NHJNbndYMnBHdTR0YkFnSDc4eStmS3dzazllYkxDeTA4Y1k0Ckt5czhzbUpLenhWN0R6U3RVR1NvZmZMaFliVVVMK3UyNU5vVXc0TG1WQU5FU0NaMTZ3aTdPQUdJMkJnNFR6TXoKdnlIUHRaaE9wTXBNV2lzM2ZnRXFzV3QxS2VLcXo0Z2M5RnJtZDZPNlQzVVAxWTRBR3VEWnNScnpiU2RQS2JxbApxekprT2tQcGtHOGo3ZjFWNkk1ZlkzblZaSWk2YW1TcTM1RTJkQzVMY0dIQXRtT1lWL0c4TEQ3OUFnTVpFUU5vCkF2R2RnV1gvbXBFVkFjMmRFQkJlcUN6WjF1aVhWUjdERld2ZDFKTEpsdVRyTm9jMUROc0xNKzNEZFFiQ2JnYVcKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= singing_key=LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRQy9JSkdEaGxLTU9SWkoKMnF0MElqY0g2TllBbWQ1cUM0OHZDSUROeEFmQm5kMUJzeFpVY1pJeXZBZU9vMjQ3M3I0eTYwNDgxRHVUS2JaMQowUzY0akVNOWltV01wdUxZSUZ1ckNwVjc1RFlocTEvOUNBSVRyajY5ZlQ5bkpKbXNjM2lxTnJ1Tlg1bDlISXdaCnVrUDNWTUZCUTVmVXdzdWZxNGFtTW51Z2ZrclBFc0p4MSt1SW9DWFN6cm5Wb3J2aVZ0ZFh4a3M4N0l1S0ZnMDIKSGdUUEs3cHNBV2EwUWN4NnJNOG5FeU1IcDVHZUdUb29TTW5HMmdURjVmTklRTXUzVEVoc2p3SWRTYmRwck1GbwpVWVdOQXNhbnkyTkJNMERIWUVHY0JWYWdMVjVIRVFucklDNzRIbTY2MXRvR2lOVUgveW04U3VndTgwWVFiVGxPCnExVjZzZGlUQWdNQkFBRUNnZ0VBVXBjZ1NsV2hGamNWQ3BVVHdmdUhERVB4TmhGSHEwdVRkQitZaVZKTWg3NVAKL2pRRlVqaEJsT3JyMlJlR2F4aTEyQXNXby9LU1MrV2Frdzd4d1kzYkFKenRoUG9Zekl3dkVKcGlQa2MvblEwUgpUYVpJUDNqc1k3WGIwQlpnMGNTVVAvbW0wbENkWXhNUzk0c21FNXJzWitkdGxPTVlXc2NrU0cxSVB2SlVJV2FZCnl3NC9kaHJ3TWRsREVZS2tSbks1aDR1dXR1dzA1Q1VzNkZWN2F5cEJRRStGM3RxVlF3QWxGbWNueXdvZTB5WjQKZW1tWkRvT1dzNUk4cGNGbjZCSW1wZjN3UEg5UWhlQXJVaXRqV3YrZmI2cWRVaHJFVDBxMzh4dTZ5M1lJNFNLYQpXME9kUng4L3FTYllXdkpzbmxscDR0aUpDWE5IdnV6MVBKSGhxOUprQVFLQmdRRCt2dHlWcVJoaEJZTmd6WWorClFCaDNLcktrNEVqTXZaSGhEelhES0pMZ2JEVmJzNExrQkYySWZGOTF0S1k3d09CNVpXcU82Z3FqVVJBVE5hc1YKOExCOGE2TEpXYVJuTklLMnJkd1FwalFYcy92OVBSYnJwc2tTbDRJdUsyZWNBMjBSQkhicW5yNHZ5ZkQ4U3BzaApSdHlTUk5CRGVsaU01Z1JDM0JKKzBZbjBVUUtCZ1FEQUVZSUp3Y2xnQloyWmNNTlh1dW1lLzUxdHBHeGppRTJpCjZ3SDNsOHNTVDN1U1A4MHdndGdHaVFsUTRsR3BmSThqWkl4N0pwcGw0Qko1ZEtuVnpIS1dqMzA3YXYxcjdpU3QKLzJOVDNobzdaYkNlYzlhMHlJU2E3dTNGZGxzZ0VPcE45dURmbG5GQVQ3ZmIrM2d4Sk9DUWp1TkFCZXZaK2pScwpZY0ZhQWhGNW93S0JnUUNGUG9HVVNsRDFGblFrWXYwL3QzalVnK0hTK1hrNmxnRkNqYmthTGhPOURQeFB6Ykl0CjM5YW9lQjFhTExZeVZPMVVzZVl0Z0Y4MkUwVnNOc3NZKzc3a0pVeU5NclVhUWs0SWpTR3BGN1h4bS9PMi9vZ0oKbEVCaDJCdUFXTFdsMWVqcldNRjJjTGVidVcyeUdMZlJqUVg3LzhCTE95Z3I4bmZTSE5nVHV6Z0VNUUtCZ0JrZgpNUjhObGNWVmRyT25LQ1hGY09FM0ZlUk5hVS9yZUJ3akdQTEZpKzR0TDBDRno5VFVpR1R5YjZHQXVLV3VnUnBrCkFHdnJOSzYyakRRT3FsZ29rYVJYeUUySlJQUmxCYThzaEZWbjY0NXhVcFNuR2lJelNBVHIwM1hNY1ViVWI1RWIKQlhhNU9yN3FybVc3a3BENi9kUnFuQmEzcjQyblNFd1V6VEYwcTh4NUFvR0FIcXdRSyt1R0NTdlNsNENJUGhyRQpDREIvcytDK2NJNXVCeFJCMHVlNjc3L2lpdGZPSU9lOUNiTHE3R0tib0w4RVg3eXhKNVRLWjlYQmh5LzNCWmVNCllydEx3M2JicTNTN2hpUGFYSmE1dXZma3BWR1RnNEdzTnBJQ3VNTEJUaXJ6M0ZRV25UNFNZbzkrREVoalhEeVQKWlVOMERtUkJVNjNjWjRLSUlXd2xWUTA9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 021824ec..a692b258 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -97,9 +97,7 @@ services: - socket_rate_limit_points=1000 - socket_rate_limit_points_disconnect=2500 - - nodered_ws_url=ws://api:3000 - saml_federation_metadata=http://api:3000/issue/FederationMetadata/2007-06/FederationMetadata.xml - - nodered_saml_entrypoint=http://localhost.openiap.io/issue - amqp_url=amqp://guest:guest@rabbitmq - mongodb_url=mongodb://mongodb:27017 - mongodb_db=openrpa From 9e97de41b39f9d0a023161110b2303b0c2b8a7d0 Mon Sep 17 00:00:00 2001 From: skadefro Date: Thu, 23 Nov 2023 14:01:19 +0100 Subject: [PATCH 10/32] add 2 cleanup jobs for housekeeping --- OpenFlow/src/Config.ts | 11 +++++- OpenFlow/src/Messages/Message.ts | 56 +++++++++++------------------- OpenFlow/src/public/Controllers.ts | 2 ++ 3 files changed, 32 insertions(+), 37 deletions(-) diff --git a/OpenFlow/src/Config.ts b/OpenFlow/src/Config.ts index fc344b03..678cfa9e 100644 --- a/OpenFlow/src/Config.ts +++ b/OpenFlow/src/Config.ts @@ -160,7 +160,11 @@ export class dbConfig extends Base { try { var msg: InsertOrUpdateOneMessage = new InsertOrUpdateOneMessage(); msg.collectionname = "config"; msg.jwt = jwt; - msg.item = conf; + msg.item = JSON.parse(JSON.stringify(conf)); + // @ts-ignore + delete msg.item.default_config; + // @ts-ignore + delete msg.item.dbConfig; msg.uniqeness = "_id"; await Config.db._InsertOrUpdateOne(msg, parent); // await Config.db.InsertOrUpdateOne(null, conf, "config", 1, true, jwt, parent); @@ -224,6 +228,8 @@ export class Config { grafana_url: "", auto_hourly_housekeeping: true, housekeeping_skip_collections: "", + housekeeping_remomve_unvalidated_user_days: 0, // if above 0, remove unvalidated users after x days + housekeeping_cleanup_openrpa_instances: false, workitem_queue_monitoring_enabled: true, workitem_queue_monitoring_interval: 10 * 1000, // 10 sec upload_max_filesize_mb: 25, @@ -483,6 +489,8 @@ export class Config { public static grafana_url:string = Config.getEnv("grafana_url"); public static auto_hourly_housekeeping: boolean = Config.parseBoolean(Config.getEnv("auto_hourly_housekeeping")); public static housekeeping_skip_collections: string = Config.getEnv("housekeeping_skip_collections"); + public static housekeeping_remomve_unvalidated_user_days: number = parseInt(Config.getEnv("housekeeping_remomve_unvalidated_user_days")); + public static housekeeping_cleanup_openrpa_instances: boolean = Config.parseBoolean(Config.getEnv("housekeeping_cleanup_openrpa_instances")); public static workitem_queue_monitoring_enabled: boolean = Config.parseBoolean(Config.getEnv("workitem_queue_monitoring_enabled")); public static workitem_queue_monitoring_interval: number = parseInt(Config.getEnv("workitem_queue_monitoring_interval")); @@ -790,6 +798,7 @@ export class Config { return metadata; } public static parseArray(s: string): string[] { + if(Array.isArray(s)) return s; let arr = s.split(","); arr = arr.map(p => p.trim()); arr = arr.filter(result => (result.trim() !== "")); diff --git a/OpenFlow/src/Messages/Message.ts b/OpenFlow/src/Messages/Message.ts index 5f3cc152..31ab78a7 100644 --- a/OpenFlow/src/Messages/Message.ts +++ b/OpenFlow/src/Messages/Message.ts @@ -3682,43 +3682,27 @@ export class Message { } catch (error) { Logger.instanse.error(error, span); } + + if(Config.housekeeping_remomve_unvalidated_user_days > 0) { + let todate = new Date(); + todate.setDate(todate.getDate() - 1); + let fromdate = new Date(); + fromdate.setMonth(fromdate.getMonth() - 1); + const jwt: string = Crypt.rootToken(); + + let query = { "validated": false, "_type": "user" } + query["_modified"] = { "$lt": todate.toISOString(), "$gt": fromdate.toISOString()} + Config.db.DeleteMany(query, null, "users", "", false, jwt, span); + } + if(Config.housekeeping_cleanup_openrpa_instances == true) { + let msg = new UpdateManyMessage(); + msg.jwt = Crypt.rootToken(); + msg.collectionname = "openrpa_instances"; + msg.query = { "state": { "$in": ["idle", "running"] } }; + msg.item = { "$set": { "state": "completed"}, "$unset": {"xml": ""}} as any; + Config.db.UpdateDocument(msg, span); + } - // if (!skipNodered) { - // Logger.instanse.debug("Get running Nodered Instances", span); - // await this.GetNoderedInstance(span); - // Logger.instanse.debug("Get users with autocreate", span); - // const users: any[] = await Config.db.db.collection("users").find({ "_type": "user", "nodered.autocreate": true }).toArray(); - // // TODO: we should get instances and compare, running ensure for each user will not scale well - // for (let i = 0; i < users.length; i++) { - // let user = users[i]; - // var doensure = false; - // if (Config.multi_tenant) { - // if (!NoderedUtil.IsNullEmpty(Config.stripe_api_secret)) { - // if (!NoderedUtil.IsNullEmpty(user.customerid)) { - // // @ts-ignore - // var customers: Customer[] = await Config.db.db.collection("users").find({ "_type": "customer", "_id": user.customerid }).toArray(); - // if (customers.length > 0 && !NoderedUtil.IsNullEmpty(customers[0].subscriptionid)) { - // doensure = true; - // } - // } - // } else { - // doensure = true; - // } - // } else { - // doensure = true; - // } - // if (doensure) { - // Logger.instanse.debug("EnsureNoderedInstance for " + user.name, span); - // var ensuremsg: EnsureNoderedInstanceMessage = new EnsureNoderedInstanceMessage(); - // ensuremsg._id = user._id; - // var msg: Message = new Message(); msg.jwt = jwt; - // msg.data = JSON.stringify(ensuremsg); - // msg.tuser = this.tuser; - // await msg.EnsureNoderedInstance(span); - // } - // } - // Logger.instanse.debug("Done processing autocreate", span); - // } } catch (error) { } diff --git a/OpenFlow/src/public/Controllers.ts b/OpenFlow/src/public/Controllers.ts index 0a39c5b7..7e92303c 100644 --- a/OpenFlow/src/public/Controllers.ts +++ b/OpenFlow/src/public/Controllers.ts @@ -7431,6 +7431,8 @@ export class ConfigCtrl extends entityCtrl { {"name": "grafana_url", "type": "string", "default": ""}, {"name": "auto_hourly_housekeeping", "type": "boolean", "default": "true"}, {"name": "housekeeping_skip_collections", "type": "string", "default": ""}, + {"name": "housekeeping_remomve_unvalidated_user_days", "type": "number", "default": "0"}, + {"name": "housekeeping_cleanup_openrpa_instances", "type": "boolean", "default": "false"}, {"name": "workitem_queue_monitoring_enabled", "type": "boolean", "default": "true"}, {"name": "workitem_queue_monitoring_interval", "type": "number", "default": "10000"}, // 10 sec {"name": "upload_max_filesize_mb", "type": "number", "default": "25"}, From 30e296581d08387e33cc3c46e1f6c0b559206a50 Mon Sep 17 00:00:00 2001 From: skadefro Date: Thu, 23 Nov 2023 15:33:37 +0100 Subject: [PATCH 11/32] fix housekeeping job / watch property to config --- OpenFlow/src/Config.ts | 13 +++++++++---- OpenFlow/src/DatabaseConnection.ts | 2 +- OpenFlow/src/LoginProvider.ts | 6 +++++- OpenFlow/src/Messages/Message.ts | 12 ++++++++++-- OpenFlow/src/index.ts | 2 +- 5 files changed, 26 insertions(+), 9 deletions(-) diff --git a/OpenFlow/src/Config.ts b/OpenFlow/src/Config.ts index 678cfa9e..66fc6cac 100644 --- a/OpenFlow/src/Config.ts +++ b/OpenFlow/src/Config.ts @@ -37,7 +37,7 @@ export class dbConfig extends Base { public compare(version: string): number { return this.version.localeCompare(version, undefined, { numeric: true, sensitivity: 'base' }); } - public static async Load(jwt: string, parent: Span): Promise { + public static async Load(jwt: string, watch:boolean, parent: Span): Promise { var conf: dbConfig = await Config.db.GetOne({ query: { "_type": "config" }, collectionname: "config", jwt }, parent); if (conf == null) { conf = new dbConfig(); } conf = Object.assign(new dbConfig(), conf); @@ -156,7 +156,7 @@ export class dbConfig extends Base { } conf._encrypt = ["stripe_api_secret", "smtp_url", "amqp_password", "cache_store_redis_password", "cookie_secret", "singing_key", "wapid_key"]; - if(updated) { + if(updated && !watch) { try { var msg: InsertOrUpdateOneMessage = new InsertOrUpdateOneMessage(); msg.collectionname = "config"; msg.jwt = jwt; @@ -177,8 +177,8 @@ export class dbConfig extends Base { await Logger.reload(); return conf; } - public static async Reload(jwt: string, parent: Span): Promise { - Config.dbConfig = await dbConfig.Load(jwt, parent); + public static async Reload(jwt: string, watch:boolean, parent: Span): Promise { + Config.dbConfig = await dbConfig.Load(jwt, watch, parent); Logger.instanse.info("Reloaded config version " + Config.dbConfig._version, parent); } @@ -217,6 +217,11 @@ export class Config { log_silly: false, log_to_exchange: false, + tls_crt: "", + tls_key: "", + tls_ca: "", + tls_passphrase: "", + heapdump_onstop: false, amqp_allow_replyto_empty_queuename: false, enable_openflow_amqp: false, diff --git a/OpenFlow/src/DatabaseConnection.ts b/OpenFlow/src/DatabaseConnection.ts index 2533022e..cb8fbe5a 100644 --- a/OpenFlow/src/DatabaseConnection.ts +++ b/OpenFlow/src/DatabaseConnection.ts @@ -410,7 +410,7 @@ export class DatabaseConnection extends events.EventEmitter { } if (collectionname === "config" && _type === "config") { discardspan = false; - await dbConfig.Reload(Crypt.rootToken(), span); + await dbConfig.Reload(Crypt.rootToken(), true, span); } } diff --git a/OpenFlow/src/LoginProvider.ts b/OpenFlow/src/LoginProvider.ts index c6d0081c..be4d507b 100644 --- a/OpenFlow/src/LoginProvider.ts +++ b/OpenFlow/src/LoginProvider.ts @@ -15,6 +15,7 @@ import { Span } from "@opentelemetry/api"; import { Logger } from "./Logger"; import { DatabaseConnection } from "./DatabaseConnection"; import { TokenRequest } from "./TokenRequest"; +import { WebServer } from "./WebServer"; var nodemailer = require('nodemailer'); var dns = require('dns'); const got = require("got"); @@ -1192,6 +1193,8 @@ export class LoginProvider { static async post_AddTokenRequest(req: any, res: any, next: any): Promise { const span: Span = Logger.otel.startSpanExpress("LoginProvider.login", req); try { + const remoteip = LoginProvider.remoteip(req); + span?.setAttribute("remoteip", remoteip); const key = req.body.key; let exists: TokenRequest = await Logger.DBHelper.FindRequestTokenID(key, span); if (!NoderedUtil.IsNullUndefinded(exists)) { @@ -1199,7 +1202,7 @@ export class LoginProvider { return res.status(500).send({ message: "Illegal key" }); } await Logger.DBHelper.AddRequestTokenID(key, {}, span); - Logger.instanse.info("Added token request " + key, span); + Logger.instanse.info("Added token request " + key + " from " + remoteip, span); res.status(200).send({ message: "ok" }); } catch (error) { Logger.instanse.error(error, span); @@ -1211,6 +1214,7 @@ export class LoginProvider { static async get_GetTokenRequest(req: any, res: any, next: any): Promise { const span: Span = Logger.otel.startSpanExpress("LoginProvider.login", req); try { + span?.setAttribute("remoteip", LoginProvider.remoteip(req)); const key = req.query.key; let exists: TokenRequest = null; exists = await Logger.DBHelper.FindRequestTokenID(key, span); diff --git a/OpenFlow/src/Messages/Message.ts b/OpenFlow/src/Messages/Message.ts index 31ab78a7..4ac66acd 100644 --- a/OpenFlow/src/Messages/Message.ts +++ b/OpenFlow/src/Messages/Message.ts @@ -3692,7 +3692,10 @@ export class Message { let query = { "validated": false, "_type": "user" } query["_modified"] = { "$lt": todate.toISOString(), "$gt": fromdate.toISOString()} - Config.db.DeleteMany(query, null, "users", "", false, jwt, span); + let count = await Config.db.DeleteMany(query, null, "users", "", false, jwt, span); + if(count > 0) { + Logger.instanse.verbose("Removed " + count + " unvalidated users", span); + } } if(Config.housekeeping_cleanup_openrpa_instances == true) { let msg = new UpdateManyMessage(); @@ -3700,7 +3703,12 @@ export class Message { msg.collectionname = "openrpa_instances"; msg.query = { "state": { "$in": ["idle", "running"] } }; msg.item = { "$set": { "state": "completed"}, "$unset": {"xml": ""}} as any; - Config.db.UpdateDocument(msg, span); + let result = await Config.db.UpdateDocument(msg, span); + if(result?.opresult?.nModified > 0) { + Logger.instanse.verbose("Updated " + result.opresult.nModified + " openrpa instances", span); + } else if (result?.opresult?.modifiedCount > 0) { + Logger.instanse.verbose("Updated " + result.opresult.modifiedCount + " openrpa instances", span); + } } } catch (error) { diff --git a/OpenFlow/src/index.ts b/OpenFlow/src/index.ts index 82d6a7f4..15dc4c36 100644 --- a/OpenFlow/src/index.ts +++ b/OpenFlow/src/index.ts @@ -72,7 +72,7 @@ async function initDatabase(parent: Span): Promise { Logger.instanse.info("Begin validating builtin roles", span); const jwt: string = Crypt.rootToken(); const rootuser = Crypt.rootUser(); - Config.dbConfig = await dbConfig.Load(jwt, span); + Config.dbConfig = await dbConfig.Load(jwt, false, span); try { var lic = Logger.License; await lic?.validate(); From 8c472f62d833a7977d74ce6f528cf6dced4c0be3 Mon Sep 17 00:00:00 2001 From: skadefro Date: Fri, 24 Nov 2023 00:37:39 +0100 Subject: [PATCH 12/32] Turn down user updates --- OpenFlow/src/WebSocketServer.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/OpenFlow/src/WebSocketServer.ts b/OpenFlow/src/WebSocketServer.ts index fe9e397f..1c92b2bb 100644 --- a/OpenFlow/src/WebSocketServer.ts +++ b/OpenFlow/src/WebSocketServer.ts @@ -386,16 +386,17 @@ export class WebSocketServer { } } - if (bulkUpdates.length > 0) { + // seconds since lastUserUpdate + const seconds: number = (Date.now() - this.lastUserUpdate) / 1000; + + if (bulkUpdates.length > 0 && Config.enable_openflow_amqp) { + amqpwrapper.Instance().send("openflow", "", { "command": "dumpwebsocketclients" }, 10000, null, "", span, 1); + } + if (bulkUpdates.length > 0 && seconds > 60) { this.lastUserUpdate = Date.now(); let ot_end: any = Logger.otel.startTimer(); var bulkresult = await Config.db.db.collection("users").bulkWrite(bulkUpdates); Logger.otel.endTimer(ot_end, DatabaseConnection.mongodb_updatemany, { collection: "users" }); - if (Config.enable_openflow_amqp) { - amqpwrapper.Instance().send("openflow", "", { "command": "dumpwebsocketclients" }, 10000, null, "", span, 1); - } else { - // WebSocketServer.DumpClients(span); - } } } catch (error) { Logger.instanse.error(error, span); From 45363db2b1dda711af298fc2183186d5f851da4e Mon Sep 17 00:00:00 2001 From: skadefro Date: Fri, 24 Nov 2023 00:38:01 +0100 Subject: [PATCH 13/32] cleanup logging and config values --- OpenFlow/src/Config.ts | 227 +++++++++++------------ OpenFlow/src/DBHelper.ts | 2 - OpenFlow/src/DatabaseConnection.ts | 104 ++++++++++- OpenFlow/src/Logger.ts | 65 +++++-- OpenFlow/src/Messages/Message.ts | 14 +- OpenFlow/src/QueueClient.ts | 1 - OpenFlow/src/SamlProvider.ts | 10 +- OpenFlow/src/WebServer.ts | 14 +- OpenFlow/src/WebSocketServerClient.ts | 2 +- OpenFlow/src/dockerdriver.ts | 22 +-- OpenFlow/src/index.ts | 12 +- OpenFlow/src/proto/client.ts | 4 +- OpenFlow/src/public/CommonControllers.ts | 2 +- OpenFlow/src/public/Config.html | 59 ++---- OpenFlow/src/public/Console.html | 3 + OpenFlow/src/public/Controllers.ts | 147 +-------------- 16 files changed, 316 insertions(+), 372 deletions(-) diff --git a/OpenFlow/src/Config.ts b/OpenFlow/src/Config.ts index 66fc6cac..0adc043a 100644 --- a/OpenFlow/src/Config.ts +++ b/OpenFlow/src/Config.ts @@ -1,7 +1,6 @@ var xml2js = require('xml2js'); import * as https from "https"; import * as http from "http"; -// import { fetch, toPassportConfig } from "passport-saml-metadata"; import * as fs from "fs"; import * as os from "os"; import * as path from "path"; @@ -11,6 +10,7 @@ import { Base, InsertOrUpdateOneMessage, NoderedUtil, Rights, WellknownIds } fro import { promiseRetry } from "./Logger"; import { Span } from "@opentelemetry/api"; + export class dbConfig extends Base { constructor() { super(); @@ -35,128 +35,136 @@ export class dbConfig extends Base { if (!NoderedUtil.IsNullEmpty(this._id)) await Config.db._UpdateOne(null, this, "config", 1, true, jwt, parent); } public compare(version: string): number { + if(this.version == null) return -1; return this.version.localeCompare(version, undefined, { numeric: true, sensitivity: 'base' }); } - public static async Load(jwt: string, watch:boolean, parent: Span): Promise { - var conf: dbConfig = await Config.db.GetOne({ query: { "_type": "config" }, collectionname: "config", jwt }, parent); - if (conf == null) { conf = new dbConfig(); } - conf = Object.assign(new dbConfig(), conf); - conf.needsupdate = false; - if (conf.compare(Config.version) == -1) { - conf.needsupdate = true; + + public static cleanAndApply(conf: dbConfig, parent: Span): Boolean { + var updated = false; + // add settings et via env variables that is not the default value + var keys = Object.keys(Config); + for(var i = 0; i < keys.length; i++) { + const key = keys[i]; + if(key == "_version") continue; + if(key.startsWith("_")) continue; + if(["db", "name", "version", "needsupdate", "updatedat"].indexOf(key) > -1 ) continue; + if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_redis_host", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval", + "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url", + "domain", "enable_openapi", "enable_openapiauth" ].indexOf(key) > -1 ) { + if(os.hostname().toLowerCase() == "nixos") { + continue; + } + } + + if (Object.prototype.hasOwnProperty.call(Config.default_config, key) && + !Object.prototype.hasOwnProperty.call(conf, key) + ) { + let _default:any = Config.default_config[key]; // envorinment variable + if(_default == null) _default = ""; + let _env:any = process.env[key]; // db value + if(_env == null || _env == "") continue; + if(typeof Config[key] === "boolean") { + _env = Config.parseBoolean(_env); + } else if(typeof Config[key] === "number") { + _env = parseInt(_env); + } else if(Array.isArray(Config[key])) { + _env = Config.parseArray(_env); + } else if(typeof Config[key] === "string") { + _env = _env; + } else { + continue; + } + if(key == "HTTP_PROXY") { + var b = true; + } + if(_env != _default) { + updated = true; + conf[key] = Config[key]; + } + } + } + // Update Config with db values var keys = Object.keys(conf); for(var i = 0; i < keys.length; i++) { const key = keys[i]; + if(key == "_version") continue; const value = conf[key]; try { if(key.startsWith("_")) continue; - if(NoderedUtil.IsNullEmpty(value)) continue; + // if(NoderedUtil.IsNullEmpty(value)) continue; if(["db", "name", "version", "needsupdate", "updatedat"].indexOf(key) > -1 ) continue; - if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval", + if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_redis_host", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval", "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url", "domain", "enable_openapi", "enable_openapiauth" ].indexOf(key) > -1 ) { if(os.hostname().toLowerCase() == "nixos") { continue; } } - if(key == "amqp_allow_replyto_empty_queuename") { - var now = Config.amqp_allow_replyto_empty_queuename; - var v = conf[key]; + if(key == "HTTP_PROXY") { var b = true; } - if (Object.prototype.hasOwnProperty.call(Config, key)) { - if(typeof Config[key] === "boolean") { - // console.log("Setting boolen " + key + " to " + conf[key]); - Config[key] = Config.parseBoolean(conf[key]); - } else if(typeof Config[key] === "number") { - // console.log("Setting number " + key + " to " + conf[key]); - Config[key] = parseInt(conf[key]); - } else if(Array.isArray(Config[key])) { - // console.log("Setting array " + key + " to " + conf[key]); - if(Array.isArray(conf[key])) { - Config[key] = conf[key]; + Config[key] = value; + + let _default:any = Config.default_config[key]; // envorinment variable + if(_default == null) _default = ""; + let _env:any = process.env[key]; // db value + if(_env != null && _env != "") { + if(typeof Config[key] === "boolean") { + _env = Config.parseBoolean(_env); + } else if(typeof Config[key] === "number") { + _env = parseInt(_env); + } else if(Array.isArray(Config[key])) { + _env = Config.parseArray(_env); + } else if(typeof Config[key] === "string") { + _env = _env; } else { - Config[key] = Config.parseArray(conf[key]); + continue; + } + if(_env != _default) { + } else if(_env == value ) { + updated = true; + delete conf[key]; } - } else if(typeof Config[key] === "string") { - // console.log("Setting string " + key + " to " + conf[key]); - Config[key] = conf[key]; } else { - // console.log("Setting Unknown " + key + " to " + conf[key]); - Config[key] = conf[key]; + if(_default == value ) { + updated = true; + delete conf[key]; + } } } - - if(key == "amqp_allow_replyto_empty_queuename") { - var now = Config.amqp_allow_replyto_empty_queuename; - var v = conf[key]; - var v2 = Config[key]; - var b = true; - } - } catch (error) { - Logger.instanse.error("Error setting config " + keys + " to " + value, parent); + Logger.instanse.error("Error setting config " + keys + " to " + value, null); } } - var keys = Object.keys(Config); - var updated = false; - for(var i = 0; i < keys.length; i++) { - const key = keys[i]; - if(key.startsWith("_")) continue; - if(["db", "name", "version", "needsupdate", "updatedat"].indexOf(key) > -1 ) continue; - if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval", - "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url", - "domain", "enable_openapi", "enable_openapiauth" ].indexOf(key) > -1 ) { - if(os.hostname().toLowerCase() == "nixos") { - continue; - } - } - if(key == "amqp_allow_replyto_empty_queuename") { - var now = Config.amqp_allow_replyto_empty_queuename; - var v = Config[key]; - var v2 = conf[key]; - var b = true; - } - - const _default = Config.default_config[key]; - const setting = Config[key]; - const dbsetting = conf[key]; - // if(_default != null && dbsetting == null) { - // Config[key] = setting; - - // if(key == "amqp_allow_replyto_empty_queuename") { - // var now = Config.amqp_allow_replyto_empty_queuename; - // var v = Config[key]; - // var v2 = conf[key]; - // var b = true; - // } - - - // continue; - // } - // console.log("Checking " + key + " " + _default + " " + setting + " " + dbsetting); - if(setting == _default) continue; // ignore if default, kee dbsettings small - if(dbsetting != null) continue; // db setting overrides env setting (yeah, a little weird) - if(setting != dbsetting) { - conf[key] = setting; - updated = true; - } + conf._encrypt = ["stripe_api_secret", "smtp_url", "amqp_password", "cache_store_redis_password", "cookie_secret", "singing_key", "wapid_key"]; + if(Config._version != conf._version) { + Config._version = conf._version; + Logger.instanse.info("Loaded config version " + conf._version, parent); + } - if(key == "amqp_allow_replyto_empty_queuename") { - var now = Config.amqp_allow_replyto_empty_queuename; - var v = Config[key]; - var v2 = conf[key]; - var b = true; + return updated; + } + public static async Load(jwt: string, watch:boolean, parent: Span): Promise { + var conf: dbConfig = await Config.db.GetOne({ query: { "_type": "config" }, collectionname: "config", jwt, decrypt: true }, parent); + // @ts-ignore + if (conf == null) { conf = new dbConfig(); } else { + if(Config._version == conf._version) { + conf = Object.assign(new dbConfig(), conf); + return conf; } - - } - conf._encrypt = ["stripe_api_secret", "smtp_url", "amqp_password", "cache_store_redis_password", "cookie_secret", "singing_key", "wapid_key"]; - if(updated && !watch) { + conf = Object.assign(new dbConfig(), conf); + conf.needsupdate = false; + if (conf.compare(Config.version) == -1) { + conf.needsupdate = true; + } + + let updated = dbConfig.cleanAndApply(conf, parent); + if(updated ) { try { var msg: InsertOrUpdateOneMessage = new InsertOrUpdateOneMessage(); msg.collectionname = "config"; msg.jwt = jwt; @@ -169,9 +177,7 @@ export class dbConfig extends Base { await Config.db._InsertOrUpdateOne(msg, parent); // await Config.db.InsertOrUpdateOne(null, conf, "config", 1, true, jwt, parent); } catch (error) { - var e = error; - console.error(error); - + Logger.instanse.error(error, null); } } await Logger.reload(); @@ -179,8 +185,6 @@ export class dbConfig extends Base { } public static async Reload(jwt: string, watch:boolean, parent: Span): Promise { Config.dbConfig = await dbConfig.Load(jwt, watch, parent); - - Logger.instanse.info("Reloaded config version " + Config.dbConfig._version, parent); } } export class Config { @@ -193,6 +197,7 @@ export class Config { cache_store_type: "memory", cache_store_max: 1000, cache_store_ttl_seconds: 300, + cache_store_redis_host: "", cache_store_redis_port: 6379, cache_workitem_queues: false, @@ -206,6 +211,7 @@ export class Config { log_webserver: false, log_database: false, log_database_queries: false, + log_database_queries_to_collection: "", log_database_queries_ms: 0, log_grafana: false, log_housekeeping: false, @@ -216,6 +222,7 @@ export class Config { log_verbose: false, log_silly: false, log_to_exchange: false, + log_all_watches: false, tls_crt: "", tls_key: "", @@ -344,7 +351,7 @@ export class Config { mongodb_minpoolsize: 25, mongodb_maxpoolsize: 25, - skip_history_collections: "audit,openrpa_instances,workflow_instances", + skip_history_collections: "audit,oauthtokens,openrpa_instances,workflow_instances,workitems,dbsize,mailhist", // "audit,openrpa_instances,workflow_instances", history_delta_count: 1000, allow_skiphistory: false, max_memory_restart_mb: 0, @@ -365,10 +372,13 @@ export class Config { agent_domain_schema: "", agent_node_selector: "", agent_apiurl: "", + agent_grpc_apihost: "", + agent_ws_apihost: "", agent_oidc_config: "", agent_oidc_client_id: "", agent_oidc_client_secret: "", agent_oidc_userinfo_endpoint: "", + agent_oidc_issuer: "", saml_federation_metadata: "", api_ws_url: "", @@ -440,6 +450,7 @@ export class Config { Config.log_oauth = false; Config.unittesting = true; } + public static _version: number = -1; public static unittesting: boolean = false; public static db: DatabaseConnection = null; public static license_key: string = Config.getEnv("license_key"); @@ -468,6 +479,7 @@ export class Config { public static log_webserver: boolean = Config.parseBoolean(Config.getEnv("log_webserver")); public static log_database: boolean = Config.parseBoolean(Config.getEnv("log_database")); public static log_database_queries: boolean = Config.parseBoolean(Config.getEnv("log_database_queries")); + public static log_database_queries_to_collection: string = Config.getEnv("log_database_queries_to_collection"); public static log_database_queries_ms: number = parseInt(Config.getEnv("log_database_queries_ms")); public static log_grafana: boolean = Config.parseBoolean(Config.getEnv("log_grafana")); @@ -479,6 +491,8 @@ export class Config { public static log_verbose: boolean = Config.parseBoolean(Config.getEnv("log_verbose")); public static log_silly: boolean = Config.parseBoolean(Config.getEnv("log_silly")); public static log_to_exchange: boolean = Config.parseBoolean(Config.getEnv("log_to_exchange")); + public static log_all_watches: boolean = Config.parseBoolean(Config.getEnv("log_all_watches")); + public static heapdump_onstop: boolean = Config.parseBoolean(Config.getEnv("heapdump_onstop")); @@ -634,7 +648,7 @@ export class Config { public static downloadtoken_expires_in: string = Config.getEnv("downloadtoken_expires_in"); public static personalnoderedtoken_expires_in: string = Config.getEnv("personalnoderedtoken_expires_in"); - public static agent_images: NoderedImage[] = JSON.parse(Config.getEnv("agent_images")); + public static agent_images: NoderedImage[] = Array.isArray(Config.getEnv("agent_images")) ? Config.getEnv("agent_images") : JSON.parse(Config.getEnv("agent_images")); public static agent_domain_schema: string = Config.getEnv("agent_domain_schema"); public static agent_node_selector:string = Config.getEnv("agent_node_selector"); @@ -728,7 +742,7 @@ export class Config { } public static getEnv(name: string): string { let value: any = process.env[name]; - if (!value || value === "") { + if (value == null || value == "") { value = this.default_config[name] } return value; @@ -753,17 +767,6 @@ export class Config { }) } public static async parse_federation_metadata(tls_ca: String, url: string): Promise { - // try { - // if (tls_ca !== null && tls_ca !== undefined && tls_ca !== "") { - // const rootCas = require('ssl-root-cas/latest').create(); - // rootCas.push(tls_ca); - // // rootCas.addFile( tls_ca ); - // https.globalAgent.options.ca = rootCas; - // require('https').globalAgent.options.ca = rootCas; - // } - // } catch (error) { - // console.error(error); - // } const metadata: any = await promiseRetry(async () => { // if (Config.saml_ignore_cert) process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"; const data: string = await Config.get(url) @@ -771,9 +774,6 @@ export class Config { if (NoderedUtil.IsNullEmpty(data)) { throw new Error("Failed getting result"); } var xml = await xml2js.parseStringPromise(data); if (xml && xml.EntityDescriptor && xml.EntityDescriptor.IDPSSODescriptor && xml.EntityDescriptor.IDPSSODescriptor.length > 0) { - // const reader: any = await fetch({ url }); - // if (NoderedUtil.IsNullUndefinded(reader)) { throw new Error("Failed getting result"); } - // const _config: any = toPassportConfig(reader); var IDPSSODescriptor = xml.EntityDescriptor.IDPSSODescriptor[0]; var identifierFormat = "urn:oasis:names:tc:SAML:2.0:attrname-format:uri"; if (IDPSSODescriptor.NameIDFormat && IDPSSODescriptor.NameIDFormat.length > 0) { @@ -785,7 +785,6 @@ export class Config { signingCerts.push(key.KeyInfo[0].X509Data[0].X509Certificate[0]); } }); - // var signingCerts = IDPSSODescriptor.KeyDescriptor[0].KeyInfo[0].X509Data[0].X509Certificate; var identityProviderUrl = IDPSSODescriptor.SingleSignOnService[0].$.Location; var logoutUrl = IDPSSODescriptor.SingleLogoutService[0].$.Location; const config = { diff --git a/OpenFlow/src/DBHelper.ts b/OpenFlow/src/DBHelper.ts index d0db17e2..5868db99 100644 --- a/OpenFlow/src/DBHelper.ts +++ b/OpenFlow/src/DBHelper.ts @@ -83,8 +83,6 @@ export class DBHelper { for (var i = 0; i < keys.length; i++) { if (keys[i] && !keys[i].startsWith("requesttoken")) { this.memoryCache.del(keys[i]); - } else { - console.log("not deleting " + keys[i]); } } Logger.instanse.debug("clearCache called with reason: " + reason, span); diff --git a/OpenFlow/src/DatabaseConnection.ts b/OpenFlow/src/DatabaseConnection.ts index cb8fbe5a..135d9d87 100644 --- a/OpenFlow/src/DatabaseConnection.ts +++ b/OpenFlow/src/DatabaseConnection.ts @@ -14,6 +14,7 @@ import { SocketMessage } from "./SocketMessage"; import { LoginProvider } from "./LoginProvider"; import { WebServer } from "./WebServer"; import { iAgent } from "./commoninterfaces"; +import * as os from "os"; // tslint:disable-next-line: typedef const safeObjectID = (s: string | number | ObjectId) => ObjectId.isValid(s) ? new ObjectId(s) : null; @@ -141,6 +142,8 @@ export class DatabaseConnection extends events.EventEmitter { } public replicat: string = null; public streams: clsstream[] = []; + public requests: any = {}; + public host: string = ""; /** * Connect to MongoDB * @returns Promise @@ -149,11 +152,15 @@ export class DatabaseConnection extends events.EventEmitter { if (this.cli !== null && this.cli !== undefined && this.isConnected) { return; } + if(this.host == "") { + this.host = os.hostname(); + } const span: Span = Logger.otel.startSubSpan("db.connect", parent); this.streams = []; span?.addEvent("connecting to mongodb"); Logger.instanse.info("Connecting to mongodb", span); const options: MongoClientOptions = { minPoolSize: Config.mongodb_minpoolsize, maxPoolSize: Config.mongodb_maxpoolsize }; + options.monitorCommands = true; this.cli = await MongoClient.connect(this.mongodburl, options); Logger.instanse.info("Connected to mongodb", span); span?.addEvent("Connected to mongodb"); @@ -173,7 +180,63 @@ export class DatabaseConnection extends events.EventEmitter { .on('error', errEvent) .on('parseError', errEvent) .on('timeout', errEvent) - .on('close', closeEvent); + .on('close', closeEvent) + // .on("commandStarted", (event) => { + // if(this.host != "nixos") return; + // let req = this.requests[event.requestId]; + // if(req != null) { + // this.requests[event.requestId] = { start: new Date(), command: event.commandName, connectionId: event.connectionId }; + // } else { + // this.requests[event.requestId] = { start: new Date(), command: event.commandName, connectionId: event.connectionId }; + // } + // if(Config.log_database_queries == true) { + // Logger.instanse.debug(event.requestId + " cmd: " + event.commandName + " started ", span, { requestId: event.requestId, connectionId: event.connectionId, command: event.commandName, cls: "DatabaseConnection" }) + // } + // }) + // .on("commandFailed", (event) => { + // if(this.host != "nixos") return; + // if(Config.log_database_queries == true) { + // Logger.instanse.debug(event.requestId + " cmd: " + event.commandName + " failed" , span, { requestId: event.requestId, ms: event.duration, connectionId: event.connectionId, command: event.commandName, failure: event.failure, cls: "DatabaseConnection" }) + // } + // // Logger.instanse.debug("Query: " + JSON.stringify({ _id: ace._id }), span, { ms, count: arr.length, collection: "users" }); + // }) + // .on("commandSucceeded", (event) => { + // if(this.host != "nixos") return; + // if(Config.log_database_queries == true) { + // // @ts-ignore + // let ns = event.reply?.cursor?.ns; + // // @ts-ignore + // let count = (event.reply?.cursor?.nextBatch?.length !== undefined && event.reply?.cursor?.nextBatch?.length !== null) ? event.reply?.cursor?.nextBatch?.length + // // @ts-ignore + // : (event.reply?.cursor?.firstBatch?.length !== undefined && event.reply?.cursor?.firstBatch?.length !== null) ? event.reply?.cursor?.firstBatch?.length + // // @ts-ignore + // : (event.reply?.cursor?.nReturned !== undefined && event.reply?.cursor?.nReturned !== null) ? event.reply?.cursor?.nReturned + // // @ts-ignore + // : (event.reply?.n !== undefined && event.reply?.n !== null) ? event.reply?.n + // // @ts-ignore + // : (event.reply?.nModified !== undefined && event.reply?.nModified !== null) ? event.reply?.nModified + // // @ts-ignore + // : (event.reply?.nUpserted !== undefined && event.reply?.nUpserted !== null) ? event.reply?.nUpserted + // // @ts-ignore + // : event.reply?.nRemoved; + + // let req = this.requests[event.requestId]; + // if(req != null) { + // req.end = new Date(); + // req.duration = req.end - req.start; + // req.count = count; + // req.ns = ns; + // // delete this.requests[event.requestId]; + // let command = event.commandName; + // if(command != req.command) command = req.command + "-" + command; + // Logger.instanse.debug(event.requestId + " cmd: " + command + " c: " + req.count + " mq:" + req.duration, span, { requestId: event.requestId, ms: event.duration, connectionId: event.connectionId, command: event.commandName, reply: event.reply, cls: "DatabaseConnection" }) + // } else { + // var b = true; + // } + // // Logger.instanse.debug(event.commandName + " " + event.requestId + " " + ns + " " + count, span, { requestId: event.requestId, ms: event.duration, connectionId: event.connectionId, command: event.commandName, reply: event.reply, cls: "DatabaseConnection" }) + // } + // // Logger.instanse.debug("Query: " + JSON.stringify({ _id: ace._id }), span, { ms, count: arr.length, collection: "users" }); + // }) this.db = this.cli.db(this._dbname); try { var topology = (this.cli as any).topology; @@ -245,7 +308,7 @@ export class DatabaseConnection extends events.EventEmitter { const wiq = queues[i]; const count = await Logger.DBHelper.GetPendingWorkitemsCount(wiq._id, null); if (count < 1) continue; - const query = { "wiqid": wiq._id, state: "new", "_type": "workitem", "nextrun": { "$lte": new Date(new Date().toISOString()) } }; + // const query = { "wiqid": wiq._id, state: "new", "_type": "workitem", "nextrun": { "$lte": new Date(new Date().toISOString()) } }; var payload = null; // const payload = await this.GetOne({ jwt, collectionname, query }, null); // if (payload == null) continue; @@ -554,7 +617,12 @@ export class DatabaseConnection extends events.EventEmitter { (stream.stream as any).on("error", err => { Logger.instanse.error(err, span, { collection: collectionname }); }); - (stream.stream as any).on("change", async (next) => { this.GlobalWatchCallback(collectionname, next) }); + (stream.stream as any).on("change", async (next) => { + if(Config.log_all_watches == true) { + Logger.instanse.debug(collectionname + " watch " + next?.fullDocument?._type + " " + next?.operationType + " " + next?.fullDocument?.name, span, { cls: "DatabaseConnection", func: "onchange", collection: collectionname }); + } + this.GlobalWatchCallback(collectionname, next) + }); this.streams.push(stream); } catch (error) { Logger.instanse.error(error, span, { collection: collectionname }); @@ -2468,7 +2536,11 @@ export class DatabaseConnection extends events.EventEmitter { if (!DatabaseConnection.hasAuthorization(user, original, Rights.update)) { throw new Error("Access denied, no authorization to UpdateOne " + q.item._type + " " + name + " to database"); } - + if(q.collectionname === "config" && q.item._type === "config") { + if (!user.HasRoleId(WellknownIds.admins)) throw new Error("Access denied, no authorization to update config"); + dbConfig.cleanAndApply(q.item as any, span); + } + await Logger.DBHelper.CheckCache(q.collectionname, q.item, false, false, span); if (q.collectionname === "agents") { @@ -3597,7 +3669,7 @@ export class DatabaseConnection extends events.EventEmitter { try { if(ids[i] != null && ids[i].trim() != "") objectids.push(safeObjectID(ids[i])) } catch (error) { - console.error(error); + Logger.instanse.error(error, span); } } } else { @@ -3605,7 +3677,7 @@ export class DatabaseConnection extends events.EventEmitter { try { if(ids[i] != null && ids[i].trim() != "") objectids.push(ids[i]) } catch (error) { - console.error(error); + Logger.instanse.error(error, span); } } } @@ -4174,6 +4246,26 @@ export class DatabaseConnection extends events.EventEmitter { } + SaveMongoDBCommand( user: TokenUser, collection: string, command: string, query: any, ms:number) { + try { + const cmd = { + _created: new Date(new Date().toISOString()), + _createdby: user.name, + _createdbyid: user._id, + collection: collection, + name: user?.username + " " + command + " " + collection + " " + ms, + command: command, + query: query, + ms: ms + } + this.db.collection("dbcommands").insertOne(cmd).then(() => { + }).catch(err => { + Logger.instanse.error(err, null); + }); + } catch (error) { + Logger.instanse.error(error, null, { collection: collection, user: user?.username, ms: ms, query: query }); + } + } async SaveUpdateDiff(q: UpdateOneMessage, user: TokenUser, parent: Span) { const span: Span = Logger.otel.startSubSpan("db.SaveUpdateDiff", parent); try { diff --git a/OpenFlow/src/Logger.ts b/OpenFlow/src/Logger.ts index 670e7ea8..d74ccfc6 100644 --- a/OpenFlow/src/Logger.ts +++ b/OpenFlow/src/Logger.ts @@ -116,7 +116,10 @@ export class Logger { } span.addEvent(obj.message, obj) } - if (obj.ms != null && obj.ms != "" && obj.func != "query" && Config.log_database_queries) { + if(Config.log_all_watches && obj.cls == "DatabaseConnection" && obj.func == "onchange") { + + } else if(Config.log_database_queries && obj.requestId != null) { + } else if (obj.ms != null && obj.ms != "" && obj.func != "query" && Config.log_database_queries) { if (obj.ms < Config.log_database_queries_ms) return; } else if (Logger.enabled[cls]) { if (Logger.enabled[cls] < lvl) return; @@ -173,8 +176,14 @@ export class Logger { obj.cls = ""; if (s.file != '') obj.cls = s.file.replace(".js", ""); } - if(options?.openapi) { - obj.cls = "OpenAIProxy"; + if(obj.func.indexOf("anonymous") > -1 || obj.func.indexOf("<") > -1 || obj.func.indexOf("[") > -1) { + obj.func = "anonymous"; + } + if(options?.cls != null && options?.cls != "") { + obj.cls = options.cls; + } + if(options?.func != null && options?.func != "") { + obj.func = options.func; } this.json(obj, span); } @@ -193,8 +202,14 @@ export class Logger { obj.cls = ""; if (s.file != '') obj.cls = s.file.replace(".js", ""); } - if(options?.openapi) { - obj.cls = "OpenAIProxy"; + if(obj.func.indexOf("anonymous") > -1 || obj.func.indexOf("<") > -1 || obj.func.indexOf("[") > -1) { + obj.func = "anonymous"; + } + if(options?.cls != null && options?.cls != "") { + obj.cls = options.cls; + } + if(options?.func != null && options?.func != "") { + obj.func = options.func; } this.json(obj, span); } @@ -212,8 +227,14 @@ export class Logger { obj.cls = ""; if (s.file != '') obj.cls = s.file.replace(".js", ""); } - if(options?.openapi) { - obj.cls = "OpenAIProxy"; + if(obj.func.indexOf("anonymous") > -1 || obj.func.indexOf("<") > -1 || obj.func.indexOf("[") > -1) { + obj.func = "anonymous"; + } + if(options?.cls != null && options?.cls != "") { + obj.cls = options.cls; + } + if(options?.func != null && options?.func != "") { + obj.func = options.func; } this.json(obj, span); } @@ -231,8 +252,14 @@ export class Logger { obj.cls = ""; if (s.file != '') obj.cls = s.file.replace(".js", ""); } - if(options?.openapi) { - obj.cls = "OpenAIProxy"; + if(obj.func.indexOf("anonymous") > -1 || obj.func.indexOf("<") > -1 || obj.func.indexOf("[") > -1) { + obj.func = "anonymous"; + } + if(options?.cls != null && options?.cls != "") { + obj.cls = options.cls; + } + if(options?.func != null && options?.func != "") { + obj.func = options.func; } this.json(obj, span); } @@ -251,8 +278,14 @@ export class Logger { obj.cls = ""; if (s.file != '') obj.cls = s.file.replace(".js", ""); } - if(options?.openapi) { - obj.cls = "OpenAIProxy"; + if(obj.func.indexOf("anonymous") > -1 || obj.func.indexOf("<") > -1 || obj.func.indexOf("[") > -1) { + obj.func = "anonymous"; + } + if(options?.cls != null && options?.cls != "") { + obj.cls = options.cls; + } + if(options?.func != null && options?.func != "") { + obj.func = options.func; } this.json(obj, span); } @@ -271,8 +304,14 @@ export class Logger { obj.cls = ""; if (s.file != '') obj.cls = s.file.replace(".js", ""); } - if(options?.openapi) { - obj.cls = "OpenAIProxy"; + if(obj.func.indexOf("anonymous") > -1 || obj.func.indexOf("<") > -1 || obj.func.indexOf("[") > -1) { + obj.func = "anonymous"; + } + if(options?.cls != null && options?.cls != "") { + obj.cls = options.cls; + } + if(options?.func != null && options?.func != "") { + obj.func = options.func; } this.json(obj, span); } diff --git a/OpenFlow/src/Messages/Message.ts b/OpenFlow/src/Messages/Message.ts index 4ac66acd..34259851 100644 --- a/OpenFlow/src/Messages/Message.ts +++ b/OpenFlow/src/Messages/Message.ts @@ -3127,7 +3127,9 @@ export class Message { if (payload && payload.limit) { url += "&limit=" + payload.limit; } - const auth = "Basic " + Buffer.from(Config.stripe_api_secret + ":").toString("base64"); + var stripe_api_secret = Config.stripe_api_secret; + if(stripe_api_secret == null || stripe_api_secret == "") throw new Error("Missing stripe_api_secret"); + const auth = "Basic " + Buffer.from(stripe_api_secret + ":").toString("base64"); const options = { headers: { @@ -4366,7 +4368,15 @@ export class Message { const dt = parseInt((new Date().getTime() / 1000).toFixed(0)) const payload: any = { "quantity": billablecount, "timestamp": dt }; if (!NoderedUtil.IsNullEmpty(config.siid) && !NoderedUtil.IsNullEmpty(c.stripeid)) { - await this.Stripe("POST", "usage_records", config.siid, payload, c.stripeid); + try { + await this.Stripe("POST", "usage_records", config.siid, payload, c.stripeid); + } catch (error) { + if (error.response && error.response.body) { + Logger.instanse.error("Update usage record error!" + error.response.body, span); + } else { + Logger.instanse.error("Update usage record error!" + error, span); + } + } } } if (c.dblocked || !c.dblocked) { diff --git a/OpenFlow/src/QueueClient.ts b/OpenFlow/src/QueueClient.ts index 2d16f3d2..cd8bac2a 100644 --- a/OpenFlow/src/QueueClient.ts +++ b/OpenFlow/src/QueueClient.ts @@ -109,7 +109,6 @@ export class QueueClient { Logger.instanse.silly("Submit request for command: " + msg.command + " queuename: " + this.queuename + " replyto: " + this.queue.queue + " correlationId: " + msg.correlationId, null) await amqpwrapper.Instance().sendWithReplyTo("", this.queuename, this.queue.queue, json, Config.openflow_amqp_expiration, msg.correlationId, "", span, priority); } catch (error) { - console.error(error); if (NoderedUtil.IsNullUndefinded(this.queue)) { Logger.instanse.warn("SendForProcessing queue is null, shutdown amqp connection", span); process.exit(406); diff --git a/OpenFlow/src/SamlProvider.ts b/OpenFlow/src/SamlProvider.ts index 659b68ca..6747ac5b 100644 --- a/OpenFlow/src/SamlProvider.ts +++ b/OpenFlow/src/SamlProvider.ts @@ -66,8 +66,10 @@ export class SamlProvider { const key: string = Buffer.from(Config.singing_key, "base64").toString("ascii"); if(cert != null && cert != "") { + let saml_issuer: string = Config.saml_issuer; + if(saml_issuer == null || saml_issuer == "") saml_issuer = "uri:" + Config.domain; const samlpoptions: any = { - issuer: Config.saml_issuer, + issuer: saml_issuer, cert: cert, key: key, getPostURL: (wtrealm: any, wreply: any, req: any, callback: any) => { @@ -126,7 +128,7 @@ export class SamlProvider { try { app.get("/issue/", samlp.auth(samlpoptions)); app.get("/issue/FederationMetadata/2007-06/FederationMetadata.xml", samlp.metadata({ - issuer: Config.saml_issuer, + issuer: saml_issuer, cert: cert, })); } catch (error) { @@ -190,8 +192,10 @@ export class SamlProvider { }); app.post('/logout', (req: any, res: any, next: any): void => { if(cert != null && cert != "") { + let saml_issuer: string = Config.saml_issuer; + if(saml_issuer == null || saml_issuer == "") saml_issuer = "uri:" + Config.domain; samlp.logout({ - issuer: Config.saml_issuer, + issuer: saml_issuer, protocolBinding: 'urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST', cert: cert, key: key diff --git a/OpenFlow/src/WebServer.ts b/OpenFlow/src/WebServer.ts index 421ea377..e3ab59e5 100644 --- a/OpenFlow/src/WebServer.ts +++ b/OpenFlow/src/WebServer.ts @@ -748,18 +748,6 @@ export class WebServer { array.push(new Array(10000000).join('x')); await new Promise(resolve => { setTimeout(resolve, 1000) }); } - // let buffer = []; - // const MB = (bytes) => Math.round(bytes/1024/1024) + 'MB' - // const memoryUsage = () => { - // const mem = process.memoryUsage(); - // return MB(mem.rss) + '\t' + MB(mem.heapTotal) + '\t' + MB(mem.external); - // } - // setInterval(()=>{ - // buffer.push(Buffer.alloc(1024 * 1024* 1024)); // Eat 1GB of RAM every second - // console.log(buffer.length + '\t' + memoryUsage()); - // }, 1000); - res.end(JSON.stringify({ "success": "true", "message": "Ok here we go, crash incomming!!!!", "remoteip": remoteip, "hostname": _hostname, dt: new Date() })); - res.end(); } static async get_heapdump(req: any, res: any, next: any): Promise { @@ -783,7 +771,7 @@ export class WebServer { // @ts-ignore span.setStatus({ code: 200 }); } catch (error) { - console.error(error); + Logger.instanse.error(error, span); // @ts-ignore span.setStatus({code: 500, message: error instanceof Error ? error.message : undefined, }); diff --git a/OpenFlow/src/WebSocketServerClient.ts b/OpenFlow/src/WebSocketServerClient.ts index 7a91d7fd..d9540461 100644 --- a/OpenFlow/src/WebSocketServerClient.ts +++ b/OpenFlow/src/WebSocketServerClient.ts @@ -711,7 +711,7 @@ export class WebSocketServerClient { } }); } catch (error) { - console.error(error); + Logger.instanse.error(error, span); } finally { subspan?.end(); } diff --git a/OpenFlow/src/dockerdriver.ts b/OpenFlow/src/dockerdriver.ts index 17e827bd..d533851f 100644 --- a/OpenFlow/src/dockerdriver.ts +++ b/OpenFlow/src/dockerdriver.ts @@ -38,29 +38,9 @@ export class dockerdriver implements i_agent_driver { } if(pull) { - console.log("Pull image " + imagename) + Logger.instanse.info("Pull image " + imagename, span); await docker.pull(imagename) } - // return new Promise((resolve, reject) => { - // docker.pull(imagename, function (err, stream) { - // if (err) - // return reject(err); - - // docker.modem.followProgress(stream, onFinished, onProgress); - - // function onFinished(err2, output) { - // Logger.instanse.debug(output, span); - // if (err2) { - // Logger.instanse.error(err2, null); - // return reject(err2); - // } - // return resolve(); - // } - // function onProgress(event) { - // Logger.instanse.debug(event, span); - // } - // }); - // }) } public async NodeLabels(parent: Span): Promise { return null; diff --git a/OpenFlow/src/index.ts b/OpenFlow/src/index.ts index 15dc4c36..903cbd11 100644 --- a/OpenFlow/src/index.ts +++ b/OpenFlow/src/index.ts @@ -3,6 +3,11 @@ function clog(message) { let dts: string = dt.getHours() + ":" + dt.getMinutes() + ":" + dt.getSeconds() + "." + dt.getMilliseconds(); console.log(dts + " " + message); } +function cerror(error) { + let dt = new Date(); + let dts: string = dt.getHours() + ":" + dt.getMinutes() + ":" + dt.getSeconds() + "." + dt.getMilliseconds(); + console.error(dts, error.message ? error.message : error); +} clog("Starting @openiap/openflow"); import { Logger } from "./Logger"; import * as http from "http"; @@ -82,7 +87,7 @@ async function initDatabase(parent: Span): Promise { try { await Logger.configure(false, true); } catch (error) { - console.error(error); + cerror(error); process.exit(404); } @@ -422,8 +427,7 @@ let OpenAIProxy: any = null; try { OpenAIProxy = require("./ee/OpenAIProxy"); } catch (error) { - console.error(error); - + cerror(error); } @@ -441,7 +445,7 @@ var server: http.Server = null; try { await Logger.configure(false, false); } catch (error) { - console.error(error); + cerror(error); process.exit(404); } Config.db = new DatabaseConnection(Config.mongodb_url, Config.mongodb_db, true); diff --git a/OpenFlow/src/proto/client.ts b/OpenFlow/src/proto/client.ts index 799a93e8..1fc27138 100644 --- a/OpenFlow/src/proto/client.ts +++ b/OpenFlow/src/proto/client.ts @@ -150,7 +150,7 @@ export class flowclient extends client { var paylad = {"command": "watchevent", "data": data} protowrap.sendMesssag(this, paylad, null, true); } catch (error) { - console.error(error); + Logger.instanse.error(error, span); } finally { } } @@ -226,7 +226,6 @@ export class flowclient extends client { this._queuescounterstr = this._queuescounter.toString(); this._queuescurrentstr = this._queuescurrent.toString(); this._queues.push(queue); - console.log(this.id + " has " + this._queues.length + " queues") } } finally { if (queue) semaphore.up(); @@ -336,7 +335,6 @@ export class flowclient extends client { } finally { semaphore.up(); Logger.otel.endSpan(span); - console.log(this.id + " has " + this._queues.length + " queues") } } public async CloseConsumers(parent: Span): Promise { diff --git a/OpenFlow/src/public/CommonControllers.ts b/OpenFlow/src/public/CommonControllers.ts index 57a6295e..2c8b45b3 100644 --- a/OpenFlow/src/public/CommonControllers.ts +++ b/OpenFlow/src/public/CommonControllers.ts @@ -759,7 +759,7 @@ export class entitiesCtrl { this.loading = false; if (this.autorefresh) { if (this.models.length >= this.pagesize || this.page > 0) { - // console.warn("Disabling auto refresh, result has more than pagesize entries"); + // Disabling auto refresh, result has more than pagesize entries } else { if (this.autorefreshpromise == null && this.searchstring === "") { this.autorefreshpromise = this.$interval(() => { diff --git a/OpenFlow/src/public/Config.html b/OpenFlow/src/public/Config.html index 29e43610..29f7656e 100644 --- a/OpenFlow/src/public/Config.html +++ b/OpenFlow/src/public/Config.html @@ -27,6 +27,10 @@ .secret { -webkit-text-security: disc; } + .dimmed { + opacity: 0.5; /* Adjust the value to control the dimness */ + color: #777; /* Adjust the color to a lighter shade */ + }
@@ -35,69 +39,28 @@ ng-click="ctrl.model.log_to_exchange = !ctrl.model.log_to_exchange; ctrl.submit()" value="1" id="ctrl.model.log_to_exchange" />enabled streaming to Console
- cache - amqp - websocket - webserver - oauth - database - grafana - housekeeping - login_provider - otel - blocked_ips - - open-api - -
database queries - ms - -
On high volume systems, these 3 can crash your system!!! - debug - verbose - silly
-
- - {{ctrl.model.version}} - - {{ctrl.model.updatedat}} -
-
- + + ng-class="{'form-control': setting.type != 'boolean', 'secret': ctrl.model._encrypt.indexOf(setting.name) > -1, 'dimmed': ctrl.model[setting.name] == null }" /> + ng-class="{'form-control': setting.type != 'boolean', 'secret': ctrl.model._encrypt.indexOf(setting.name) > -1, 'dimmed': ctrl.model[setting.name] == null}" /> - +
diff --git a/OpenFlow/src/public/Console.html b/OpenFlow/src/public/Console.html index e34f475f..3062f296 100644 --- a/OpenFlow/src/public/Console.html +++ b/OpenFlow/src/public/Console.html @@ -86,6 +86,9 @@ ng-click="ctrl.model.log_database_queries = !ctrl.model.log_database_queries; ctrl.submit()" value="1" id="ctrl.model.log_database_queries" />database queries ms + all watches +
On high volume systems, these 3 can crash your system!!! debug diff --git a/OpenFlow/src/public/Controllers.ts b/OpenFlow/src/public/Controllers.ts index 7e92303c..f0fe7148 100644 --- a/OpenFlow/src/public/Controllers.ts +++ b/OpenFlow/src/public/Controllers.ts @@ -1320,7 +1320,6 @@ export class RPAWorkflowCtrl extends entityCtrl { this.queuename = await NoderedUtil.RegisterQueue({ callback: (data: QueueMessage, ack: any) => { ack(); - console.debug(data); if (data.data.command == undefined && data.data.data != null) data.data = data.data.data; this.messages += data.data.command + "\n"; if (data.data.command == "invokecompleted") { @@ -1341,7 +1340,6 @@ export class RPAWorkflowCtrl extends entityCtrl { setTimeout(this.RegisterQueue.bind(this), (Math.floor(Math.random() * 6) + 1) * 500); } }); - console.debug("queuename: " + this.queuename); } catch (error) { this.queuename = ""; console.debug("register queue failed, start reconnect. " + error.message ? error.message : error) @@ -1627,7 +1625,6 @@ export class ReportsCtrl extends entitiesCtrl { this.userdata.data.ReportsCtrl.run(this.userdata.data.ReportsCtrl.points); } async processData(): Promise { - console.debug('processData'); this.userdata.data.ReportsCtrl.run = this.processData.bind(this); this.userdata.data.ReportsCtrl.points = null; this.loading = true; @@ -1696,7 +1693,6 @@ export class ReportsCtrl extends entitiesCtrl { if (!this.$scope.$$phase) { this.$scope.$apply(); } } async robotsclick(points, evt): Promise { - console.debug('robotsclick'); this.userdata.data.ReportsCtrl.run = this.robotsclick.bind(this); this.userdata.data.ReportsCtrl.points = points; if (points.length > 0) { @@ -1823,7 +1819,6 @@ export class ReportsCtrl extends entitiesCtrl { } async robotclick(points, evt): Promise { - console.debug('robotclick'); if (points.length > 0) { } else { return; } const userid = this.charts[0].ids[points[0]._index]; @@ -1860,7 +1855,6 @@ export class ReportsCtrl extends entitiesCtrl { } async workflowclick(points, evt): Promise { - console.debug('workflowclick'); if (points.length > 0) { } else { return; } @@ -2047,7 +2041,6 @@ export class LoginCtrl { fs.root.getFile(filename, { create: true }, function (fileEntry) { fileEntry.createWriter(function (fileWriter) { fileWriter.onwriteend = function (e) { - console.debug('Write completed.'); resolve(); }; fileWriter.onerror = function (e) { @@ -2068,7 +2061,6 @@ export class LoginCtrl { scanForQRScanner() { try { if (QRScanner !== undefined) { - console.debug("Found QRScanner!!!!"); this.qrcodescan = true; if (!this.$scope.$$phase) { this.$scope.$apply(); } } else { @@ -2082,7 +2074,6 @@ export class LoginCtrl { } Scan() { try { - console.debug("Scan"); if (this.scanning) { this.scanning = false; QRScanner.destroy(); @@ -2100,12 +2091,10 @@ export class LoginCtrl { } async QRScannerHit(err, value) { try { - console.debug("QRScannerHit"); if (err) { console.error(err); return; } - console.debug(value); QRScanner.hide(); QRScanner.destroy(); @@ -2117,7 +2106,6 @@ export class LoginCtrl { console.debug("QRCode value: " + value); const config = JSON.parse(value); if (config.url !== null || config.url !== undefined || config.url !== "" || config.loginurl !== null || config.loginurl !== undefined || config.loginurl !== "") { - console.debug("set mobiledomain to " + value); await this.writefile("mobiledomain.txt", value); window.location.replace(config.url); } @@ -2133,7 +2121,6 @@ export class LoginCtrl { async submit(): Promise { this.message = ""; try { - console.debug("signing in with username/password"); const result: SigninMessage = await NoderedUtil.SigninWithUsername({ username: this.username, password: this.password }); if (result.user == null) { return; } this.setCookie("jwt", result.jwt, 365); @@ -2313,7 +2300,6 @@ export class UsersCtrl extends entitiesCtrl { haderror = true; } if (haderror) { - console.debug("loading stripe script") await jsutil.loadScript('//js.stripe.com/v3/'); this.stripe = Stripe(this.WebSocketClientService.stripe_api_key); } @@ -2621,12 +2607,10 @@ export class UserCtrl extends entityCtrl { if (roles.length > 0) { role = roles[0]; if (role.members === null || role.members === undefined) { - console.log("role.members is null", role); continue; } const exists = role.members.filter(x => x._id == this.model._id); if (exists.length > 0) { - console.log("Updating role", role.name); role.members = role.members.filter(x => x._id != this.model._id); try { await NoderedUtil.UpdateOne({ collectionname: "users", item: role }); @@ -2936,23 +2920,19 @@ export class FilesCtrl extends entitiesCtrl { const xhr = new XMLHttpRequest(); xhr.onload = () => { if (xhr.status >= 200 && xhr.status < 300) { - console.debug("upload complete"); // we done! if (!this.$scope.$$phase) { this.$scope.$apply(); } this.loadData(); } }; - console.debug("open"); xhr.open('POST', '/upload', true); - console.debug("send"); xhr.send(fd); } async Upload_usingapi() { try { const filename = (this.$scope as any).filename; const mimeType = (this.$scope as any).type; - console.debug("filename: " + filename + " mimeType: " + mimeType); this.loading = true; if (!this.$scope.$$phase) { this.$scope.$apply(); } const lastp: number = 0; @@ -3006,7 +2986,6 @@ export class EntitiesCtrl extends entitiesCtrl { return; } } - console.debug("path: " + this.$location.path()); if (NoderedUtil.IsNullEmpty(this.collection)) { this.$location.path("/Entities/entities"); if (!this.$scope.$$phase) { this.$scope.$apply(); } @@ -3424,7 +3403,6 @@ export class FormCtrl extends entityCtrl { WebSocketClientService.onSignedin(async (user: TokenUser) => { await jsutil.ensureJQuery(); await this.RegisterQueue(); - console.debug("queuename: " + this.queuename); if (this.id !== null && this.id !== undefined && this.id !== "") { this.basequery = { _id: this.id }; this.loadData(); @@ -3441,7 +3419,6 @@ export class FormCtrl extends entityCtrl { const result = await NoderedUtil.RegisterExchange( { exchangename: exchange, algorithm: "direct", callback: async (msg: QueueMessage, ack: any) => { - console.debug(msg); ack(); if (NoderedUtil.IsNullEmpty(msg.routingkey) || msg.routingkey == this.instanceid) { // this.loadData(); @@ -3468,14 +3445,11 @@ export class FormCtrl extends entityCtrl { } }); this.localexchangequeue = result.queuename; - console.debug("Register exchange for " + exchange + " with queue " + this.localexchangequeue); - } async RegisterQueue() { this.queuename = await NoderedUtil.RegisterQueue({ callback: (data: QueueMessage, ack: any) => { ack(); - console.debug(data); if (data.queuename == this.queuename) { if (data && data.data && data.data.command == "timeout") { this.errormessage = "No \"workflow in\" node listening or message timed out, is nodered running?"; @@ -3510,8 +3484,6 @@ export class FormCtrl extends entityCtrl { }); } async hideFormElements() { - console.debug("hideFormElements"); - $('input[ref="component"]').prop("disabled", true); $('#workflowform :input').prop("disabled", true); $('#workflowform :button').prop("disabled", true); @@ -3556,7 +3528,6 @@ export class FormCtrl extends entityCtrl { if (this.model.form === "none" || this.model.form === "") { - console.debug("workflow_instances has no form set, state " + this.model.state); if (this.model.state != "failed") { this.$location.path("/main"); } else { @@ -3580,7 +3551,6 @@ export class FormCtrl extends entityCtrl { if (!this.$scope.$$phase) { this.$scope.$apply(); } return; } else if (this.model.form === "unknown") { - console.debug("Form is unknown for instance, send empty message"); this.Save(); return; } else if (this.model.form !== "") { @@ -3601,8 +3571,6 @@ export class FormCtrl extends entityCtrl { this.renderform(); } else { try { - console.debug("No instance id found, send empty message"); - console.debug("SendOne: " + this.workflow._id + " / " + this.workflow.queue); await this.SendOne(this.workflow.queue, {}); } catch (error) { this.errormessage = error.message ? error.message : error; @@ -3691,14 +3659,10 @@ export class FormCtrl extends entityCtrl { for (let x = 0; x < item.components.length; x++) { obj[item.components[x].key] = ""; } - console.debug("add default array for " + item.key, obj); this.model.payload[item.key] = [obj]; } else { - console.debug("payload already have values for " + item.key); - console.debug("isArray: " + Array.isArray(this.model.payload[item.key])) if (Array.isArray(this.model.payload[item.key])) { } else { - console.debug("convert payload for " + item.key + " from object to array"); const keys = Object.keys(this.model.payload[item.key]); const arr: any[] = []; for (let x = 0; x < keys.length; x++) { @@ -3726,7 +3690,6 @@ export class FormCtrl extends entityCtrl { const item = components[y]; if (item.key == keys[i]) { if (Array.isArray(values)) { - console.debug("handle " + item.key + " as array"); const obj2: any = {}; for (let x = 0; x < values.length; x++) { obj2[x] = values[x]; @@ -3738,7 +3701,6 @@ export class FormCtrl extends entityCtrl { item.values = values; } } else { - console.debug("handle " + item.key + " as an object"); if (item.data != null && item.data != undefined) { item.data.values = values; item.data.json = JSON.stringify(values); @@ -3788,7 +3750,6 @@ export class FormCtrl extends entityCtrl { if ((this.form.fbeditor as any) == "false") this.form.fbeditor = false; if (!this.$scope.$$phase) { this.$scope.$apply(); } if (this.form.fbeditor === true) { - console.debug("renderform fbeditor"); const roles: any = {}; WebSocketClient.instance.user.roles.forEach(role => { roles[role._id] = role.name; @@ -3860,7 +3821,6 @@ export class FormCtrl extends entityCtrl { } this.formRender = ele.formRender(formRenderOpts); } else { - console.debug("renderform formio", this.form.schema?.components); if (!this.form.schema || !this.form.schema.components || this.form.schema.components.length == 0) { if (this.form.formData && this.form.formData.components && this.form.formData.components.length > 0) { console.warn("schema has no components, but forData does, using form formData.components instead") @@ -4180,8 +4140,6 @@ export class EntityCtrl extends entityCtrl { if (idx <= 0) { idx = 0; } else { idx--; } - console.debug("idx: " + idx); - // this.searchtext = this.searchFilteredList[idx].name; this.searchSelectedItem = this.searchFilteredList[idx]; return; } @@ -4189,8 +4147,6 @@ export class EntityCtrl extends entityCtrl { if (idx >= this.searchFilteredList.length) { idx = this.searchFilteredList.length - 1; } else { idx++; } - console.debug("idx: " + idx); - // this.searchtext = this.searchFilteredList[idx].name; this.searchSelectedItem = this.searchFilteredList[idx]; return; } @@ -4592,8 +4548,6 @@ export class AuditlogsCtrl extends entitiesCtrl { this.searchfields = ["name", "impostorname", "clientagent", "type"]; console.debug("AuditlogsCtrl"); this.pagesize = 20; - // this.basequery = { _type: "role" }; - // this.orderby = { "_created": -1 }; this.collection = "audit"; this.postloadData = this.processdata; WebSocketClientService.onSignedin(async (user: TokenUser) => { @@ -4706,7 +4660,6 @@ export class CredentialsCtrl extends entitiesCtrl { var query = { _type: "role", "$or": [{ name: name + "noderedadmins" }, { name: name + "nodered api users" }] } const list = await NoderedUtil.Query({ collectionname: "users", query, top: 4 }); for (var i = 0; i < list.length; i++) { - console.debug("Deleting " + list[i].name) await NoderedUtil.DeleteOne({ collectionname: "users", id: list[i]._id }); } @@ -4835,8 +4788,6 @@ export class CredentialCtrl extends entityCtrl { if (idx <= 0) { idx = 0; } else { idx--; } - console.debug("idx: " + idx); - // this.searchtext = this.searchFilteredList[idx].name; this.searchSelectedItem = this.searchFilteredList[idx]; return; } @@ -4844,8 +4795,6 @@ export class CredentialCtrl extends entityCtrl { if (idx >= this.searchFilteredList.length) { idx = this.searchFilteredList.length - 1; } else { idx++; } - console.debug("idx: " + idx); - // this.searchtext = this.searchFilteredList[idx].name; this.searchSelectedItem = this.searchFilteredList[idx]; return; } @@ -5348,7 +5297,6 @@ export class DeletedCtrl extends entitiesCtrl { return; } } - console.debug("path: " + this.$location.path()); if (NoderedUtil.IsNullEmpty(this.collection)) { this.$location.path("/Deleted/entities"); if (!this.$scope.$$phase) { this.$scope.$apply(); } @@ -5501,13 +5449,11 @@ export class CustomerCtrl extends entityCtrl { haderror = true; } if (haderror) { - console.debug("loading stripe script") await jsutil.loadScript('//js.stripe.com/v3/'); this.stripe = Stripe(this.WebSocketClientService.stripe_api_key); } } if (this.id !== null && this.id !== undefined && this.id != "new") { - console.debug("Loading customer id " + this.id); this.loading = false; this.loadData(); return; @@ -5516,7 +5462,6 @@ export class CustomerCtrl extends entityCtrl { if (user.customerid != null && this.id != "new") { this.id = user.customerid; this.basequery = { _id: this.id }; - console.debug("Loading customer id " + this.id); this.loading = false; this.loadData(); return; @@ -5529,7 +5474,6 @@ export class CustomerCtrl extends entityCtrl { }); if (results.length > 0) { this.model = results[0]; - console.debug("Loaded customer " + this.model._id); } } @@ -5558,7 +5502,6 @@ export class CustomerCtrl extends entityCtrl { if (this.model.email && this.model.email.indexOf("@") == -1) { this.model.email = (WebSocketClient.instance.user as any).username + "@domain.com"; } - console.debug("Create new customer"); } this.loading = false; @@ -5607,11 +5550,8 @@ export class CustomerCtrl extends entityCtrl { async processdata() { try { if (this.model._type != "customer") { - console.log("Not customer!", this.model); return; } - console.log(this.model); - console.debug("processdata"); this.loading = true; if (!this.$scope.$$phase) { this.$scope.$apply(); } this.errormessage = ""; @@ -5628,9 +5568,7 @@ export class CustomerCtrl extends entityCtrl { await NoderedUtil.EnsureCustomer({ customer: this.model }); } this.Resources = await NoderedUtil.Query({ collectionname: "config", query: { "_type": "resource", "target": "customer", "allowdirectassign": true }, orderby: { _created: -1 } }); - console.debug("Resources", this.Resources); this.Assigned = await NoderedUtil.Query({ collectionname: "config", query: { "_type": "resourceusage", "customerid": this.model._id, "userid": { "$exists": false } }, orderby: { _created: -1 } }); - console.debug("Assigned", this.Assigned); for (var res of this.Resources) { res.products = res.products.filter(x => x.allowdirectassign == true); for (var prod of res.products) { @@ -5664,8 +5602,6 @@ export class CustomerCtrl extends entityCtrl { } } } - console.debug("Assigned", this.Assigned); - console.debug("UserAssigned", this.UserAssigned); this.support = []; this.licenses = []; for (let a of this.Assigned) { @@ -5681,7 +5617,6 @@ export class CustomerCtrl extends entityCtrl { this.errormessage = error; } this.loading = false; - console.debug("processdata::end"); if (!this.$scope.$$phase) { this.$scope.$apply(); } } ToggleNextInvoiceModal() { @@ -5881,7 +5816,6 @@ export class CustomerCtrl extends entityCtrl { } this.domain = domain; const res:string = await NoderedUtil.CustomCommand({ command: "issuelicense", data: payload }); - console.log(res); // @ts-ignore this.licensekey = res; this.licensekeydecoded = atob(res); @@ -6065,10 +5999,8 @@ export class EntityRestrictionsCtrl extends entitiesCtrl { (model as any).collection = collection; (model as any).paths = paths; if (model._id) { - console.debug("updating " + name); await NoderedUtil.UpdateOne({ collectionname: this.collection, item: model }); } else { - console.debug("adding " + name); await NoderedUtil.InsertOne({ collectionname: this.collection, item: model }); } } @@ -6216,7 +6148,6 @@ export class EntityRestrictionCtrl extends entityCtrl { if (idx <= 0) { idx = 0; } else { idx--; } - console.debug("idx: " + idx); this.searchSelectedItem = this.searchFilteredList[idx]; return; } @@ -6224,7 +6155,6 @@ export class EntityRestrictionCtrl extends entityCtrl { if (idx >= this.searchFilteredList.length) { idx = this.searchFilteredList.length - 1; } else { idx++; } - console.debug("idx: " + idx); this.searchSelectedItem = this.searchFilteredList[idx]; return; } @@ -6508,10 +6438,8 @@ export class ResourcesCtrl extends entitiesCtrl { Base.addRight(model, "5a17f157c4815318c8536c21", "users", [2]); } if (model._id) { - console.debug("updating " + name); return await NoderedUtil.UpdateOne({ collectionname: this.collection, item: model }); } else { - console.debug("adding " + name); return await NoderedUtil.InsertOne({ collectionname: this.collection, item: model }); } } @@ -6612,8 +6540,6 @@ export class WorkitemsCtrl extends entitiesCtrl { this.basequery = { _type: "workitem", "wiq": this.queue }; } if (!this.$scope.$$phase) { this.$scope.$apply(); } - console.log(this.workitemqueues); - console.log(this.queue); this.loadData(); } SetState(state) { @@ -6927,7 +6853,6 @@ export class WorkitemQueueCtrl extends entityCtrl { this.projects.unshift({ "_id": "", "name": "", "display": "(no project)" } as any); let queryas: string = null; if (this.model != null) queryas = this.model.robotqueue; - console.log("queryas", queryas) this.workflows = await NoderedUtil.Query({ collectionname: "openrpa", query: { "_type": "workflow" }, projection: { "name": 1, "projectandname": 1 }, orderby: "name", top: 500, queryas }); this.workflows.forEach((e: any) => { e.display = e.projectandname }); this.workflows.unshift({ "_id": "", "name": "", "projectandname": "", "display": "(no workflow)" } as any); @@ -6947,10 +6872,8 @@ export class WorkitemQueueCtrl extends entityCtrl { this.workitemqueues.unshift({ "_id": "", "name": "", "display": "(no workitem queue)" } as any); this.agents = await NoderedUtil.Query({ collectionname: "agents", query: { "_type": "agent" }, orderby: "name", projection: { "slug": 1, "name": 1 } }); - console.log(this.agents); this.agents.unshift({ "name": "" } as any) this.packages = await NoderedUtil.Query({ collectionname: "agents", query: { "_type": "package" }, orderby: "name", projection: { "name": 1 } }); - console.log(this.packages); this.packages.unshift({ "name": "" } as any) if (!this.$scope.$$phase) { this.$scope.$apply(); } @@ -7280,7 +7203,6 @@ export class ConsoleCtrl extends entityCtrl { if (this.messages.length >= lines) this.messages.splice(lines - 1); if (!this.$scope.$$phase) { this.$scope.$apply(); } }, closedcallback: (msg) => { - console.debug("rabbitmq disconnected, start reconnect") setTimeout(this.RegisterQueue.bind(this), (Math.floor(Math.random() * 6) + 1) * 500); } }); @@ -7289,12 +7211,9 @@ export class ConsoleCtrl extends entityCtrl { } this.watchid = await NoderedUtil.Watch({ aggregates: [{ "$match": { "fullDocument._type": "config" } }], collectionname: "config", callback: (data) => { - console.log(data); this.loadData(); } }) - console.debug("exchange: ", this.exchange); - console.debug("watchid: ", this.watchid); } catch (error) { console.debug("register queue failed, start reconnect. " + error.message ? error.message : error) setTimeout(this.RegisterQueue.bind(this), (Math.floor(Math.random() * 6) + 1) * 500); @@ -7412,6 +7331,7 @@ export class ConfigCtrl extends entityCtrl { {"name": "enable_openapiauth", "type": "boolean", "default": "true"}, {"name": "openai_token", "type": "string", "default": ""}, {"name": "log_with_colors", "type": "boolean", "default": "true"}, + {"name": "log_database_queries_to_collection", "type": "string", "default": ""}, {"name": "cache_store_type", "type": "string", "default": "memory"}, {"name": "cache_store_max", "type": "number", "default": "1000"}, {"name": "cache_store_ttl_seconds", "type": "number", "default": "300"}, @@ -7617,44 +7537,36 @@ export class ConfigCtrl extends entityCtrl { ] WebSocketClientService.onSignedin(async (_user: TokenUser) => { await this.RegisterWatch(); - console.log("Load data") this.loading = false; this.loadData(); this.$scope.$on('signin', (event, data) => { this.RegisterWatch(); - console.log("Load data") this.loading = false; this.loadData(); }); }); } async RegisterWatch() { - console.log("RegisterWatch") try { if (!NoderedUtil.IsNullEmpty(this.watchid)) { await NoderedUtil.UnWatch({ id: this.watchid }); } this.watchid = await NoderedUtil.Watch({ aggregates: [{ "$match": { "fullDocument._type": "config" } }], collectionname: "config", callback: (data) => { - console.log(data); this.loading = false; this.loadData(); } }) - console.debug("watchid: ", this.watchid); } catch (error) { console.debug("register queue failed, start reconnect. " + error.message ? error.message : error) setTimeout(this.RegisterWatch.bind(this), (Math.floor(Math.random() * 6) + 1) * 500); } } + deletekey(key: string) { + delete this.model[key]; + this.submit(); + } processdata() { - console.log("processdata") - // console.log(this.model); - // for(let i = 0; i < this.settings.length; i++) { - // if (this.model[this.settings[i].name] == null) { - // this.model[this.settings[i].name] = this.settings[i].default; - // } - // } const ids: string[] = []; if (this.collection == "files") { for (let i: number = 0; i < (this.model as any).metadata._acl.length; i++) { @@ -7672,13 +7584,6 @@ export class ConfigCtrl extends entityCtrl { } async submit(): Promise { try { - for(let i = 0; i < this.settings.length; i++) { - if (this.model[this.settings[i].name] == null || this.model[this.settings[i].name] == "" - || this.model[this.settings[i].name] == this.settings[i].default) { - // delete this.model[this.settings[i].name]; - } - } - if (this.model._id) { await NoderedUtil.UpdateOne({ collectionname: this.collection, item: this.model }); } else { @@ -7732,9 +7637,7 @@ export class ConfigCtrl extends entityCtrl { delayedUpdate() { if(this.loading == true) return; if(this.delayhandler ! = null) return; - console.log("delayedUpdate", this.loading) this.delayhandler = setTimeout(() => { - console.log("do it") this.delayhandler = null; this.submit(); }, 500); @@ -7801,19 +7704,16 @@ export class AgentsCtrl extends entitiesCtrl { if(model.image == null) { var cli = this.clients.filter(x=> x.user?._id ==model.runas && (x.agent == "nodeagent" || x.agent == "assistant")); if(cli != null && cli.length > 0) { - console.log(cli[0], model); model.status = "online" } } else if(model.image.indexOf("nodered") > -1) { var cli = this.clients.filter(x=> x.user?._id ==model.runas && x.agent == "nodered"); if(cli != null && cli.length > 0) { - console.log(cli[0], model); model.status = "online" } } else if(model.image.indexOf("agent") > -1) { var cli = this.clients.filter(x=> x.user?._id ==model.runas && (x.agent == "nodeagent" || x.agent == "python") ); if(cli != null && cli.length > 0) { - console.log(cli[0], model); model.status = "online" } } @@ -8079,7 +7979,6 @@ export class AgentCtrl extends entityCtrl { } } async ImageUpdated() { - console.log("ImageUpdated") this.sizewarningtitle = "" this.sizewarning = "" @@ -8098,20 +7997,10 @@ export class AgentCtrl extends entityCtrl { } if (languages == null || languages.length == 0) { this.packages = []; - console.log("languages empty", this.packages) } else { - console.log("languages", languages[0]) this.packages = this.allpackages.filter(x => languages.indexOf(x.language) > -1) - console.log("filtered", this.packages) if (!haschromium && !haschrome) { this.packages = this.packages.filter(x => x.chrome != true && x.chromium != true) - console.log("filtered again", this.packages) - } - if (this.model._id == null || this.model._id == "") { - // this.model.package = ""; - // if ((this.model.package == null || this.model.package == "") && this.packages.length > 0) { - // this.model.package = this.packages[0]._id; - // } } this.packages.unshift({ _id: "", name: "None" }) } @@ -8194,8 +8083,8 @@ export class AgentCtrl extends entityCtrl { this.refreshtimer = setTimeout(() => { this.refreshtimer = null; var path = this.$location.path(); - if (path == null && path == undefined) { console.debug("getagent, path is null"); return false; } - if (!path.toLowerCase().startsWith("/agent/") && path.toLowerCase() != "/agent") { console.debug("getagent, path is no longer /Agent"); return false; } + if (path == null && path == undefined) { return false; } + if (!path.toLowerCase().startsWith("/agent/") && path.toLowerCase() != "/agent") { return false; } this.loadInstances(); }, 2000); } @@ -8406,12 +8295,9 @@ export class AgentCtrl extends entityCtrl { var tmp = await NoderedUtil.InsertOne({ collectionname: this.collection, item: this.model }); if (this.model) { this.model = tmp; - console.log("insertone", this.model) this.id = this.model._id this.basequery = { _id: this.id }; - console.log("startagent", this.model.slug) await NoderedUtil.CustomCommand({ command: "startagent", id: this.model._id, name: this.model.slug }) - console.log("load data") } } this.loading = false; @@ -8679,16 +8565,12 @@ export class PackageCtrl extends entityCtrl { const filename = e.files[0].name // @ts-ignore var result = await NoderedUtil.SaveFile({ filename, mimeType, file: buffer, compressed: false, metadata: { _type: "package" } }); - // console.log(result) - console.log(result.id) // @ts-ignore this.oldfileid = this.model.fileid; // @ts-ignore this.model.fileid = result.id; e.value = null; if (!this.$scope.$$phase) { this.$scope.$apply(); } - } else { - console.log("no files in form") } } catch (error) { @@ -8787,17 +8669,9 @@ export class RunPackageCtrl extends entityCtrl { this.haschromium = (_a.chromium == true) this.packages = this.allpackages.filter(x => this.languages.indexOf(x.language) > -1) - console.log("filtered", this.packages) if (!this.haschromium && !this.haschrome) { this.packages = this.packages.filter(x => x.chrome != true && x.chromium != true) - console.log("filtered again", this.packages) } - // if(this.packages.find(x => x._id == this.package) == null) { - // this.package = ""; - // } - // if(this.package == "" && this.packages.length > 0) { - // this.package = this.packages[0]._id; - // } if (!this.$scope.$$phase) { this.$scope.$apply(); } } async addprocess(streamid:string, schedulename:string = undefined): Promise { @@ -8838,7 +8712,6 @@ export class RunPackageCtrl extends entityCtrl { killbutton.id = streamid + "_kill"; killbutton.onclick = async () => { try { - console.log("kill", streamid) await NoderedUtil.Queue({ data: { command: "kill", "id": streamid }, queuename: _a.slug + "agent", correlationId: streamid }) } catch (error) { console.error(error); @@ -8855,7 +8728,6 @@ export class RunPackageCtrl extends entityCtrl { runs.prepend(div); await NoderedUtil.Queue({ data: payload, queuename: _a.slug + "agent", correlationId: streamid }) - console.log("submitted", payload) } async submit(): Promise { var _a = this.agents.find(x => x._id == this.id); @@ -8868,20 +8740,17 @@ export class RunPackageCtrl extends entityCtrl { "queuename": this.queuename } await NoderedUtil.Queue({ data: payload, queuename: _a.slug + "agent", correlationId: streamid }) - console.log("submitted", payload) } async RegisterQueue() { if(this.queuename != "") return; this.queuename = await NoderedUtil.RegisterQueue({ callback: (_data: QueueMessage, ack: any) => { ack(); - console.log(_data.data) if(_data == null) return; var correlationId = _data.correlationId; var data: any = _data; while(data.data != null && data.data != "") data = data.data; if(data.command == "listprocesses") { - console.log("listprocesses !!!") for(var i = 0; i < data.processes.length; i++) { console.log("add process " + data.processes[i].id) this.addprocess(data.processes[i].id, data.processes[i].schedulename); @@ -8917,9 +8786,7 @@ export class RunPackageCtrl extends entityCtrl { setTimeout(this.RegisterQueue.bind(this), (Math.floor(Math.random() * 6) + 1) * 500); } }); - console.debug("registed queue", this.queuename); var _a = this.agents.find(x => x._id == this.id); await NoderedUtil.Queue({ data: {"command": "addcommandstreamid"}, queuename: _a.slug + "agent" }); - console.debug("Added streamid to command streams for " + _a.slug + "agent") } } \ No newline at end of file From 90e6bbc8bc7db93870698b01c7093d7b6fe1db09 Mon Sep 17 00:00:00 2001 From: skadefro Date: Fri, 24 Nov 2023 00:57:49 +0100 Subject: [PATCH 14/32] fix settings list --- OpenFlow/src/public/Controllers.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/OpenFlow/src/public/Controllers.ts b/OpenFlow/src/public/Controllers.ts index f0fe7148..2663de0c 100644 --- a/OpenFlow/src/public/Controllers.ts +++ b/OpenFlow/src/public/Controllers.ts @@ -7325,6 +7325,10 @@ export class ConfigCtrl extends entityCtrl { this.messages = []; this.postloadData = this.processdata; this.settings = [ + // {"name": "supports_watch", "type": "boolean", "default": "false"}, + // {"name": "NODE_ENV", "type": "string", "default": "development"}, + // {"name": "log_to_exchange", "type": "boolean", "default": "false"}, // called straming handled elsewere + // {"name": "aes_secret", "type": "string", "default": ""}, // ONLY envoriment variable {"name": "license_key", "type": "string", "default": ""}, {"name": "enable_openai", "type": "boolean", "default": "false"}, {"name": "enable_openapi", "type": "boolean", "default": "true"}, @@ -7342,8 +7346,7 @@ export class ConfigCtrl extends entityCtrl { {"name": "heapdump_onstop", "type": "boolean", "default": "false"}, {"name": "amqp_allow_replyto_empty_queuename", "type": "boolean", "default": "false"}, {"name": "enable_openflow_amqp", "type": "boolean", "default": "false"}, - // {"name": "log_to_exchange", "type": "boolean", "default": "false"}, - {"name": "openflow_amqp_expiration", "type": "number", "default": "1500000"}, // 25 min in milliseconds + {"name": "openflow_amqp_expiration", "type": "number", "default": "1500000"}, {"name": "amqp_prefetch", "type": "number", "default": "25"}, {"name": "enable_entity_restriction", "type": "boolean", "default": "false"}, {"name": "enable_web_tours", "type": "boolean", "default": "true"}, @@ -7354,10 +7357,9 @@ export class ConfigCtrl extends entityCtrl { {"name": "housekeeping_remomve_unvalidated_user_days", "type": "number", "default": "0"}, {"name": "housekeeping_cleanup_openrpa_instances", "type": "boolean", "default": "false"}, {"name": "workitem_queue_monitoring_enabled", "type": "boolean", "default": "true"}, - {"name": "workitem_queue_monitoring_interval", "type": "number", "default": "10000"}, // 10 sec + {"name": "workitem_queue_monitoring_interval", "type": "number", "default": "10000"}, {"name": "upload_max_filesize_mb", "type": "number", "default": "25"}, {"name": "getting_started_url", "type": "string", "default": ""}, - {"name": "NODE_ENV", "type": "string", "default": "development"}, {"name": "HTTP_PROXY", "type": "string", "default": ""}, {"name": "HTTPS_PROXY", "type": "string", "default": ""}, {"name": "NO_PROXY", "type": "string", "default": ""}, @@ -7369,7 +7371,6 @@ export class ConfigCtrl extends entityCtrl { {"name": "stripe_force_vat", "type": "boolean", "default": "false"}, {"name": "stripe_force_checkout", "type": "boolean", "default": "false"}, {"name": "stripe_allow_promotion_codes", "type": "boolean", "default": "true"}, - {"name": "supports_watch", "type": "boolean", "default": "false"}, {"name": "ensure_indexes", "type": "boolean", "default": "true"}, {"name": "text_index_name_fields", "type": "string[]", "default": "name,_names"}, {"name": "auto_create_users", "type": "boolean", "default": "false"}, @@ -7469,7 +7470,6 @@ export class ConfigCtrl extends entityCtrl { {"name": "allow_skiphistory", "type": "boolean", "default": "false"}, {"name": "max_memory_restart_mb", "type": "number", "default": "0"}, {"name": "saml_issuer", "type": "string", "default": "the-issuer"}, - // {"name": "aes_secret", "type": "string", "default": ""}, {"name": "signing_crt", "type": "string", "default": ""}, {"name": "singing_key", "type": "string", "default": ""}, {"name": "wapid_mail", "type": "string", "default": ""}, From 8f6922b3255925d19f52c63ace82a3aba2b1d6ac Mon Sep 17 00:00:00 2001 From: skadefro Date: Fri, 24 Nov 2023 00:59:30 +0100 Subject: [PATCH 15/32] cross link Console and Config --- OpenFlow/src/public/Config.html | 4 ++-- OpenFlow/src/public/Console.html | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/OpenFlow/src/public/Config.html b/OpenFlow/src/public/Config.html index 29f7656e..90da280c 100644 --- a/OpenFlow/src/public/Config.html +++ b/OpenFlow/src/public/Config.html @@ -35,8 +35,8 @@
- enabled streaming to + Console

diff --git a/OpenFlow/src/public/Console.html b/OpenFlow/src/public/Console.html index 3062f296..a60646b0 100644 --- a/OpenFlow/src/public/Console.html +++ b/OpenFlow/src/public/Console.html @@ -98,7 +98,7 @@ ng-click="ctrl.model.log_silly = !ctrl.model.log_silly; ctrl.submit()" value="1" id="ctrl.model.log_silly" />silly -
Click on message to copy log object with all properties to clipboard +
Click on message to copy log object with all properties to clipboard Config page
search
From 5a383bdea55d729da3e695e3e90c989bf9d9db96 Mon Sep 17 00:00:00 2001 From: skadefro Date: Fri, 24 Nov 2023 01:05:43 +0100 Subject: [PATCH 16/32] Remove supports_watch --- OpenFlow/src/Config.ts | 2 -- OpenFlow/src/DatabaseConnection.ts | 30 +++++++++++++++--------------- OpenFlow/src/LoginProvider.ts | 2 +- OpenFlow/src/Messages/Message.ts | 18 +++++++++--------- 4 files changed, 25 insertions(+), 27 deletions(-) diff --git a/OpenFlow/src/Config.ts b/OpenFlow/src/Config.ts index 0adc043a..fc418a8d 100644 --- a/OpenFlow/src/Config.ts +++ b/OpenFlow/src/Config.ts @@ -257,7 +257,6 @@ export class Config { stripe_force_checkout: false, stripe_allow_promotion_codes: true, - supports_watch: false, ensure_indexes: true, text_index_name_fields: ["name", "_names"], auto_create_users: false, @@ -531,7 +530,6 @@ export class Config { public static stripe_force_checkout: boolean = Config.parseBoolean(Config.getEnv("stripe_force_checkout")); public static stripe_allow_promotion_codes: boolean = Config.parseBoolean(Config.getEnv("stripe_allow_promotion_codes")); - public static supports_watch: boolean = Config.parseBoolean(Config.getEnv("supports_watch")); public static ensure_indexes: boolean = Config.parseBoolean(Config.getEnv("ensure_indexes")); public static text_index_name_fields: string[] = Config.parseArray(Config.getEnv("text_index_name_fields")); diff --git a/OpenFlow/src/DatabaseConnection.ts b/OpenFlow/src/DatabaseConnection.ts index 135d9d87..28e307f5 100644 --- a/OpenFlow/src/DatabaseConnection.ts +++ b/OpenFlow/src/DatabaseConnection.ts @@ -238,18 +238,18 @@ export class DatabaseConnection extends events.EventEmitter { // // Logger.instanse.debug("Query: " + JSON.stringify({ _id: ace._id }), span, { ms, count: arr.length, collection: "users" }); // }) this.db = this.cli.db(this._dbname); - try { - var topology = (this.cli as any).topology; - if (topology.s.description.type == "Single" || topology.s.description.type == "single") { - Config.supports_watch = false; - } else { - Config.supports_watch = true; - } - } catch (error) { - Logger.instanse.error(error, span); - } - Logger.instanse.debug("supports_watch: " + Config.supports_watch, span); - if (Config.supports_watch && this.registerGlobalWatches) { + // try { + // var topology = (this.cli as any).topology; + // if (topology.s.description.type == "Single" || topology.s.description.type == "single") { + // Config.supports_watch = false; + // } else { + // Config.supports_watch = true; + // } + // } catch (error) { + // Logger.instanse.error(error, span); + // } + // Logger.instanse.debug("supports_watch: " + Config.supports_watch, span); + // if (Config.supports_watch && this.registerGlobalWatches) { // let collections = await DatabaseConnection.toArray(this.db.listCollections()); let collections = await Logger.DBHelper.GetCollections(span); collections = collections.filter(x => x.name.indexOf("system.") === -1); @@ -259,7 +259,7 @@ export class DatabaseConnection extends events.EventEmitter { if (collections[c].name == "fs.files" || collections[c].name == "fs.chunks") continue; this.registerGlobalWatch(collections[c].name, span); } - } + // } this.ensureQueueMonitoring(); this.isConnected = true; Logger.otel.endSpan(span); @@ -4836,14 +4836,14 @@ export class DatabaseConnection extends events.EventEmitter { collections = await DatabaseConnection.toArray(this.db.listCollections()); collections = collections.filter(x => x.name.indexOf("system.") === -1); - if (Config.supports_watch) { + // if (Config.supports_watch) { Logger.instanse.info("Register global watches for each collection", span); for (var c = 0; c < collections.length; c++) { if (collections[c].type != "collection") continue; if (collections[c].name == "fs.files" || collections[c].name == "fs.chunks") continue; this.registerGlobalWatch(collections[c].name, span); } - } + // } DatabaseConnection.timeseries_collections = []; DatabaseConnection.collections_with_text_index = []; diff --git a/OpenFlow/src/LoginProvider.ts b/OpenFlow/src/LoginProvider.ts index be4d507b..8934dec1 100644 --- a/OpenFlow/src/LoginProvider.ts +++ b/OpenFlow/src/LoginProvider.ts @@ -1156,7 +1156,7 @@ export class LoginProvider { validate_user_form: Config.validate_user_form, validate_emails: Config.validate_emails, forgot_pass_emails: Config.forgot_pass_emails, - supports_watch: Config.supports_watch, + supports_watch: true, agent_images: Config.agent_images, amqp_enabled_exchange: Config.amqp_enabled_exchange, multi_tenant: Config.multi_tenant, diff --git a/OpenFlow/src/Messages/Message.ts b/OpenFlow/src/Messages/Message.ts index 34259851..79d48623 100644 --- a/OpenFlow/src/Messages/Message.ts +++ b/OpenFlow/src/Messages/Message.ts @@ -1356,11 +1356,11 @@ export class Message { msg = WatchMessage.assign(this.data); if (NoderedUtil.IsNullEmpty(msg.jwt)) { msg.jwt = this.jwt; } if (NoderedUtil.IsNullEmpty(msg.jwt)) { msg.jwt = cli.jwt; } - if (Config.supports_watch) { + //if (Config.supports_watch) { await cli.UnWatch(msg.id, msg.jwt); - } else { - msg.error = "Watch is not supported by this openflow"; - } + // } else { + // msg.error = "Watch is not supported by this openflow"; + // } msg.result = null; delete msg.jwt; this.data = JSON.stringify(msg); @@ -1372,14 +1372,14 @@ export class Message { if (NoderedUtil.IsNullEmpty(msg.jwt)) { msg.jwt = this.jwt; } if (NoderedUtil.IsNullEmpty(msg.jwt)) { msg.jwt = cli.jwt; } msg.id = null; - if (Config.supports_watch) { + // if (Config.supports_watch) { msg.id = await cli.Watch(msg.aggregates, msg.collectionname, msg.jwt); if(msg.collectionname != null && msg.collectionname != "") { Config.db.registerGlobalWatch(msg.collectionname, null); } - } else { - msg.error = "Watch is not supported by this openflow"; - } + // } else { + // msg.error = "Watch is not supported by this openflow"; + // } msg.result = msg.id; delete msg.jwt; this.data = JSON.stringify(msg); @@ -1945,7 +1945,7 @@ export class Message { } else { Logger.instanse.debug(tuser.username + " was validated in using " + tokentype, span); } - msg.supports_watch = Config.supports_watch; + msg.supports_watch = true; var keys = Object.keys(UpdateDoc.$set); if (keys.length > 0 || UpdateDoc.$unset || NoderedUtil.IsNullEmpty(user.lastseen)) { // ping will handle this, if no new information needs to be added From 2e2f746b7351aad52cd5087734b8a43a78aa5175 Mon Sep 17 00:00:00 2001 From: skadefro Date: Fri, 24 Nov 2023 01:36:48 +0100 Subject: [PATCH 17/32] Add set/unset button to config --- OpenFlow/src/Config.ts | 29 ++++++++++++++++++++--------- OpenFlow/src/public/Config.html | 15 ++++++++++++++- OpenFlow/src/public/Controllers.ts | 20 ++++++++++++++++---- 3 files changed, 50 insertions(+), 14 deletions(-) diff --git a/OpenFlow/src/Config.ts b/OpenFlow/src/Config.ts index fc418a8d..cffaabef 100644 --- a/OpenFlow/src/Config.ts +++ b/OpenFlow/src/Config.ts @@ -17,12 +17,13 @@ export class dbConfig extends Base { this._type = "config"; this.name = "Base configuration"; this.version = "0.0.1"; - this._encrypt = ["stripe_api_secret", "smtp_url", "amqp_password", "cache_store_redis_password", "cookie_secret", "singing_key", "wapid_key"]; + this._encrypt = ["mongodb_url", "stripe_api_secret", "smtp_url", "amqp_password", "cache_store_redis_password", "cookie_secret", "singing_key", "wapid_key"]; } public version: string; public needsupdate: boolean; public updatedat: Date; + public async Save(jwt: string, parent: Span): Promise { if (this.needsupdate = true) { @@ -38,6 +39,19 @@ export class dbConfig extends Base { if(this.version == null) return -1; return this.version.localeCompare(version, undefined, { numeric: true, sensitivity: 'base' }); } + public static areEqual(a, b) { + if (a === b) return true; + if (a == null || b == null) return false; + try { + var _a = JSON.stringify(a); + var _b = JSON.stringify(b); + } catch (error) { + console.error("areEqual failed to stringify") + return false; + } + if (_a !== _b) return false; + return true; + } public static cleanAndApply(conf: dbConfig, parent: Span): Boolean { var updated = false; @@ -77,7 +91,7 @@ export class dbConfig extends Base { if(key == "HTTP_PROXY") { var b = true; } - if(_env != _default) { + if(!dbConfig.areEqual(_env, _default)) { updated = true; conf[key] = Config[key]; } @@ -103,9 +117,6 @@ export class dbConfig extends Base { continue; } } - if(key == "HTTP_PROXY") { - var b = true; - } if (Object.prototype.hasOwnProperty.call(Config, key)) { Config[key] = value; @@ -125,12 +136,12 @@ export class dbConfig extends Base { continue; } if(_env != _default) { - } else if(_env == value ) { + } else if(dbConfig.areEqual(_env, value)) { updated = true; delete conf[key]; } } else { - if(_default == value ) { + if(dbConfig.areEqual(_default, value)) { updated = true; delete conf[key]; } @@ -140,7 +151,7 @@ export class dbConfig extends Base { Logger.instanse.error("Error setting config " + keys + " to " + value, null); } } - conf._encrypt = ["stripe_api_secret", "smtp_url", "amqp_password", "cache_store_redis_password", "cookie_secret", "singing_key", "wapid_key"]; + conf._encrypt = ["mongodb_url", "stripe_api_secret", "smtp_url", "amqp_password", "cache_store_redis_password", "cookie_secret", "singing_key", "wapid_key"]; if(Config._version != conf._version) { Config._version = conf._version; Logger.instanse.info("Loaded config version " + conf._version, parent); @@ -319,7 +330,7 @@ export class Config { migrate_audit_to_ts: true, websocket_package_size: 25000, - websocket_max_package_count: 25000, + websocket_max_package_count: 1048576, websocket_message_callback_timeout: 3600, websocket_disconnect_out_of_sync: false, protocol: "http", diff --git a/OpenFlow/src/public/Config.html b/OpenFlow/src/public/Config.html index 90da280c..e27beca7 100644 --- a/OpenFlow/src/public/Config.html +++ b/OpenFlow/src/public/Config.html @@ -38,13 +38,26 @@ Console + + all + + + set + + + unset + +

-
+