diff --git a/OpenFlow/src/Config.ts b/OpenFlow/src/Config.ts index c8804527..869ae7e2 100644 --- a/OpenFlow/src/Config.ts +++ b/OpenFlow/src/Config.ts @@ -1,7 +1,6 @@ var xml2js = require('xml2js'); import * as https from "https"; import * as http from "http"; -// import { fetch, toPassportConfig } from "passport-saml-metadata"; import * as fs from "fs"; import * as os from "os"; import * as path from "path"; @@ -11,18 +10,20 @@ import { Base, InsertOrUpdateOneMessage, NoderedUtil, Rights, WellknownIds } fro import { promiseRetry } from "./Logger"; import { Span } from "@opentelemetry/api"; + export class dbConfig extends Base { constructor() { super(); this._type = "config"; this.name = "Base configuration"; this.version = "0.0.1"; - this._encrypt = ["stripe_api_secret", "smtp_url", "amqp_password", "cache_store_redis_password", "cookie_secret", "singing_key", "wapid_key"]; + this._encrypt = ["mongodb_url", "stripe_api_secret", "smtp_url", "amqp_password", "cache_store_redis_password", "cookie_secret", "singing_key", "wapid_key"]; } public version: string; public needsupdate: boolean; public updatedat: Date; + public async Save(jwt: string, parent: Span): Promise { if (this.needsupdate = true) { @@ -35,148 +36,172 @@ export class dbConfig extends Base { if (!NoderedUtil.IsNullEmpty(this._id)) await Config.db._UpdateOne(null, this, "config", 1, true, jwt, parent); } public compare(version: string): number { + if(this.version == null) return -1; return this.version.localeCompare(version, undefined, { numeric: true, sensitivity: 'base' }); } - public static async Load(jwt: string, parent: Span): Promise { - var conf: dbConfig = await Config.db.GetOne({ query: { "_type": "config" }, collectionname: "config", jwt }, parent); - if (conf == null) { conf = new dbConfig(); } - conf = Object.assign(new dbConfig(), conf); - conf.needsupdate = false; - if (conf.compare(Config.version) == -1) { - conf.needsupdate = true; + public static areEqual(a, b) { + if (a === b) return true; + if (a == null || b == null) return false; + try { + var _a = JSON.stringify(a); + var _b = JSON.stringify(b); + } catch (error) { + console.error("areEqual failed to stringify") + return false; + } + if (_a !== _b) return false; + return true; + } + + public static cleanAndApply(conf: dbConfig, parent: Span): Boolean { + if(Config.disable_db_config) return false; + var updated = false; + // add settings et via env variables that is not the default value + var keys = Object.keys(Config); + for(var i = 0; i < keys.length; i++) { + const key = keys[i]; + if(key == "_version") continue; + if(key.startsWith("_")) continue; + if(key == "disable_db_config") continue; + + if(["db", "name", "version", "needsupdate", "updatedat"].indexOf(key) > -1 ) continue; + if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_redis_host", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval", + "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url", + "domain", "enable_openapi", "enable_openapiauth", "ping_clients_interval", "tls_crt", "tls_key", "tls_ca", + "otel_metric_url", "otel_trace_url", "multi_tenant" ].indexOf(key) > -1 ) { + if(os.hostname().toLowerCase() == "nixos") { + continue; + } + } + + if (Object.prototype.hasOwnProperty.call(Config.default_config, key) && + !Object.prototype.hasOwnProperty.call(conf, key) + ) { + let _default:any = Config.default_config[key]; // envorinment variable + if(_default == null) _default = ""; + let _env:any = process.env[key]; // db value + if(_env == null || _env == "") continue; + if(typeof Config[key] === "boolean") { + _env = Config.parseBoolean(_env); + } else if(typeof Config[key] === "number") { + _env = parseInt(_env); + } else if(Array.isArray(Config[key])) { + _env = Config.parseArray(_env); + } else if(typeof Config[key] === "string") { + _env = _env; + } else { + continue; + } + if(key == "HTTP_PROXY") { + var b = true; + } + if(!dbConfig.areEqual(_env, _default)) { + updated = true; + conf[key] = Config[key]; + } + } + } + // Update Config with db values var keys = Object.keys(conf); for(var i = 0; i < keys.length; i++) { const key = keys[i]; + if(key == "_version") continue; + if(key == "disable_db_config") continue; const value = conf[key]; try { if(key.startsWith("_")) continue; - if(NoderedUtil.IsNullEmpty(value)) continue; + // if(NoderedUtil.IsNullEmpty(value)) continue; if(["db", "name", "version", "needsupdate", "updatedat"].indexOf(key) > -1 ) continue; - if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval", - "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url", "nodered_domain_schema", - "domain" ].indexOf(key) > -1 ) { + if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_redis_host", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval", + "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url", + "domain", "enable_openapi", "enable_openapiauth", "ping_clients_interval", "tls_crt", "tls_key", "tls_ca", + "otel_metric_url", "otel_trace_url", "multi_tenant" ].indexOf(key) > -1 ) { if(os.hostname().toLowerCase() == "nixos") { continue; } } - if(key == "amqp_allow_replyto_empty_queuename") { - var now = Config.amqp_allow_replyto_empty_queuename; - var v = conf[key]; - var b = true; - } - if (Object.prototype.hasOwnProperty.call(Config, key)) { - if(typeof Config[key] === "boolean") { - // console.log("Setting boolen " + key + " to " + conf[key]); - Config[key] = Config.parseBoolean(conf[key]); - } else if(typeof Config[key] === "number") { - // console.log("Setting number " + key + " to " + conf[key]); - Config[key] = parseInt(conf[key]); - } else if(Array.isArray(Config[key])) { - // console.log("Setting array " + key + " to " + conf[key]); - if(Array.isArray(conf[key])) { - Config[key] = conf[key]; + Config[key] = value; + + let _default:any = Config.default_config[key]; // envorinment variable + if(_default == null) _default = ""; + let _env:any = process.env[key]; // db value + if(_env != null && _env != "") { + if(typeof Config[key] === "boolean") { + _env = Config.parseBoolean(_env); + } else if(typeof Config[key] === "number") { + _env = parseInt(_env); + } else if(Array.isArray(Config[key])) { + _env = Config.parseArray(_env); + } else if(typeof Config[key] === "string") { + _env = _env; } else { - Config[key] = Config.parseArray(conf[key]); + continue; + } + if(_env != _default) { + } else if(dbConfig.areEqual(_env, value)) { + updated = true; + delete conf[key]; } - } else if(typeof Config[key] === "string") { - // console.log("Setting string " + key + " to " + conf[key]); - Config[key] = conf[key]; } else { - // console.log("Setting Unknown " + key + " to " + conf[key]); - Config[key] = conf[key]; + if(dbConfig.areEqual(_default, value)) { + updated = true; + delete conf[key]; + } } } - - if(key == "amqp_allow_replyto_empty_queuename") { - var now = Config.amqp_allow_replyto_empty_queuename; - var v = conf[key]; - var v2 = Config[key]; - var b = true; - } - } catch (error) { - Logger.instanse.error("Error setting config " + keys + " to " + value, parent); + Logger.instanse.error("Error setting config " + keys + " to " + value, null); } } - var keys = Object.keys(Config); - var updated = false; - for(var i = 0; i < keys.length; i++) { - const key = keys[i]; - if(key.startsWith("_")) continue; - if(["db", "name", "version", "needsupdate", "updatedat"].indexOf(key) > -1 ) continue; - if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval", - "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url", "nodered_domain_schema", - "domain" ].indexOf(key) > -1 ) { - if(os.hostname().toLowerCase() == "nixos") { - continue; - } - } - if(key == "amqp_allow_replyto_empty_queuename") { - var now = Config.amqp_allow_replyto_empty_queuename; - var v = Config[key]; - var v2 = conf[key]; - var b = true; - } - - const _default = Config.default_config[key]; - const setting = Config[key]; - const dbsetting = conf[key]; - // if(_default != null && dbsetting == null) { - // Config[key] = setting; - - // if(key == "amqp_allow_replyto_empty_queuename") { - // var now = Config.amqp_allow_replyto_empty_queuename; - // var v = Config[key]; - // var v2 = conf[key]; - // var b = true; - // } - - - // continue; - // } - // console.log("Checking " + key + " " + _default + " " + setting + " " + dbsetting); - if(setting == _default) continue; // ignore if default, kee dbsettings small - if(dbsetting != null) continue; // db setting overrides env setting (yeah, a little weird) - if(setting != dbsetting) { - conf[key] = setting; - updated = true; - } + conf._encrypt = ["mongodb_url", "stripe_api_secret", "smtp_url", "amqp_password", "cache_store_redis_password", "cookie_secret", "singing_key", "wapid_key"]; + if(Config._version != conf._version) { + Config._version = conf._version; + Logger.instanse.info("Loaded config version " + conf._version, parent); + } - if(key == "amqp_allow_replyto_empty_queuename") { - var now = Config.amqp_allow_replyto_empty_queuename; - var v = Config[key]; - var v2 = conf[key]; - var b = true; + return updated; + } + public static async Load(jwt: string, watch:boolean, parent: Span): Promise { + var conf: dbConfig = await Config.db.GetOne({ query: { "_type": "config" }, collectionname: "config", jwt, decrypt: true }, parent); + // @ts-ignore + if (conf == null) { conf = new dbConfig(); } else { + if(Config._version == conf._version) { + conf = Object.assign(new dbConfig(), conf); + return conf; } - - } - conf._encrypt = ["stripe_api_secret", "smtp_url", "amqp_password", "cache_store_redis_password", "cookie_secret", "singing_key", "wapid_key"]; - if(updated) { + conf = Object.assign(new dbConfig(), conf); + conf.needsupdate = false; + if (conf.compare(Config.version) == -1) { + conf.needsupdate = true; + } + + let updated = dbConfig.cleanAndApply(conf, parent); + if(updated ) { try { var msg: InsertOrUpdateOneMessage = new InsertOrUpdateOneMessage(); msg.collectionname = "config"; msg.jwt = jwt; - msg.item = conf; + msg.item = JSON.parse(JSON.stringify(conf)); + // @ts-ignore + delete msg.item.default_config; + // @ts-ignore + delete msg.item.dbConfig; msg.uniqeness = "_id"; await Config.db._InsertOrUpdateOne(msg, parent); // await Config.db.InsertOrUpdateOne(null, conf, "config", 1, true, jwt, parent); } catch (error) { - var e = error; - console.error(error); - + Logger.instanse.error(error, null); } } await Logger.reload(); return conf; } - public static async Reload(jwt: string, parent: Span): Promise { - Config.dbConfig = await dbConfig.Load(jwt, parent); - - Logger.instanse.info("Reloaded config version " + Config.dbConfig._version, parent); + public static async Reload(jwt: string, watch:boolean, parent: Span): Promise { + Config.dbConfig = await dbConfig.Load(jwt, watch, parent); } } export class Config { @@ -184,11 +209,12 @@ export class Config { public static default_config: dbConfig = { enable_openai: false, enable_openapi: true, - enable_openaiauth: true, + enable_openapiauth: true, log_with_colors: true, cache_store_type: "memory", cache_store_max: 1000, cache_store_ttl_seconds: 300, + cache_store_redis_host: "", cache_store_redis_port: 6379, cache_workitem_queues: false, @@ -202,6 +228,7 @@ export class Config { log_webserver: false, log_database: false, log_database_queries: false, + log_database_queries_to_collection: "", log_database_queries_ms: 0, log_grafana: false, log_housekeeping: false, @@ -212,6 +239,12 @@ export class Config { log_verbose: false, log_silly: false, log_to_exchange: false, + log_all_watches: false, + + tls_crt: "", + tls_key: "", + tls_ca: "", + tls_passphrase: "", heapdump_onstop: false, amqp_allow_replyto_empty_queuename: false, @@ -224,6 +257,8 @@ export class Config { grafana_url: "", auto_hourly_housekeeping: true, housekeeping_skip_collections: "", + housekeeping_remomve_unvalidated_user_days: 0, // if above 0, remove unvalidated users after x days + housekeeping_cleanup_openrpa_instances: false, workitem_queue_monitoring_enabled: true, workitem_queue_monitoring_interval: 10 * 1000, // 10 sec upload_max_filesize_mb: 25, @@ -239,11 +274,11 @@ export class Config { stripe_force_checkout: false, stripe_allow_promotion_codes: true, - supports_watch: false, ensure_indexes: true, text_index_name_fields: ["name", "_names"], auto_create_users: false, auto_create_user_from_jwt: false, + allow_signin_with_expired_jwt: false, auto_create_domains: [], persist_user_impersonation: false, ping_clients_interval: 10000, // 10 seconds @@ -296,19 +331,19 @@ export class Config { cleanup_on_delete_customer: false, cleanup_on_delete_user: false, api_bypass_perm_check: false, - ignore_expiration: false, + disable_db_config: false, force_audit_ts: false, force_dbusage_ts: false, migrate_audit_to_ts: true, websocket_package_size: 25000, - websocket_max_package_count: 25000, + websocket_max_package_count: 1048576, websocket_message_callback_timeout: 3600, websocket_disconnect_out_of_sync: false, protocol: "http", port: 80, domain: "localhost.openiap.io", - cookie_secret: "", + cookie_secret: "NLgUIsozJaxO38ze0WuHthfj2eb1eIEu", max_ace_count: 128, amqp_reply_expiration: 60 * 1000, // 1 min @@ -328,12 +363,12 @@ export class Config { amqp_requeue_time: 1000, // 1 seconds amqp_dlx: "openflow-dlx", // Dead letter exchange, used to pickup dead or timeout messages - // mongodb_url: "mongodb://localhost:27017", - // mongodb_db: "openflow", - // mongodb_minpoolsize: 25, - // mongodb_maxpoolsize: 25, + mongodb_url: "mongodb://localhost:27017", + mongodb_db: "openflow", + mongodb_minpoolsize: 25, + mongodb_maxpoolsize: 25, - skip_history_collections: "audit,openrpa_instances,workflow_instances", + skip_history_collections: "audit,oauthtokens,openrpa_instances,workflow_instances,workitems,mailhist", // "audit,openrpa_instances,workflow_instances", history_delta_count: 1000, allow_skiphistory: false, max_memory_restart_mb: 0, @@ -354,29 +389,24 @@ export class Config { agent_domain_schema: "", agent_node_selector: "", agent_apiurl: "", + agent_grpc_apihost: "", + agent_ws_apihost: "", agent_oidc_config: "", agent_oidc_client_id: "", agent_oidc_client_secret: "", agent_oidc_userinfo_endpoint: "", + agent_oidc_issuer: "", saml_federation_metadata: "", api_ws_url: "", - nodered_ws_url: "", - nodered_saml_entrypoint: "", agent_docker_entrypoints: "web", agent_docker_use_project: false, agent_docker_certresolver: "", namespace: "", - nodered_domain_schema: "", - nodered_initial_liveness_delay: 60, - nodered_allow_nodeselector: false, - nodered_liveness_failurethreshold: 5, - nodered_liveness_timeoutseconds: 5, - noderedcatalogues: "", + agent_allow_nodeselector: false, otel_measure_nodeid: false, otel_measure_queued_messages: false, otel_measure__mongodb_watch: false, - otel_measure_onlineuser: false, enable_analytics: true, enable_detailed_analytic: false, otel_debug_log: false, @@ -437,268 +467,263 @@ export class Config { Config.log_oauth = false; Config.unittesting = true; } + public static _version: number = -1; public static unittesting: boolean = false; public static db: DatabaseConnection = null; - public static license_key: string = Config.getEnv("license_key", ""); - public static enable_openai: boolean = Config.parseBoolean(Config.getEnv("enable_openai", "false")); - public static enable_openapi: boolean = Config.parseBoolean(Config.getEnv("enable_openapi", "true")); - public static enable_openaiauth: boolean = Config.parseBoolean(Config.getEnv("enable_openaiauth", "true")); - public static openai_token: string = Config.getEnv("openai_token", ""); + public static license_key: string = Config.getEnv("license_key"); + public static enable_openai: boolean = Config.parseBoolean(Config.getEnv("enable_openai")); + public static enable_openapi: boolean = Config.parseBoolean(Config.getEnv("enable_openapi")); + public static enable_openapiauth: boolean = Config.parseBoolean(Config.getEnv("enable_openapiauth")); + public static openai_token: string = Config.getEnv("openai_token"); public static version: string = Config.getversion(); - public static log_with_colors: boolean = Config.parseBoolean(Config.getEnv("log_with_colors", "true")); - - public static cache_store_type: string = Config.getEnv("cache_store_type", "memory"); - public static cache_store_max: number = parseInt(Config.getEnv("cache_store_max", "1000")); - public static cache_store_ttl_seconds: number = parseInt(Config.getEnv("cache_store_ttl_seconds", "300")); - public static cache_store_redis_host: string = Config.getEnv("cache_store_redis_host", ""); - public static cache_store_redis_port: number = parseInt(Config.getEnv("cache_store_redis_port", "6379")); - public static cache_store_redis_password: string = Config.getEnv("cache_store_redis_password", ""); - public static cache_workitem_queues: boolean = Config.parseBoolean(Config.getEnv("cache_workitem_queues", "false")); - - public static log_cache: boolean = Config.parseBoolean(Config.getEnv("log_cache", "false")); - public static log_amqp: boolean = Config.parseBoolean(Config.getEnv("log_amqp", "false")); - public static log_openapi: boolean = Config.parseBoolean(Config.getEnv("log_openapi", "false")); - public static log_login_provider: boolean = Config.parseBoolean(Config.getEnv("log_login_provider", "false")); - public static log_with_trace: boolean = Config.parseBoolean(Config.getEnv("log_with_trace", "false")); - public static log_websocket: boolean = Config.parseBoolean(Config.getEnv("log_websocket", "false")); - public static log_oauth: boolean = Config.parseBoolean(Config.getEnv("log_oauth", "false")); - public static log_webserver: boolean = Config.parseBoolean(Config.getEnv("log_webserver", "false")); - public static log_database: boolean = Config.parseBoolean(Config.getEnv("log_database", "false")); - public static log_database_queries: boolean = Config.parseBoolean(Config.getEnv("log_database_queries", "false")); - public static log_database_queries_ms: number = parseInt(Config.getEnv("log_database_queries_ms", "0")); - - public static log_grafana: boolean = Config.parseBoolean(Config.getEnv("log_grafana", "false")); - public static log_housekeeping: boolean = Config.parseBoolean(Config.getEnv("log_housekeeping", "false")); - public static log_otel: boolean = Config.parseBoolean(Config.getEnv("log_otel", "false")); - public static log_blocked_ips: boolean = Config.parseBoolean(Config.getEnv("log_blocked_ips", "true")); - public static log_information: boolean = Config.parseBoolean(Config.getEnv("log_information", "true")); - public static log_debug: boolean = Config.parseBoolean(Config.getEnv("log_debug", "false")); - public static log_verbose: boolean = Config.parseBoolean(Config.getEnv("log_verbose", "false")); - public static log_silly: boolean = Config.parseBoolean(Config.getEnv("log_silly", "false")); - public static log_to_exchange: boolean = Config.parseBoolean(Config.getEnv("log_to_exchange", "false")); - - public static heapdump_onstop: boolean = Config.parseBoolean(Config.getEnv("heapdump_onstop", "false")); - - public static amqp_allow_replyto_empty_queuename: boolean = Config.parseBoolean(Config.getEnv("amqp_allow_replyto_empty_queuename", "false")); - - public static openflow_uniqueid: string = Config.getEnv("openflow_uniqueid", ""); - public static enable_openflow_amqp: boolean = Config.parseBoolean(Config.getEnv("enable_openflow_amqp", "false")); - public static openflow_amqp_expiration: number = parseInt(Config.getEnv("openflow_amqp_expiration", (60 * 1000 * 25).toString())); // 25 min - public static amqp_prefetch: number = parseInt(Config.getEnv("amqp_prefetch", "25")); - public static enable_entity_restriction: boolean = Config.parseBoolean(Config.getEnv("enable_entity_restriction", "false")); - public static enable_web_tours: boolean = Config.parseBoolean(Config.getEnv("enable_web_tours", "true")); - public static enable_nodered_tours: boolean = Config.parseBoolean(Config.getEnv("enable_nodered_tours", "true")); - public static grafana_url:string = Config.getEnv("grafana_url", ""); - public static auto_hourly_housekeeping: boolean = Config.parseBoolean(Config.getEnv("auto_hourly_housekeeping", "true")); - public static housekeeping_skip_collections: string = Config.getEnv("housekeeping_skip_collections", ""); - public static workitem_queue_monitoring_enabled: boolean = Config.parseBoolean(Config.getEnv("workitem_queue_monitoring_enabled", "true")); - public static workitem_queue_monitoring_interval: number = parseInt(Config.getEnv("workitem_queue_monitoring_interval", (10 * 1000).toString())); // 10 sec - - public static upload_max_filesize_mb: number = parseInt(Config.getEnv("upload_max_filesize_mb", "25")); - - public static getting_started_url: string = Config.getEnv("getting_started_url", ""); - - public static NODE_ENV: string = Config.getEnv("NODE_ENV", "development"); - public static HTTP_PROXY: string = Config.getEnv("HTTP_PROXY", ""); - public static HTTPS_PROXY: string = Config.getEnv("HTTPS_PROXY", ""); - public static NO_PROXY: string = Config.getEnv("NO_PROXY", ""); - public static agent_HTTP_PROXY: string = Config.getEnv("agent_HTTP_PROXY", ""); - public static agent_HTTPS_PROXY: string = Config.getEnv("agent_HTTPS_PROXY", ""); - public static agent_NO_PROXY: string = Config.getEnv("agent_NO_PROXY", ""); - - public static stripe_api_key: string = Config.getEnv("stripe_api_key", ""); - public static stripe_api_secret: string = Config.getEnv("stripe_api_secret", ""); - public static stripe_force_vat: boolean = Config.parseBoolean(Config.getEnv("stripe_force_vat", "false")); - public static stripe_force_checkout: boolean = Config.parseBoolean(Config.getEnv("stripe_force_checkout", "false")); - public static stripe_allow_promotion_codes: boolean = Config.parseBoolean(Config.getEnv("stripe_allow_promotion_codes", "true")); - - public static supports_watch: boolean = Config.parseBoolean(Config.getEnv("supports_watch", "false")); - public static ensure_indexes: boolean = Config.parseBoolean(Config.getEnv("ensure_indexes", "true")); - public static text_index_name_fields: string[] = Config.parseArray(Config.getEnv("text_index_name_fields", "name,_names")); - - public static auto_create_users: boolean = Config.parseBoolean(Config.getEnv("auto_create_users", "false")); - public static auto_create_user_from_jwt: boolean = Config.parseBoolean(Config.getEnv("auto_create_user_from_jwt", "false")); - public static auto_create_domains: string[] = Config.parseArray(Config.getEnv("auto_create_domains", "")); - public static persist_user_impersonation: boolean = Config.parseBoolean(Config.getEnv("persist_user_impersonation", "false")); - public static ping_clients_interval: number = parseInt(Config.getEnv("ping_clients_interval", (10000).toString())); // 10 seconds - - public static use_ingress_beta1_syntax: boolean = Config.parseBoolean(Config.getEnv("use_ingress_beta1_syntax", "false")); - public static use_openshift_routes: boolean = Config.parseBoolean(Config.getEnv("use_openshift_routes", "false")); - public static agent_image_pull_secrets: string[] = Config.parseArray(Config.getEnv("agent_image_pull_secrets", "")); - public static auto_create_personal_nodered_group: boolean = Config.parseBoolean(Config.getEnv("auto_create_personal_nodered_group", "false")); - public static auto_create_personal_noderedapi_group: boolean = Config.parseBoolean(Config.getEnv("auto_create_personal_noderedapi_group", "false")); - public static force_add_admins: boolean = Config.parseBoolean(Config.getEnv("force_add_admins", "true")); - public static validate_emails: boolean = Config.parseBoolean(Config.getEnv("validate_emails", "false")); - public static forgot_pass_emails: boolean = Config.parseBoolean(Config.getEnv("forgot_pass_emails", "false")); - public static smtp_service: string = Config.getEnv("smtp_service", ""); - public static smtp_from: string = Config.getEnv("smtp_from", ""); - public static smtp_user: string = Config.getEnv("smtp_user", ""); - public static smtp_pass: string = Config.getEnv("smtp_pass", ""); - public static smtp_url: string = Config.getEnv("smtp_url", ""); - public static debounce_lookup: boolean = Config.parseBoolean(Config.getEnv("debounce_lookup", "false")); - public static validate_emails_disposable: boolean = Config.parseBoolean(Config.getEnv("validate_emails_disposable", "false")); - - public static tls_crt: string = Config.getEnv("tls_crt", ""); - public static tls_key: string = Config.getEnv("tls_key", ""); - public static tls_ca: string = Config.getEnv("tls_ca", ""); - public static tls_passphrase: string = Config.getEnv("tls_passphrase", ""); - - public static oidc_access_token_ttl: number = parseInt(Config.getEnv("oidc_access_token_ttl", "480")); // 8 hours - public static oidc_authorization_code_ttl: number = parseInt(Config.getEnv("oidc_authorization_code_ttl", "480")); // 8 hours - public static oidc_client_credentials_ttl: number = parseInt(Config.getEnv("oidc_client_credentials_ttl", "480")); // 8 hours - public static oidc_refresh_token_ttl: number = parseInt(Config.getEnv("oidc_refresh_token_ttl", "20160")); // 14 days in seconds - public static oidc_session_ttl: number = parseInt(Config.getEnv("oidc_session_ttl", "20160")); // 14 days in seconds - - public static oidc_cookie_key: string = Config.getEnv("oidc_cookie_key", "Y6SPiXCxDhAJbN7cbydMw5eX1wIrdy8PiWApqEcguss="); - public static api_rate_limit: boolean = Config.parseBoolean(Config.getEnv("api_rate_limit", "true")); - public static api_rate_limit_points: number = parseInt(Config.getEnv("api_rate_limit_points", "20")); - public static api_rate_limit_duration: number = parseInt(Config.getEnv("api_rate_limit_duration", "1")); - public static socket_rate_limit: boolean = Config.parseBoolean(Config.getEnv("socket_rate_limit", "true")); - public static socket_rate_limit_points: number = parseInt(Config.getEnv("socket_rate_limit_points", "30")); - public static socket_rate_limit_points_disconnect: number = parseInt(Config.getEnv("socket_rate_limit_points_disconnect", "100")); - public static socket_rate_limit_duration: number = parseInt(Config.getEnv("socket_rate_limit_duration", "1")); - public static socket_error_rate_limit_points: number = parseInt(Config.getEnv("socket_error_rate_limit_points", "30")); - public static socket_error_rate_limit_duration: number = parseInt(Config.getEnv("socket_error_rate_limit_duration", "1")); - - public static client_heartbeat_timeout: number = parseInt(Config.getEnv("client_heartbeat_timeout", "60")); - public static client_signin_timeout: number = parseInt(Config.getEnv("client_signin_timeout", "120")); - public static client_disconnect_signin_error: boolean = Config.parseBoolean(Config.getEnv("client_disconnect_signin_error", "false")); - - public static expected_max_roles: number = parseInt(Config.getEnv("expected_max_roles", "20000")); - public static decorate_roles_fetching_all_roles = Config.parseBoolean(Config.getEnv("decorate_roles_fetching_all_roles", "true")); - public static max_recursive_group_depth: number = parseInt(Config.getEnv("max_recursive_group_depth", "2")); - public static update_acl_based_on_groups: boolean = Config.parseBoolean(Config.getEnv("update_acl_based_on_groups", "true")); - public static allow_merge_acl: boolean = Config.parseBoolean(Config.getEnv("allow_merge_acl", "false")); - - public static multi_tenant: boolean = Config.parseBoolean(Config.getEnv("multi_tenant", "false")); - public static cleanup_on_delete_customer: boolean = Config.parseBoolean(Config.getEnv("cleanup_on_delete_customer", "false")); - public static cleanup_on_delete_user: boolean = Config.parseBoolean(Config.getEnv("cleanup_on_delete_user", "false")); - public static api_bypass_perm_check: boolean = Config.parseBoolean(Config.getEnv("api_bypass_perm_check", "false")); - public static ignore_expiration: boolean = Config.parseBoolean(Config.getEnv("ignore_expiration", "false")); - public static force_audit_ts: boolean = Config.parseBoolean(Config.getEnv("force_audit_ts", "false")); - public static force_dbusage_ts: boolean = Config.parseBoolean(Config.getEnv("force_dbusage_ts", "false")); - public static migrate_audit_to_ts: boolean = Config.parseBoolean(Config.getEnv("migrate_audit_to_ts", "true")); - - public static websocket_package_size: number = parseInt(Config.getEnv("websocket_package_size", "25000"), 10); - public static websocket_max_package_count: number = parseInt(Config.getEnv("websocket_max_package_count", "25000"), 10); - public static websocket_message_callback_timeout: number = parseInt(Config.getEnv("websocket_message_callback_timeout", "3600"), 10); - public static websocket_disconnect_out_of_sync: boolean = Config.parseBoolean(Config.getEnv("websocket_disconnect_out_of_sync", "false")); - public static protocol: string = Config.getEnv("protocol", "http"); // used by personal nodered and baseurl() - public static port: number = parseInt(Config.getEnv("port", "80")); - public static domain: string = Config.getEnv("domain", "localhost"); // sent to website and used in baseurl() - public static cookie_secret: string = Config.getEnv("cookie_secret", "NLgUIsozJaxO38ze0WuHthfj2eb1eIEu"); // Used to protect cookies - public static max_ace_count: number = parseInt(Config.getEnv("max_ace_count", "128"), 10); - - public static amqp_reply_expiration: number = parseInt(Config.getEnv("amqp_reply_expiration", (60 * 1000).toString())); // 1 min - public static amqp_force_queue_prefix: boolean = Config.parseBoolean(Config.getEnv("amqp_force_queue_prefix", "false")); - public static amqp_force_exchange_prefix: boolean = Config.parseBoolean(Config.getEnv("amqp_force_exchange_prefix", "false")); - public static amqp_force_sender_has_read: boolean = Config.parseBoolean(Config.getEnv("amqp_force_sender_has_read", "true")); - public static amqp_force_sender_has_invoke: boolean = Config.parseBoolean(Config.getEnv("amqp_force_sender_has_invoke", "false")); - public static amqp_force_consumer_has_update: boolean = Config.parseBoolean(Config.getEnv("amqp_force_consumer_has_update", "false")); - public static amqp_enabled_exchange: boolean = Config.parseBoolean(Config.getEnv("amqp_enabled_exchange", "false")); - public static amqp_url: string = Config.getEnv("amqp_url", "amqp://localhost"); // used to register queues and by personal nodered - public static amqp_username: string = Config.getEnv("amqp_username", "guest"); // used to talk wth rabbitmq api - public static amqp_password: string = Config.getEnv("amqp_password", "guest"); // used to talk wth rabbitmq api - - public static amqp_check_for_consumer: boolean = Config.parseBoolean(Config.getEnv("amqp_check_for_consumer", "true")); - public static amqp_check_for_consumer_count: boolean = Config.parseBoolean(Config.getEnv("amqp_check_for_consumer_count", "false")); - public static amqp_default_expiration: number = parseInt(Config.getEnv("amqp_default_expiration", (60 * 1000).toString())); // 1 min - public static amqp_requeue_time: number = parseInt(Config.getEnv("amqp_requeue_time", "1000")); // 1 seconds - public static amqp_dlx: string = Config.getEnv("amqp_dlx", "openflow-dlx"); // Dead letter exchange, used to pickup dead or timeout messages - - public static mongodb_url: string = Config.getEnv("mongodb_url", "mongodb://localhost:27017"); - public static mongodb_db: string = Config.getEnv("mongodb_db", "openflow"); - public static mongodb_minpoolsize: number = parseInt(Config.getEnv("mongodb_minpoolsize", "25")); - public static mongodb_maxpoolsize: number = parseInt(Config.getEnv("mongodb_maxpoolsize", "25")); - - public static skip_history_collections: string = Config.getEnv("skip_history_collections", "audit,openrpa_instances,workflow_instances"); - public static history_delta_count: number = parseInt(Config.getEnv("history_delta_count", "1000")); - public static allow_skiphistory: boolean = Config.parseBoolean(Config.getEnv("allow_skiphistory", "false")); - public static max_memory_restart_mb: number = parseInt(Config.getEnv("max_memory_restart_mb", "0")); - - public static saml_issuer: string = Config.getEnv("saml_issuer", "the-issuer"); // define uri of STS, also sent to personal nodereds - public static aes_secret: string = Config.getEnv("aes_secret", ""); - public static signing_crt: string = Config.getEnv("signing_crt", ""); - public static singing_key: string = Config.getEnv("singing_key", ""); - public static wapid_mail: string = Config.getEnv("wapid_mail", ""); - public static wapid_pub: string = Config.getEnv("wapid_pub", ""); - public static wapid_key: string = Config.getEnv("wapid_key", ""); - - public static shorttoken_expires_in: string = Config.getEnv("shorttoken_expires_in", "5m"); - public static longtoken_expires_in: string = Config.getEnv("longtoken_expires_in", "365d"); - public static downloadtoken_expires_in: string = Config.getEnv("downloadtoken_expires_in", "15m"); - public static personalnoderedtoken_expires_in: string = Config.getEnv("personalnoderedtoken_expires_in", "365d"); - - public static agent_images: NoderedImage[] = JSON.parse(Config.getEnv("agent_images", - JSON.stringify([{"name":"Agent", "image":"openiap/nodeagent", "languages": ["nodejs", "python"]}, {"name":"Agent+Chromium", "image":"openiap/nodechromiumagent", "chromium": true, "languages": ["nodejs", "python"]}, {"name":"NodeRED", "image":"openiap/noderedagent", "port": 3000}, {"name":"DotNet 6", "image":"openiap/dotnetagent", "languages": ["dotnet"]} , {"name":"PowerShell 7.3", "image":"openiap/nodeagent:pwsh", "languages": ["powershell"]} ]) - )); - public static agent_domain_schema: string = Config.getEnv("agent_domain_schema", ""); - public static agent_node_selector:string = Config.getEnv("agent_node_selector", ""); - - public static agent_grpc_apihost: string = Config.getEnv("agent_grpc_apihost", ""); - public static agent_ws_apihost: string = Config.getEnv("agent_ws_apihost", ""); - public static agent_oidc_config: string = Config.getEnv("agent_oidc_config", ""); - public static agent_oidc_client_id: string = Config.getEnv("agent_oidc_client_id", ""); - public static agent_oidc_client_secret: string = Config.getEnv("agent_oidc_client_secret", ""); - public static agent_oidc_userinfo_endpoint: string = Config.getEnv("agent_oidc_userinfo_endpoint", ""); - public static agent_oidc_issuer: string = Config.getEnv("agent_oidc_issuer", ""); - public static agent_oidc_authorization_endpoint: string = Config.getEnv("agent_oidc_authorization_endpoint", ""); - public static agent_oidc_token_endpoint: string = Config.getEnv("agent_oidc_token_endpoint", ""); - - public static saml_federation_metadata: string = Config.getEnv("saml_federation_metadata", ""); - public static api_ws_url: string = Config.getEnv("api_ws_url", ""); - public static nodered_ws_url: string = Config.getEnv("nodered_ws_url", ""); - public static nodered_saml_entrypoint: string = Config.getEnv("nodered_saml_entrypoint", ""); - - public static agent_docker_entrypoints: string = Config.getEnv("agent_docker_entrypoints", "web"); - public static agent_docker_use_project: boolean = Config.parseBoolean(Config.getEnv("agent_docker_use_project", "false")); - public static agent_docker_certresolver: string = Config.getEnv("agent_docker_certresolver", ""); - - public static namespace: string = Config.getEnv("namespace", ""); // also sent to website - public static nodered_domain_schema: string = Config.getEnv("nodered_domain_schema", ""); // also sent to website - public static nodered_initial_liveness_delay: number = parseInt(Config.getEnv("nodered_initial_liveness_delay", "60")); - public static nodered_allow_nodeselector: boolean = Config.parseBoolean(Config.getEnv("nodered_allow_nodeselector", "false")); - public static nodered_liveness_failurethreshold: number = parseInt(Config.getEnv("nodered_liveness_failurethreshold", "5")); - public static nodered_liveness_timeoutseconds: number = parseInt(Config.getEnv("nodered_liveness_timeoutseconds", "5")); - public static noderedcatalogues: string = Config.getEnv("noderedcatalogues", ""); - - public static otel_measure_nodeid: boolean = Config.parseBoolean(Config.getEnv("otel_measure_nodeid", "false")); - public static otel_measure_queued_messages: boolean = Config.parseBoolean(Config.getEnv("otel_measure_queued_messages", "false")); - public static otel_measure__mongodb_watch: boolean = Config.parseBoolean(Config.getEnv("otel_measure__mongodb_watch", "false")); - public static otel_measure_onlineuser: boolean = Config.parseBoolean(Config.getEnv("otel_measure_onlineuser", "false")); - public static enable_analytics: boolean = Config.parseBoolean(Config.getEnv("enable_analytics", "true")); - public static enable_detailed_analytic: boolean = Config.parseBoolean(Config.getEnv("enable_detailed_analytic", "false")); - public static otel_debug_log: boolean = Config.parseBoolean(Config.getEnv("otel_debug_log", "false")); - public static otel_warn_log: boolean = Config.parseBoolean(Config.getEnv("otel_warn_log", "false")); - public static otel_err_log: boolean = Config.parseBoolean(Config.getEnv("otel_err_log", "false")); - public static otel_trace_url: string = Config.getEnv("otel_trace_url", ""); - public static otel_metric_url: string = Config.getEnv("otel_metric_url", ""); - public static otel_trace_interval: number = parseInt(Config.getEnv("otel_trace_interval", "5000")); - public static otel_metric_interval: number = parseInt(Config.getEnv("otel_metric_interval", "5000")); - public static otel_trace_pingclients: boolean = Config.parseBoolean(Config.getEnv("otel_trace_pingclients", "false")); - public static otel_trace_dashboardauth: boolean = Config.parseBoolean(Config.getEnv("otel_trace_dashboardauth", "false")); - public static otel_trace_include_query: boolean = Config.parseBoolean(Config.getEnv("otel_trace_include_query", "false")); - public static otel_trace_connection_ips: boolean = Config.parseBoolean(Config.getEnv("otel_trace_connection_ips", "false")); - public static otel_trace_mongodb_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_per_users", "false")); - public static otel_trace_mongodb_query_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_query_per_users", "false")); - public static otel_trace_mongodb_count_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_count_per_users", "false")); - public static otel_trace_mongodb_aggregate_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_aggregate_per_users", "false")); - public static otel_trace_mongodb_insert_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_insert_per_users", "false")); - public static otel_trace_mongodb_update_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_update_per_users", "false")); - public static otel_trace_mongodb_delete_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_delete_per_users", "false")); - - public static grpc_keepalive_time_ms: number = parseInt(Config.getEnv("grpc_keepalive_time_ms", "-1")); - public static grpc_keepalive_timeout_ms: number = parseInt(Config.getEnv("grpc_keepalive_timeout_ms", "-1")); - public static grpc_http2_min_ping_interval_without_data_ms: number = parseInt(Config.getEnv("grpc_http2_min_ping_interval_without_data_ms", "-1")); - public static grpc_max_connection_idle_ms: number = parseInt(Config.getEnv("grpc_max_connection_idle_ms", "-1")); - public static grpc_max_connection_age_ms: number = parseInt(Config.getEnv("grpc_max_connection_age_ms", "-1")); - public static grpc_max_connection_age_grace_ms: number = parseInt(Config.getEnv("grpc_max_connection_age_grace_ms", "-1")); - public static grpc_http2_max_pings_without_data: number = parseInt(Config.getEnv("grpc_http2_max_pings_without_data", "-1")); - public static grpc_keepalive_permit_without_calls: number = parseInt(Config.getEnv("grpc_keepalive_permit_without_calls", "-1")); - public static grpc_max_receive_message_length: number = parseInt(Config.getEnv("grpc_max_receive_message_length", "-1")); - public static grpc_max_send_message_length: number = parseInt(Config.getEnv("grpc_max_send_message_length", "-1")); - - public static validate_user_form: string = Config.getEnv("validate_user_form", ""); + public static log_with_colors: boolean = Config.parseBoolean(Config.getEnv("log_with_colors")); + + public static cache_store_type: string = Config.getEnv("cache_store_type"); + public static cache_store_max: number = parseInt(Config.getEnv("cache_store_max")); + public static cache_store_ttl_seconds: number = parseInt(Config.getEnv("cache_store_ttl_seconds")); + public static cache_store_redis_host: string = Config.getEnv("cache_store_redis_host"); + public static cache_store_redis_port: number = parseInt(Config.getEnv("cache_store_redis_port")); + public static cache_store_redis_password: string = Config.getEnv("cache_store_redis_password"); + public static cache_workitem_queues: boolean = Config.parseBoolean(Config.getEnv("cache_workitem_queues")); + + public static log_cache: boolean = Config.parseBoolean(Config.getEnv("log_cache")); + public static log_amqp: boolean = Config.parseBoolean(Config.getEnv("log_amqp")); + public static log_openapi: boolean = Config.parseBoolean(Config.getEnv("log_openapi")); + public static log_login_provider: boolean = Config.parseBoolean(Config.getEnv("log_login_provider")); + public static log_with_trace: boolean = Config.parseBoolean(Config.getEnv("log_with_trace")); + public static log_websocket: boolean = Config.parseBoolean(Config.getEnv("log_websocket")); + public static log_oauth: boolean = Config.parseBoolean(Config.getEnv("log_oauth")); + public static log_webserver: boolean = Config.parseBoolean(Config.getEnv("log_webserver")); + public static log_database: boolean = Config.parseBoolean(Config.getEnv("log_database")); + public static log_database_queries: boolean = Config.parseBoolean(Config.getEnv("log_database_queries")); + public static log_database_queries_to_collection: string = Config.getEnv("log_database_queries_to_collection"); + public static log_database_queries_ms: number = parseInt(Config.getEnv("log_database_queries_ms")); + + public static log_grafana: boolean = Config.parseBoolean(Config.getEnv("log_grafana")); + public static log_housekeeping: boolean = Config.parseBoolean(Config.getEnv("log_housekeeping")); + public static log_otel: boolean = Config.parseBoolean(Config.getEnv("log_otel")); + public static log_blocked_ips: boolean = Config.parseBoolean(Config.getEnv("log_blocked_ips")); + public static log_information: boolean = Config.parseBoolean(Config.getEnv("log_information")); + public static log_debug: boolean = Config.parseBoolean(Config.getEnv("log_debug")); + public static log_verbose: boolean = Config.parseBoolean(Config.getEnv("log_verbose")); + public static log_silly: boolean = Config.parseBoolean(Config.getEnv("log_silly")); + public static log_to_exchange: boolean = Config.parseBoolean(Config.getEnv("log_to_exchange")); + public static log_all_watches: boolean = Config.parseBoolean(Config.getEnv("log_all_watches")); + + + public static heapdump_onstop: boolean = Config.parseBoolean(Config.getEnv("heapdump_onstop")); + + public static amqp_allow_replyto_empty_queuename: boolean = Config.parseBoolean(Config.getEnv("amqp_allow_replyto_empty_queuename")); + + public static openflow_uniqueid: string = Config.getEnv("openflow_uniqueid"); + public static enable_openflow_amqp: boolean = Config.parseBoolean(Config.getEnv("enable_openflow_amqp")); + public static openflow_amqp_expiration: number = parseInt(Config.getEnv("openflow_amqp_expiration")); + public static amqp_prefetch: number = parseInt(Config.getEnv("amqp_prefetch")); + public static enable_entity_restriction: boolean = Config.parseBoolean(Config.getEnv("enable_entity_restriction")); + public static enable_web_tours: boolean = Config.parseBoolean(Config.getEnv("enable_web_tours")); + public static enable_nodered_tours: boolean = Config.parseBoolean(Config.getEnv("enable_nodered_tours")); + public static grafana_url:string = Config.getEnv("grafana_url"); + public static auto_hourly_housekeeping: boolean = Config.parseBoolean(Config.getEnv("auto_hourly_housekeeping")); + public static housekeeping_skip_collections: string = Config.getEnv("housekeeping_skip_collections"); + public static housekeeping_remomve_unvalidated_user_days: number = parseInt(Config.getEnv("housekeeping_remomve_unvalidated_user_days")); + public static housekeeping_cleanup_openrpa_instances: boolean = Config.parseBoolean(Config.getEnv("housekeeping_cleanup_openrpa_instances")); + public static workitem_queue_monitoring_enabled: boolean = Config.parseBoolean(Config.getEnv("workitem_queue_monitoring_enabled")); + public static workitem_queue_monitoring_interval: number = parseInt(Config.getEnv("workitem_queue_monitoring_interval")); + + public static upload_max_filesize_mb: number = parseInt(Config.getEnv("upload_max_filesize_mb")); + + public static getting_started_url: string = Config.getEnv("getting_started_url"); + + public static NODE_ENV: string = Config.getEnv("NODE_ENV"); + public static HTTP_PROXY: string = Config.getEnv("HTTP_PROXY"); + public static HTTPS_PROXY: string = Config.getEnv("HTTPS_PROXY"); + public static NO_PROXY: string = Config.getEnv("NO_PROXY"); + public static agent_HTTP_PROXY: string = Config.getEnv("agent_HTTP_PROXY"); + public static agent_HTTPS_PROXY: string = Config.getEnv("agent_HTTPS_PROXY"); + public static agent_NO_PROXY: string = Config.getEnv("agent_NO_PROXY"); + + public static stripe_api_key: string = Config.getEnv("stripe_api_key"); + public static stripe_api_secret: string = Config.getEnv("stripe_api_secret"); + public static stripe_force_vat: boolean = Config.parseBoolean(Config.getEnv("stripe_force_vat")); + public static stripe_force_checkout: boolean = Config.parseBoolean(Config.getEnv("stripe_force_checkout")); + public static stripe_allow_promotion_codes: boolean = Config.parseBoolean(Config.getEnv("stripe_allow_promotion_codes")); + + public static ensure_indexes: boolean = Config.parseBoolean(Config.getEnv("ensure_indexes")); + public static text_index_name_fields: string[] = Config.parseArray(Config.getEnv("text_index_name_fields")); + + public static auto_create_users: boolean = Config.parseBoolean(Config.getEnv("auto_create_users")); + public static auto_create_user_from_jwt: boolean = Config.parseBoolean(Config.getEnv("auto_create_user_from_jwt")); + public static allow_signin_with_expired_jwt: boolean = Config.parseBoolean(Config.getEnv("allow_signin_with_expired_jwt")); + public static auto_create_domains: string[] = Config.parseArray(Config.getEnv("auto_create_domains")); + public static persist_user_impersonation: boolean = Config.parseBoolean(Config.getEnv("persist_user_impersonation")); + public static ping_clients_interval: number = parseInt(Config.getEnv("ping_clients_interval")); // 10 seconds + + public static use_ingress_beta1_syntax: boolean = Config.parseBoolean(Config.getEnv("use_ingress_beta1_syntax")); + public static use_openshift_routes: boolean = Config.parseBoolean(Config.getEnv("use_openshift_routes")); + public static agent_image_pull_secrets: string[] = Config.parseArray(Config.getEnv("agent_image_pull_secrets")); + public static auto_create_personal_nodered_group: boolean = Config.parseBoolean(Config.getEnv("auto_create_personal_nodered_group")); + public static auto_create_personal_noderedapi_group: boolean = Config.parseBoolean(Config.getEnv("auto_create_personal_noderedapi_group")); + public static force_add_admins: boolean = Config.parseBoolean(Config.getEnv("force_add_admins")); + public static validate_emails: boolean = Config.parseBoolean(Config.getEnv("validate_emails")); + public static forgot_pass_emails: boolean = Config.parseBoolean(Config.getEnv("forgot_pass_emails")); + public static smtp_service: string = Config.getEnv("smtp_service"); + public static smtp_from: string = Config.getEnv("smtp_from"); + public static smtp_user: string = Config.getEnv("smtp_user"); + public static smtp_pass: string = Config.getEnv("smtp_pass"); + public static smtp_url: string = Config.getEnv("smtp_url"); + public static debounce_lookup: boolean = Config.parseBoolean(Config.getEnv("debounce_lookup")); + public static validate_emails_disposable: boolean = Config.parseBoolean(Config.getEnv("validate_emails_disposable")); + + public static tls_crt: string = Config.getEnv("tls_crt"); + public static tls_key: string = Config.getEnv("tls_key"); + public static tls_ca: string = Config.getEnv("tls_ca"); + public static tls_passphrase: string = Config.getEnv("tls_passphrase"); + + public static oidc_access_token_ttl: number = parseInt(Config.getEnv("oidc_access_token_ttl")); + public static oidc_authorization_code_ttl: number = parseInt(Config.getEnv("oidc_authorization_code_ttl")); + public static oidc_client_credentials_ttl: number = parseInt(Config.getEnv("oidc_client_credentials_ttl")); + public static oidc_refresh_token_ttl: number = parseInt(Config.getEnv("oidc_refresh_token_ttl")); + public static oidc_session_ttl: number = parseInt(Config.getEnv("oidc_session_ttl")); + public static oidc_cookie_key: string = Config.getEnv("oidc_cookie_key"); + + public static api_rate_limit: boolean = Config.parseBoolean(Config.getEnv("api_rate_limit")); + public static api_rate_limit_points: number = parseInt(Config.getEnv("api_rate_limit_points")); + public static api_rate_limit_duration: number = parseInt(Config.getEnv("api_rate_limit_duration")); + public static socket_rate_limit: boolean = Config.parseBoolean(Config.getEnv("socket_rate_limit")); + public static socket_rate_limit_points: number = parseInt(Config.getEnv("socket_rate_limit_points")); + public static socket_rate_limit_points_disconnect: number = parseInt(Config.getEnv("socket_rate_limit_points_disconnect")); + public static socket_rate_limit_duration: number = parseInt(Config.getEnv("socket_rate_limit_duration")); + public static socket_error_rate_limit_points: number = parseInt(Config.getEnv("socket_error_rate_limit_points")); + public static socket_error_rate_limit_duration: number = parseInt(Config.getEnv("socket_error_rate_limit_duration")); + + public static client_heartbeat_timeout: number = parseInt(Config.getEnv("client_heartbeat_timeout")); + public static client_signin_timeout: number = parseInt(Config.getEnv("client_signin_timeout")); + public static client_disconnect_signin_error: boolean = Config.parseBoolean(Config.getEnv("client_disconnect_signin_error")); + + public static expected_max_roles: number = parseInt(Config.getEnv("expected_max_roles")); + public static decorate_roles_fetching_all_roles = Config.parseBoolean(Config.getEnv("decorate_roles_fetching_all_roles")); + public static max_recursive_group_depth: number = parseInt(Config.getEnv("max_recursive_group_depth")); + public static update_acl_based_on_groups: boolean = Config.parseBoolean(Config.getEnv("update_acl_based_on_groups")); + public static allow_merge_acl: boolean = Config.parseBoolean(Config.getEnv("allow_merge_acl")); + + public static multi_tenant: boolean = Config.parseBoolean(Config.getEnv("multi_tenant")); + public static cleanup_on_delete_customer: boolean = Config.parseBoolean(Config.getEnv("cleanup_on_delete_customer")); + public static cleanup_on_delete_user: boolean = Config.parseBoolean(Config.getEnv("cleanup_on_delete_user")); + public static api_bypass_perm_check: boolean = Config.parseBoolean(Config.getEnv("api_bypass_perm_check")); + public static disable_db_config: boolean = Config.parseBoolean(Config.getEnv("disable_db_config")); + public static force_audit_ts: boolean = Config.parseBoolean(Config.getEnv("force_audit_ts")); + public static force_dbusage_ts: boolean = Config.parseBoolean(Config.getEnv("force_dbusage_ts")); + public static migrate_audit_to_ts: boolean = Config.parseBoolean(Config.getEnv("migrate_audit_to_ts")); + + public static websocket_package_size: number = parseInt(Config.getEnv("websocket_package_size"), 10); + public static websocket_max_package_count: number = parseInt(Config.getEnv("websocket_max_package_count"), 10); + public static websocket_message_callback_timeout: number = parseInt(Config.getEnv("websocket_message_callback_timeout"), 10); + public static websocket_disconnect_out_of_sync: boolean = Config.parseBoolean(Config.getEnv("websocket_disconnect_out_of_sync")); + public static protocol: string = Config.getEnv("protocol"); + public static port: number = parseInt(Config.getEnv("port")); + public static domain: string = Config.getEnv("domain"); + public static cookie_secret: string = Config.getEnv("cookie_secret"); + public static max_ace_count: number = parseInt(Config.getEnv("max_ace_count"), 10); + + public static amqp_reply_expiration: number = parseInt(Config.getEnv("amqp_reply_expiration")); + public static amqp_force_queue_prefix: boolean = Config.parseBoolean(Config.getEnv("amqp_force_queue_prefix")); + public static amqp_force_exchange_prefix: boolean = Config.parseBoolean(Config.getEnv("amqp_force_exchange_prefix")); + public static amqp_force_sender_has_read: boolean = Config.parseBoolean(Config.getEnv("amqp_force_sender_has_read")); + public static amqp_force_sender_has_invoke: boolean = Config.parseBoolean(Config.getEnv("amqp_force_sender_has_invoke")); + public static amqp_force_consumer_has_update: boolean = Config.parseBoolean(Config.getEnv("amqp_force_consumer_has_update")); + public static amqp_enabled_exchange: boolean = Config.parseBoolean(Config.getEnv("amqp_enabled_exchange")); + public static amqp_url: string = Config.getEnv("amqp_url"); + public static amqp_username: string = Config.getEnv("amqp_username"); + public static amqp_password: string = Config.getEnv("amqp_password"); + + public static amqp_check_for_consumer: boolean = Config.parseBoolean(Config.getEnv("amqp_check_for_consumer")); + public static amqp_check_for_consumer_count: boolean = Config.parseBoolean(Config.getEnv("amqp_check_for_consumer_count")); + public static amqp_default_expiration: number = parseInt(Config.getEnv("amqp_default_expiration")); + public static amqp_requeue_time: number = parseInt(Config.getEnv("amqp_requeue_time")); + public static amqp_dlx: string = Config.getEnv("amqp_dlx"); + + public static mongodb_url: string = Config.getEnv("mongodb_url"); + public static mongodb_db: string = Config.getEnv("mongodb_db"); + public static mongodb_minpoolsize: number = parseInt(Config.getEnv("mongodb_minpoolsize")); + public static mongodb_maxpoolsize: number = parseInt(Config.getEnv("mongodb_maxpoolsize")); + + public static skip_history_collections: string = Config.getEnv("skip_history_collections"); + public static history_delta_count: number = parseInt(Config.getEnv("history_delta_count")); + public static allow_skiphistory: boolean = Config.parseBoolean(Config.getEnv("allow_skiphistory")); + public static max_memory_restart_mb: number = parseInt(Config.getEnv("max_memory_restart_mb")); + + public static saml_issuer: string = Config.getEnv("saml_issuer"); + public static aes_secret: string = Config.getEnv("aes_secret"); + public static signing_crt: string = Config.getEnv("signing_crt"); + public static singing_key: string = Config.getEnv("singing_key"); + public static wapid_mail: string = Config.getEnv("wapid_mail"); + public static wapid_pub: string = Config.getEnv("wapid_pub"); + public static wapid_key: string = Config.getEnv("wapid_key"); + + public static shorttoken_expires_in: string = Config.getEnv("shorttoken_expires_in"); + public static longtoken_expires_in: string = Config.getEnv("longtoken_expires_in"); + public static downloadtoken_expires_in: string = Config.getEnv("downloadtoken_expires_in"); + public static personalnoderedtoken_expires_in: string = Config.getEnv("personalnoderedtoken_expires_in"); + + public static agent_images: NoderedImage[] = Array.isArray(Config.getEnv("agent_images")) ? Config.getEnv("agent_images") : JSON.parse(Config.getEnv("agent_images")); + public static agent_domain_schema: string = Config.getEnv("agent_domain_schema"); + public static agent_node_selector:string = Config.getEnv("agent_node_selector"); + + public static agent_grpc_apihost: string = Config.getEnv("agent_grpc_apihost"); + public static agent_ws_apihost: string = Config.getEnv("agent_ws_apihost"); + public static agent_oidc_config: string = Config.getEnv("agent_oidc_config"); + public static agent_oidc_client_id: string = Config.getEnv("agent_oidc_client_id"); + public static agent_oidc_client_secret: string = Config.getEnv("agent_oidc_client_secret"); + public static agent_oidc_userinfo_endpoint: string = Config.getEnv("agent_oidc_userinfo_endpoint"); + public static agent_oidc_issuer: string = Config.getEnv("agent_oidc_issuer"); + public static agent_oidc_authorization_endpoint: string = Config.getEnv("agent_oidc_authorization_endpoint"); + public static agent_oidc_token_endpoint: string = Config.getEnv("agent_oidc_token_endpoint"); + + public static saml_federation_metadata: string = Config.getEnv("saml_federation_metadata"); + public static api_ws_url: string = Config.getEnv("api_ws_url"); + + public static agent_docker_entrypoints: string = Config.getEnv("agent_docker_entrypoints"); + public static agent_docker_use_project: boolean = Config.parseBoolean(Config.getEnv("agent_docker_use_project")); + public static agent_docker_certresolver: string = Config.getEnv("agent_docker_certresolver"); + + public static namespace: string = Config.getEnv("namespace"); + public static agent_allow_nodeselector: boolean = Config.parseBoolean(Config.getEnv("agent_allow_nodeselector")); + + public static otel_measure_queued_messages: boolean = Config.parseBoolean(Config.getEnv("otel_measure_queued_messages")); + public static otel_measure__mongodb_watch: boolean = Config.parseBoolean(Config.getEnv("otel_measure__mongodb_watch")); + public static enable_analytics: boolean = Config.parseBoolean(Config.getEnv("enable_analytics")); + public static enable_detailed_analytic: boolean = Config.parseBoolean(Config.getEnv("enable_detailed_analytic")); + public static otel_debug_log: boolean = Config.parseBoolean(Config.getEnv("otel_debug_log")); + public static otel_warn_log: boolean = Config.parseBoolean(Config.getEnv("otel_warn_log")); + public static otel_err_log: boolean = Config.parseBoolean(Config.getEnv("otel_err_log")); + public static otel_trace_url: string = Config.getEnv("otel_trace_url"); + public static otel_metric_url: string = Config.getEnv("otel_metric_url"); + public static otel_trace_interval: number = parseInt(Config.getEnv("otel_trace_interval")); + public static otel_metric_interval: number = parseInt(Config.getEnv("otel_metric_interval")); + public static otel_trace_pingclients: boolean = Config.parseBoolean(Config.getEnv("otel_trace_pingclients")); + public static otel_trace_dashboardauth: boolean = Config.parseBoolean(Config.getEnv("otel_trace_dashboardauth")); + public static otel_trace_include_query: boolean = Config.parseBoolean(Config.getEnv("otel_trace_include_query")); + public static otel_trace_connection_ips: boolean = Config.parseBoolean(Config.getEnv("otel_trace_connection_ips")); + public static otel_trace_mongodb_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_per_users")); + public static otel_trace_mongodb_query_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_query_per_users")); + public static otel_trace_mongodb_count_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_count_per_users")); + public static otel_trace_mongodb_aggregate_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_aggregate_per_users")); + public static otel_trace_mongodb_insert_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_insert_per_users")); + public static otel_trace_mongodb_update_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_update_per_users")); + public static otel_trace_mongodb_delete_per_users: boolean = Config.parseBoolean(Config.getEnv("otel_trace_mongodb_delete_per_users")); + + public static grpc_keepalive_time_ms: number = parseInt(Config.getEnv("grpc_keepalive_time_ms")); + public static grpc_keepalive_timeout_ms: number = parseInt(Config.getEnv("grpc_keepalive_timeout_ms")); + public static grpc_http2_min_ping_interval_without_data_ms: number = parseInt(Config.getEnv("grpc_http2_min_ping_interval_without_data_ms")); + public static grpc_max_connection_idle_ms: number = parseInt(Config.getEnv("grpc_max_connection_idle_ms")); + public static grpc_max_connection_age_ms: number = parseInt(Config.getEnv("grpc_max_connection_age_ms")); + public static grpc_max_connection_age_grace_ms: number = parseInt(Config.getEnv("grpc_max_connection_age_grace_ms")); + public static grpc_http2_max_pings_without_data: number = parseInt(Config.getEnv("grpc_http2_max_pings_without_data")); + public static grpc_keepalive_permit_without_calls: number = parseInt(Config.getEnv("grpc_keepalive_permit_without_calls")); + public static grpc_max_receive_message_length: number = parseInt(Config.getEnv("grpc_max_receive_message_length")); + public static grpc_max_send_message_length: number = parseInt(Config.getEnv("grpc_max_send_message_length")); + + public static validate_user_form: string = Config.getEnv("validate_user_form"); public static externalbaseurl(): string { @@ -732,9 +757,11 @@ export class Config { } else { result = result + "/"; } return result; } - public static getEnv(name: string, defaultvalue: string): string { + public static getEnv(name: string): string { let value: any = process.env[name]; - if (!value || value === "") { value = defaultvalue; } + if (value == null || value == "") { + value = this.default_config[name] + } return value; } public static get(url: string): Promise { @@ -757,17 +784,6 @@ export class Config { }) } public static async parse_federation_metadata(tls_ca: String, url: string): Promise { - // try { - // if (tls_ca !== null && tls_ca !== undefined && tls_ca !== "") { - // const rootCas = require('ssl-root-cas/latest').create(); - // rootCas.push(tls_ca); - // // rootCas.addFile( tls_ca ); - // https.globalAgent.options.ca = rootCas; - // require('https').globalAgent.options.ca = rootCas; - // } - // } catch (error) { - // console.error(error); - // } const metadata: any = await promiseRetry(async () => { // if (Config.saml_ignore_cert) process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"; const data: string = await Config.get(url) @@ -775,9 +791,6 @@ export class Config { if (NoderedUtil.IsNullEmpty(data)) { throw new Error("Failed getting result"); } var xml = await xml2js.parseStringPromise(data); if (xml && xml.EntityDescriptor && xml.EntityDescriptor.IDPSSODescriptor && xml.EntityDescriptor.IDPSSODescriptor.length > 0) { - // const reader: any = await fetch({ url }); - // if (NoderedUtil.IsNullUndefinded(reader)) { throw new Error("Failed getting result"); } - // const _config: any = toPassportConfig(reader); var IDPSSODescriptor = xml.EntityDescriptor.IDPSSODescriptor[0]; var identifierFormat = "urn:oasis:names:tc:SAML:2.0:attrname-format:uri"; if (IDPSSODescriptor.NameIDFormat && IDPSSODescriptor.NameIDFormat.length > 0) { @@ -789,7 +802,6 @@ export class Config { signingCerts.push(key.KeyInfo[0].X509Data[0].X509Certificate[0]); } }); - // var signingCerts = IDPSSODescriptor.KeyDescriptor[0].KeyInfo[0].X509Data[0].X509Certificate; var identityProviderUrl = IDPSSODescriptor.SingleSignOnService[0].$.Location; var logoutUrl = IDPSSODescriptor.SingleLogoutService[0].$.Location; const config = { @@ -807,6 +819,7 @@ export class Config { return metadata; } public static parseArray(s: string): string[] { + if(Array.isArray(s)) return s; let arr = s.split(","); arr = arr.map(p => p.trim()); arr = arr.filter(result => (result.trim() !== "")); diff --git a/OpenFlow/src/Crypt.ts b/OpenFlow/src/Crypt.ts index d0d754d6..a5be97ca 100644 --- a/OpenFlow/src/Crypt.ts +++ b/OpenFlow/src/Crypt.ts @@ -123,7 +123,7 @@ export class Crypt { throw new Error('jwt must be provided'); } if (NoderedUtil.IsNullEmpty(Crypt.encryption_key)) Crypt.encryption_key = Config.aes_secret.substring(0, 32); - if(ignoreExpiration == false && Config.ignore_expiration == true) ignoreExpiration = true; + if(Config.allow_signin_with_expired_jwt == false) ignoreExpiration = false; const o: any = jsonwebtoken.verify(token, Crypt.encryption_key, { ignoreExpiration: ignoreExpiration }); let impostor: string = null; if (!NoderedUtil.IsNullUndefinded(o) && !NoderedUtil.IsNullUndefinded(o.data) && !NoderedUtil.IsNullEmpty(o.data._id)) { @@ -162,6 +162,6 @@ export class Crypt { } static decryptToken(token: string): any { if (NoderedUtil.IsNullEmpty(Crypt.encryption_key)) Crypt.encryption_key = Config.aes_secret.substring(0, 32); - return jsonwebtoken.verify(token, Crypt.encryption_key, { ignoreExpiration: Config.ignore_expiration }); + return jsonwebtoken.verify(token, Crypt.encryption_key, { ignoreExpiration: Config.allow_signin_with_expired_jwt }); } } \ No newline at end of file diff --git a/OpenFlow/src/DBHelper.ts b/OpenFlow/src/DBHelper.ts index d0db17e2..7c411d1d 100644 --- a/OpenFlow/src/DBHelper.ts +++ b/OpenFlow/src/DBHelper.ts @@ -83,8 +83,6 @@ export class DBHelper { for (var i = 0; i < keys.length; i++) { if (keys[i] && !keys[i].startsWith("requesttoken")) { this.memoryCache.del(keys[i]); - } else { - console.log("not deleting " + keys[i]); } } Logger.instanse.debug("clearCache called with reason: " + reason, span); @@ -106,6 +104,12 @@ export class DBHelper { } else { keys = await this.memoryCache.keys(); } + } else if(this.memoryCache &&this.memoryCache.store && this.memoryCache.store.keys) { + if(this.memoryCache.store.keys.get) { + keys = await this.memoryCache.store.keys.get(); + } else { + keys = await this.memoryCache.store.keys(); + } } } catch (error) { diff --git a/OpenFlow/src/DatabaseConnection.ts b/OpenFlow/src/DatabaseConnection.ts index a1f546d3..3acb0666 100644 --- a/OpenFlow/src/DatabaseConnection.ts +++ b/OpenFlow/src/DatabaseConnection.ts @@ -14,6 +14,7 @@ import { SocketMessage } from "./SocketMessage"; import { LoginProvider } from "./LoginProvider"; import { WebServer } from "./WebServer"; import { iAgent } from "./commoninterfaces"; +import * as os from "os"; // tslint:disable-next-line: typedef const safeObjectID = (s: string | number | ObjectId) => ObjectId.isValid(s) ? new ObjectId(s) : null; @@ -141,6 +142,8 @@ export class DatabaseConnection extends events.EventEmitter { } public replicat: string = null; public streams: clsstream[] = []; + public requests: any = {}; + public host: string = ""; /** * Connect to MongoDB * @returns Promise @@ -149,11 +152,15 @@ export class DatabaseConnection extends events.EventEmitter { if (this.cli !== null && this.cli !== undefined && this.isConnected) { return; } + if(this.host == "") { + this.host = os.hostname(); + } const span: Span = Logger.otel.startSubSpan("db.connect", parent); this.streams = []; span?.addEvent("connecting to mongodb"); Logger.instanse.info("Connecting to mongodb", span); const options: MongoClientOptions = { minPoolSize: Config.mongodb_minpoolsize, maxPoolSize: Config.mongodb_maxpoolsize }; + // options.monitorCommands = true; this.cli = await MongoClient.connect(this.mongodburl, options); Logger.instanse.info("Connected to mongodb", span); span?.addEvent("Connected to mongodb"); @@ -173,20 +180,76 @@ export class DatabaseConnection extends events.EventEmitter { .on('error', errEvent) .on('parseError', errEvent) .on('timeout', errEvent) - .on('close', closeEvent); + .on('close', closeEvent) + // .on("commandStarted", (event) => { + // if(this.host != "nixos") return; + // let req = this.requests[event.requestId]; + // if(req != null) { + // this.requests[event.requestId] = { start: new Date(), command: event.commandName, connectionId: event.connectionId }; + // } else { + // this.requests[event.requestId] = { start: new Date(), command: event.commandName, connectionId: event.connectionId }; + // } + // if(Config.log_database_queries == true) { + // Logger.instanse.debug(event.requestId + " cmd: " + event.commandName + " started ", span, { requestId: event.requestId, connectionId: event.connectionId, command: event.commandName, cls: "DatabaseConnection" }) + // } + // }) + // .on("commandFailed", (event) => { + // if(this.host != "nixos") return; + // if(Config.log_database_queries == true) { + // Logger.instanse.debug(event.requestId + " cmd: " + event.commandName + " failed" , span, { requestId: event.requestId, ms: event.duration, connectionId: event.connectionId, command: event.commandName, failure: event.failure, cls: "DatabaseConnection" }) + // } + // // Logger.instanse.debug("Query: " + JSON.stringify({ _id: ace._id }), span, { ms, count: arr.length, collection: "users" }); + // }) + // .on("commandSucceeded", (event) => { + // if(this.host != "nixos") return; + // if(Config.log_database_queries == true) { + // // @ts-ignore + // let ns = event.reply?.cursor?.ns; + // // @ts-ignore + // let count = (event.reply?.cursor?.nextBatch?.length !== undefined && event.reply?.cursor?.nextBatch?.length !== null) ? event.reply?.cursor?.nextBatch?.length + // // @ts-ignore + // : (event.reply?.cursor?.firstBatch?.length !== undefined && event.reply?.cursor?.firstBatch?.length !== null) ? event.reply?.cursor?.firstBatch?.length + // // @ts-ignore + // : (event.reply?.cursor?.nReturned !== undefined && event.reply?.cursor?.nReturned !== null) ? event.reply?.cursor?.nReturned + // // @ts-ignore + // : (event.reply?.n !== undefined && event.reply?.n !== null) ? event.reply?.n + // // @ts-ignore + // : (event.reply?.nModified !== undefined && event.reply?.nModified !== null) ? event.reply?.nModified + // // @ts-ignore + // : (event.reply?.nUpserted !== undefined && event.reply?.nUpserted !== null) ? event.reply?.nUpserted + // // @ts-ignore + // : event.reply?.nRemoved; + + // let req = this.requests[event.requestId]; + // if(req != null) { + // req.end = new Date(); + // req.duration = req.end - req.start; + // req.count = count; + // req.ns = ns; + // // delete this.requests[event.requestId]; + // let command = event.commandName; + // if(command != req.command) command = req.command + "-" + command; + // Logger.instanse.debug(event.requestId + " cmd: " + command + " c: " + req.count + " mq:" + req.duration, span, { requestId: event.requestId, ms: event.duration, connectionId: event.connectionId, command: event.commandName, reply: event.reply, cls: "DatabaseConnection" }) + // } else { + // var b = true; + // } + // // Logger.instanse.debug(event.commandName + " " + event.requestId + " " + ns + " " + count, span, { requestId: event.requestId, ms: event.duration, connectionId: event.connectionId, command: event.commandName, reply: event.reply, cls: "DatabaseConnection" }) + // } + // // Logger.instanse.debug("Query: " + JSON.stringify({ _id: ace._id }), span, { ms, count: arr.length, collection: "users" }); + // }) this.db = this.cli.db(this._dbname); - try { - var topology = (this.cli as any).topology; - if (topology.s.description.type == "Single" || topology.s.description.type == "single") { - Config.supports_watch = false; - } else { - Config.supports_watch = true; - } - } catch (error) { - Logger.instanse.error(error, span); - } - Logger.instanse.debug("supports_watch: " + Config.supports_watch, span); - if (Config.supports_watch && this.registerGlobalWatches) { + // try { + // var topology = (this.cli as any).topology; + // if (topology.s.description.type == "Single" || topology.s.description.type == "single") { + // Config.supports_watch = false; + // } else { + // Config.supports_watch = true; + // } + // } catch (error) { + // Logger.instanse.error(error, span); + // } + // Logger.instanse.debug("supports_watch: " + Config.supports_watch, span); + // if (Config.supports_watch && this.registerGlobalWatches) { // let collections = await DatabaseConnection.toArray(this.db.listCollections()); let collections = await Logger.DBHelper.GetCollections(span); collections = collections.filter(x => x.name.indexOf("system.") === -1); @@ -196,7 +259,7 @@ export class DatabaseConnection extends events.EventEmitter { if (collections[c].name == "fs.files" || collections[c].name == "fs.chunks") continue; this.registerGlobalWatch(collections[c].name, span); } - } + // } this.ensureQueueMonitoring(); this.isConnected = true; Logger.otel.endSpan(span); @@ -245,7 +308,7 @@ export class DatabaseConnection extends events.EventEmitter { const wiq = queues[i]; const count = await Logger.DBHelper.GetPendingWorkitemsCount(wiq._id, null); if (count < 1) continue; - const query = { "wiqid": wiq._id, state: "new", "_type": "workitem", "nextrun": { "$lte": new Date(new Date().toISOString()) } }; + // const query = { "wiqid": wiq._id, state: "new", "_type": "workitem", "nextrun": { "$lte": new Date(new Date().toISOString()) } }; var payload = null; // const payload = await this.GetOne({ jwt, collectionname, query }, null); // if (payload == null) continue; @@ -410,7 +473,7 @@ export class DatabaseConnection extends events.EventEmitter { } if (collectionname === "config" && _type === "config") { discardspan = false; - await dbConfig.Reload(Crypt.rootToken(), span); + await dbConfig.Reload(Crypt.rootToken(), true, span); } } @@ -554,7 +617,12 @@ export class DatabaseConnection extends events.EventEmitter { (stream.stream as any).on("error", err => { Logger.instanse.error(err, span, { collection: collectionname }); }); - (stream.stream as any).on("change", async (next) => { this.GlobalWatchCallback(collectionname, next) }); + (stream.stream as any).on("change", async (next) => { + if(Config.log_all_watches == true) { + Logger.instanse.debug(collectionname + " watch " + next?.fullDocument?._type + " " + next?.operationType + " " + next?.fullDocument?.name, span, { cls: "DatabaseConnection", func: "onchange", collection: collectionname }); + } + this.GlobalWatchCallback(collectionname, next) + }); this.streams.push(stream); } catch (error) { Logger.instanse.error(error, span, { collection: collectionname }); @@ -2468,7 +2536,11 @@ export class DatabaseConnection extends events.EventEmitter { if (!DatabaseConnection.hasAuthorization(user, original, Rights.update)) { throw new Error("Access denied, no authorization to UpdateOne " + q.item._type + " " + name + " to database"); } - + if(q.collectionname === "config" && q.item._type === "config") { + if (!user.HasRoleId(WellknownIds.admins)) throw new Error("Access denied, no authorization to update config"); + dbConfig.cleanAndApply(q.item as any, span); + } + await Logger.DBHelper.CheckCache(q.collectionname, q.item, false, false, span); if (q.collectionname === "agents") { @@ -3597,7 +3669,7 @@ export class DatabaseConnection extends events.EventEmitter { try { if(ids[i] != null && ids[i].trim() != "") objectids.push(safeObjectID(ids[i])) } catch (error) { - console.error(error); + Logger.instanse.error(error, span); } } } else { @@ -3605,7 +3677,7 @@ export class DatabaseConnection extends events.EventEmitter { try { if(ids[i] != null && ids[i].trim() != "") objectids.push(ids[i]) } catch (error) { - console.error(error); + Logger.instanse.error(error, span); } } } @@ -3812,7 +3884,9 @@ export class DatabaseConnection extends events.EventEmitter { newObj[key] = value; } } catch (error) { - Logger.instanse.error(error, null); + // Logger.instanse.error(error, null); + var errormessage = error.message ? error.message : error; + Logger.instanse.debug(errormessage, null); newObj[key] = value; } return newObj; @@ -3830,7 +3904,7 @@ export class DatabaseConnection extends events.EventEmitter { _decryptentity(item, newObj, key) { const value: any = item[key]; try { - if (this._shouldEncryptValue(item._encrypt, key, value) && value != null) { + if (this._shouldEncryptValue(item._encrypt, key, value) && value != null && value != "") { let newvalue = Crypt.decrypt(value); if (newvalue.indexOf("{") === 0 || newvalue.indexOf("[") === 0) { try { @@ -3843,7 +3917,9 @@ export class DatabaseConnection extends events.EventEmitter { newObj[key] = value; } } catch (error) { - Logger.instanse.error(error, null); + // Logger.instanse.error(error, null); + var errormessage = error.message ? error.message : error; + Logger.instanse.debug(errormessage, null); newObj[key] = value; } return newObj; @@ -4170,6 +4246,26 @@ export class DatabaseConnection extends events.EventEmitter { } + SaveMongoDBCommand( user: TokenUser, collection: string, command: string, query: any, ms:number) { + try { + const cmd = { + _created: new Date(new Date().toISOString()), + _createdby: user.name, + _createdbyid: user._id, + collection: collection, + name: user?.username + " " + command + " " + collection + " " + ms, + command: command, + query: query, + ms: ms + } + this.db.collection("dbcommands").insertOne(cmd).then(() => { + }).catch(err => { + Logger.instanse.error(err, null); + }); + } catch (error) { + Logger.instanse.error(error, null, { collection: collection, user: user?.username, ms: ms, query: query }); + } + } async SaveUpdateDiff(q: UpdateOneMessage, user: TokenUser, parent: Span) { const span: Span = Logger.otel.startSubSpan("db.SaveUpdateDiff", parent); try { @@ -4380,8 +4476,10 @@ export class DatabaseConnection extends events.EventEmitter { return new Promise((resolve, reject) => { try { Logger.instanse.info("Adding index " + name + " to " + collectionname, span, { collection: collectionname }); - if (NoderedUtil.IsNullUndefinded(options)) options = {}; - options["name"] = name; + if(typeof keypath === "string") keypath = JSON.parse(keypath); + if (NoderedUtil.IsNullEmpty(options)) options = {}; + if(typeof options === "string") options = JSON.parse(options); + if (!NoderedUtil.IsNullEmpty(name)) options["name"] = name; this.db.collection(collectionname).createIndex(keypath, options, (err, name) => { if (err) { Logger.instanse.error(err, span); @@ -4738,14 +4836,14 @@ export class DatabaseConnection extends events.EventEmitter { collections = await DatabaseConnection.toArray(this.db.listCollections()); collections = collections.filter(x => x.name.indexOf("system.") === -1); - if (Config.supports_watch) { + // if (Config.supports_watch) { Logger.instanse.info("Register global watches for each collection", span); for (var c = 0; c < collections.length; c++) { if (collections[c].type != "collection") continue; if (collections[c].name == "fs.files" || collections[c].name == "fs.chunks") continue; this.registerGlobalWatch(collections[c].name, span); } - } + // } DatabaseConnection.timeseries_collections = []; DatabaseConnection.collections_with_text_index = []; diff --git a/OpenFlow/src/Logger.ts b/OpenFlow/src/Logger.ts index 2f7c85f5..d74ccfc6 100644 --- a/OpenFlow/src/Logger.ts +++ b/OpenFlow/src/Logger.ts @@ -116,7 +116,10 @@ export class Logger { } span.addEvent(obj.message, obj) } - if (obj.ms != null && obj.ms != "" && obj.func != "query" && Config.log_database_queries) { + if(Config.log_all_watches && obj.cls == "DatabaseConnection" && obj.func == "onchange") { + + } else if(Config.log_database_queries && obj.requestId != null) { + } else if (obj.ms != null && obj.ms != "" && obj.func != "query" && Config.log_database_queries) { if (obj.ms < Config.log_database_queries_ms) return; } else if (Logger.enabled[cls]) { if (Logger.enabled[cls] < lvl) return; @@ -152,7 +155,7 @@ export class Logger { return JSON.stringify(plainObject, filter, space); }; if (Config.log_to_exchange && !Config.unittesting) { - if (NoderedUtil.IsNullEmpty(Logger._hostname)) Logger._hostname = (Config.getEnv("HOSTNAME", undefined) || os.hostname()) || "unknown"; + if (NoderedUtil.IsNullEmpty(Logger._hostname)) Logger._hostname = (process.env.HOSTNAME || os.hostname()) || "unknown"; if (amqpwrapper.Instance() && amqpwrapper.Instance().connected && amqpwrapper.Instance().of_logger_ready) { if (typeof obj.message == "object") obj.message = JSON.parse(stringifyError(obj.message, null, 2)); amqpwrapper.Instance().send("openflow_logs", "", { ...obj, host: Logger._hostname }, 500, null, "", span, 1); @@ -173,8 +176,14 @@ export class Logger { obj.cls = ""; if (s.file != '') obj.cls = s.file.replace(".js", ""); } - if(options?.openapi) { - obj.cls = "OpenAIProxy"; + if(obj.func.indexOf("anonymous") > -1 || obj.func.indexOf("<") > -1 || obj.func.indexOf("[") > -1) { + obj.func = "anonymous"; + } + if(options?.cls != null && options?.cls != "") { + obj.cls = options.cls; + } + if(options?.func != null && options?.func != "") { + obj.func = options.func; } this.json(obj, span); } @@ -193,8 +202,14 @@ export class Logger { obj.cls = ""; if (s.file != '') obj.cls = s.file.replace(".js", ""); } - if(options?.openapi) { - obj.cls = "OpenAIProxy"; + if(obj.func.indexOf("anonymous") > -1 || obj.func.indexOf("<") > -1 || obj.func.indexOf("[") > -1) { + obj.func = "anonymous"; + } + if(options?.cls != null && options?.cls != "") { + obj.cls = options.cls; + } + if(options?.func != null && options?.func != "") { + obj.func = options.func; } this.json(obj, span); } @@ -212,8 +227,14 @@ export class Logger { obj.cls = ""; if (s.file != '') obj.cls = s.file.replace(".js", ""); } - if(options?.openapi) { - obj.cls = "OpenAIProxy"; + if(obj.func.indexOf("anonymous") > -1 || obj.func.indexOf("<") > -1 || obj.func.indexOf("[") > -1) { + obj.func = "anonymous"; + } + if(options?.cls != null && options?.cls != "") { + obj.cls = options.cls; + } + if(options?.func != null && options?.func != "") { + obj.func = options.func; } this.json(obj, span); } @@ -231,8 +252,14 @@ export class Logger { obj.cls = ""; if (s.file != '') obj.cls = s.file.replace(".js", ""); } - if(options?.openapi) { - obj.cls = "OpenAIProxy"; + if(obj.func.indexOf("anonymous") > -1 || obj.func.indexOf("<") > -1 || obj.func.indexOf("[") > -1) { + obj.func = "anonymous"; + } + if(options?.cls != null && options?.cls != "") { + obj.cls = options.cls; + } + if(options?.func != null && options?.func != "") { + obj.func = options.func; } this.json(obj, span); } @@ -251,8 +278,14 @@ export class Logger { obj.cls = ""; if (s.file != '') obj.cls = s.file.replace(".js", ""); } - if(options?.openapi) { - obj.cls = "OpenAIProxy"; + if(obj.func.indexOf("anonymous") > -1 || obj.func.indexOf("<") > -1 || obj.func.indexOf("[") > -1) { + obj.func = "anonymous"; + } + if(options?.cls != null && options?.cls != "") { + obj.cls = options.cls; + } + if(options?.func != null && options?.func != "") { + obj.func = options.func; } this.json(obj, span); } @@ -271,8 +304,14 @@ export class Logger { obj.cls = ""; if (s.file != '') obj.cls = s.file.replace(".js", ""); } - if(options?.openapi) { - obj.cls = "OpenAIProxy"; + if(obj.func.indexOf("anonymous") > -1 || obj.func.indexOf("<") > -1 || obj.func.indexOf("[") > -1) { + obj.func = "anonymous"; + } + if(options?.cls != null && options?.cls != "") { + obj.cls = options.cls; + } + if(options?.func != null && options?.func != "") { + obj.func = options.func; } this.json(obj, span); } diff --git a/OpenFlow/src/LoginProvider.ts b/OpenFlow/src/LoginProvider.ts index cc7c77d0..8934dec1 100644 --- a/OpenFlow/src/LoginProvider.ts +++ b/OpenFlow/src/LoginProvider.ts @@ -15,6 +15,7 @@ import { Span } from "@opentelemetry/api"; import { Logger } from "./Logger"; import { DatabaseConnection } from "./DatabaseConnection"; import { TokenRequest } from "./TokenRequest"; +import { WebServer } from "./WebServer"; var nodemailer = require('nodemailer'); var dns = require('dns'); const got = require("got"); @@ -1126,10 +1127,6 @@ export class LoginProvider { let _url = Config.basewsurl(); if (!NoderedUtil.IsNullEmpty(Config.api_ws_url)) _url = Config.api_ws_url; if (!_url.endsWith("/")) _url += "/"; - let nodered_domain_schema = Config.nodered_domain_schema; - if (NoderedUtil.IsNullEmpty(nodered_domain_schema)) { - nodered_domain_schema = "$nodered_id$." + Config.domain; - } let agent_domain_schema = Config.agent_domain_schema; if (NoderedUtil.IsNullEmpty(agent_domain_schema)) { agent_domain_schema = "$slug$." + Config.domain; @@ -1151,7 +1148,6 @@ export class LoginProvider { auto_create_personal_nodered_group: Config.auto_create_personal_nodered_group, auto_create_personal_noderedapi_group: Config.auto_create_personal_noderedapi_group, namespace: Config.namespace, - nodered_domain_schema: nodered_domain_schema, agent_domain_schema: agent_domain_schema, websocket_package_size: Config.websocket_package_size, version: Config.version, @@ -1160,7 +1156,7 @@ export class LoginProvider { validate_user_form: Config.validate_user_form, validate_emails: Config.validate_emails, forgot_pass_emails: Config.forgot_pass_emails, - supports_watch: Config.supports_watch, + supports_watch: true, agent_images: Config.agent_images, amqp_enabled_exchange: Config.amqp_enabled_exchange, multi_tenant: Config.multi_tenant, @@ -1197,6 +1193,8 @@ export class LoginProvider { static async post_AddTokenRequest(req: any, res: any, next: any): Promise { const span: Span = Logger.otel.startSpanExpress("LoginProvider.login", req); try { + const remoteip = LoginProvider.remoteip(req); + span?.setAttribute("remoteip", remoteip); const key = req.body.key; let exists: TokenRequest = await Logger.DBHelper.FindRequestTokenID(key, span); if (!NoderedUtil.IsNullUndefinded(exists)) { @@ -1204,7 +1202,7 @@ export class LoginProvider { return res.status(500).send({ message: "Illegal key" }); } await Logger.DBHelper.AddRequestTokenID(key, {}, span); - Logger.instanse.info("Added token request " + key, span); + Logger.instanse.info("Added token request " + key + " from " + remoteip, span); res.status(200).send({ message: "ok" }); } catch (error) { Logger.instanse.error(error, span); @@ -1216,6 +1214,7 @@ export class LoginProvider { static async get_GetTokenRequest(req: any, res: any, next: any): Promise { const span: Span = Logger.otel.startSpanExpress("LoginProvider.login", req); try { + span?.setAttribute("remoteip", LoginProvider.remoteip(req)); const key = req.query.key; let exists: TokenRequest = null; exists = await Logger.DBHelper.FindRequestTokenID(key, span); @@ -1477,11 +1476,6 @@ export class LoginProvider { } } - Config.smtp_service = Config.getEnv("smtp_service", ""); - Config.smtp_from = Config.getEnv("smtp_from", ""); - Config.smtp_user = Config.getEnv("smtp_user", ""); - Config.smtp_pass = Config.getEnv("smtp_service", ""); - if (Config.validate_emails) { let email: string = tuser.username; if (tuser.email && tuser.email.indexOf("@") > -1) email = tuser.email; diff --git a/OpenFlow/src/Messages/Message.ts b/OpenFlow/src/Messages/Message.ts index 3aff3203..7f4edc3c 100644 --- a/OpenFlow/src/Messages/Message.ts +++ b/OpenFlow/src/Messages/Message.ts @@ -590,6 +590,36 @@ export class Message { case "deleteworkitem": await this.DeleteWorkitem(span); break; + case "startagent": + await this.ControlAgent(span); + break; + case "stopagent": + await this.ControlAgent(span); + break; + case "deleteagentpod": + await this.ControlAgent(span); + break; + case "getagentlog": + await this.ControlAgent(span); + break; + case "getagentpods": + await this.ControlAgent(span); + break; + case "deleteagent": + await this.ControlAgent(span); + break; + case "createindex": + await this.CreateIndex(span); + break; + case "deletepackage": + await this.DeletePackage(span); + break; + case "issuelicense": + await this.IssueLicense(cli, span); + break; + case "invokeopenra": + await this.InvokeOpenRPA(cli, span); + break; case "customcommand": await this.CustomCommand(cli, span); break; @@ -628,6 +658,145 @@ export class Message { } }); } + async ControlAgent(parent: Span) { + this.Reply(); + let msg: any = this.data + if( typeof this.data == "string") { + msg = JSON.stringify(this.data) + } + msg =JSON.parse(JSON.stringify(msg)); + try { + console.log(this.data); + if (Logger.agentdriver == null) throw new Error("No agentdriver is loaded") + var agent = null; + if((msg.agentid == null || msg.agentid == "")) { + if(this.command != "getagentpods") throw new Error("No agentid is specified"); + } else { + agent = await Config.db.GetOne({ query: { _id: msg.agentid }, collectionname: "agents", jwt:this.jwt }, parent); + if(agent == null) throw new Error("Access denied"); + if (!DatabaseConnection.hasAuthorization(this.tuser, agent, Rights.invoke)) { + throw new Error(`[${this.tuser.name}] Access denied, missing invoke permission on ${agent.name}`); + } + if(agent.image == null || agent.image == "") return; + } + + + if(this.command == "startagent") { + await Logger.agentdriver.EnsureInstance(this.tuser, this.jwt, agent, parent); + } else if(this.command == "stopagent") { + await Logger.agentdriver.RemoveInstance(this.tuser, this.jwt, agent, false, parent); + } else if(this.command == "getagentlog") { + msg.result = await Logger.agentdriver.GetInstanceLog(this.tuser, this.jwt, agent, msg.podname, parent); + } else if(this.command == "getagentpods") { + var getstats = false; + if(!NoderedUtil.IsNullEmpty(msg.stats)) getstats = msg.stats; + msg.results = await Logger.agentdriver.GetInstancePods(this.tuser, this.jwt, agent, msg.podname, parent); + // msg.results = JSON.stringify(await Logger.agentdriver.GetInstancePods(this.tuser, this.jwt, agent, msg.podname, parent)); + var b = true; + } else if(this.command == "deleteagentpod") { + await Logger.agentdriver.RemoveInstancePod(this.tuser, this.jwt, agent, msg.podname, parent); + } else if(this.command == "deleteagent") { + if (!DatabaseConnection.hasAuthorization(this.tuser, agent, Rights.delete)) { + throw new Error(`[${this.tuser.name}] Access denied, missing delete permission on ${agent.name}`); + } + await Logger.agentdriver.RemoveInstance(this.tuser, this.jwt, agent, true, parent); + Config.db.DeleteOne(agent._id, "agents", false, this.jwt, parent); + } + } finally { + this.data = JSON.stringify(msg); + } + } + async CreateIndex(parent: Span) { + this.Reply(); + let msg: any = this.data + if( typeof this.data == "string") { + msg = JSON.stringify(this.data) + } + msg = JSON.parse(JSON.stringify(msg)); + try { + if (!this.tuser.HasRoleId(WellknownIds.admins)) throw new Error("Access denied"); + msg.result = await Config.db.createIndex(msg.collectionname, msg.name, msg.index, msg.options, parent); + } finally { + this.data = JSON.stringify(msg); + } + } + async DeletePackage(parent: Span) { + this.Reply(); + let msg: any = this.data + if( typeof this.data == "string") { + msg = JSON.stringify(this.data) + } + try { + var pack = await Config.db.GetOne({ query: { _id: msg.id, "_type": "package" }, collectionname: "agents", jwt: this.jwt }, parent); + if(pack == null) throw new Error("Access denied or package not found"); + if (!DatabaseConnection.hasAuthorization(this.tuser, pack, Rights.delete)) { + throw new Error(`[${this.tuser.name}] Access denied, missing delete permission on ${pack.name}`); + } + if(pack.fileid != null && pack.fileid != "") { + const rootjwt = Crypt.rootToken(); + let query = { _id: pack.fileid }; + const item = await Config.db.GetOne({ query, collectionname: "fs.files", jwt: rootjwt }, parent); + if(item != null) { + await Config.db.DeleteOne(pack.fileid, "files", false, this.jwt, parent); + } + } + await Config.db.DeleteOne(pack._id, "agents", false, this.jwt, parent); + } finally { + this.data = JSON.stringify(msg); + } + } + async IssueLicense(cli: WebSocketServerClient, parent: Span) { + this.Reply(); + let msg: any = this.data + if( typeof this.data == "string") { + msg = JSON.stringify(this.data) + } + try { + let _lic_require: any = null; + try { + _lic_require = require("../ee/license-file"); + } catch (error) { + } + if (_lic_require == null) { + throw new Error("License module not found"); + } + Logger.License = new _lic_require.LicenseFile(); + // @ts-ignore + var data = msg.data; + try { + data = JSON.parse(data); + } catch (error) { + } + if(data == null || data == "") throw new Error("No data found"); + var domain = data.domain; + if (!this.tuser.HasRoleId(WellknownIds.admins)) { + delete data.months; + } + var exists = await Config.db.GetOne({ query: { domains: domain, "_type": "resourceusage"}, collectionname: "config", jwt:this.jwt }, parent); + if (!this.tuser.HasRoleId(WellknownIds.admins)) { + if(exists == null) throw new Error("Access denied"); + } + if(data.months == null || data.months == "") { + if(exists != null && exists.issuemonths != null) data.months = parseInt(exists.issuemonths); + } + // throw new Error("Access denied"); + msg.result = await Logger.License.generate2(data, cli?.remoteip, this.tuser, parent); + } finally { + this.data = JSON.stringify(msg); + } + } + async InvokeOpenRPA(cli: WebSocketServerClient, parent: Span) { + this.Reply(); + let msg: any = this.data + if( typeof this.data == "string") { + msg = JSON.stringify(this.data) + } + try { + throw new Error("Not implemented, only available using OpenAPI"); + } finally { + this.data = JSON.stringify(msg); + } + } async RegisterExchange(cli: WebSocketServerClient, parent: Span) { this.Reply(); let msg: RegisterExchangeMessage; @@ -977,8 +1146,8 @@ export class Message { return; } this.data = "{\"message\": \"Unknown command " + this.command + "\"}"; - this.Reply("error"); Logger.instanse.error(`UnknownCommand ${this.command}`, null); + this.Reply("error"); } private static collectionCache: any = {}; private static collectionCachetime: Date = new Date(); @@ -1187,11 +1356,11 @@ export class Message { msg = WatchMessage.assign(this.data); if (NoderedUtil.IsNullEmpty(msg.jwt)) { msg.jwt = this.jwt; } if (NoderedUtil.IsNullEmpty(msg.jwt)) { msg.jwt = cli.jwt; } - if (Config.supports_watch) { + //if (Config.supports_watch) { await cli.UnWatch(msg.id, msg.jwt); - } else { - msg.error = "Watch is not supported by this openflow"; - } + // } else { + // msg.error = "Watch is not supported by this openflow"; + // } msg.result = null; delete msg.jwt; this.data = JSON.stringify(msg); @@ -1203,14 +1372,14 @@ export class Message { if (NoderedUtil.IsNullEmpty(msg.jwt)) { msg.jwt = this.jwt; } if (NoderedUtil.IsNullEmpty(msg.jwt)) { msg.jwt = cli.jwt; } msg.id = null; - if (Config.supports_watch) { + // if (Config.supports_watch) { msg.id = await cli.Watch(msg.aggregates, msg.collectionname, msg.jwt); if(msg.collectionname != null && msg.collectionname != "") { Config.db.registerGlobalWatch(msg.collectionname, null); } - } else { - msg.error = "Watch is not supported by this openflow"; - } + // } else { + // msg.error = "Watch is not supported by this openflow"; + // } msg.result = msg.id; delete msg.jwt; this.data = JSON.stringify(msg); @@ -1776,7 +1945,7 @@ export class Message { } else { Logger.instanse.debug(tuser.username + " was validated in using " + tokentype, span); } - msg.supports_watch = Config.supports_watch; + msg.supports_watch = true; var keys = Object.keys(UpdateDoc.$set); if (keys.length > 0 || UpdateDoc.$unset || NoderedUtil.IsNullEmpty(user.lastseen)) { // ping will handle this, if no new information needs to be added @@ -2353,7 +2522,7 @@ export class Message { // throw new Error("Customer has no billing information, please update with vattype and vatnumber"); } if (Config.stripe_force_vat) { - if (NoderedUtil.IsNullEmpty(customer.stripeid)) throw new Error("Customer has no billing information, please update with vattype and vatnumber"); + if (NoderedUtil.IsNullEmpty(customer.stripeid)) throw new Error("Customer " + customer.name + " has no billing information, please update with vattype and vatnumber"); } @@ -2958,7 +3127,9 @@ export class Message { if (payload && payload.limit) { url += "&limit=" + payload.limit; } - const auth = "Basic " + Buffer.from(Config.stripe_api_secret + ":").toString("base64"); + var stripe_api_secret = Config.stripe_api_secret; + if(stripe_api_secret == null || stripe_api_secret == "") throw new Error("Missing stripe_api_secret"); + const auth = "Basic " + Buffer.from(stripe_api_secret + ":").toString("base64"); const options = { headers: { @@ -3513,43 +3684,35 @@ export class Message { } catch (error) { Logger.instanse.error(error, span); } + + if(Config.housekeeping_remomve_unvalidated_user_days > 0) { + let todate = new Date(); + todate.setDate(todate.getDate() - 1); + let fromdate = new Date(); + fromdate.setMonth(fromdate.getMonth() - 1); + const jwt: string = Crypt.rootToken(); + + let query = { "validated": false, "_type": "user", "_id": { "$ne": WellknownIds.root } }; + query["_modified"] = { "$lt": todate.toISOString(), "$gt": fromdate.toISOString()} + let count = await Config.db.DeleteMany(query, null, "users", "", false, jwt, span); + if(count > 0) { + Logger.instanse.verbose("Removed " + count + " unvalidated users", span); + } + } + if(Config.housekeeping_cleanup_openrpa_instances == true) { + let msg = new UpdateManyMessage(); + msg.jwt = Crypt.rootToken(); + msg.collectionname = "openrpa_instances"; + msg.query = { "state": { "$in": ["idle", "running"] } }; + msg.item = { "$set": { "state": "completed"}, "$unset": {"xml": ""}} as any; + let result = await Config.db.UpdateDocument(msg, span); + if(result?.opresult?.nModified > 0) { + Logger.instanse.verbose("Updated " + result.opresult.nModified + " openrpa instances", span); + } else if (result?.opresult?.modifiedCount > 0) { + Logger.instanse.verbose("Updated " + result.opresult.modifiedCount + " openrpa instances", span); + } + } - // if (!skipNodered) { - // Logger.instanse.debug("Get running Nodered Instances", span); - // await this.GetNoderedInstance(span); - // Logger.instanse.debug("Get users with autocreate", span); - // const users: any[] = await Config.db.db.collection("users").find({ "_type": "user", "nodered.autocreate": true }).toArray(); - // // TODO: we should get instances and compare, running ensure for each user will not scale well - // for (let i = 0; i < users.length; i++) { - // let user = users[i]; - // var doensure = false; - // if (Config.multi_tenant) { - // if (!NoderedUtil.IsNullEmpty(Config.stripe_api_secret)) { - // if (!NoderedUtil.IsNullEmpty(user.customerid)) { - // // @ts-ignore - // var customers: Customer[] = await Config.db.db.collection("users").find({ "_type": "customer", "_id": user.customerid }).toArray(); - // if (customers.length > 0 && !NoderedUtil.IsNullEmpty(customers[0].subscriptionid)) { - // doensure = true; - // } - // } - // } else { - // doensure = true; - // } - // } else { - // doensure = true; - // } - // if (doensure) { - // Logger.instanse.debug("EnsureNoderedInstance for " + user.name, span); - // var ensuremsg: EnsureNoderedInstanceMessage = new EnsureNoderedInstanceMessage(); - // ensuremsg._id = user._id; - // var msg: Message = new Message(); msg.jwt = jwt; - // msg.data = JSON.stringify(ensuremsg); - // msg.tuser = this.tuser; - // await msg.EnsureNoderedInstance(span); - // } - // } - // Logger.instanse.debug("Done processing autocreate", span); - // } } catch (error) { } @@ -4010,13 +4173,6 @@ export class Message { cursor = Config.db.db.collection("users").find({ "_type": "user", lastseen: { "$gte": fivedaysago } }); } else { cursor = Config.db.db.collection("users").find({ "_type": "user", lastseen: { "$gte": fivedaysago } }); - // if (Config.nodered_domain_schema == "$nodered_id$.app.openiap.io") { - // // cursor = Config.db.db.collection("users").find({ "_type": "user", "dbusage": { "$gte": 15815993 } }) - // cursor = Config.db.db.collection("users").find({ "_type": "user", "dblocked": true }) - // } else { - // cursor = Config.db.db.collection("users").find({ "_type": "user" }) - // } - // While debugging, also update users who has not been online the last 24 hours } for await (const u of cursor) { if (u.dbusage == null) u.dbusage = 0; @@ -4212,7 +4368,15 @@ export class Message { const dt = parseInt((new Date().getTime() / 1000).toFixed(0)) const payload: any = { "quantity": billablecount, "timestamp": dt }; if (!NoderedUtil.IsNullEmpty(config.siid) && !NoderedUtil.IsNullEmpty(c.stripeid)) { - await this.Stripe("POST", "usage_records", config.siid, payload, c.stripeid); + try { + await this.Stripe("POST", "usage_records", config.siid, payload, c.stripeid); + } catch (error) { + if (error.response && error.response.body) { + Logger.instanse.error("Update usage record error!" + error.response.body, span); + } else { + Logger.instanse.error("Update usage record error!" + error, span); + } + } } } if (c.dblocked || !c.dblocked) { diff --git a/OpenFlow/src/OAuthProvider.ts b/OpenFlow/src/OAuthProvider.ts index c5be3a4a..968f9bc4 100644 --- a/OpenFlow/src/OAuthProvider.ts +++ b/OpenFlow/src/OAuthProvider.ts @@ -168,7 +168,7 @@ export class OAuthProvider { defaultrole : "Viewer", rolemappings : { "admins": "Admin" }, clientId: "openai",client_id: "openai", - client_secret: "secret", + client_secret: "openai", token_endpoint_auth_method: "client_secret_post", response_types: ['code', 'id_token', 'code id_token'], grant_types: ['implicit', 'authorization_code'], diff --git a/OpenFlow/src/QueueClient.ts b/OpenFlow/src/QueueClient.ts index 2d16f3d2..cd8bac2a 100644 --- a/OpenFlow/src/QueueClient.ts +++ b/OpenFlow/src/QueueClient.ts @@ -109,7 +109,6 @@ export class QueueClient { Logger.instanse.silly("Submit request for command: " + msg.command + " queuename: " + this.queuename + " replyto: " + this.queue.queue + " correlationId: " + msg.correlationId, null) await amqpwrapper.Instance().sendWithReplyTo("", this.queuename, this.queue.queue, json, Config.openflow_amqp_expiration, msg.correlationId, "", span, priority); } catch (error) { - console.error(error); if (NoderedUtil.IsNullUndefinded(this.queue)) { Logger.instanse.warn("SendForProcessing queue is null, shutdown amqp connection", span); process.exit(406); diff --git a/OpenFlow/src/SamlProvider.ts b/OpenFlow/src/SamlProvider.ts index 659b68ca..6747ac5b 100644 --- a/OpenFlow/src/SamlProvider.ts +++ b/OpenFlow/src/SamlProvider.ts @@ -66,8 +66,10 @@ export class SamlProvider { const key: string = Buffer.from(Config.singing_key, "base64").toString("ascii"); if(cert != null && cert != "") { + let saml_issuer: string = Config.saml_issuer; + if(saml_issuer == null || saml_issuer == "") saml_issuer = "uri:" + Config.domain; const samlpoptions: any = { - issuer: Config.saml_issuer, + issuer: saml_issuer, cert: cert, key: key, getPostURL: (wtrealm: any, wreply: any, req: any, callback: any) => { @@ -126,7 +128,7 @@ export class SamlProvider { try { app.get("/issue/", samlp.auth(samlpoptions)); app.get("/issue/FederationMetadata/2007-06/FederationMetadata.xml", samlp.metadata({ - issuer: Config.saml_issuer, + issuer: saml_issuer, cert: cert, })); } catch (error) { @@ -190,8 +192,10 @@ export class SamlProvider { }); app.post('/logout', (req: any, res: any, next: any): void => { if(cert != null && cert != "") { + let saml_issuer: string = Config.saml_issuer; + if(saml_issuer == null || saml_issuer == "") saml_issuer = "uri:" + Config.domain; samlp.logout({ - issuer: Config.saml_issuer, + issuer: saml_issuer, protocolBinding: 'urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST', cert: cert, key: key diff --git a/OpenFlow/src/WebServer.ts b/OpenFlow/src/WebServer.ts index 71258da0..e3ab59e5 100644 --- a/OpenFlow/src/WebServer.ts +++ b/OpenFlow/src/WebServer.ts @@ -385,6 +385,35 @@ export class WebServer { }); }); } + + public static async ProcessMessage(req: any, tuser: TokenUser, jwt: string): Promise { + const client:any = {user: tuser, jwt: jwt}; + const msg = new Message(); + const urlPath = req.path; + msg.command = urlPath.replace("/rest/v1/", "").toLowerCase(); + + if(msg.command == "updatedocument") { + msg.command = "updatemany" // new command to new + } + if(msg.command == "unregisterqueue") { + msg.command = "closequeue" // new command to new + } + if(msg.command == "pushworkitem") { + msg.command = "addworkitem" // new command to new + } + if(msg.command == "pushworkitems") { + msg.command = "addworkitems" // new command to new + } + + msg.id = NoderedUtil.GetUniqueIdentifier(); + msg.jwt = jwt; + msg.data = req.body; + msg.tuser = tuser; + msg.clientagent = req.headers["user-agent"]; + msg.clientversion = "0.0.1"; + var result = await msg.Process(client); + return result + } public static async onMessage(client: flowclient, message: any) { let command, msg, reply; try { @@ -719,18 +748,6 @@ export class WebServer { array.push(new Array(10000000).join('x')); await new Promise(resolve => { setTimeout(resolve, 1000) }); } - // let buffer = []; - // const MB = (bytes) => Math.round(bytes/1024/1024) + 'MB' - // const memoryUsage = () => { - // const mem = process.memoryUsage(); - // return MB(mem.rss) + '\t' + MB(mem.heapTotal) + '\t' + MB(mem.external); - // } - // setInterval(()=>{ - // buffer.push(Buffer.alloc(1024 * 1024* 1024)); // Eat 1GB of RAM every second - // console.log(buffer.length + '\t' + memoryUsage()); - // }, 1000); - res.end(JSON.stringify({ "success": "true", "message": "Ok here we go, crash incomming!!!!", "remoteip": remoteip, "hostname": _hostname, dt: new Date() })); - res.end(); } static async get_heapdump(req: any, res: any, next: any): Promise { @@ -748,13 +765,13 @@ export class WebServer { let span = Logger.otel.startSpanExpress("get_livenessprobe", req) try { const [traceId, spanId] = Logger.otel.GetTraceSpanId(span); - if (NoderedUtil.IsNullEmpty(_hostname)) _hostname = (Config.getEnv("HOSTNAME", undefined) || os.hostname()) || "unknown"; + if (NoderedUtil.IsNullEmpty(_hostname)) _hostname = (process.env.HOSTNAME || os.hostname()) || "unknown"; res.end(JSON.stringify({ "success": "true", "hostname": _hostname, dt: new Date(), traceId, spanId })); res.end(); // @ts-ignore span.setStatus({ code: 200 }); } catch (error) { - console.error(error); + Logger.instanse.error(error, span); // @ts-ignore span.setStatus({code: 500, message: error instanceof Error ? error.message : undefined, }); diff --git a/OpenFlow/src/WebSocketServer.ts b/OpenFlow/src/WebSocketServer.ts index 5d1e7696..1c92b2bb 100644 --- a/OpenFlow/src/WebSocketServer.ts +++ b/OpenFlow/src/WebSocketServer.ts @@ -219,7 +219,7 @@ export class WebSocketServer { try { WebSocketServer._remoteclients = []; - const hostname = (Config.getEnv("HOSTNAME", undefined) || os.hostname()) || "unknown"; + const hostname = (process.env.HOSTNAME || os.hostname()) || "unknown"; const clients: Base[] = []; for (let i = WebSocketServer._clients.length - 1; i >= 0; i--) { const cli: WebSocketServerClient = WebSocketServer._clients[i]; @@ -386,16 +386,17 @@ export class WebSocketServer { } } - if (bulkUpdates.length > 0) { + // seconds since lastUserUpdate + const seconds: number = (Date.now() - this.lastUserUpdate) / 1000; + + if (bulkUpdates.length > 0 && Config.enable_openflow_amqp) { + amqpwrapper.Instance().send("openflow", "", { "command": "dumpwebsocketclients" }, 10000, null, "", span, 1); + } + if (bulkUpdates.length > 0 && seconds > 60) { this.lastUserUpdate = Date.now(); let ot_end: any = Logger.otel.startTimer(); var bulkresult = await Config.db.db.collection("users").bulkWrite(bulkUpdates); Logger.otel.endTimer(ot_end, DatabaseConnection.mongodb_updatemany, { collection: "users" }); - if (Config.enable_openflow_amqp) { - amqpwrapper.Instance().send("openflow", "", { "command": "dumpwebsocketclients" }, 10000, null, "", span, 1); - } else { - // WebSocketServer.DumpClients(span); - } } } catch (error) { Logger.instanse.error(error, span); diff --git a/OpenFlow/src/WebSocketServerClient.ts b/OpenFlow/src/WebSocketServerClient.ts index 51a75790..d9540461 100644 --- a/OpenFlow/src/WebSocketServerClient.ts +++ b/OpenFlow/src/WebSocketServerClient.ts @@ -550,7 +550,7 @@ export class WebSocketServerClient { result.priority = first.priority; if (result.command != "ping" && result.command != "pong") { result.Process(this).then(msg=> { - this.Send(msg); + if(msg != null) this.Send(msg); }) .catch((error) => { Logger.instanse.error(error, span, Logger.parsecli(this)); }); @@ -711,7 +711,7 @@ export class WebSocketServerClient { } }); } catch (error) { - console.error(error); + Logger.instanse.error(error, span); } finally { subspan?.end(); } diff --git a/OpenFlow/src/dockerdriver.ts b/OpenFlow/src/dockerdriver.ts index 17e827bd..d533851f 100644 --- a/OpenFlow/src/dockerdriver.ts +++ b/OpenFlow/src/dockerdriver.ts @@ -38,29 +38,9 @@ export class dockerdriver implements i_agent_driver { } if(pull) { - console.log("Pull image " + imagename) + Logger.instanse.info("Pull image " + imagename, span); await docker.pull(imagename) } - // return new Promise((resolve, reject) => { - // docker.pull(imagename, function (err, stream) { - // if (err) - // return reject(err); - - // docker.modem.followProgress(stream, onFinished, onProgress); - - // function onFinished(err2, output) { - // Logger.instanse.debug(output, span); - // if (err2) { - // Logger.instanse.error(err2, null); - // return reject(err2); - // } - // return resolve(); - // } - // function onProgress(event) { - // Logger.instanse.debug(event, span); - // } - // }); - // }) } public async NodeLabels(parent: Span): Promise { return null; diff --git a/OpenFlow/src/index.ts b/OpenFlow/src/index.ts index f4446d13..903cbd11 100644 --- a/OpenFlow/src/index.ts +++ b/OpenFlow/src/index.ts @@ -3,6 +3,11 @@ function clog(message) { let dts: string = dt.getHours() + ":" + dt.getMinutes() + ":" + dt.getSeconds() + "." + dt.getMilliseconds(); console.log(dts + " " + message); } +function cerror(error) { + let dt = new Date(); + let dts: string = dt.getHours() + ":" + dt.getMinutes() + ":" + dt.getSeconds() + "." + dt.getMilliseconds(); + console.error(dts, error.message ? error.message : error); +} clog("Starting @openiap/openflow"); import { Logger } from "./Logger"; import * as http from "http"; @@ -72,7 +77,7 @@ async function initDatabase(parent: Span): Promise { Logger.instanse.info("Begin validating builtin roles", span); const jwt: string = Crypt.rootToken(); const rootuser = Crypt.rootUser(); - Config.dbConfig = await dbConfig.Load(jwt, span); + Config.dbConfig = await dbConfig.Load(jwt, false, span); try { var lic = Logger.License; await lic?.validate(); @@ -82,7 +87,7 @@ async function initDatabase(parent: Span): Promise { try { await Logger.configure(false, true); } catch (error) { - console.error(error); + cerror(error); process.exit(404); } @@ -422,7 +427,7 @@ let OpenAIProxy: any = null; try { OpenAIProxy = require("./ee/OpenAIProxy"); } catch (error) { - + cerror(error); } @@ -440,7 +445,7 @@ var server: http.Server = null; try { await Logger.configure(false, false); } catch (error) { - console.error(error); + cerror(error); process.exit(404); } Config.db = new DatabaseConnection(Config.mongodb_url, Config.mongodb_db, true); diff --git a/OpenFlow/src/proto/client.ts b/OpenFlow/src/proto/client.ts index 799a93e8..1fc27138 100644 --- a/OpenFlow/src/proto/client.ts +++ b/OpenFlow/src/proto/client.ts @@ -150,7 +150,7 @@ export class flowclient extends client { var paylad = {"command": "watchevent", "data": data} protowrap.sendMesssag(this, paylad, null, true); } catch (error) { - console.error(error); + Logger.instanse.error(error, span); } finally { } } @@ -226,7 +226,6 @@ export class flowclient extends client { this._queuescounterstr = this._queuescounter.toString(); this._queuescurrentstr = this._queuescurrent.toString(); this._queues.push(queue); - console.log(this.id + " has " + this._queues.length + " queues") } } finally { if (queue) semaphore.up(); @@ -336,7 +335,6 @@ export class flowclient extends client { } finally { semaphore.up(); Logger.otel.endSpan(span); - console.log(this.id + " has " + this._queues.length + " queues") } } public async CloseConsumers(parent: Span): Promise { diff --git a/OpenFlow/src/public/Agents.html b/OpenFlow/src/public/Agents.html index c4670c71..80936e62 100644 --- a/OpenFlow/src/public/Agents.html +++ b/OpenFlow/src/public/Agents.html @@ -1,3 +1,13 @@ + @@ -10,17 +20,20 @@

agents

reload - - + + + + + - - + + - - + + - - + + packages addagent @@ -33,7 +46,9 @@

agents

name image os - arch + arch + $$ + createdby status @@ -45,12 +60,14 @@

agents

{{model.name | limitTo: 20}} {{model.name | limitTo: 20}} - {{ctrl.trimimage(model.image)}} - {{model.os | limitTo: 20}} - {{model.arch | limitTo: 20}} + {{ctrl.trimimage(model.image) }} + {{model.os }} + {{model.arch}} +
*
{{model._createdby | limitTo: 20}} {{model.status | limitTo: 20}} +
-
Click on message to copy log object with all properties to clipboard +
Click on message to copy log object with all properties to clipboard Config page
search
diff --git a/OpenFlow/src/public/Controllers.ts b/OpenFlow/src/public/Controllers.ts index 5c5ade78..3c3ce75b 100644 --- a/OpenFlow/src/public/Controllers.ts +++ b/OpenFlow/src/public/Controllers.ts @@ -1320,7 +1320,6 @@ export class RPAWorkflowCtrl extends entityCtrl { this.queuename = await NoderedUtil.RegisterQueue({ callback: (data: QueueMessage, ack: any) => { ack(); - console.debug(data); if (data.data.command == undefined && data.data.data != null) data.data = data.data.data; this.messages += data.data.command + "\n"; if (data.data.command == "invokecompleted") { @@ -1341,7 +1340,6 @@ export class RPAWorkflowCtrl extends entityCtrl { setTimeout(this.RegisterQueue.bind(this), (Math.floor(Math.random() * 6) + 1) * 500); } }); - console.debug("queuename: " + this.queuename); } catch (error) { this.queuename = ""; console.debug("register queue failed, start reconnect. " + error.message ? error.message : error) @@ -1627,7 +1625,6 @@ export class ReportsCtrl extends entitiesCtrl { this.userdata.data.ReportsCtrl.run(this.userdata.data.ReportsCtrl.points); } async processData(): Promise { - console.debug('processData'); this.userdata.data.ReportsCtrl.run = this.processData.bind(this); this.userdata.data.ReportsCtrl.points = null; this.loading = true; @@ -1696,7 +1693,6 @@ export class ReportsCtrl extends entitiesCtrl { if (!this.$scope.$$phase) { this.$scope.$apply(); } } async robotsclick(points, evt): Promise { - console.debug('robotsclick'); this.userdata.data.ReportsCtrl.run = this.robotsclick.bind(this); this.userdata.data.ReportsCtrl.points = points; if (points.length > 0) { @@ -1823,7 +1819,6 @@ export class ReportsCtrl extends entitiesCtrl { } async robotclick(points, evt): Promise { - console.debug('robotclick'); if (points.length > 0) { } else { return; } const userid = this.charts[0].ids[points[0]._index]; @@ -1860,7 +1855,6 @@ export class ReportsCtrl extends entitiesCtrl { } async workflowclick(points, evt): Promise { - console.debug('workflowclick'); if (points.length > 0) { } else { return; } @@ -2047,7 +2041,6 @@ export class LoginCtrl { fs.root.getFile(filename, { create: true }, function (fileEntry) { fileEntry.createWriter(function (fileWriter) { fileWriter.onwriteend = function (e) { - console.debug('Write completed.'); resolve(); }; fileWriter.onerror = function (e) { @@ -2068,7 +2061,6 @@ export class LoginCtrl { scanForQRScanner() { try { if (QRScanner !== undefined) { - console.debug("Found QRScanner!!!!"); this.qrcodescan = true; if (!this.$scope.$$phase) { this.$scope.$apply(); } } else { @@ -2082,7 +2074,6 @@ export class LoginCtrl { } Scan() { try { - console.debug("Scan"); if (this.scanning) { this.scanning = false; QRScanner.destroy(); @@ -2100,12 +2091,10 @@ export class LoginCtrl { } async QRScannerHit(err, value) { try { - console.debug("QRScannerHit"); if (err) { console.error(err); return; } - console.debug(value); QRScanner.hide(); QRScanner.destroy(); @@ -2117,7 +2106,6 @@ export class LoginCtrl { console.debug("QRCode value: " + value); const config = JSON.parse(value); if (config.url !== null || config.url !== undefined || config.url !== "" || config.loginurl !== null || config.loginurl !== undefined || config.loginurl !== "") { - console.debug("set mobiledomain to " + value); await this.writefile("mobiledomain.txt", value); window.location.replace(config.url); } @@ -2133,7 +2121,6 @@ export class LoginCtrl { async submit(): Promise { this.message = ""; try { - console.debug("signing in with username/password"); const result: SigninMessage = await NoderedUtil.SigninWithUsername({ username: this.username, password: this.password }); if (result.user == null) { return; } this.setCookie("jwt", result.jwt, 365); @@ -2313,7 +2300,6 @@ export class UsersCtrl extends entitiesCtrl { haderror = true; } if (haderror) { - console.debug("loading stripe script") await jsutil.loadScript('//js.stripe.com/v3/'); this.stripe = Stripe(this.WebSocketClientService.stripe_api_key); } @@ -2621,12 +2607,10 @@ export class UserCtrl extends entityCtrl { if (roles.length > 0) { role = roles[0]; if (role.members === null || role.members === undefined) { - console.log("role.members is null", role); continue; } const exists = role.members.filter(x => x._id == this.model._id); if (exists.length > 0) { - console.log("Updating role", role.name); role.members = role.members.filter(x => x._id != this.model._id); try { await NoderedUtil.UpdateOne({ collectionname: "users", item: role }); @@ -2936,23 +2920,19 @@ export class FilesCtrl extends entitiesCtrl { const xhr = new XMLHttpRequest(); xhr.onload = () => { if (xhr.status >= 200 && xhr.status < 300) { - console.debug("upload complete"); // we done! if (!this.$scope.$$phase) { this.$scope.$apply(); } this.loadData(); } }; - console.debug("open"); xhr.open('POST', '/upload', true); - console.debug("send"); xhr.send(fd); } async Upload_usingapi() { try { const filename = (this.$scope as any).filename; const mimeType = (this.$scope as any).type; - console.debug("filename: " + filename + " mimeType: " + mimeType); this.loading = true; if (!this.$scope.$$phase) { this.$scope.$apply(); } const lastp: number = 0; @@ -3006,7 +2986,6 @@ export class EntitiesCtrl extends entitiesCtrl { return; } } - console.debug("path: " + this.$location.path()); if (NoderedUtil.IsNullEmpty(this.collection)) { this.$location.path("/Entities/entities"); if (!this.$scope.$$phase) { this.$scope.$apply(); } @@ -3424,7 +3403,6 @@ export class FormCtrl extends entityCtrl { WebSocketClientService.onSignedin(async (user: TokenUser) => { await jsutil.ensureJQuery(); await this.RegisterQueue(); - console.debug("queuename: " + this.queuename); if (this.id !== null && this.id !== undefined && this.id !== "") { this.basequery = { _id: this.id }; this.loadData(); @@ -3441,7 +3419,6 @@ export class FormCtrl extends entityCtrl { const result = await NoderedUtil.RegisterExchange( { exchangename: exchange, algorithm: "direct", callback: async (msg: QueueMessage, ack: any) => { - console.debug(msg); ack(); if (NoderedUtil.IsNullEmpty(msg.routingkey) || msg.routingkey == this.instanceid) { // this.loadData(); @@ -3468,14 +3445,11 @@ export class FormCtrl extends entityCtrl { } }); this.localexchangequeue = result.queuename; - console.debug("Register exchange for " + exchange + " with queue " + this.localexchangequeue); - } async RegisterQueue() { this.queuename = await NoderedUtil.RegisterQueue({ callback: (data: QueueMessage, ack: any) => { ack(); - console.debug(data); if (data.queuename == this.queuename) { if (data && data.data && data.data.command == "timeout") { this.errormessage = "No \"workflow in\" node listening or message timed out, is nodered running?"; @@ -3510,8 +3484,6 @@ export class FormCtrl extends entityCtrl { }); } async hideFormElements() { - console.debug("hideFormElements"); - $('input[ref="component"]').prop("disabled", true); $('#workflowform :input').prop("disabled", true); $('#workflowform :button').prop("disabled", true); @@ -3556,7 +3528,6 @@ export class FormCtrl extends entityCtrl { if (this.model.form === "none" || this.model.form === "") { - console.debug("workflow_instances has no form set, state " + this.model.state); if (this.model.state != "failed") { this.$location.path("/main"); } else { @@ -3580,7 +3551,6 @@ export class FormCtrl extends entityCtrl { if (!this.$scope.$$phase) { this.$scope.$apply(); } return; } else if (this.model.form === "unknown") { - console.debug("Form is unknown for instance, send empty message"); this.Save(); return; } else if (this.model.form !== "") { @@ -3601,8 +3571,6 @@ export class FormCtrl extends entityCtrl { this.renderform(); } else { try { - console.debug("No instance id found, send empty message"); - console.debug("SendOne: " + this.workflow._id + " / " + this.workflow.queue); await this.SendOne(this.workflow.queue, {}); } catch (error) { this.errormessage = error.message ? error.message : error; @@ -3691,14 +3659,10 @@ export class FormCtrl extends entityCtrl { for (let x = 0; x < item.components.length; x++) { obj[item.components[x].key] = ""; } - console.debug("add default array for " + item.key, obj); this.model.payload[item.key] = [obj]; } else { - console.debug("payload already have values for " + item.key); - console.debug("isArray: " + Array.isArray(this.model.payload[item.key])) if (Array.isArray(this.model.payload[item.key])) { } else { - console.debug("convert payload for " + item.key + " from object to array"); const keys = Object.keys(this.model.payload[item.key]); const arr: any[] = []; for (let x = 0; x < keys.length; x++) { @@ -3726,7 +3690,6 @@ export class FormCtrl extends entityCtrl { const item = components[y]; if (item.key == keys[i]) { if (Array.isArray(values)) { - console.debug("handle " + item.key + " as array"); const obj2: any = {}; for (let x = 0; x < values.length; x++) { obj2[x] = values[x]; @@ -3738,7 +3701,6 @@ export class FormCtrl extends entityCtrl { item.values = values; } } else { - console.debug("handle " + item.key + " as an object"); if (item.data != null && item.data != undefined) { item.data.values = values; item.data.json = JSON.stringify(values); @@ -3788,7 +3750,6 @@ export class FormCtrl extends entityCtrl { if ((this.form.fbeditor as any) == "false") this.form.fbeditor = false; if (!this.$scope.$$phase) { this.$scope.$apply(); } if (this.form.fbeditor === true) { - console.debug("renderform fbeditor"); const roles: any = {}; WebSocketClient.instance.user.roles.forEach(role => { roles[role._id] = role.name; @@ -3860,7 +3821,6 @@ export class FormCtrl extends entityCtrl { } this.formRender = ele.formRender(formRenderOpts); } else { - console.debug("renderform formio", this.form.schema?.components); if (!this.form.schema || !this.form.schema.components || this.form.schema.components.length == 0) { if (this.form.formData && this.form.formData.components && this.form.formData.components.length > 0) { console.warn("schema has no components, but forData does, using form formData.components instead") @@ -4180,8 +4140,6 @@ export class EntityCtrl extends entityCtrl { if (idx <= 0) { idx = 0; } else { idx--; } - console.debug("idx: " + idx); - // this.searchtext = this.searchFilteredList[idx].name; this.searchSelectedItem = this.searchFilteredList[idx]; return; } @@ -4189,8 +4147,6 @@ export class EntityCtrl extends entityCtrl { if (idx >= this.searchFilteredList.length) { idx = this.searchFilteredList.length - 1; } else { idx++; } - console.debug("idx: " + idx); - // this.searchtext = this.searchFilteredList[idx].name; this.searchSelectedItem = this.searchFilteredList[idx]; return; } @@ -4554,7 +4510,6 @@ export class ClientsCtrl extends entitiesCtrl { let name = model.username; name = name.toLowerCase(); name = name.replace(/([^a-z0-9]+){1,63}/gi, ""); - // const noderedurl = "//" + this.WebSocketClientService.nodered_domain_schema.replace("$nodered_id$", name); const noderedurl = "//" + this.WebSocketClientService.agent_domain_schema.replace("$slug$", name); window.open(noderedurl); } @@ -4593,8 +4548,6 @@ export class AuditlogsCtrl extends entitiesCtrl { this.searchfields = ["name", "impostorname", "clientagent", "type"]; console.debug("AuditlogsCtrl"); this.pagesize = 20; - // this.basequery = { _type: "role" }; - // this.orderby = { "_created": -1 }; this.collection = "audit"; this.postloadData = this.processdata; WebSocketClientService.onSignedin(async (user: TokenUser) => { @@ -4633,6 +4586,16 @@ export class AuditlogsCtrl extends entitiesCtrl { this.loading = false; if (!this.$scope.$$phase) { this.$scope.$apply(); } } + ToogleType(logtype) { + if(logtype == null || logtype == "") { + this.basequery = {} + } else { + this.basequery['type'] = logtype; + } + this.page = 0; + this.loading = false; + this.loadData() + } ToggleModal() { var modal = document.getElementById("exampleModal"); modal.classList.toggle("show"); @@ -4707,7 +4670,6 @@ export class CredentialsCtrl extends entitiesCtrl { var query = { _type: "role", "$or": [{ name: name + "noderedadmins" }, { name: name + "nodered api users" }] } const list = await NoderedUtil.Query({ collectionname: "users", query, top: 4 }); for (var i = 0; i < list.length; i++) { - console.debug("Deleting " + list[i].name) await NoderedUtil.DeleteOne({ collectionname: "users", id: list[i]._id }); } @@ -4836,8 +4798,6 @@ export class CredentialCtrl extends entityCtrl { if (idx <= 0) { idx = 0; } else { idx--; } - console.debug("idx: " + idx); - // this.searchtext = this.searchFilteredList[idx].name; this.searchSelectedItem = this.searchFilteredList[idx]; return; } @@ -4845,8 +4805,6 @@ export class CredentialCtrl extends entityCtrl { if (idx >= this.searchFilteredList.length) { idx = this.searchFilteredList.length - 1; } else { idx++; } - console.debug("idx: " + idx); - // this.searchtext = this.searchFilteredList[idx].name; this.searchSelectedItem = this.searchFilteredList[idx]; return; } @@ -5349,7 +5307,6 @@ export class DeletedCtrl extends entitiesCtrl { return; } } - console.debug("path: " + this.$location.path()); if (NoderedUtil.IsNullEmpty(this.collection)) { this.$location.path("/Deleted/entities"); if (!this.$scope.$$phase) { this.$scope.$apply(); } @@ -5502,13 +5459,11 @@ export class CustomerCtrl extends entityCtrl { haderror = true; } if (haderror) { - console.debug("loading stripe script") await jsutil.loadScript('//js.stripe.com/v3/'); this.stripe = Stripe(this.WebSocketClientService.stripe_api_key); } } if (this.id !== null && this.id !== undefined && this.id != "new") { - console.debug("Loading customer id " + this.id); this.loading = false; this.loadData(); return; @@ -5517,7 +5472,6 @@ export class CustomerCtrl extends entityCtrl { if (user.customerid != null && this.id != "new") { this.id = user.customerid; this.basequery = { _id: this.id }; - console.debug("Loading customer id " + this.id); this.loading = false; this.loadData(); return; @@ -5530,7 +5484,6 @@ export class CustomerCtrl extends entityCtrl { }); if (results.length > 0) { this.model = results[0]; - console.debug("Loaded customer " + this.model._id); } } @@ -5559,7 +5512,6 @@ export class CustomerCtrl extends entityCtrl { if (this.model.email && this.model.email.indexOf("@") == -1) { this.model.email = (WebSocketClient.instance.user as any).username + "@domain.com"; } - console.debug("Create new customer"); } this.loading = false; @@ -5608,11 +5560,8 @@ export class CustomerCtrl extends entityCtrl { async processdata() { try { if (this.model._type != "customer") { - console.log("Not customer!", this.model); return; } - console.log(this.model); - console.debug("processdata"); this.loading = true; if (!this.$scope.$$phase) { this.$scope.$apply(); } this.errormessage = ""; @@ -5629,9 +5578,7 @@ export class CustomerCtrl extends entityCtrl { await NoderedUtil.EnsureCustomer({ customer: this.model }); } this.Resources = await NoderedUtil.Query({ collectionname: "config", query: { "_type": "resource", "target": "customer", "allowdirectassign": true }, orderby: { _created: -1 } }); - console.debug("Resources", this.Resources); this.Assigned = await NoderedUtil.Query({ collectionname: "config", query: { "_type": "resourceusage", "customerid": this.model._id, "userid": { "$exists": false } }, orderby: { _created: -1 } }); - console.debug("Assigned", this.Assigned); for (var res of this.Resources) { res.products = res.products.filter(x => x.allowdirectassign == true); for (var prod of res.products) { @@ -5665,8 +5612,6 @@ export class CustomerCtrl extends entityCtrl { } } } - console.debug("Assigned", this.Assigned); - console.debug("UserAssigned", this.UserAssigned); this.support = []; this.licenses = []; for (let a of this.Assigned) { @@ -5682,7 +5627,6 @@ export class CustomerCtrl extends entityCtrl { this.errormessage = error; } this.loading = false; - console.debug("processdata::end"); if (!this.$scope.$$phase) { this.$scope.$apply(); } } ToggleNextInvoiceModal() { @@ -5882,7 +5826,6 @@ export class CustomerCtrl extends entityCtrl { } this.domain = domain; const res:string = await NoderedUtil.CustomCommand({ command: "issuelicense", data: payload }); - console.log(res); // @ts-ignore this.licensekey = res; this.licensekeydecoded = atob(res); @@ -6066,10 +6009,8 @@ export class EntityRestrictionsCtrl extends entitiesCtrl { (model as any).collection = collection; (model as any).paths = paths; if (model._id) { - console.debug("updating " + name); await NoderedUtil.UpdateOne({ collectionname: this.collection, item: model }); } else { - console.debug("adding " + name); await NoderedUtil.InsertOne({ collectionname: this.collection, item: model }); } } @@ -6217,7 +6158,6 @@ export class EntityRestrictionCtrl extends entityCtrl { if (idx <= 0) { idx = 0; } else { idx--; } - console.debug("idx: " + idx); this.searchSelectedItem = this.searchFilteredList[idx]; return; } @@ -6225,7 +6165,6 @@ export class EntityRestrictionCtrl extends entityCtrl { if (idx >= this.searchFilteredList.length) { idx = this.searchFilteredList.length - 1; } else { idx++; } - console.debug("idx: " + idx); this.searchSelectedItem = this.searchFilteredList[idx]; return; } @@ -6509,10 +6448,8 @@ export class ResourcesCtrl extends entitiesCtrl { Base.addRight(model, "5a17f157c4815318c8536c21", "users", [2]); } if (model._id) { - console.debug("updating " + name); return await NoderedUtil.UpdateOne({ collectionname: this.collection, item: model }); } else { - console.debug("adding " + name); return await NoderedUtil.InsertOne({ collectionname: this.collection, item: model }); } } @@ -6613,11 +6550,10 @@ export class WorkitemsCtrl extends entitiesCtrl { this.basequery = { _type: "workitem", "wiq": this.queue }; } if (!this.$scope.$$phase) { this.$scope.$apply(); } - console.log(this.workitemqueues); - console.log(this.queue); this.loadData(); } SetState(state) { + this.page = 0; if (state == "" || state == null) { delete this.basequery["state"]; } else { @@ -6928,7 +6864,6 @@ export class WorkitemQueueCtrl extends entityCtrl { this.projects.unshift({ "_id": "", "name": "", "display": "(no project)" } as any); let queryas: string = null; if (this.model != null) queryas = this.model.robotqueue; - console.log("queryas", queryas) this.workflows = await NoderedUtil.Query({ collectionname: "openrpa", query: { "_type": "workflow" }, projection: { "name": 1, "projectandname": 1 }, orderby: "name", top: 500, queryas }); this.workflows.forEach((e: any) => { e.display = e.projectandname }); this.workflows.unshift({ "_id": "", "name": "", "projectandname": "", "display": "(no workflow)" } as any); @@ -6948,10 +6883,8 @@ export class WorkitemQueueCtrl extends entityCtrl { this.workitemqueues.unshift({ "_id": "", "name": "", "display": "(no workitem queue)" } as any); this.agents = await NoderedUtil.Query({ collectionname: "agents", query: { "_type": "agent" }, orderby: "name", projection: { "slug": 1, "name": 1 } }); - console.log(this.agents); this.agents.unshift({ "name": "" } as any) this.packages = await NoderedUtil.Query({ collectionname: "agents", query: { "_type": "package" }, orderby: "name", projection: { "name": 1 } }); - console.log(this.packages); this.packages.unshift({ "name": "" } as any) if (!this.$scope.$$phase) { this.$scope.$apply(); } @@ -7281,7 +7214,6 @@ export class ConsoleCtrl extends entityCtrl { if (this.messages.length >= lines) this.messages.splice(lines - 1); if (!this.$scope.$$phase) { this.$scope.$apply(); } }, closedcallback: (msg) => { - console.debug("rabbitmq disconnected, start reconnect") setTimeout(this.RegisterQueue.bind(this), (Math.floor(Math.random() * 6) + 1) * 500); } }); @@ -7290,12 +7222,9 @@ export class ConsoleCtrl extends entityCtrl { } this.watchid = await NoderedUtil.Watch({ aggregates: [{ "$match": { "fullDocument._type": "config" } }], collectionname: "config", callback: (data) => { - console.log(data); this.loadData(); } }) - console.debug("exchange: ", this.exchange); - console.debug("watchid: ", this.watchid); } catch (error) { console.debug("register queue failed, start reconnect. " + error.message ? error.message : error) setTimeout(this.RegisterQueue.bind(this), (Math.floor(Math.random() * 6) + 1) * 500); @@ -7389,6 +7318,7 @@ export class ConfigCtrl extends entityCtrl { public func: boolean = true; public searchstring: string = ""; public settings: any[] = []; + public show: "all" | "set" | "unset" = "set"; constructor( public $rootScope: ng.IRootScopeService, public $scope: ng.IScope, @@ -7407,12 +7337,19 @@ export class ConfigCtrl extends entityCtrl { this.messages = []; this.postloadData = this.processdata; this.settings = [ + // {"name": "supports_watch", "type": "boolean", "default": "false"}, + // {"name": "NODE_ENV", "type": "string", "default": "development"}, + // {"name": "log_to_exchange", "type": "boolean", "default": "false"}, // called straming handled elsewere + // {"name": "aes_secret", "type": "string", "default": ""}, // ONLY envoriment variable + // {"name": "signing_crt", "type": "string", "default": ""}, // ONLY envoriment variable + // {"name": "singing_key", "type": "string", "default": ""}, // ONLY envoriment variable {"name": "license_key", "type": "string", "default": ""}, {"name": "enable_openai", "type": "boolean", "default": "false"}, {"name": "enable_openapi", "type": "boolean", "default": "true"}, - {"name": "enable_openaiauth", "type": "boolean", "default": "true"}, + {"name": "enable_openapiauth", "type": "boolean", "default": "true"}, {"name": "openai_token", "type": "string", "default": ""}, {"name": "log_with_colors", "type": "boolean", "default": "true"}, + {"name": "log_database_queries_to_collection", "type": "string", "default": ""}, {"name": "cache_store_type", "type": "string", "default": "memory"}, {"name": "cache_store_max", "type": "number", "default": "1000"}, {"name": "cache_store_ttl_seconds", "type": "number", "default": "300"}, @@ -7423,8 +7360,7 @@ export class ConfigCtrl extends entityCtrl { {"name": "heapdump_onstop", "type": "boolean", "default": "false"}, {"name": "amqp_allow_replyto_empty_queuename", "type": "boolean", "default": "false"}, {"name": "enable_openflow_amqp", "type": "boolean", "default": "false"}, - // {"name": "log_to_exchange", "type": "boolean", "default": "false"}, - {"name": "openflow_amqp_expiration", "type": "number", "default": "1500000"}, // 25 min in milliseconds + {"name": "openflow_amqp_expiration", "type": "number", "default": "1500000"}, {"name": "amqp_prefetch", "type": "number", "default": "25"}, {"name": "enable_entity_restriction", "type": "boolean", "default": "false"}, {"name": "enable_web_tours", "type": "boolean", "default": "true"}, @@ -7432,11 +7368,12 @@ export class ConfigCtrl extends entityCtrl { {"name": "grafana_url", "type": "string", "default": ""}, {"name": "auto_hourly_housekeeping", "type": "boolean", "default": "true"}, {"name": "housekeeping_skip_collections", "type": "string", "default": ""}, + {"name": "housekeeping_remomve_unvalidated_user_days", "type": "number", "default": "0"}, + {"name": "housekeeping_cleanup_openrpa_instances", "type": "boolean", "default": "false"}, {"name": "workitem_queue_monitoring_enabled", "type": "boolean", "default": "true"}, - {"name": "workitem_queue_monitoring_interval", "type": "number", "default": "10000"}, // 10 sec + {"name": "workitem_queue_monitoring_interval", "type": "number", "default": "10000"}, {"name": "upload_max_filesize_mb", "type": "number", "default": "25"}, {"name": "getting_started_url", "type": "string", "default": ""}, - {"name": "NODE_ENV", "type": "string", "default": "development"}, {"name": "HTTP_PROXY", "type": "string", "default": ""}, {"name": "HTTPS_PROXY", "type": "string", "default": ""}, {"name": "NO_PROXY", "type": "string", "default": ""}, @@ -7448,7 +7385,6 @@ export class ConfigCtrl extends entityCtrl { {"name": "stripe_force_vat", "type": "boolean", "default": "false"}, {"name": "stripe_force_checkout", "type": "boolean", "default": "false"}, {"name": "stripe_allow_promotion_codes", "type": "boolean", "default": "true"}, - {"name": "supports_watch", "type": "boolean", "default": "false"}, {"name": "ensure_indexes", "type": "boolean", "default": "true"}, {"name": "text_index_name_fields", "type": "string[]", "default": "name,_names"}, {"name": "auto_create_users", "type": "boolean", "default": "false"}, @@ -7458,7 +7394,7 @@ export class ConfigCtrl extends entityCtrl { {"name": "ping_clients_interval", "type": "number", "default": "10000"}, // 10 seconds {"name": "use_ingress_beta1_syntax", "type": "boolean", "default": "false"}, {"name": "use_openshift_routes", "type": "boolean", "default": "false"}, - {"name": "agent_image_pull_secrets", "type": "string[]", "default": ""}, + {"name": "agent_image_pull_secrets", "type": "string[]", "default": "[]"}, {"name": "auto_create_personal_nodered_group", "type": "boolean", "default": "false"}, {"name": "auto_create_personal_noderedapi_group", "type": "boolean", "default": "false"}, {"name": "force_add_admins", "type": "boolean", "default": "true"}, @@ -7507,12 +7443,12 @@ export class ConfigCtrl extends entityCtrl { {"name": "cleanup_on_delete_customer", "type": "boolean", "default": "false"}, {"name": "cleanup_on_delete_user", "type": "boolean", "default": "false"}, {"name": "api_bypass_perm_check", "type": "boolean", "default": "false"}, - {"name": "ignore_expiration", "type": "boolean", "default": "false"}, + {"name": "allow_signin_with_expired_jwt", "type": "boolean", "default": "false"}, {"name": "force_audit_ts", "type": "boolean", "default": "false"}, {"name": "force_dbusage_ts", "type": "boolean", "default": "false"}, {"name": "migrate_audit_to_ts", "type": "boolean", "default": "true"}, {"name": "websocket_package_size", "type": "number", "default": "25000"}, - {"name": "websocket_max_package_count", "type": "number", "default": "25000"}, + {"name": "websocket_max_package_count", "type": "number", "default": "1048576"}, {"name": "websocket_message_callback_timeout", "type": "number", "default": "3600"}, {"name": "websocket_disconnect_out_of_sync", "type": "boolean", "default": "false"}, {"name": "protocol", "type": "string", "default": "http"}, @@ -7543,14 +7479,11 @@ export class ConfigCtrl extends entityCtrl { {"name": "mongodb_minpoolsize", "type": "number", "default": "25"}, {"name": "mongodb_maxpoolsize", "type": "number", "default": "25"}, - {"name": "skip_history_collections", "type": "string", "default": "audit,openrpa_instances,workflow_instances"}, + {"name": "skip_history_collections", "type": "string", "default": "audit,oauthtokens,openrpa_instances,workflow_instances,workitems,mailhist"}, {"name": "history_delta_count", "type": "number", "default": "1000"}, {"name": "allow_skiphistory", "type": "boolean", "default": "false"}, {"name": "max_memory_restart_mb", "type": "number", "default": "0"}, {"name": "saml_issuer", "type": "string", "default": "the-issuer"}, - // {"name": "aes_secret", "type": "string", "default": ""}, - {"name": "signing_crt", "type": "string", "default": ""}, - {"name": "singing_key", "type": "string", "default": ""}, {"name": "wapid_mail", "type": "string", "default": ""}, {"name": "wapid_pub", "type": "string", "default": ""}, {"name": "wapid_key", "type": "string", "default": ""}, @@ -7572,23 +7505,15 @@ export class ConfigCtrl extends entityCtrl { {"name": "agent_oidc_token_endpoint", "type": "string", "default": ""}, {"name": "saml_federation_metadata", "type": "string", "default": ""}, {"name": "api_ws_url", "type": "string", "default": ""}, - {"name": "nodered_ws_url", "type": "string", "default": ""}, - {"name": "nodered_saml_entrypoint", "type": "string", "default": ""}, {"name": "agent_docker_entrypoints", "type": "string", "default": "web"}, {"name": "agent_docker_use_project", "type": "boolean", "default": "false"}, {"name": "agent_docker_certresolver", "type": "string", "default": ""}, {"name": "namespace", "type": "string", "default": ""}, - {"name": "nodered_domain_schema", "type": "string", "default": ""}, - {"name": "nodered_initial_liveness_delay", "type": "number", "default": "60"}, - {"name": "nodered_allow_nodeselector", "type": "boolean", "default": "false"}, - {"name": "nodered_liveness_failurethreshold", "type": "number", "default": "5"}, - {"name": "nodered_liveness_timeoutseconds", "type": "number", "default": "5"}, - {"name": "noderedcatalogues", "type": "string", "default": ""}, + {"name": "agent_allow_nodeselector", "type": "boolean", "default": "false"}, {"name": "otel_measure_nodeid", "type": "boolean", "default": "false"}, {"name": "otel_measure_queued_messages", "type": "boolean", "default": "false"}, {"name": "otel_measure__mongodb_watch", "type": "boolean", "default": "false"}, - {"name": "otel_measure_onlineuser", "type": "boolean", "default": "false"}, {"name": "enable_analytics", "type": "boolean", "default": "true"}, {"name": "enable_detailed_analytic", "type": "boolean", "default": "false"}, {"name": "otel_debug_log", "type": "boolean", "default": "false"}, @@ -7624,44 +7549,47 @@ export class ConfigCtrl extends entityCtrl { ] WebSocketClientService.onSignedin(async (_user: TokenUser) => { await this.RegisterWatch(); - console.log("Load data") this.loading = false; this.loadData(); this.$scope.$on('signin', (event, data) => { this.RegisterWatch(); - console.log("Load data") this.loading = false; this.loadData(); }); }); } + SetState(state: "all" | "set" | "unset") { + this.show = state; + if (!this.$scope.$$phase) { this.$scope.$apply(); } + } + Show(setting) { + if(this.show == "all") return true; + var isset = this.model[setting.name] != null; + if(this.show == "set" && isset) return true; + if(this.show == "unset" && !isset) return true; + return false; + } async RegisterWatch() { - console.log("RegisterWatch") try { if (!NoderedUtil.IsNullEmpty(this.watchid)) { await NoderedUtil.UnWatch({ id: this.watchid }); } this.watchid = await NoderedUtil.Watch({ aggregates: [{ "$match": { "fullDocument._type": "config" } }], collectionname: "config", callback: (data) => { - console.log(data); this.loading = false; this.loadData(); } }) - console.debug("watchid: ", this.watchid); } catch (error) { console.debug("register queue failed, start reconnect. " + error.message ? error.message : error) setTimeout(this.RegisterWatch.bind(this), (Math.floor(Math.random() * 6) + 1) * 500); } } + deletekey(key: string) { + delete this.model[key]; + this.submit(); + } processdata() { - console.log("processdata") - // console.log(this.model); - // for(let i = 0; i < this.settings.length; i++) { - // if (this.model[this.settings[i].name] == null) { - // this.model[this.settings[i].name] = this.settings[i].default; - // } - // } const ids: string[] = []; if (this.collection == "files") { for (let i: number = 0; i < (this.model as any).metadata._acl.length; i++) { @@ -7679,13 +7607,6 @@ export class ConfigCtrl extends entityCtrl { } async submit(): Promise { try { - for(let i = 0; i < this.settings.length; i++) { - if (this.model[this.settings[i].name] == null || this.model[this.settings[i].name] == "" - || this.model[this.settings[i].name] == this.settings[i].default) { - // delete this.model[this.settings[i].name]; - } - } - if (this.model._id) { await NoderedUtil.UpdateOne({ collectionname: this.collection, item: this.model }); } else { @@ -7739,9 +7660,7 @@ export class ConfigCtrl extends entityCtrl { delayedUpdate() { if(this.loading == true) return; if(this.delayhandler ! = null) return; - console.log("delayedUpdate", this.loading) this.delayhandler = setTimeout(() => { - console.log("do it") this.delayhandler = null; this.submit(); }, 500); @@ -7769,7 +7688,7 @@ export class AgentsCtrl extends entitiesCtrl { this.postloadData = this.processdata; this.searchfields = ["name", "slug"]; this.orderby = { "_modified": -1 } - this.baseprojection = { _type: 1, name: 1, _created: 1, _modified: 1, image: 1, webserver: 1, runas: 1, _createdby: 1, slug: 1, arch: 1, os:1 }; + this.baseprojection = { _type: 1, name: 1, _created: 1, _modified: 1, image: 1, stripeprice: 1, webserver: 1, runas: 1, _createdby: 1, slug: 1, arch: 1, os:1 }; if (this.userdata.data.AgentsCtrl) { this.basequery = this.userdata.data.AgentsCtrl.basequery; this.collection = this.userdata.data.AgentsCtrl.collection; @@ -7808,19 +7727,16 @@ export class AgentsCtrl extends entitiesCtrl { if(model.image == null) { var cli = this.clients.filter(x=> x.user?._id ==model.runas && (x.agent == "nodeagent" || x.agent == "assistant")); if(cli != null && cli.length > 0) { - console.log(cli[0], model); model.status = "online" } } else if(model.image.indexOf("nodered") > -1) { var cli = this.clients.filter(x=> x.user?._id ==model.runas && x.agent == "nodered"); if(cli != null && cli.length > 0) { - console.log(cli[0], model); model.status = "online" } } else if(model.image.indexOf("agent") > -1) { var cli = this.clients.filter(x=> x.user?._id ==model.runas && (x.agent == "nodeagent" || x.agent == "python") ); if(cli != null && cli.length > 0) { - console.log(cli[0], model); model.status = "online" } } @@ -7838,11 +7754,15 @@ export class AgentsCtrl extends entitiesCtrl { // } return image; } - preLoad() { - if (this.show != "all") { + async preLoad() { + if (this.show == "pods") { + this.knownpods = await NoderedUtil.CustomCommand({ command: "getagentpods" }) + const slugs = this.knownpods.map(x => x.metadata.labels.slug).filter(x => x != null); + this.basequery = { _type: "agent", slug: { $in: slugs } }; + } else if (this.show != "all" ) { this.basequery = { _type: "agent" }; this.basequery[this.show] = true; - }else { + } else { this.basequery = { _type: "agent" }; } } @@ -7857,6 +7777,16 @@ export class AgentsCtrl extends entitiesCtrl { this.userdata.data.AgentsCtrl.skipcustomerfilter = this.skipcustomerfilter; if (!this.$scope.$$phase) { this.$scope.$apply(); } + for (var i = 0; i < this.models.length; i++) { + var model = this.models[i]; + var user = await NoderedUtil.Query({ collectionname: "users", query: { _id: (model as any).runas }, top: 1 }); + if(user != null && user.length > 0) { + // @ts-ignore + model.customerid = user[0].customerid; + } + } + if (!this.$scope.$$phase) { this.$scope.$apply(); } + this.knownpods = await NoderedUtil.CustomCommand({ command: "getagentpods" }) this.clients = await NoderedUtil.CustomCommand({ "command": "getclients" }); @@ -7880,7 +7810,6 @@ export class AgentsCtrl extends entitiesCtrl { if (!this.$scope.$$phase) { this.$scope.$apply(); } } weburl(model) { - // return "//" + this.WebSocketClientService.nodered_domain_schema.replace("$nodered_id$", model.slug) return "//" + this.WebSocketClientService.agent_domain_schema.replace("$slug$", model.slug) } async DeleteAgent(model: any): Promise { @@ -8083,7 +8012,6 @@ export class AgentCtrl extends entityCtrl { } } async ImageUpdated() { - console.log("ImageUpdated") this.sizewarningtitle = "" this.sizewarning = "" @@ -8102,20 +8030,10 @@ export class AgentCtrl extends entityCtrl { } if (languages == null || languages.length == 0) { this.packages = []; - console.log("languages empty", this.packages) } else { - console.log("languages", languages[0]) this.packages = this.allpackages.filter(x => languages.indexOf(x.language) > -1) - console.log("filtered", this.packages) if (!haschromium && !haschrome) { this.packages = this.packages.filter(x => x.chrome != true && x.chromium != true) - console.log("filtered again", this.packages) - } - if (this.model._id == null || this.model._id == "") { - // this.model.package = ""; - // if ((this.model.package == null || this.model.package == "") && this.packages.length > 0) { - // this.model.package = this.packages[0]._id; - // } } this.packages.unshift({ _id: "", name: "None" }) } @@ -8198,8 +8116,8 @@ export class AgentCtrl extends entityCtrl { this.refreshtimer = setTimeout(() => { this.refreshtimer = null; var path = this.$location.path(); - if (path == null && path == undefined) { console.debug("getagent, path is null"); return false; } - if (!path.toLowerCase().startsWith("/agent/") && path.toLowerCase() != "/agent") { console.debug("getagent, path is no longer /Agent"); return false; } + if (path == null && path == undefined) { return false; } + if (!path.toLowerCase().startsWith("/agent/") && path.toLowerCase() != "/agent") { return false; } this.loadInstances(); }, 2000); } @@ -8410,12 +8328,9 @@ export class AgentCtrl extends entityCtrl { var tmp = await NoderedUtil.InsertOne({ collectionname: this.collection, item: this.model }); if (this.model) { this.model = tmp; - console.log("insertone", this.model) this.id = this.model._id this.basequery = { _id: this.id }; - console.log("startagent", this.model.slug) await NoderedUtil.CustomCommand({ command: "startagent", id: this.model._id, name: this.model.slug }) - console.log("load data") } } this.loading = false; @@ -8683,16 +8598,12 @@ export class PackageCtrl extends entityCtrl { const filename = e.files[0].name // @ts-ignore var result = await NoderedUtil.SaveFile({ filename, mimeType, file: buffer, compressed: false, metadata: { _type: "package" } }); - // console.log(result) - console.log(result.id) // @ts-ignore this.oldfileid = this.model.fileid; // @ts-ignore this.model.fileid = result.id; e.value = null; if (!this.$scope.$$phase) { this.$scope.$apply(); } - } else { - console.log("no files in form") } } catch (error) { @@ -8791,17 +8702,9 @@ export class RunPackageCtrl extends entityCtrl { this.haschromium = (_a.chromium == true) this.packages = this.allpackages.filter(x => this.languages.indexOf(x.language) > -1) - console.log("filtered", this.packages) if (!this.haschromium && !this.haschrome) { this.packages = this.packages.filter(x => x.chrome != true && x.chromium != true) - console.log("filtered again", this.packages) } - // if(this.packages.find(x => x._id == this.package) == null) { - // this.package = ""; - // } - // if(this.package == "" && this.packages.length > 0) { - // this.package = this.packages[0]._id; - // } if (!this.$scope.$$phase) { this.$scope.$apply(); } } async addprocess(streamid:string, schedulename:string = undefined): Promise { @@ -8842,7 +8745,6 @@ export class RunPackageCtrl extends entityCtrl { killbutton.id = streamid + "_kill"; killbutton.onclick = async () => { try { - console.log("kill", streamid) await NoderedUtil.Queue({ data: { command: "kill", "id": streamid }, queuename: _a.slug + "agent", correlationId: streamid }) } catch (error) { console.error(error); @@ -8859,7 +8761,6 @@ export class RunPackageCtrl extends entityCtrl { runs.prepend(div); await NoderedUtil.Queue({ data: payload, queuename: _a.slug + "agent", correlationId: streamid }) - console.log("submitted", payload) } async submit(): Promise { var _a = this.agents.find(x => x._id == this.id); @@ -8872,20 +8773,17 @@ export class RunPackageCtrl extends entityCtrl { "queuename": this.queuename } await NoderedUtil.Queue({ data: payload, queuename: _a.slug + "agent", correlationId: streamid }) - console.log("submitted", payload) } async RegisterQueue() { if(this.queuename != "") return; this.queuename = await NoderedUtil.RegisterQueue({ callback: (_data: QueueMessage, ack: any) => { ack(); - console.log(_data.data) if(_data == null) return; var correlationId = _data.correlationId; var data: any = _data; while(data.data != null && data.data != "") data = data.data; if(data.command == "listprocesses") { - console.log("listprocesses !!!") for(var i = 0; i < data.processes.length; i++) { console.log("add process " + data.processes[i].id) this.addprocess(data.processes[i].id, data.processes[i].schedulename); @@ -8921,9 +8819,7 @@ export class RunPackageCtrl extends entityCtrl { setTimeout(this.RegisterQueue.bind(this), (Math.floor(Math.random() * 6) + 1) * 500); } }); - console.debug("registed queue", this.queuename); var _a = this.agents.find(x => x._id == this.id); await NoderedUtil.Queue({ data: {"command": "addcommandstreamid"}, queuename: _a.slug + "agent" }); - console.debug("Added streamid to command streams for " + _a.slug + "agent") } } \ No newline at end of file diff --git a/OpenFlow/src/public/WebSocketClientService.ts b/OpenFlow/src/public/WebSocketClientService.ts index c1f702ba..cb77ebc7 100644 --- a/OpenFlow/src/public/WebSocketClientService.ts +++ b/OpenFlow/src/public/WebSocketClientService.ts @@ -28,7 +28,6 @@ export class WebSocketClientService { this.auto_create_personal_nodered_group = data.auto_create_personal_nodered_group; this.namespace = data.namespace; - this.nodered_domain_schema = data.nodered_domain_schema; this.agent_domain_schema = data.agent_domain_schema; this.websocket_package_size = data.websocket_package_size; @@ -180,7 +179,6 @@ export class WebSocketClientService { public allow_user_registration: boolean = false; public auto_create_personal_nodered_group: boolean = false; public namespace: string = ""; - public nodered_domain_schema: string = ""; public agent_domain_schema: string = ""; public websocket_package_size: number = 25000; public stripe_api_key: string = ""; diff --git a/docker-compose.yml b/docker-compose.yml index ee68de8e..3d47ab75 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -67,11 +67,8 @@ services: # - enable_nodered_tours=false # - enable_openflow_amqp=true - # - nodered_ws_url=ws://api:3000 - # - domain=localhost.openiap.io # - saml_federation_metadata=http://api:3000/issue/FederationMetadata/2007-06/FederationMetadata.xml - # - nodered_saml_entrypoint=http://localhost.openiap.io/issue # - saml_issuer=uri:localhost.openiap.io # - aes_secret=7TXsxf7cn9EkUqm5h4MEWGjzkxkNCk2K diff --git a/openflow.env b/openflow.env index bc18a7bf..a12a284f 100644 --- a/openflow.env +++ b/openflow.env @@ -1,7 +1,6 @@ mongodb_url=mongodb://localhost:27017 mongodb_db=openflow namespace=openflow -# nodered_domain_schema=$nodered_id$.localhost.openiap.io port=80 signing_crt=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURqRENDQW5TZ0F3SUJBZ0lKQUp5N0tIMTE1dkQ4TUEwR0NTcUdTSWIzRFFFQkN3VUFNRnN4Q3pBSkJnTlYKQkFZVEFrUkxNUk13RVFZRFZRUUlEQXBUYjIxbExWTjBZWFJsTVNFd0h3WURWUVFLREJoSmJuUmxjbTVsZENCWAphV1JuYVhSeklGQjBlU0JNZEdReEZEQVNCZ05WQkFNTUMzTnBaMjVwYm1kalpYSjBNQjRYRFRFNU1EUXdOekUwCk5ETXpORm9YRFRJNU1EUXdOREUwTkRNek5Gb3dXekVMTUFrR0ExVUVCaE1DUkVzeEV6QVJCZ05WQkFnTUNsTnYKYldVdFUzUmhkR1V4SVRBZkJnTlZCQW9NR0VsdWRHVnlibVYwSUZkcFpHZHBkSE1nVUhSNUlFeDBaREVVTUJJRwpBMVVFQXd3TGMybG5ibWx1WjJObGNuUXdnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCRHdBd2dnRUtBb0lCCkFRQy9JSkdEaGxLTU9SWkoycXQwSWpjSDZOWUFtZDVxQzQ4dkNJRE54QWZCbmQxQnN4WlVjWkl5dkFlT28yNDcKM3I0eTYwNDgxRHVUS2JaMTBTNjRqRU05aW1XTXB1TFlJRnVyQ3BWNzVEWWhxMS85Q0FJVHJqNjlmVDluSkptcwpjM2lxTnJ1Tlg1bDlISXdadWtQM1ZNRkJRNWZVd3N1ZnE0YW1NbnVnZmtyUEVzSngxK3VJb0NYU3pyblZvcnZpClZ0ZFh4a3M4N0l1S0ZnMDJIZ1RQSzdwc0FXYTBRY3g2ck04bkV5TUhwNUdlR1Rvb1NNbkcyZ1RGNWZOSVFNdTMKVEVoc2p3SWRTYmRwck1Gb1VZV05Bc2FueTJOQk0wREhZRUdjQlZhZ0xWNUhFUW5ySUM3NEhtNjYxdG9HaU5VSAoveW04U3VndTgwWVFiVGxPcTFWNnNkaVRBZ01CQUFHalV6QlJNQjBHQTFVZERnUVdCQlMrYWc1NGtITllKZ29pCm9yRnlia293THR5R3ZqQWZCZ05WSFNNRUdEQVdnQlMrYWc1NGtITllKZ29pb3JGeWJrb3dMdHlHdmpBUEJnTlYKSFJNQkFmOEVCVEFEQVFIL01BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQTZpZHBYdzdSR2pzcUpNanI2c1YvaQpXeXFsL2xkSy9sa1NCdnNBSENieDdQYi9rVUd2NHJNbndYMnBHdTR0YkFnSDc4eStmS3dzazllYkxDeTA4Y1k0Ckt5czhzbUpLenhWN0R6U3RVR1NvZmZMaFliVVVMK3UyNU5vVXc0TG1WQU5FU0NaMTZ3aTdPQUdJMkJnNFR6TXoKdnlIUHRaaE9wTXBNV2lzM2ZnRXFzV3QxS2VLcXo0Z2M5RnJtZDZPNlQzVVAxWTRBR3VEWnNScnpiU2RQS2JxbApxekprT2tQcGtHOGo3ZjFWNkk1ZlkzblZaSWk2YW1TcTM1RTJkQzVMY0dIQXRtT1lWL0c4TEQ3OUFnTVpFUU5vCkF2R2RnV1gvbXBFVkFjMmRFQkJlcUN6WjF1aVhWUjdERld2ZDFKTEpsdVRyTm9jMUROc0xNKzNEZFFiQ2JnYVcKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= singing_key=LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRQy9JSkdEaGxLTU9SWkoKMnF0MElqY0g2TllBbWQ1cUM0OHZDSUROeEFmQm5kMUJzeFpVY1pJeXZBZU9vMjQ3M3I0eTYwNDgxRHVUS2JaMQowUzY0akVNOWltV01wdUxZSUZ1ckNwVjc1RFlocTEvOUNBSVRyajY5ZlQ5bkpKbXNjM2lxTnJ1Tlg1bDlISXdaCnVrUDNWTUZCUTVmVXdzdWZxNGFtTW51Z2ZrclBFc0p4MSt1SW9DWFN6cm5Wb3J2aVZ0ZFh4a3M4N0l1S0ZnMDIKSGdUUEs3cHNBV2EwUWN4NnJNOG5FeU1IcDVHZUdUb29TTW5HMmdURjVmTklRTXUzVEVoc2p3SWRTYmRwck1GbwpVWVdOQXNhbnkyTkJNMERIWUVHY0JWYWdMVjVIRVFucklDNzRIbTY2MXRvR2lOVUgveW04U3VndTgwWVFiVGxPCnExVjZzZGlUQWdNQkFBRUNnZ0VBVXBjZ1NsV2hGamNWQ3BVVHdmdUhERVB4TmhGSHEwdVRkQitZaVZKTWg3NVAKL2pRRlVqaEJsT3JyMlJlR2F4aTEyQXNXby9LU1MrV2Frdzd4d1kzYkFKenRoUG9Zekl3dkVKcGlQa2MvblEwUgpUYVpJUDNqc1k3WGIwQlpnMGNTVVAvbW0wbENkWXhNUzk0c21FNXJzWitkdGxPTVlXc2NrU0cxSVB2SlVJV2FZCnl3NC9kaHJ3TWRsREVZS2tSbks1aDR1dXR1dzA1Q1VzNkZWN2F5cEJRRStGM3RxVlF3QWxGbWNueXdvZTB5WjQKZW1tWkRvT1dzNUk4cGNGbjZCSW1wZjN3UEg5UWhlQXJVaXRqV3YrZmI2cWRVaHJFVDBxMzh4dTZ5M1lJNFNLYQpXME9kUng4L3FTYllXdkpzbmxscDR0aUpDWE5IdnV6MVBKSGhxOUprQVFLQmdRRCt2dHlWcVJoaEJZTmd6WWorClFCaDNLcktrNEVqTXZaSGhEelhES0pMZ2JEVmJzNExrQkYySWZGOTF0S1k3d09CNVpXcU82Z3FqVVJBVE5hc1YKOExCOGE2TEpXYVJuTklLMnJkd1FwalFYcy92OVBSYnJwc2tTbDRJdUsyZWNBMjBSQkhicW5yNHZ5ZkQ4U3BzaApSdHlTUk5CRGVsaU01Z1JDM0JKKzBZbjBVUUtCZ1FEQUVZSUp3Y2xnQloyWmNNTlh1dW1lLzUxdHBHeGppRTJpCjZ3SDNsOHNTVDN1U1A4MHdndGdHaVFsUTRsR3BmSThqWkl4N0pwcGw0Qko1ZEtuVnpIS1dqMzA3YXYxcjdpU3QKLzJOVDNobzdaYkNlYzlhMHlJU2E3dTNGZGxzZ0VPcE45dURmbG5GQVQ3ZmIrM2d4Sk9DUWp1TkFCZXZaK2pScwpZY0ZhQWhGNW93S0JnUUNGUG9HVVNsRDFGblFrWXYwL3QzalVnK0hTK1hrNmxnRkNqYmthTGhPOURQeFB6Ykl0CjM5YW9lQjFhTExZeVZPMVVzZVl0Z0Y4MkUwVnNOc3NZKzc3a0pVeU5NclVhUWs0SWpTR3BGN1h4bS9PMi9vZ0oKbEVCaDJCdUFXTFdsMWVqcldNRjJjTGVidVcyeUdMZlJqUVg3LzhCTE95Z3I4bmZTSE5nVHV6Z0VNUUtCZ0JrZgpNUjhObGNWVmRyT25LQ1hGY09FM0ZlUk5hVS9yZUJ3akdQTEZpKzR0TDBDRno5VFVpR1R5YjZHQXVLV3VnUnBrCkFHdnJOSzYyakRRT3FsZ29rYVJYeUUySlJQUmxCYThzaEZWbjY0NXhVcFNuR2lJelNBVHIwM1hNY1ViVWI1RWIKQlhhNU9yN3FybVc3a3BENi9kUnFuQmEzcjQyblNFd1V6VEYwcTh4NUFvR0FIcXdRSyt1R0NTdlNsNENJUGhyRQpDREIvcytDK2NJNXVCeFJCMHVlNjc3L2lpdGZPSU9lOUNiTHE3R0tib0w4RVg3eXhKNVRLWjlYQmh5LzNCWmVNCllydEx3M2JicTNTN2hpUGFYSmE1dXZma3BWR1RnNEdzTnBJQ3VNTEJUaXJ6M0ZRV25UNFNZbzkrREVoalhEeVQKWlVOMERtUkJVNjNjWjRLSUlXd2xWUTA9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K diff --git a/package.json b/package.json index c36a9572..b7f13f06 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@openiap/openflow", - "version": "1.5.2", + "version": "1.5.3", "description": "Simple wrapper around NodeRed, RabbitMQ and MongoDB to support a more scaleable NodeRed implementation.\r Also the \"backend\" for [OpenRPA](https://github.com/skadefro/OpenRPA)", "main": "index.js", "scripts": { @@ -12,6 +12,7 @@ "latest": "gulp latest", "rebuild": "docker-compose up registry -d && docker-compose build && docker-compose push --ignore-push-failures && docker-compose up -d", "build": "docker-compose up registry -d && docker-compose push --ignore-push-failures && docker-compose up -d", + "updateapilocal": "npm uninstall @openiap/nodeapi && npm i ../nodeapi", "updateapidev": "npm uninstall @openiap/nodeapi && npm i openiap/nodeapi", "updateapi": "npm uninstall @openiap/nodeapi && npm i @openiap/nodeapi" }, @@ -33,7 +34,7 @@ "@fortawesome/fontawesome-free": "^5.15.3", "@kubernetes/client-node": "^0.19.0", "@node-saml/passport-saml": "^4.0.1", - "@openiap/nodeapi": "^0.0.27", + "@openiap/nodeapi": "^0.0.30", "@openiap/openflow-api": "^2.1.11", "@opentelemetry/exporter-metrics-otlp-grpc": "^0.43.0", "@opentelemetry/sdk-node": "^0.43.0", diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 021824ec..a692b258 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -97,9 +97,7 @@ services: - socket_rate_limit_points=1000 - socket_rate_limit_points_disconnect=2500 - - nodered_ws_url=ws://api:3000 - saml_federation_metadata=http://api:3000/issue/FederationMetadata/2007-06/FederationMetadata.xml - - nodered_saml_entrypoint=http://localhost.openiap.io/issue - amqp_url=amqp://guest:guest@rabbitmq - mongodb_url=mongodb://mongodb:27017 - mongodb_db=openrpa