diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 34d8f02f0..7e780a580 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -128,6 +128,7 @@ fworch_log_dir: "/var/log/{{ product_name }}" fworch_log_lock_dir: "/var/{{ product_name }}/lock" fworch_mw_lockfile: "{{ fworch_log_lock_dir }}/FWO.Middleware.Server_log.lock" fworch_ui_lockfile: "{{ fworch_log_lock_dir }}/FWO.Ui_log.lock" +fworch_api_importer_lockfile: "{{ fworch_log_lock_dir }}/importer_api_log.lock" ### apache: both package and dir name (needed both on ui and api hosts): webserver_package_name: apache2 diff --git a/roles/common/tasks/install_syslog.yml b/roles/common/tasks/install_syslog.yml index 4606ea5a9..192deb1ba 100644 --- a/roles/common/tasks/install_syslog.yml +++ b/roles/common/tasks/install_syslog.yml @@ -138,7 +138,21 @@ {{ fworch_log_dir }}/database.log {{ fworch_log_dir }}/ldap.log {{ fworch_log_dir }}/api.log - {{ fworch_log_dir }}/importer-api.log + {{ fworch_log_dir }}/importer-api.log { + compress + maxage 7 + rotate 99 + maxsize 4096k + missingok + copytruncate + sharedscripts + prerotate + {{ fworch_home }}/scripts/acquire_lock.py {{ fworch_api_importer_lockfile }} >/dev/null 2>&1 + endscript + postrotate + {{ fworch_home }}/scripts/release_lock.py {{ fworch_api_importer_lockfile }} >/dev/null 2>&1 + endscript + } {{ fworch_log_dir }}/audit.log {{ fworch_log_dir }}/alert.log {{ fworch_log_dir }}/webhook.log diff --git a/roles/importer/files/importer/fwo_log.py b/roles/importer/files/importer/fwo_log.py index 708029947..0c59508b8 100644 --- a/roles/importer/files/importer/fwo_log.py +++ b/roles/importer/files/importer/fwo_log.py @@ -1,31 +1,107 @@ -import logging -from sys import stdout +import sys import fwo_globals -#from fwo_globals import global_debug_level +import logging +import asyncio +import time +from asyncio import Semaphore + +class LogLock: + semaphore = Semaphore(value=1) + + async def handle_log_lock(): + # Initialize values + lock_file_path = "/var/fworch/lock/importer_api_log.lock" + log_owned_by_external = False + stopwatch = time.time() + + while True: + try: + with open(lock_file_path, "a+") as file: + # Jump to the beginning of the file + file.seek(0) + # Read the file content + lock_file_content = file.read().strip() + # Forcefully release lock after timeout + if log_owned_by_external and time.time() - stopwatch > 10: + file.write("FORCEFULLY RELEASED\n") + stopwatch = -1 + LogLock.semaphore.release() + log_owned_by_external = False + # GRANTED - lock was granted by us + elif lock_file_content.endswith("GRANTED"): + # Request lock if it is not already requested by us + # (in case of restart with log already granted) + if not log_owned_by_external: + await LogLock.semaphore.acquire() + stopwatch = time.time() + log_owned_by_external = True + # REQUESTED - lock was requested by log swap process + elif lock_file_content.endswith("REQUESTED"): + # only request lock if it is not already requested by us + if not log_owned_by_external: + await LogLock.semaphore.acquire() + stopwatch = time.time() + log_owned_by_external = True + file.write("GRANTED\n") + # RELEASED - lock was released by log swap process + elif lock_file_content.endswith("RELEASED"): + # only release lock if it was formerly requested by us + if log_owned_by_external: + stopwatch = -1 + LogLock.semaphore.release() + log_owned_by_external = False + except Exception as e: + pass + # Wait a second + time.sleep(1) + +# Used to accquire lock before log processing +class LogFilter(logging.Filter): + async def acquire_lock(): + LogLock.semaphore.acquire() + def filter(self, record): + # Acquire lock + asyncio.run(LogFilter.acquire_lock()) + # Return True to allow the log record to be processed + return True +# Used to release lock after log processing +class LogHandler(logging.StreamHandler): + async def release_lock(): + LogLock.semaphore.release() + def emit(self, record): + # Call the parent class's emit method to perform the actual logging + super().emit(record) + # Release lock + asyncio.run(LogHandler.release_lock()) def getFwoLogger(): - debug_level=int(fwo_globals.debug_level) - if debug_level>=1: - llevel = logging.DEBUG + debug_level = int(fwo_globals.debug_level) + if debug_level >= 1: + log_level = logging.DEBUG else: - llevel = logging.INFO + log_level = logging.INFO - logger = logging.getLogger() # use root logger - logHandler = logging.StreamHandler(stream=stdout) - logformat = "%(asctime)s [%(levelname)-5.5s] [%(filename)-10.10s:%(funcName)-10.10s:%(lineno)4d] %(message)s" - logHandler.setLevel(llevel) - handlers = [logHandler] - logging.basicConfig(format=logformat, datefmt="%Y-%m-%dT%H:%M:%S%z", handlers=handlers, level=llevel) - logger.setLevel(llevel) + logger = logging.getLogger() + log_handler = LogHandler(stream=sys.stdout) + log_filter = LogFilter() - # set log level for noisy requests/connectionpool module to WARNING: + log_format = "%(asctime)s [%(levelname)-5.5s] [%(filename)-10.10s:%(funcName)-10.10s:%(lineno)4d] %(message)s" + log_handler.setLevel(log_level) + log_handler.addFilter(log_filter) + handlers = [log_handler] + + logging.basicConfig(format=log_format, datefmt="%Y-%m-%dT%H:%M:%S%z", handlers=handlers, level=log_level) + logger.setLevel(log_level) + + # Set log level for noisy requests/connectionpool module to WARNING: connection_log = logging.getLogger("urllib3.connectionpool") connection_log.setLevel(logging.WARNING) connection_log.propagate = True - - if debug_level>8: - logger.debug ("debug_level=" + str(debug_level) ) + + if debug_level > 8: + logger.debug("debug_level=" + str(debug_level)) + return logger @@ -37,10 +113,14 @@ def getFwoAlertLogger(debug_level=0): llevel = logging.INFO logger = logging.getLogger() # use root logger - logHandler = logging.StreamHandler(stream=stdout) + log_handler = LogHandler(stream=sys.stdout) + log_filter = LogFilter() + logformat = "%(asctime)s %(message)s" - logHandler.setLevel(llevel) - handlers = [logHandler] + log_handler.setLevel(llevel) + log_handler.addFilter(log_filter) + handlers = [log_handler] + logging.basicConfig(format=logformat, datefmt="", handlers=handlers, level=llevel) logger.setLevel(llevel) diff --git a/roles/importer/files/importer/import-main-loop.py b/roles/importer/files/importer/import-main-loop.py index f619a1283..0128a2282 100755 --- a/roles/importer/files/importer/import-main-loop.py +++ b/roles/importer/files/importer/import-main-loop.py @@ -3,6 +3,7 @@ # run import loop every x seconds (adjust sleep time per management depending on the change frequency ) import signal +import asyncio import traceback import argparse import sys @@ -11,7 +12,7 @@ import requests, warnings import fwo_api# common # from current working dir from common import import_management -from fwo_log import getFwoLogger +from fwo_log import getFwoLogger, LogLock import fwo_globals, fwo_config from fwo_const import base_dir, importer_base_dir from fwo_exception import FwoApiLoginFailed, FwoApiFailedLockImport, FwLoginFailed @@ -28,6 +29,12 @@ def __init__(self): def exit_gracefully(self, *args): self.kill_now = True +# Store all background tasks in a set to avoid garbage collection +background_tasks = set() + +async def log_lock_task(): + # Start the log lock task in the background + background_tasks.add(asyncio.create_task(LogLock.handle_log_lock())) if __name__ == '__main__': parser = argparse.ArgumentParser( @@ -45,6 +52,9 @@ def exit_gracefully(self, *args): args = parser.parse_args() + # Log locking + asyncio.run(log_lock_task()) + fwo_config = fwo_config.readConfig() fwo_globals.setGlobalValues(verify_certs_in=args.verify_certificates, suppress_cert_warnings_in=args.suppress_certificate_warnings, diff --git a/roles/lib/files/FWO.Report/Display/RuleDisplayHtml.cs b/roles/lib/files/FWO.Report/Display/RuleDisplayHtml.cs index a1dbaddee..c84f0d2b0 100644 --- a/roles/lib/files/FWO.Report/Display/RuleDisplayHtml.cs +++ b/roles/lib/files/FWO.Report/Display/RuleDisplayHtml.cs @@ -92,10 +92,9 @@ public string DisplayLastRecertifier(Rule rule) return string.Join("", Array.ConvertAll(rule.Metadata.RuleRecertification.ToArray(), recert => getLastRecertifierDisplayString(countString(rule.Metadata.RuleRecertification.Count > 1, ++count), recert).ToString())); } - protected string constructLink(string type, string symbol, long id, string name, OutputLocation location, int mgmtId, string style, ReportType reportType = ReportType.Rules) + protected string ConstructLink(string type, string symbol, long id, string name, OutputLocation location, int mgmtId, string style, ReportType reportType = ReportType.Rules) { - string genPath = "/generation"; - string link = location == OutputLocation.export ? $"#" : $"{location.ToString()}{genPath}#goto-report-m{mgmtId}-"; + string link = location == OutputLocation.export ? $"#" : $"{location}/generation#goto-report-m{mgmtId}-"; return $" {name}"; } @@ -103,16 +102,16 @@ protected string NetworkLocationToHtml(NetworkLocation networkLocation, int mgmt { return DisplayNetworkLocation(networkLocation, reportType, reportType.IsResolvedReport() || networkLocation.User == null ? null : - constructLink("user", ReportBase.GetIconClass(ObjCategory.user, networkLocation.User?.Type.Name), networkLocation.User!.Id, networkLocation.User.Name, location, mgmtId, style, reportType), + ConstructLink("user", ReportBase.GetIconClass(ObjCategory.user, networkLocation.User?.Type.Name), networkLocation.User!.Id, networkLocation.User.Name, location, mgmtId, style, reportType), reportType.IsResolvedReport() ? null : - constructLink("nwobj", ReportBase.GetIconClass(ObjCategory.nobj, networkLocation.Object.Type.Name), networkLocation.Object.Id, networkLocation.Object.Name, location, mgmtId, style, reportType) + ConstructLink("nwobj", ReportBase.GetIconClass(ObjCategory.nobj, networkLocation.Object.Type.Name), networkLocation.Object.Id, networkLocation.Object.Name, location, mgmtId, style, reportType) ).ToString(); } protected string ServiceToHtml(NetworkService service, int mgmtId, OutputLocation location, string style, ReportType reportType) { return DisplayService(service, reportType, reportType.IsResolvedReport() ? null : - constructLink("svc", ReportBase.GetIconClass(ObjCategory.nsrv, service.Type.Name), service.Id, service.Name, location, mgmtId, style, reportType)).ToString(); + ConstructLink("svc", ReportBase.GetIconClass(ObjCategory.nsrv, service.Type.Name), service.Id, service.Name, location, mgmtId, style, reportType)).ToString(); } private string DisplaySourceOrDestination(Rule rule, OutputLocation location, ReportType reportType, string style, bool isSource) diff --git a/roles/ui/files/FWO.UI/wwwroot/js/scrollIntoView.js b/roles/ui/files/FWO.UI/wwwroot/js/scrollIntoView.js index 189fbc9e9..5b1c4e088 100644 --- a/roles/ui/files/FWO.UI/wwwroot/js/scrollIntoView.js +++ b/roles/ui/files/FWO.UI/wwwroot/js/scrollIntoView.js @@ -7,5 +7,5 @@ function scrollIntoRSBView(htmlObjId) { obj.classList.add("fade-bg"); obj.classList.add("temp-highlight"); setTimeout(() => obj.classList.remove("temp-highlight"), 800) - return obj.offsetParent !== null; // alement visible? + return obj.offsetParent !== null; // element visible? } \ No newline at end of file