-
Notifications
You must be signed in to change notification settings - Fork 708
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Restored metric logging to third-party loggers #2489
base: main
Are you sure you want to change the base?
Restored metric logging to third-party loggers #2489
Conversation
Signed-off-by: AdityaSinghDevs <[email protected]>
Signed-off-by: AdityaSinghDevs <[email protected]>
1d292c5
to
35b360c
Compare
Hi @ashwinvaidya17 , @samet-akcay , |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks for the PR! I have a few questions.
Signed-off-by: AdityaSinghDevs <[email protected]>
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Try to avoid multiple lines of try/except for imports. I've added a few lines of code as suggestion. Feel free to use them as a reference for refactoring.
Additionally, I would recommend manually testing whether the script runs without any backends (mlflow, tensorboard, etc) in your environment, and when some of them are present. There might be some edge cases here
@@ -89,8 +125,44 @@ def run( | |||
**test_results[0], | |||
} | |||
logger.info(f"Completed with result {output}") | |||
# Logging metrics to External Loggers (excluding TensorBoard) | |||
trainer = engine.trainer() |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Are you sure that you can call trainer?
# Import external loggers | ||
AVAILABLE_LOGGERS: dict[str, Any] = {} | ||
|
||
try: | ||
from anomalib.loggers import AnomalibCometLogger | ||
|
||
AVAILABLE_LOGGERS["comet"] = AnomalibCometLogger | ||
except ImportError: | ||
logger.debug("Comet logger not available. Install using `pip install comet-ml`") | ||
try: | ||
from anomalib.loggers import AnomalibMLFlowLogger | ||
|
||
AVAILABLE_LOGGERS["mlflow"] = AnomalibMLFlowLogger | ||
except ImportError: | ||
logger.debug("MLflow logger not available. Install using `pip install mlflow`") | ||
try: | ||
from anomalib.loggers import AnomalibTensorBoardLogger | ||
|
||
AVAILABLE_LOGGERS["tensorboard"] = AnomalibTensorBoardLogger | ||
except ImportError: | ||
logger.debug("TensorBoard logger not available. Install using `pip install tensorboard`") | ||
try: | ||
from anomalib.loggers import AnomalibWandbLogger | ||
|
||
AVAILABLE_LOGGERS["wandb"] = AnomalibWandbLogger | ||
except ImportError: | ||
logger.debug("Weights & Biases logger not available. Install using `pip install wandb`") | ||
|
||
LOGGERS_AVAILABLE = len(AVAILABLE_LOGGERS) > 0 | ||
|
||
if LOGGERS_AVAILABLE: | ||
logger.info(f"Available loggers: {', '.join(AVAILABLE_LOGGERS.keys())}") | ||
else: | ||
logger.warning("No external loggers available. Install required packages using `anomalib install -v`") | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
# Import external loggers | |
AVAILABLE_LOGGERS: dict[str, Any] = {} | |
try: | |
from anomalib.loggers import AnomalibCometLogger | |
AVAILABLE_LOGGERS["comet"] = AnomalibCometLogger | |
except ImportError: | |
logger.debug("Comet logger not available. Install using `pip install comet-ml`") | |
try: | |
from anomalib.loggers import AnomalibMLFlowLogger | |
AVAILABLE_LOGGERS["mlflow"] = AnomalibMLFlowLogger | |
except ImportError: | |
logger.debug("MLflow logger not available. Install using `pip install mlflow`") | |
try: | |
from anomalib.loggers import AnomalibTensorBoardLogger | |
AVAILABLE_LOGGERS["tensorboard"] = AnomalibTensorBoardLogger | |
except ImportError: | |
logger.debug("TensorBoard logger not available. Install using `pip install tensorboard`") | |
try: | |
from anomalib.loggers import AnomalibWandbLogger | |
AVAILABLE_LOGGERS["wandb"] = AnomalibWandbLogger | |
except ImportError: | |
logger.debug("Weights & Biases logger not available. Install using `pip install wandb`") | |
LOGGERS_AVAILABLE = len(AVAILABLE_LOGGERS) > 0 | |
if LOGGERS_AVAILABLE: | |
logger.info(f"Available loggers: {', '.join(AVAILABLE_LOGGERS.keys())}") | |
else: | |
logger.warning("No external loggers available. Install required packages using `anomalib install -v`") | |
def try_create_logger(logger_class: str, config: dict[str, dict[str, Any]]) -> Logger: | |
"""Try to import a logger class. | |
Args: | |
logger_class (str): The name of the logger class to import. | |
config (dict[str, dict[str, Any]]): The configuration for the logger. | |
Returns: | |
Logger: The logger instance. | |
""" | |
try: | |
module = importlib.import_module("anomalib.loggers") | |
logger_class = getattr(module, f"Anomalib{logger_class}Logger") | |
return logger_class(**config) | |
except (ImportError, ModuleNotFoundError): | |
logger.info( | |
f"{logger_class} logger not available. Please install the respective package.", | |
) | |
return None |
@@ -69,6 +104,7 @@ def run( | |||
accelerator=self.accelerator, | |||
devices=devices, | |||
default_root_dir=temp_dir, | |||
logger=self._initialize_loggers(self.flat_cfg or {}) if LOGGERS_AVAILABLE else [], |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
logger=self._initialize_loggers(self.flat_cfg or {}) if LOGGERS_AVAILABLE else [], | |
logger=self._initialize_loggers(self.flat_cfg), |
# Logging metrics to External Loggers (excluding TensorBoard) | ||
trainer = engine.trainer() | ||
for logger_instance in trainer.loggers: | ||
if any( | ||
isinstance(logger_instance, AVAILABLE_LOGGERS.get(name, object)) | ||
for name in ["comet", "wandb", "mlflow"] | ||
): | ||
logger_instance.log_metrics(test_results[0]) | ||
logger.debug(f"Successfully logged metrics to {logger_instance.__class__.__name__}") |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
# Logging metrics to External Loggers (excluding TensorBoard) | |
trainer = engine.trainer() | |
for logger_instance in trainer.loggers: | |
if any( | |
isinstance(logger_instance, AVAILABLE_LOGGERS.get(name, object)) | |
for name in ["comet", "wandb", "mlflow"] | |
): | |
logger_instance.log_metrics(test_results[0]) | |
logger.debug(f"Successfully logged metrics to {logger_instance.__class__.__name__}") | |
# Logging metrics to External Loggers (excluding TensorBoard) | |
for logger_instance in engine.trainer.loggers: | |
logger_instance.log_metrics(test_results[0]) | |
logger.info(f"Successfully logged metrics to {logger_instance.__class__.__name__}") |
@staticmethod | ||
def _initialize_loggers(logger_configs: dict[str, dict[str, Any]]) -> list[Any]: | ||
"""Initialize configured external loggers. | ||
|
||
Args: | ||
logger_configs: Dictionary mapping logger names to their configurations. | ||
|
||
Returns: | ||
List of initialized loggers. | ||
""" | ||
active_loggers = [] | ||
default_configs = { | ||
"tensorboard": {"save_dir": "logs/benchmarks"}, | ||
"comet": {"project_name": "anomalib"}, | ||
"wandb": {"project": "anomalib"}, | ||
"mlflow": {"experiment_name": "anomalib"}, | ||
} | ||
|
||
for logger_name, logger_class in AVAILABLE_LOGGERS.items(): | ||
# Use provided config or fall back to defaults | ||
config = logger_configs.get(logger_name, default_configs.get(logger_name, {})) | ||
logger_instance = logger_class(**config) | ||
active_loggers.append(logger_instance) | ||
logger.info(f"Successfully initialized {logger_name} logger") | ||
|
||
return active_loggers |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@staticmethod | |
def _initialize_loggers(logger_configs: dict[str, dict[str, Any]]) -> list[Any]: | |
"""Initialize configured external loggers. | |
Args: | |
logger_configs: Dictionary mapping logger names to their configurations. | |
Returns: | |
List of initialized loggers. | |
""" | |
active_loggers = [] | |
default_configs = { | |
"tensorboard": {"save_dir": "logs/benchmarks"}, | |
"comet": {"project_name": "anomalib"}, | |
"wandb": {"project": "anomalib"}, | |
"mlflow": {"experiment_name": "anomalib"}, | |
} | |
for logger_name, logger_class in AVAILABLE_LOGGERS.items(): | |
# Use provided config or fall back to defaults | |
config = logger_configs.get(logger_name, default_configs.get(logger_name, {})) | |
logger_instance = logger_class(**config) | |
active_loggers.append(logger_instance) | |
logger.info(f"Successfully initialized {logger_name} logger") | |
return active_loggers | |
@staticmethod | |
def _initialize_loggers(training_config: dict[str, dict[str, Any]]) -> list[Logger]: | |
"""Initialize configured external loggers. | |
Args: | |
training_config: Dictionary mapping logger names to their configurations. | |
Returns: | |
List of initialized loggers. | |
""" | |
active_loggers: list[Logger] = [] | |
default_configs = { | |
"TensorBoard": {"save_dir": "logs/benchmarks"}, | |
"Comet": {"project_name": "anomalib"}, | |
"Wandb": {"project": "anomalib"}, | |
"MLFlow": {"experiment_name": "anomalib"}, | |
} | |
for logger_name, default_config in default_configs.items(): | |
# Use provided config or fall back to defaults | |
config = training_config.get(logger_name.lower(), default_config) | |
logger_instance = try_create_logger(logger_name, config) | |
if logger_instance is None: | |
continue | |
active_loggers.append(logger_instance) | |
logger.info(f"Successfully initialized {logger_name} logger") | |
return active_loggers |
📝 Description
✨ Changes
Select what type of change your PR is:
✅ Checklist
Before you submit your pull request, please make sure you have completed the following steps:
For more information about code review checklists, see the Code Review Checklist.