diff --git a/docs/source/contrib/handlers.rst b/docs/source/contrib/handlers.rst index 1635b7d5bb9..e9d125e50b8 100644 --- a/docs/source/contrib/handlers.rst +++ b/docs/source/contrib/handlers.rst @@ -25,40 +25,8 @@ Time profilers [deprecated] Use :class:`~ignite.handlers.time_profilers.BasicTimeProfiler` instead, will be removed in version 0.6.0. Use :class:`~ignite.handlers.time_profilers.HandlersTimeProfiler` instead, will be removed in version 0.6.0. -Loggers -------- +Loggers [deprecated] +-------------------- -.. currentmodule:: ignite.contrib.handlers - -.. autosummary:: - :nosignatures: - :toctree: ../generated - :recursive: - - base_logger - clearml_logger - mlflow_logger - neptune_logger - polyaxon_logger - tensorboard_logger - tqdm_logger - - visdom_logger - wandb_logger - -.. seealso:: - - Below are a comprehensive list of examples of various loggers. - - * See `tensorboardX mnist example `_ - and `CycleGAN and EfficientNet notebooks `_ for detailed usage. - - * See `visdom mnist example `_ for detailed usage. - - * See `neptune mnist example `_ for detailed usage. - - * See `tqdm mnist example `_ for detailed usage. - - * See `wandb mnist example `_ for detailed usage. - - * See `clearml mnist example `_ for detailed usage. +.. deprecated:: 0.4.14 + Loggers moved to :ref:`Loggers`. diff --git a/docs/source/handlers.rst b/docs/source/handlers.rst index edf7f5e54f3..348639ccf2a 100644 --- a/docs/source/handlers.rst +++ b/docs/source/handlers.rst @@ -1,8 +1,8 @@ ignite.handlers =============== -Complete list of handlers -------------------------- +Complete list of generic handlers +---------------------------------- .. currentmodule:: ignite.handlers @@ -33,6 +33,46 @@ Complete list of handlers param_scheduler.ParamScheduler state_param_scheduler.StateParamScheduler + +Loggers +-------- + +.. currentmodule:: ignite.handlers + +.. autosummary:: + :nosignatures: + :toctree: generated + :recursive: + + base_logger + clearml_logger + mlflow_logger + neptune_logger + polyaxon_logger + tensorboard_logger + tqdm_logger + + visdom_logger + wandb_logger + +.. seealso:: + + Below are a comprehensive list of examples of various loggers. + + * See `tensorboardX mnist example `_ + and `CycleGAN and EfficientNet notebooks `_ for detailed usage. + + * See `visdom mnist example `_ for detailed usage. + + * See `neptune mnist example `_ for detailed usage. + + * See `tqdm mnist example `_ for detailed usage. + + * See `wandb mnist example `_ for detailed usage. + + * See `clearml mnist example `_ for detailed usage. + + .. _param-scheduler-label: Parameter scheduler @@ -396,7 +436,7 @@ Example with :class:`ignite.handlers.param_scheduler.ReduceLROnPlateauScheduler` init_lr = 0.1 lr_values = np.array(ReduceLROnPlateauScheduler.simulate_values( - num_events, metric_values, init_lr, + num_events, metric_values, init_lr, factor=0.5, patience=1, mode='max', threshold=0.01, threshold_mode='abs' ) ) diff --git a/ignite/contrib/engines/common.py b/ignite/contrib/engines/common.py index 9cb9a1915c4..0bc52a0ea02 100644 --- a/ignite/contrib/engines/common.py +++ b/ignite/contrib/engines/common.py @@ -15,21 +15,24 @@ from torch.optim.lr_scheduler import _LRScheduler as PyTorchLRScheduler import ignite.distributed as idist -from ignite.contrib.handlers import ( +from ignite.contrib.metrics import GpuInfo +from ignite.engine import Engine, Events +from ignite.handlers import ( + Checkpoint, ClearMLLogger, + DiskSaver, + EarlyStopping, global_step_from_engine, MLflowLogger, NeptuneLogger, PolyaxonLogger, ProgressBar, TensorboardLogger, + TerminateOnNan, VisdomLogger, WandBLogger, ) -from ignite.contrib.handlers.base_logger import BaseLogger -from ignite.contrib.metrics import GpuInfo -from ignite.engine import Engine, Events -from ignite.handlers import Checkpoint, DiskSaver, EarlyStopping, TerminateOnNan +from ignite.handlers.base_logger import BaseLogger from ignite.handlers.checkpoint import BaseSaveHandler from ignite.handlers.param_scheduler import ParamScheduler from ignite.metrics import RunningAverage @@ -361,7 +364,7 @@ def setup_tb_logging( kwargs: optional keyword args to be passed to construct the logger. Returns: - :class:`~ignite.contrib.handlers.tensorboard_logger.TensorboardLogger` + :class:`~ignite.handlers.tensorboard_logger.TensorboardLogger` """ logger = TensorboardLogger(log_dir=output_path, **kwargs) _setup_logging(logger, trainer, optimizers, evaluators, log_every_iters) @@ -392,7 +395,7 @@ def setup_visdom_logging( kwargs: optional keyword args to be passed to construct the logger. Returns: - :class:`~ignite.contrib.handlers.visdom_logger.VisdomLogger` + :class:`~ignite.handlers.visdom_logger.VisdomLogger` """ logger = VisdomLogger(**kwargs) _setup_logging(logger, trainer, optimizers, evaluators, log_every_iters) @@ -423,7 +426,7 @@ def setup_mlflow_logging( kwargs: optional keyword args to be passed to construct the logger. Returns: - :class:`~ignite.contrib.handlers.mlflow_logger.MLflowLogger` + :class:`~ignite.handlers.mlflow_logger.MLflowLogger` """ logger = MLflowLogger(**kwargs) _setup_logging(logger, trainer, optimizers, evaluators, log_every_iters) @@ -454,7 +457,7 @@ def setup_neptune_logging( kwargs: optional keyword args to be passed to construct the logger. Returns: - :class:`~ignite.contrib.handlers.neptune_logger.NeptuneLogger` + :class:`~ignite.handlers.neptune_logger.NeptuneLogger` """ logger = NeptuneLogger(**kwargs) _setup_logging(logger, trainer, optimizers, evaluators, log_every_iters) @@ -485,7 +488,7 @@ def setup_wandb_logging( kwargs: optional keyword args to be passed to construct the logger. Returns: - :class:`~ignite.contrib.handlers.wandb_logger.WandBLogger` + :class:`~ignite.handlers.wandb_logger.WandBLogger` """ logger = WandBLogger(**kwargs) _setup_logging(logger, trainer, optimizers, evaluators, log_every_iters) @@ -516,7 +519,7 @@ def setup_plx_logging( kwargs: optional keyword args to be passed to construct the logger. Returns: - :class:`~ignite.contrib.handlers.polyaxon_logger.PolyaxonLogger` + :class:`~ignite.handlers.polyaxon_logger.PolyaxonLogger` """ logger = PolyaxonLogger(**kwargs) _setup_logging(logger, trainer, optimizers, evaluators, log_every_iters) @@ -547,7 +550,7 @@ def setup_clearml_logging( kwargs: optional keyword args to be passed to construct the logger. Returns: - :class:`~ignite.contrib.handlers.clearml_logger.ClearMLLogger` + :class:`~ignite.handlers.clearml_logger.ClearMLLogger` """ logger = ClearMLLogger(**kwargs) _setup_logging(logger, trainer, optimizers, evaluators, log_every_iters) diff --git a/ignite/contrib/handlers/__init__.py b/ignite/contrib/handlers/__init__.py index 2db80fd2fd9..dcefa8b2b60 100644 --- a/ignite/contrib/handlers/__init__.py +++ b/ignite/contrib/handlers/__init__.py @@ -1,14 +1,19 @@ -from ignite.contrib.handlers.clearml_logger import ClearMLLogger -from ignite.contrib.handlers.mlflow_logger import MLflowLogger -from ignite.contrib.handlers.neptune_logger import NeptuneLogger -from ignite.contrib.handlers.polyaxon_logger import PolyaxonLogger -from ignite.contrib.handlers.tensorboard_logger import TensorboardLogger -from ignite.contrib.handlers.tqdm_logger import ProgressBar - -from ignite.contrib.handlers.visdom_logger import VisdomLogger -from ignite.contrib.handlers.wandb_logger import WandBLogger -from ignite.handlers import EpochOutputStore, global_step_from_engine # ref # ref +from ignite.handlers import ( # ref # ref + clearml_logger, + EpochOutputStore, + global_step_from_engine, + mlflow_logger, + neptune_logger, + polyaxon_logger, + tensorboard_logger, + tqdm_logger, + visdom_logger, + wandb_logger, +) +from ignite.handlers.clearml_logger import ClearMLLogger from ignite.handlers.lr_finder import FastaiLRFinder +from ignite.handlers.mlflow_logger import MLflowLogger +from ignite.handlers.neptune_logger import NeptuneLogger from ignite.handlers.param_scheduler import ( ConcatScheduler, CosineAnnealingScheduler, @@ -18,4 +23,10 @@ ParamGroupScheduler, PiecewiseLinear, ) +from ignite.handlers.polyaxon_logger import PolyaxonLogger +from ignite.handlers.tensorboard_logger import TensorboardLogger from ignite.handlers.time_profilers import BasicTimeProfiler, HandlersTimeProfiler +from ignite.handlers.tqdm_logger import ProgressBar + +from ignite.handlers.visdom_logger import VisdomLogger +from ignite.handlers.wandb_logger import WandBLogger diff --git a/ignite/contrib/handlers/base_logger.py b/ignite/contrib/handlers/base_logger.py index e25849554de..d2d6cdaef26 100644 --- a/ignite/contrib/handlers/base_logger.py +++ b/ignite/contrib/handlers/base_logger.py @@ -1,294 +1,38 @@ -"""Base logger and its helper handlers.""" +""" ``ignite.contrib.handlers.base_logger`` was moved to ``ignite.handlers.base_logger``. +Note: + ``ignite.contrib.handlers.base_logger`` was moved to ``ignite.handlers.base_logger``. + Please refer to :mod:`~ignite.handlers.base_logger`. +""" -import numbers import warnings -from abc import ABCMeta, abstractmethod -from collections import OrderedDict -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union -import torch -import torch.nn as nn -from torch.optim import Optimizer - -from ignite.engine import Engine, Events, EventsList, State -from ignite.engine.events import CallableEventWithFilter, RemovableEventHandle - - -class BaseHandler(metaclass=ABCMeta): - """Base handler for defining various useful handlers.""" - - @abstractmethod - def __call__(self, engine: Engine, logger: Any, event_name: Union[str, Events]) -> None: - pass - - -class BaseWeightsHandler(BaseHandler): - """ - Base handler for logging weights or their gradients. - """ - - def __init__( - self, - model: nn.Module, - tag: Optional[str] = None, - whitelist: Optional[Union[List[str], Callable[[str, nn.Parameter], bool]]] = None, - ): - if not isinstance(model, torch.nn.Module): - raise TypeError(f"Argument model should be of type torch.nn.Module, but given {type(model)}") - - self.model = model - self.tag = tag - - weights = {} - if whitelist is None: - weights = dict(model.named_parameters()) - elif callable(whitelist): - for n, p in model.named_parameters(): - if whitelist(n, p): - weights[n] = p - else: - for n, p in model.named_parameters(): - for item in whitelist: - if n.startswith(item): - weights[n] = p - - self.weights = weights.items() - - -class BaseOptimizerParamsHandler(BaseHandler): - """ - Base handler for logging optimizer parameters - """ - - def __init__(self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None): - if not ( - isinstance(optimizer, Optimizer) - or (hasattr(optimizer, "param_groups") and isinstance(optimizer.param_groups, Sequence)) - ): - raise TypeError( - "Argument optimizer should be torch.optim.Optimizer or has attribute 'param_groups' as list/tuple, " - f"but given {type(optimizer)}" - ) - - self.optimizer = optimizer - self.param_name = param_name - self.tag = tag - - -class BaseOutputHandler(BaseHandler): - """ - Helper handler to log engine's output and/or metrics - """ - - def __init__( - self, - tag: str, - metric_names: Optional[Union[str, List[str]]] = None, - output_transform: Optional[Callable] = None, - global_step_transform: Optional[Callable[[Engine, Union[str, Events]], int]] = None, - state_attributes: Optional[List[str]] = None, - ): - if metric_names is not None: - if not (isinstance(metric_names, list) or (isinstance(metric_names, str) and metric_names == "all")): - raise TypeError( - f"metric_names should be either a list or equal 'all', got {type(metric_names)} instead." - ) - - if output_transform is not None and not callable(output_transform): - raise TypeError(f"output_transform should be a function, got {type(output_transform)} instead.") - - if output_transform is None and metric_names is None and state_attributes is None: - raise ValueError("Either metric_names, output_transform or state_attributes should be defined") - - if global_step_transform is not None and not callable(global_step_transform): - raise TypeError(f"global_step_transform should be a function, got {type(global_step_transform)} instead.") - - if global_step_transform is None: - - def global_step_transform(engine: Engine, event_name: Union[str, Events]) -> int: - return engine.state.get_event_attrib_value(event_name) - - self.tag = tag - self.metric_names = metric_names - self.output_transform = output_transform - self.global_step_transform = global_step_transform - self.state_attributes = state_attributes - - def _setup_output_metrics_state_attrs( - self, engine: Engine, log_text: Optional[bool] = False, key_tuple: Optional[bool] = True - ) -> Dict[Any, Any]: - """Helper method to setup metrics and state attributes to log""" - metrics_state_attrs = OrderedDict() - if self.metric_names is not None: - if isinstance(self.metric_names, str) and self.metric_names == "all": - metrics_state_attrs = OrderedDict(engine.state.metrics) - else: - for name in self.metric_names: - if name not in engine.state.metrics: - warnings.warn( - f"Provided metric name '{name}' is missing " - f"in engine's state metrics: {list(engine.state.metrics.keys())}" - ) - continue - metrics_state_attrs[name] = engine.state.metrics[name] - - if self.output_transform is not None: - output_dict = self.output_transform(engine.state.output) - - if not isinstance(output_dict, dict): - output_dict = {"output": output_dict} - - metrics_state_attrs.update(output_dict) - - if self.state_attributes is not None: - metrics_state_attrs.update({name: getattr(engine.state, name, None) for name in self.state_attributes}) - - metrics_state_attrs_dict: Dict[Any, Union[str, float, numbers.Number]] = OrderedDict() - - def key_tuple_tf(tag: str, name: str, *args: str) -> Tuple[str, ...]: - return (tag, name) + args - - def key_str_tf(tag: str, name: str, *args: str) -> str: - return "/".join((tag, name) + args) - - key_tf = key_tuple_tf if key_tuple else key_str_tf - - for name, value in metrics_state_attrs.items(): - if isinstance(value, numbers.Number): - metrics_state_attrs_dict[key_tf(self.tag, name)] = value - elif isinstance(value, torch.Tensor) and value.ndimension() == 0: - metrics_state_attrs_dict[key_tf(self.tag, name)] = value.item() - elif isinstance(value, torch.Tensor) and value.ndimension() == 1: - for i, v in enumerate(value): - metrics_state_attrs_dict[key_tf(self.tag, name, str(i))] = v.item() - else: - if isinstance(value, str) and log_text: - metrics_state_attrs_dict[key_tf(self.tag, name)] = value - else: - warnings.warn(f"Logger output_handler can not log metrics value type {type(value)}") - return metrics_state_attrs_dict - - -class BaseWeightsScalarHandler(BaseWeightsHandler): - """ - Helper handler to log model's weights or gradients as scalars. - """ - - def __init__( - self, - model: nn.Module, - reduction: Callable[[torch.Tensor], Union[float, torch.Tensor]] = torch.norm, - tag: Optional[str] = None, - whitelist: Optional[Union[List[str], Callable[[str, nn.Parameter], bool]]] = None, - ): - super(BaseWeightsScalarHandler, self).__init__(model, tag=tag, whitelist=whitelist) - - if not callable(reduction): - raise TypeError(f"Argument reduction should be callable, but given {type(reduction)}") - - def _is_0D_tensor(t: Any) -> bool: - return isinstance(t, torch.Tensor) and t.ndimension() == 0 - - # Test reduction function on a tensor - o = reduction(torch.ones(4, 2)) - if not (isinstance(o, numbers.Number) or _is_0D_tensor(o)): - raise TypeError(f"Output of the reduction function should be a scalar, but got {type(o)}") - - self.reduction = reduction - - -class BaseLogger(metaclass=ABCMeta): - """ - Base logger handler. See implementations: TensorboardLogger, VisdomLogger, PolyaxonLogger, MLflowLogger, ... - - """ - - def attach( - self, - engine: Engine, - log_handler: Callable, - event_name: Union[str, Events, CallableEventWithFilter, EventsList], - *args: Any, - **kwargs: Any, - ) -> RemovableEventHandle: - """Attach the logger to the engine and execute `log_handler` function at `event_name` events. - - Args: - engine: engine object. - log_handler: a logging handler to execute - event_name: event to attach the logging handler to. Valid events are from - :class:`~ignite.engine.events.Events` or :class:`~ignite.engine.events.EventsList` or any `event_name` - added by :meth:`~ignite.engine.engine.Engine.register_events`. - args: args forwarded to the `log_handler` method - kwargs: kwargs forwarded to the `log_handler` method - - Returns: - :class:`~ignite.engine.events.RemovableEventHandle`, which can be used to remove the handler. - """ - if isinstance(event_name, EventsList): - for name in event_name: - if name not in State.event_to_attr: - raise RuntimeError(f"Unknown event name '{name}'") - engine.add_event_handler(name, log_handler, self, name) - - return RemovableEventHandle(event_name, log_handler, engine) - - else: - if event_name not in State.event_to_attr: - raise RuntimeError(f"Unknown event name '{event_name}'") - - return engine.add_event_handler(event_name, log_handler, self, event_name, *args, **kwargs) - - def attach_output_handler(self, engine: Engine, event_name: Any, *args: Any, **kwargs: Any) -> RemovableEventHandle: - """Shortcut method to attach `OutputHandler` to the logger. - - Args: - engine: engine object. - event_name: event to attach the logging handler to. Valid events are from - :class:`~ignite.engine.events.Events` or any `event_name` added by - :meth:`~ignite.engine.engine.Engine.register_events`. - args: args to initialize `OutputHandler` - kwargs: kwargs to initialize `OutputHandler` - - Returns: - :class:`~ignite.engine.events.RemovableEventHandle`, which can be used to remove the handler. - """ - return self.attach(engine, self._create_output_handler(*args, **kwargs), event_name=event_name) - - def attach_opt_params_handler( - self, engine: Engine, event_name: Any, *args: Any, **kwargs: Any - ) -> RemovableEventHandle: - """Shortcut method to attach `OptimizerParamsHandler` to the logger. - - Args: - engine: engine object. - event_name: event to attach the logging handler to. Valid events are from - :class:`~ignite.engine.events.Events` or any `event_name` added by - :meth:`~ignite.engine.engine.Engine.register_events`. - args: args to initialize `OptimizerParamsHandler` - kwargs: kwargs to initialize `OptimizerParamsHandler` - - Returns: - :class:`~ignite.engine.events.RemovableEventHandle`, which can be used to remove the handler. - - .. versionchanged:: 0.4.3 - Added missing return statement. - """ - return self.attach(engine, self._create_opt_params_handler(*args, **kwargs), event_name=event_name) - - @abstractmethod - def _create_output_handler(self, engine: Engine, *args: Any, **kwargs: Any) -> Callable: - pass - - @abstractmethod - def _create_opt_params_handler(self, *args: Any, **kwargs: Any) -> Callable: - pass - - def __enter__(self) -> "BaseLogger": - return self - - def __exit__(self, type: Any, value: Any, traceback: Any) -> None: - self.close() - - def close(self) -> None: - pass +removed_in = "0.6.0" +deprecation_warning = ( + f"{__file__} has been moved to /ignite/handlers/base_logger.py" + + (f" and will be removed in version {removed_in}" if removed_in else "") + + ".\n Please refer to the documentation for more details." +) +warnings.warn(deprecation_warning, DeprecationWarning, stacklevel=2) +from ignite.handlers.base_logger import ( + BaseHandler, + BaseLogger, + BaseOptimizerParamsHandler, + BaseOutputHandler, + BaseWeightsHandler, + BaseWeightsScalarHandler, +) + +__all__ = [ + "BaseHandler", + "BaseWeightsHandler", + "BaseOptimizerParamsHandler", + "BaseOutputHandler", + "BaseWeightsScalarHandler", + "BaseLogger", +] +BaseHandler = BaseHandler +BaseWeightsHandler = BaseWeightsHandler +BaseOptimizerParamsHandler = BaseOptimizerParamsHandler +BaseOutputHandler = BaseOutputHandler +BaseWeightsScalarHandler = BaseWeightsScalarHandler +BaseLogger = BaseLogger diff --git a/ignite/contrib/handlers/clearml_logger.py b/ignite/contrib/handlers/clearml_logger.py index 6b2c30dee85..51af7c6fccb 100644 --- a/ignite/contrib/handlers/clearml_logger.py +++ b/ignite/contrib/handlers/clearml_logger.py @@ -1,26 +1,29 @@ -"""ClearML logger and its helper handlers.""" +""" ``ignite.contrib.handlers.clearml_logger`` was moved to ``ignite.handlers.clearml_logger``. +Note: + ``ignite.contrib.handlers.clearml_logger`` was moved to ``ignite.handlers.clearml_logger``. + Please refer to :mod:`~ignite.handlers.clearml_logger`. +""" -import os -import tempfile import warnings -from collections import defaultdict -from datetime import datetime -from enum import Enum -from typing import Any, Callable, DefaultDict, List, Mapping, Optional, Tuple, Type, Union -from torch.optim import Optimizer - -import ignite.distributed as idist -from ignite.contrib.handlers.base_logger import ( - BaseLogger, - BaseOptimizerParamsHandler, - BaseOutputHandler, - BaseWeightsHandler, - BaseWeightsScalarHandler, +removed_in = "0.6.0" +deprecation_warning = ( + f"{__file__} has been moved to /ignite/handlers/clearml_logger.py" + + (f" and will be removed in version {removed_in}" if removed_in else "") + + ".\n Please refer to the documentation for more details." +) +warnings.warn(deprecation_warning, DeprecationWarning, stacklevel=2) +from ignite.handlers.clearml_logger import ( + ClearMLLogger, + ClearMLSaver, + GradsHistHandler, + GradsScalarHandler, + OptimizerParamsHandler, + OutputHandler, + WeightsHistHandler, + WeightsScalarHandler, ) -from ignite.engine import Engine, Events -from ignite.handlers import global_step_from_engine -from ignite.handlers.checkpoint import DiskSaver +from ignite.handlers.utils import global_step_from_engine # noqa __all__ = [ "ClearMLLogger", @@ -31,961 +34,12 @@ "WeightsHistHandler", "GradsScalarHandler", "GradsHistHandler", - "global_step_from_engine", ] - - -class ClearMLLogger(BaseLogger): - """ - `ClearML `_ handler to log metrics, text, model/optimizer parameters, - plots during training and validation. - Also supports model checkpoints logging and upload to the storage solution of your choice (i.e. ClearML File server, - S3 bucket etc.) - - .. code-block:: bash - - pip install clearml - clearml-init - - Args: - kwargs: Keyword arguments accepted from ``Task.init`` method. - All arguments are optional. If a ClearML Task has already been created, - kwargs will be ignored and the current ClearML Task will be used. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.clearml_logger import * - - # Create a logger - - clearml_logger = ClearMLLogger( - project_name="pytorch-ignite-integration", - task_name="cnn-mnist" - ) - - # Attach the logger to the trainer to log training loss at each iteration - clearml_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - output_transform=lambda loss: {"loss": loss} - ) - - # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch - # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch - # of the `trainer` instead of `train_evaluator`. - clearml_logger.attach_output_handler( - train_evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="training", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer), - ) - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the - # `trainer` instead of `evaluator`. - clearml_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer)), - ) - - # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration - clearml_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer, - param_name='lr' # optional - ) - - # Attach the logger to the trainer to log model's weights norm after each iteration - clearml_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsScalarHandler(model) - ) - - """ - - def __init__(self, **kwargs: Any): - try: - from clearml import Task - from clearml.binding.frameworks.tensorflow_bind import WeightsGradientHistHelper - except ImportError: - raise ModuleNotFoundError( - "This contrib module requires clearml to be installed. " - "You may install clearml using: \n pip install clearml \n" - ) - - experiment_kwargs = {k: v for k, v in kwargs.items() if k not in ("project_name", "task_name", "task_type")} - - if self.bypass_mode(): - warnings.warn("ClearMLSaver: running in bypass mode") - - # Try to retrieve current the ClearML Task before trying to create a new one - self._task = Task.current_task() - - if self._task is None: - self._task = Task.init( - project_name=kwargs.get("project_name"), - task_name=kwargs.get("task_name"), - task_type=kwargs.get("task_type", Task.TaskTypes.training), - **experiment_kwargs, - ) - - self.clearml_logger = self._task.get_logger() - - self.grad_helper = WeightsGradientHistHelper(logger=self.clearml_logger, report_freq=1) - - @classmethod - def set_bypass_mode(cls, bypass: bool) -> None: - """ - Set ``clearml.Task`` to offline mode. - Will bypass all outside communication, and will save all data and logs to a local session folder. - Should only be used in "standalone mode", when there is no access to the *clearml-server*. - - Args: - bypass: If ``True``, all outside communication is skipped. - Data and logs will be stored in a local session folder. - For more information, please refer to `ClearML docs - `_. - """ - from clearml import Task - - setattr(cls, "_bypass", bypass) - Task.set_offline(offline_mode=bypass) - - @classmethod - def bypass_mode(cls) -> bool: - """ - Returns the bypass mode state. - - Note: - `GITHUB_ACTIONS` env will automatically set bypass_mode to ``True`` - unless overridden specifically with ``ClearMLLogger.set_bypass_mode(False)``. - For more information, please refer to `ClearML docs - `_. - - Return: - If True, ``clearml.Task`` is on offline mode, and all outside communication is skipped. - """ - return getattr(cls, "_bypass", bool(os.environ.get("CI"))) - - def __getattr__(self, attr: Any) -> Any: - """ - Calls the corresponding method of ``clearml.Logger``. - - Args: - attr: methods of the ``clearml.Logger`` class. - """ - return getattr(self.clearml_logger, attr) - - def get_task(self) -> Any: - """ - Returns the task context that the logger is reporting. - - Return: - Returns the current task, equivalent to ``clearml.Task.current_task()``. - """ - return self._task - - def close(self) -> None: - self.clearml_logger.flush() - - def _create_output_handler(self, *args: Any, **kwargs: Any) -> "OutputHandler": - return OutputHandler(*args, **kwargs) - - def _create_opt_params_handler(self, *args: Any, **kwargs: Any) -> "OptimizerParamsHandler": - return OptimizerParamsHandler(*args, **kwargs) - - -class OutputHandler(BaseOutputHandler): - """Helper handler to log engine's output and/or metrics - - Args: - tag: common title for all produced plots. For example, "training" - metric_names: list of metric names to plot or a string "all" to plot all available - metrics. - output_transform: output transform function to prepare `engine.state.output` as a number. - For example, `output_transform = lambda output: output` - This function can also return a dictionary, e.g `{"loss": loss1, "another_loss": loss2}` to label the plot - with corresponding keys. - global_step_transform: global step transform function to output a desired global step. - Input of the function is `(engine, event_name)`. Output of function should be an integer. - Default is None, global_step based on attached engine. If provided, - uses function output as global_step. To setup global step from another engine, please use - :meth:`~ignite.contrib.handlers.clearml_logger.global_step_from_engine`. - state_attributes: list of attributes of the ``trainer.state`` to plot. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.clearml_logger import * - - # Create a logger - - clearml_logger = ClearMLLogger( - project_name="pytorch-ignite-integration", - task_name="cnn-mnist" - ) - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch - # of the `trainer`: - clearml_logger.attach( - evaluator, - log_handler=OutputHandler( - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer) - ), - event_name=Events.EPOCH_COMPLETED - ) - # or equivalently - clearml_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer) - ) - - Another example, where model is evaluated every 500 iterations: - - .. code-block:: python - - from ignite.contrib.handlers.clearml_logger import * - - @trainer.on(Events.ITERATION_COMPLETED(every=500)) - def evaluate(engine): - evaluator.run(validation_set, max_epochs=1) - - # Create a logger - - clearml_logger = ClearMLLogger( - project_name="pytorch-ignite-integration", - task_name="cnn-mnist" - ) - - def global_step_transform(*args, **kwargs): - return trainer.state.iteration - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # every 500 iterations. Since evaluator engine does not have access to the training iteration, we - # provide a global_step_transform to return the trainer.state.iteration for the global_step, each time - # evaluator metrics are plotted on ClearML. - - clearml_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metrics=["nll", "accuracy"], - global_step_transform=global_step_transform - ) - - Another example where the State Attributes ``trainer.state.alpha`` and ``trainer.state.beta`` - are also logged along with the NLL and Accuracy after each iteration: - - .. code-block:: python - - clearml_logger.attach( - trainer, - log_handler=OutputHandler( - tag="training", - metric_names=["nll", "accuracy"], - state_attributes=["alpha", "beta"], - ), - event_name=Events.ITERATION_COMPLETED - ) - - Example of `global_step_transform` - - .. code-block:: python - - def global_step_transform(engine, event_name): - return engine.state.get_event_attrib_value(event_name) - - .. versionchanged:: 0.4.7 - accepts an optional list of `state_attributes` - """ - - def __init__( - self, - tag: str, - metric_names: Optional[List[str]] = None, - output_transform: Optional[Callable] = None, - global_step_transform: Optional[Callable[[Engine, Union[str, Events]], int]] = None, - state_attributes: Optional[List[str]] = None, - ): - super(OutputHandler, self).__init__( - tag, metric_names, output_transform, global_step_transform, state_attributes - ) - - def __call__(self, engine: Engine, logger: ClearMLLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, ClearMLLogger): - raise RuntimeError("Handler OutputHandler works only with ClearMLLogger") - - metrics = self._setup_output_metrics_state_attrs(engine) - - global_step = self.global_step_transform(engine, event_name) - - if not isinstance(global_step, int): - raise TypeError( - f"global_step must be int, got {type(global_step)}." - " Please check the output of global_step_transform." - ) - - for key, value in metrics.items(): - if len(key) == 2: - logger.clearml_logger.report_scalar(title=key[0], series=key[1], iteration=global_step, value=value) - elif len(key) == 3: - logger.clearml_logger.report_scalar( - title=f"{key[0]}/{key[1]}", series=key[2], iteration=global_step, value=value - ) - - -class OptimizerParamsHandler(BaseOptimizerParamsHandler): - """Helper handler to log optimizer parameters - - Args: - optimizer: torch optimizer or any object with attribute ``param_groups`` - as a sequence. - param_name: parameter name - tag: common title for all produced plots. For example, "generator" - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.clearml_logger import * - - # Create a logger - - clearml_logger = ClearMLLogger( - project_name="pytorch-ignite-integration", - task_name="cnn-mnist" - ) - - # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration - clearml_logger.attach( - trainer, - log_handler=OptimizerParamsHandler(optimizer), - event_name=Events.ITERATION_STARTED - ) - # or equivalently - clearml_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer - ) - """ - - def __init__(self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None): - super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) - - def __call__(self, engine: Engine, logger: ClearMLLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, ClearMLLogger): - raise RuntimeError("Handler OptimizerParamsHandler works only with ClearMLLogger") - - global_step = engine.state.get_event_attrib_value(event_name) - tag_prefix = f"{self.tag}/" if self.tag else "" - params = { - str(i): float(param_group[self.param_name]) for i, param_group in enumerate(self.optimizer.param_groups) - } - - for k, v in params.items(): - logger.clearml_logger.report_scalar( - title=f"{tag_prefix}{self.param_name}", series=k, value=v, iteration=global_step - ) - - -class WeightsScalarHandler(BaseWeightsScalarHandler): - """Helper handler to log model's weights as scalars. - Handler, upon construction, iterates over named parameters of the model and keep - reference to ones permitted by `whitelist`. Then at every call, applies - reduction function to each parameter, produces a scalar and logs it. - - Args: - model: model to log weights - reduction: function to reduce parameters into scalar - tag: common title for all produced plots. For example, "generator" - whitelist: specific weights to log. Should be list of model's submodules - or parameters names, or a callable which gets weight along with its name - and determines if it should be logged. Names should be fully-qualified. - For more information please refer to `PyTorch docs - `_. - If not given, all of model's weights are logged. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.clearml_logger import * - - # Create a logger - - clearml_logger = ClearMLLogger( - project_name="pytorch-ignite-integration", - task_name="cnn-mnist" - ) - - # Attach the logger to the trainer to log model's weights norm after each iteration - clearml_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsScalarHandler(model, reduction=torch.norm) - ) - - .. code-block:: python - - from ignite.contrib.handlers.clearml_logger import * - - clearml_logger = ClearMLLogger( - project_name="pytorch-ignite-integration", - task_name="cnn-mnist" - ) - - # Log only `fc` weights - clearml_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsScalarHandler( - model, - whitelist=['fc'] - ) - ) - - .. code-block:: python - - from ignite.contrib.handlers.clearml_logger import * - - clearml_logger = ClearMLLogger( - project_name="pytorch-ignite-integration", - task_name="cnn-mnist" - ) - - # Log weights which have `bias` in their names - def has_bias_in_name(n, p): - return 'bias' in n - - clearml_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsScalarHandler(model, whitelist=has_bias_in_name) - ) - - .. versionchanged:: 0.4.9 - optional argument `whitelist` added. - """ - - def __call__(self, engine: Engine, logger: ClearMLLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, ClearMLLogger): - raise RuntimeError("Handler WeightsScalarHandler works only with ClearMLLogger") - - global_step = engine.state.get_event_attrib_value(event_name) - tag_prefix = f"{self.tag}/" if self.tag else "" - for name, p in self.weights: - title_name, _, series_name = name.partition(".") - logger.clearml_logger.report_scalar( - title=f"{tag_prefix}weights_{self.reduction.__name__}/{title_name}", - series=series_name, - value=self.reduction(p.data), - iteration=global_step, - ) - - -class WeightsHistHandler(BaseWeightsHandler): - """Helper handler to log model's weights as histograms. - - Args: - model: model to log weights - tag: common title for all produced plots. For example, 'generator' - whitelist: specific weights to log. Should be list of model's submodules - or parameters names, or a callable which gets weight along with its name - and determines if it should be logged. Names should be fully-qualified. - For more information please refer to `PyTorch docs - `_. - If not given, all of model's weights are logged. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.clearml_logger import * - - # Create a logger - - clearml_logger = ClearMLLogger( - project_name="pytorch-ignite-integration", - task_name="cnn-mnist" - ) - - # Attach the logger to the trainer to log model's weights norm after each iteration - clearml_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsHistHandler(model) - ) - - .. code-block:: python - - from ignite.contrib.handlers.clearml_logger import * - - clearml_logger = ClearMLLogger( - project_name="pytorch-ignite-integration", - task_name="cnn-mnist" - ) - - # Log weights of `fc` layer - weights = ['fc'] - - # Attach the logger to the trainer to log weights norm after each iteration - clearml_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsHistHandler(model, whitelist=weights) - ) - - .. code-block:: python - - from ignite.contrib.handlers.clearml_logger import * - - clearml_logger = ClearMLLogger( - project_name="pytorch-ignite-integration", - task_name="cnn-mnist" - ) - - # Log weights which name include 'conv'. - weight_selector = lambda name, p: 'conv' in name - - # Attach the logger to the trainer to log weights norm after each iteration - clearml_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsHistHandler(model, whitelist=weight_selector) - ) - - .. versionchanged:: 0.4.9 - optional argument `whitelist` added. - """ - - def __call__(self, engine: Engine, logger: ClearMLLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, ClearMLLogger): - raise RuntimeError("Handler 'WeightsHistHandler' works only with ClearMLLogger") - - global_step = engine.state.get_event_attrib_value(event_name) - tag_prefix = f"{self.tag}/" if self.tag else "" - for name, p in self.weights: - title_name, _, series_name = name.partition(".") - - logger.grad_helper.add_histogram( - title=f"{tag_prefix}weights_{title_name}", - series=series_name, - step=global_step, - hist_data=p.data.cpu().numpy(), - ) - - -class GradsScalarHandler(BaseWeightsScalarHandler): - """Helper handler to log model's gradients as scalars. - Handler, upon construction, iterates over named parameters of the model and keep - reference to ones permitted by the `whitelist`. Then at every call, applies - reduction function to each parameter's gradient, produces a scalar and logs it. - - Args: - model: model to log weights - reduction: function to reduce parameters into scalar - tag: common title for all produced plots. For example, "generator" - whitelist: specific gradients to log. Should be list of model's submodules - or parameters names, or a callable which gets weight along with its name - and determines if its gradient should be logged. Names should be - fully-qualified. For more information please refer to `PyTorch docs - `_. - If not given, all of model's gradients are logged. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.clearml_logger import * - - # Create a logger - - clearml_logger = ClearMLLogger( - project_name="pytorch-ignite-integration", - task_name="cnn-mnist" - ) - - # Attach the logger to the trainer to log model's weights norm after each iteration - clearml_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=GradsScalarHandler(model, reduction=torch.norm) - ) - - .. code-block:: python - - from ignite.contrib.handlers.clearml_logger import * - - clearml_logger = ClearMLLogger( - project_name="pytorch-ignite-integration", - task_name="cnn-mnist" - ) - - # Log gradient of `base` - clearml_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=GradsScalarHandler( - model, - reduction=torch.norm, - whitelist=['base'] - ) - ) - - .. code-block:: python - - from ignite.contrib.handlers.clearml_logger import * - - clearml_logger = ClearMLLogger( - project_name="pytorch-ignite-integration", - task_name="cnn-mnist" - ) - - # Log gradient of weights which belong to a `fc` layer - def is_in_fc_layer(n, p): - return 'fc' in n - - clearml_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=GradsScalarHandler(model, whitelist=is_in_fc_layer) - ) - - .. versionchanged:: 0.4.9 - optional argument `whitelist` added. - """ - - def __call__(self, engine: Engine, logger: ClearMLLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, ClearMLLogger): - raise RuntimeError("Handler GradsScalarHandler works only with ClearMLLogger") - - global_step = engine.state.get_event_attrib_value(event_name) - tag_prefix = f"{self.tag}/" if self.tag else "" - for name, p in self.weights: - if p.grad is None: - continue - - title_name, _, series_name = name.partition(".") - logger.clearml_logger.report_scalar( - title=f"{tag_prefix}grads_{self.reduction.__name__}/{title_name}", - series=series_name, - value=self.reduction(p.grad), - iteration=global_step, - ) - - -class GradsHistHandler(BaseWeightsHandler): - """Helper handler to log model's gradients as histograms. - - Args: - model: model to log weights - tag: common title for all produced plots. For example, 'generator' - whitelist: specific gradients to log. Should be list of model's submodules - or parameters names, or a callable which gets weight along with its name - and determines if its gradient should be logged. Names should be - fully-qualified. For more information please refer to `PyTorch docs - `_. - If not given, all of model's gradients are logged. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.clearml_logger import * - - # Create a logger - - clearml_logger = ClearMLLogger( - project_name="pytorch-ignite-integration", - task_name="cnn-mnist" - ) - - # Attach the logger to the trainer to log model's weights norm after each iteration - clearml_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=GradsHistHandler(model) - ) - - .. code-block:: python - - from ignite.contrib.handlers.clearml_logger import * - - clearml_logger = ClearMLLogger( - project_name="pytorch-ignite-integration", - task_name="cnn-mnist" - ) - - # Log gradient of `fc.bias` - clearml_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=GradsHistHandler(model, whitelist=['fc.bias']) - ) - - .. code-block:: python - - from ignite.contrib.handlers.clearml_logger import * - - clearml_logger = ClearMLLogger( - project_name="pytorch-ignite-integration", - task_name="cnn-mnist" - ) - - # Log gradient of weights which have shape (2, 1) - def has_shape_2_1(n, p): - return p.shape == (2,1) - - clearml_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=GradsHistHandler(model, whitelist=has_shape_2_1) - ) - - .. versionchanged:: 0.4.9 - optional argument `whitelist` added. - """ - - def __call__(self, engine: Engine, logger: ClearMLLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, ClearMLLogger): - raise RuntimeError("Handler 'GradsHistHandler' works only with ClearMLLogger") - - global_step = engine.state.get_event_attrib_value(event_name) - tag_prefix = f"{self.tag}/" if self.tag else "" - for name, p in self.weights: - if p.grad is None: - continue - - title_name, _, series_name = name.partition(".") - logger.grad_helper.add_histogram( - title=f"{tag_prefix}grads_{title_name}", - series=series_name, - step=global_step, - hist_data=p.grad.cpu().numpy(), - ) - - -class ClearMLSaver(DiskSaver): - """ - Handler that saves input checkpoint as ClearML artifacts - - Args: - logger: An instance of :class:`~ignite.contrib.handlers.clearml_logger.ClearMLLogger`, - ensuring a valid ClearML ``Task`` has been initialized. If not provided, and a ClearML Task - has not been manually initialized, a runtime error will be raised. - output_uri: The default location for output models and other artifacts uploaded by ClearML. For - more information, see ``clearml.Task.init``. - dirname: Directory path where the checkpoint will be saved. If not provided, a temporary - directory will be created. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.clearml_logger import * - from ignite.handlers import Checkpoint - - clearml_logger = ClearMLLogger( - project_name="pytorch-ignite-integration", - task_name="cnn-mnist" - ) - - to_save = {"model": model} - - handler = Checkpoint( - to_save, - ClearMLSaver(), - n_saved=1, - score_function=lambda e: 123, - score_name="acc", - filename_prefix="best", - global_step_transform=global_step_from_engine(trainer) - ) - - validation_evaluator.add_event_handler(Events.EVENT_COMPLETED, handler) - - """ - - def __init__( - self, - logger: Optional[ClearMLLogger] = None, - output_uri: Optional[str] = None, - dirname: Optional[str] = None, - *args: Any, - **kwargs: Any, - ): - self._setup_check_clearml(logger, output_uri) - - if not dirname: - dirname = "" - if idist.get_rank() == 0: - dirname = tempfile.mkdtemp(prefix=f"ignite_checkpoints_{datetime.now().strftime('%Y_%m_%d_%H_%M_%S_')}") - if idist.get_world_size() > 1: - dirname = idist.all_gather(dirname)[0] # type: ignore[index, assignment] - - warnings.warn(f"ClearMLSaver created a temporary checkpoints directory: {dirname}") - idist.barrier() - - # Let's set non-atomic tmp dir saving behaviour - if "atomic" not in kwargs: - kwargs["atomic"] = False - - self._checkpoint_slots: DefaultDict[Union[str, Tuple[str, str]], List[Any]] = defaultdict(list) - - super(ClearMLSaver, self).__init__(dirname=dirname, *args, **kwargs) # type: ignore[misc] - - @idist.one_rank_only() - def _setup_check_clearml(self, logger: ClearMLLogger, output_uri: str) -> None: - try: - from clearml import Task - except ImportError: - try: - # Backwards-compatibility for legacy Trains SDK - from trains import Task - except ImportError: - raise ModuleNotFoundError( - "This contrib module requires clearml to be installed. " - "You may install clearml using: \n pip install clearml \n" - ) - - if logger and not isinstance(logger, ClearMLLogger): - raise TypeError("logger must be an instance of ClearMLLogger") - - self._task = Task.current_task() - if not self._task: - raise RuntimeError( - "ClearMLSaver requires a ClearML Task to be initialized. " - "Please use the `logger` argument or call `clearml.Task.init()`." - ) - - if output_uri: - self._task.output_uri = output_uri - - class _CallbacksContext: - def __init__( - self, - callback_type: Type[Enum], - slots: List, - checkpoint_key: str, - filename: str, - basename: str, - metadata: Optional[Mapping] = None, - ) -> None: - self._callback_type = callback_type - self._slots = slots - self._checkpoint_key = str(checkpoint_key) - self._filename = filename - self._basename = basename - self._metadata = metadata - - def pre_callback(self, action: str, model_info: Any) -> Any: - if action != self._callback_type.save: # type: ignore[attr-defined] - return model_info - - try: - slot = self._slots.index(None) - self._slots[slot] = model_info.upload_filename - except ValueError: - self._slots.append(model_info.upload_filename) - slot = len(self._slots) - 1 - - model_info.upload_filename = f"{self._basename}_{slot}{os.path.splitext(self._filename)[1]}" - model_info.local_model_id = f"{self._checkpoint_key}:{model_info.upload_filename}" - return model_info - - def post_callback(self, action: str, model_info: Any) -> Any: - if action != self._callback_type.save: # type: ignore[attr-defined] - return model_info - - model_info.model.name = f"{model_info.task.name}: {self._filename}" - prefix = "Checkpoint Metadata: " - metadata_items = ", ".join(f"{k}={v}" for k, v in self._metadata.items()) if self._metadata else "none" - metadata = f"{prefix}{metadata_items}" - comment = "\n".join( - metadata if line.startswith(prefix) else line for line in (model_info.model.comment or "").split("\n") - ) - if prefix not in comment: - comment += "\n" + metadata - model_info.model.comment = comment - - return model_info - - def __call__(self, checkpoint: Mapping, filename: str, metadata: Optional[Mapping] = None) -> None: - try: - from clearml.binding.frameworks import WeightsFileHandler - except ImportError: - try: - # Backwards-compatibility for legacy Trains SDK - from trains.binding.frameworks import WeightsFileHandler - except ImportError: - raise ModuleNotFoundError( - "This contrib module requires clearml to be installed. " - "You may install clearml using: \n pip install clearml \n" - ) - - try: - basename = metadata["basename"] # type: ignore[index] - except (TypeError, KeyError): - warnings.warn("Checkpoint metadata missing or basename cannot be found") - basename = "checkpoint" - - checkpoint_key = (str(self.dirname), basename) - - cb_context = self._CallbacksContext( - callback_type=WeightsFileHandler.CallbackType, - slots=self._checkpoint_slots[checkpoint_key], - checkpoint_key=str(checkpoint_key), - filename=filename, - basename=basename, - metadata=metadata, - ) - - pre_cb_id = WeightsFileHandler.add_pre_callback(cb_context.pre_callback) - post_cb_id = WeightsFileHandler.add_post_callback(cb_context.post_callback) - - try: - super(ClearMLSaver, self).__call__(checkpoint, filename, metadata) - finally: - WeightsFileHandler.remove_pre_callback(pre_cb_id) - WeightsFileHandler.remove_post_callback(post_cb_id) - - @idist.one_rank_only() - def get_local_copy(self, filename: str) -> Optional[str]: - """Get artifact local copy. - - .. warning:: - - In distributed configuration this method should be called on rank 0 process. - - Args: - filename: artifact name. - - Returns: - a local path to a downloaded copy of the artifact - """ - artifact = self._task.artifacts.get(filename) - if artifact: - return artifact.get_local_copy() - self._task.get_logger().report_text(f"Can not find artifact {filename}") - - return None - - @idist.one_rank_only() - def remove(self, filename: str) -> None: - super(ClearMLSaver, self).remove(filename) - for slots in self._checkpoint_slots.values(): - try: - slots[slots.index(filename)] = None - except ValueError: - pass - else: - break +ClearMLLogger = ClearMLLogger +ClearMLSaver = ClearMLSaver +OptimizerParamsHandler = OptimizerParamsHandler +OutputHandler = OutputHandler +WeightsScalarHandler = WeightsScalarHandler +WeightsHistHandler = WeightsHistHandler +GradsScalarHandler = GradsScalarHandler +GradsHistHandler = GradsHistHandler diff --git a/ignite/contrib/handlers/mlflow_logger.py b/ignite/contrib/handlers/mlflow_logger.py index 16514e987d4..648f15458b0 100644 --- a/ignite/contrib/handlers/mlflow_logger.py +++ b/ignite/contrib/handlers/mlflow_logger.py @@ -1,312 +1,22 @@ -"""MLflow logger and its helper handlers.""" +""" ``ignite.contrib.handlers.mlflow_logger`` was moved to ``ignite.handlers.mlflow_logger``. +Note: + ``ignite.contrib.handlers.mlflow_logger`` was moved to ``ignite.handlers.mlflow_logger``. + Please refer to :mod:`~ignite.handlers.mlflow_logger`. +""" import warnings -from typing import Any, Callable, List, Optional, Union -from torch.optim import Optimizer - -from ignite.contrib.handlers.base_logger import BaseLogger, BaseOptimizerParamsHandler, BaseOutputHandler -from ignite.engine import Engine, Events -from ignite.handlers import global_step_from_engine - -__all__ = ["MLflowLogger", "OutputHandler", "OptimizerParamsHandler", "global_step_from_engine"] - - -class MLflowLogger(BaseLogger): - """ - `MLflow `_ tracking client handler to log parameters and metrics during the training - and validation. - - This class requires `mlflow package `_ to be installed: - - .. code-block:: bash - - pip install mlflow - - Args: - tracking_uri: MLflow tracking uri. See MLflow docs for more details - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.mlflow_logger import * - - # Create a logger - mlflow_logger = MLflowLogger() - - # Log experiment parameters: - mlflow_logger.log_params({ - "seed": seed, - "batch_size": batch_size, - "model": model.__class__.__name__, - - "pytorch version": torch.__version__, - "ignite version": ignite.__version__, - "cuda version": torch.version.cuda, - "device name": torch.cuda.get_device_name(0) - }) - - # Attach the logger to the trainer to log training loss at each iteration - mlflow_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - output_transform=lambda loss: {'loss': loss} - ) - - # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch - # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch - # of the `trainer` instead of `train_evaluator`. - mlflow_logger.attach_output_handler( - train_evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="training", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer), - ) - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the - # `trainer` instead of `evaluator`. - mlflow_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer)), - ) - - # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration - mlflow_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer, - param_name='lr' # optional - ) - """ - - def __init__(self, tracking_uri: Optional[str] = None): - try: - import mlflow - except ImportError: - raise ModuleNotFoundError( - "This contrib module requires mlflow to be installed. " - "Please install it with command: \n pip install mlflow" - ) - - if tracking_uri is not None: - mlflow.set_tracking_uri(tracking_uri) - - self.active_run = mlflow.active_run() - if self.active_run is None: - self.active_run = mlflow.start_run() - - def __getattr__(self, attr: Any) -> Any: - import mlflow - - return getattr(mlflow, attr) - - def close(self) -> None: - import mlflow - - mlflow.end_run() - - def _create_output_handler(self, *args: Any, **kwargs: Any) -> "OutputHandler": - return OutputHandler(*args, **kwargs) - - def _create_opt_params_handler(self, *args: Any, **kwargs: Any) -> "OptimizerParamsHandler": - return OptimizerParamsHandler(*args, **kwargs) - - -class OutputHandler(BaseOutputHandler): - """Helper handler to log engine's output and/or metrics. - - Args: - tag: common title for all produced plots. For example, 'training' - metric_names: list of metric names to plot or a string "all" to plot all available - metrics. - output_transform: output transform function to prepare `engine.state.output` as a number. - For example, `output_transform = lambda output: output` - This function can also return a dictionary, e.g `{'loss': loss1, 'another_loss': loss2}` to label the plot - with corresponding keys. - global_step_transform: global step transform function to output a desired global step. - Input of the function is `(engine, event_name)`. Output of function should be an integer. - Default is None, global_step based on attached engine. If provided, - uses function output as global_step. To setup global step from another engine, please use - :meth:`~ignite.contrib.handlers.mlflow_logger.global_step_from_engine`. - state_attributes: list of attributes of the ``trainer.state`` to plot. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.mlflow_logger import * - - # Create a logger - mlflow_logger = MLflowLogger() - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch - # of the `trainer`: - mlflow_logger.attach( - evaluator, - log_handler=OutputHandler( - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer) - ), - event_name=Events.EPOCH_COMPLETED - ) - # or equivalently - mlflow_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer) - ) - - Another example, where model is evaluated every 500 iterations: - - .. code-block:: python - - from ignite.contrib.handlers.mlflow_logger import * - - @trainer.on(Events.ITERATION_COMPLETED(every=500)) - def evaluate(engine): - evaluator.run(validation_set, max_epochs=1) - - mlflow_logger = MLflowLogger() - - def global_step_transform(*args, **kwargs): - return trainer.state.iteration - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # every 500 iterations. Since evaluator engine does not have access to the training iteration, we - # provide a global_step_transform to return the trainer.state.iteration for the global_step, each time - # evaluator metrics are plotted on MLflow. - - mlflow_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metrics=["nll", "accuracy"], - global_step_transform=global_step_transform - ) - - Another example where the State Attributes ``trainer.state.alpha`` and ``trainer.state.beta`` - are also logged along with the NLL and Accuracy after each iteration: - - .. code-block:: python - - mlflow_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - metrics=["nll", "accuracy"], - state_attributes=["alpha", "beta"], - ) - - Example of `global_step_transform`: - - .. code-block:: python - - def global_step_transform(engine, event_name): - return engine.state.get_event_attrib_value(event_name) - - .. versionchanged:: 0.4.7 - accepts an optional list of `state_attributes` - """ - - def __init__( - self, - tag: str, - metric_names: Optional[Union[str, List[str]]] = None, - output_transform: Optional[Callable] = None, - global_step_transform: Optional[Callable[[Engine, Union[str, Events]], int]] = None, - state_attributes: Optional[List[str]] = None, - ) -> None: - super(OutputHandler, self).__init__( - tag, metric_names, output_transform, global_step_transform, state_attributes - ) - - def __call__(self, engine: Engine, logger: MLflowLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, MLflowLogger): - raise TypeError("Handler 'OutputHandler' works only with MLflowLogger") - - rendered_metrics = self._setup_output_metrics_state_attrs(engine) - - global_step = self.global_step_transform(engine, event_name) - - if not isinstance(global_step, int): - raise TypeError( - f"global_step must be int, got {type(global_step)}." - " Please check the output of global_step_transform." - ) - - # Additionally recheck metric names as MLflow rejects non-valid names with MLflowException - from mlflow.utils.validation import _VALID_PARAM_AND_METRIC_NAMES - - metrics = {} - for keys, value in rendered_metrics.items(): - key = " ".join(keys) - metrics[key] = value - - for key in list(metrics.keys()): - if not _VALID_PARAM_AND_METRIC_NAMES.match(key): - warnings.warn( - f"MLflowLogger output_handler encountered an invalid metric name '{key}' that " - "will be ignored and not logged to MLflow" - ) - del metrics[key] - - logger.log_metrics(metrics, step=global_step) - - -class OptimizerParamsHandler(BaseOptimizerParamsHandler): - """Helper handler to log optimizer parameters - - Args: - optimizer: torch optimizer or any object with attribute ``param_groups`` - as a sequence. - param_name: parameter name - tag: common title for all produced plots. For example, 'generator' - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.mlflow_logger import * - - # Create a logger - mlflow_logger = MLflowLogger() - # Optionally, user can specify tracking_uri with corresponds to MLFLOW_TRACKING_URI - # mlflow_logger = MLflowLogger(tracking_uri="uri") - - # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration - mlflow_logger.attach( - trainer, - log_handler=OptimizerParamsHandler(optimizer), - event_name=Events.ITERATION_STARTED - ) - # or equivalently - mlflow_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer - ) - """ - - def __init__(self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None): - super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) - - def __call__(self, engine: Engine, logger: MLflowLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, MLflowLogger): - raise TypeError("Handler OptimizerParamsHandler works only with MLflowLogger") - - global_step = engine.state.get_event_attrib_value(event_name) - tag_prefix = f"{self.tag} " if self.tag else "" - params = { - f"{tag_prefix}{self.param_name} group_{i}": float(param_group[self.param_name]) - for i, param_group in enumerate(self.optimizer.param_groups) - } - - logger.log_metrics(params, step=global_step) +removed_in = "0.6.0" +deprecation_warning = ( + f"{__file__} has been moved to /ignite/handlers/mlflow_logger.py" + + (f" and will be removed in version {removed_in}" if removed_in else "") + + ".\n Please refer to the documentation for more details." +) +warnings.warn(deprecation_warning, DeprecationWarning, stacklevel=2) +from ignite.handlers.mlflow_logger import MLflowLogger, OptimizerParamsHandler, OutputHandler +from ignite.handlers.utils import global_step_from_engine # noqa + +__all__ = ["MLflowLogger", "OutputHandler", "OptimizerParamsHandler"] +MLflowLogger = MLflowLogger +OutputHandler = OutputHandler +OptimizerParamsHandler = OptimizerParamsHandler diff --git a/ignite/contrib/handlers/neptune_logger.py b/ignite/contrib/handlers/neptune_logger.py index cac4b4fbbed..8a279560f15 100644 --- a/ignite/contrib/handlers/neptune_logger.py +++ b/ignite/contrib/handlers/neptune_logger.py @@ -1,23 +1,28 @@ -"""Neptune logger and its helper handlers.""" +""" ``ignite.contrib.handlers.neptune_logger`` was moved to ``ignite.handlers.neptune_logger``. +Note: + ``ignite.contrib.handlers.neptune_logger`` was moved to ``ignite.handlers.neptune_logger``. + Please refer to :mod:`~ignite.handlers.neptune_logger`. +""" -import tempfile import warnings -from typing import Any, Callable, List, Mapping, Optional, Union -import torch -from torch.optim import Optimizer - -import ignite.distributed as idist -from ignite import __version__ -from ignite.contrib.handlers.base_logger import ( - BaseLogger, - BaseOptimizerParamsHandler, - BaseOutputHandler, - BaseWeightsScalarHandler, +removed_in = "0.6.0" +deprecation_warning = ( + f"{__file__} has been moved to /ignite/handlers/neptune_logger.py" + + (f" and will be removed in version {removed_in}" if removed_in else "") + + ".\n Please refer to the documentation for more details." +) +warnings.warn(deprecation_warning, DeprecationWarning, stacklevel=2) +from ignite.handlers.neptune_logger import ( + _INTEGRATION_VERSION_KEY, + GradsScalarHandler, + NeptuneLogger, + NeptuneSaver, + OptimizerParamsHandler, + OutputHandler, + WeightsScalarHandler, ) -from ignite.engine import Engine, Events -from ignite.handlers import global_step_from_engine -from ignite.handlers.checkpoint import BaseSaveHandler +from ignite.handlers.utils import global_step_from_engine # noqa __all__ = [ "NeptuneLogger", @@ -26,679 +31,11 @@ "OutputHandler", "WeightsScalarHandler", "GradsScalarHandler", - "global_step_from_engine", ] - -_INTEGRATION_VERSION_KEY = "source_code/integrations/neptune-pytorch-ignite" - - -class NeptuneLogger(BaseLogger): - """ - `Neptune `_ handler to log metrics, model/optimizer parameters and gradients during training - and validation. It can also log model checkpoints to Neptune. - - .. code-block:: bash - - pip install neptune - - Args: - api_token: Neptune API token, found on https://neptune.ai -> User menu -> "Get your API token". - If None, the value of the NEPTUNE_API_TOKEN environment variable is used. To keep your token - secure, you should set it to the environment variable rather than including it in your code. - project: Name of a Neptune project, in the form "workspace-name/project-name". - For example "tom/mnist-classification". - If None, the value of the NEPTUNE_PROJECT environment variable is used. - **kwargs: Other arguments to be passed to the `init_run()` function. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.neptune_logger import * - - # Create a logger - # Note: We are using the API token for anonymous logging. You can pass your own token, or save it as an - # environment variable and leave out the api_token argument. - - npt_logger = NeptuneLogger( - api_token="ANONYMOUS", - project="common/pytorch-ignite-integration", - name="cnn-mnist", # Optional, - tags=["pytorch-ignite", "minst"], # Optional - ) - - # Attach the logger to the trainer to log training loss at each iteration. - npt_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - output_transform=lambda loss: {"loss": loss}, - ) - - # Attach the logger to the evaluator on the training dataset and log NLL - # and accuracy metrics after each epoch. - # We set up `global_step_transform=global_step_from_engine(trainer)` to take the epoch - # of the `trainer` instead of `train_evaluator`. - npt_logger.attach_output_handler( - train_evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="training", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer), - ) - - # Attach the logger to the evaluator on the validation dataset and log NLL and accuracy metrics after - # each epoch. We set up `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the - # `trainer` instead of `evaluator`. - npt_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer), - ) - - # Attach the logger to the trainer to log optimizer parameters, such as learning rate at each iteration. - npt_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer, - param_name="lr", # optional - ) - - # Attach the logger to the trainer to log model's weights norm after each iteration. - npt_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsScalarHandler(model), - ) - - Explore runs with Neptune tracking here: - https://app.neptune.ai/o/common/org/pytorch-ignite-integration/ - - You can also save model checkpoints to a Neptune: - - .. code-block:: python - - from ignite.handlers import Checkpoint - - - def score_function(engine): - return engine.state.metrics["accuracy"] - - - to_save = {"model": model} - handler = Checkpoint( - to_save, - NeptuneSaver(npt_logger), n_saved=2, - filename_prefix="best", - score_function=score_function, - score_name="validation_accuracy", - global_step_transform=global_step_from_engine(trainer), - ) - validation_evaluator.add_event_handler(Events.COMPLETED, handler) - - It is also possible to use the logger as a context manager: - - .. code-block:: python - - from ignite.contrib.handlers.neptune_logger import * - - with NeptuneLogger() as npt_logger: - trainer = Engine(update_fn) - # Attach the logger to the trainer to log training loss at each iteration - npt_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - output_transform=lambda loss: {"loss": loss}, - ) - - """ - - def __getattr__(self, attr: Any) -> Any: - return getattr(self.experiment, attr) - - def __getitem__(self, key: str) -> Any: - return self.experiment[key] - - def __setitem__(self, key: str, val: Any) -> Any: - self.experiment[key] = val - - def __init__(self, api_token: Optional[str] = None, project: Optional[str] = None, **kwargs: Any) -> None: - try: - try: - # neptune-client<1.0.0 package structure - with warnings.catch_warnings(): - # ignore the deprecation warnings - warnings.simplefilter("ignore") - import neptune.new as neptune - except ImportError: - # neptune>=1.0.0 package structure - import neptune - except ImportError: - raise ModuleNotFoundError( - "This contrib module requires the Neptune client library to be installed. " - "Install neptune with the command: \n pip install neptune \n" - ) - - run = neptune.init_run( - api_token=api_token, - project=project, - **kwargs, - ) - run[_INTEGRATION_VERSION_KEY] = __version__ - - self.experiment = run - - def close(self) -> None: - self.experiment.stop() - - def _create_output_handler(self, *args: Any, **kwargs: Any) -> "OutputHandler": - return OutputHandler(*args, **kwargs) - - def _create_opt_params_handler(self, *args: Any, **kwargs: Any) -> "OptimizerParamsHandler": - return OptimizerParamsHandler(*args, **kwargs) - - -class OutputHandler(BaseOutputHandler): - """Helper handler to log engine's output and/or metrics. - - Args: - tag: common title for all produced plots. For example, "training" - metric_names: list of metric names to plot or a string "all" to plot all available - metrics. - output_transform: output transform function to prepare `engine.state.output` as a number. - For example, `output_transform = lambda output: output` - This function can also return a dictionary, e.g `{"loss": loss1, "another_loss": loss2}` to label the plot - with corresponding keys. - global_step_transform: global step transform function to output a desired global step. - Input of the function is `(engine, event_name)`. Output of function should be an integer. - Default is None, global_step based on attached engine. If provided, - uses function output as global_step. To setup global step from another engine, please use - :meth:`~ignite.contrib.handlers.neptune_logger.global_step_from_engine`. - state_attributes: list of attributes of the ``trainer.state`` to plot. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.neptune_logger import * - - # Create a logger - # We are using the api_token for the anonymous user neptuner but you can use your own. - - npt_logger = NeptuneLogger( - api_token="ANONYMOUS", - project_name="shared/pytorch-ignite-integration", - experiment_name="cnn-mnist", # Optional, - params={"max_epochs": 10}, # Optional, - tags=["pytorch-ignite","minst"] # Optional - ) - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch - # of the `trainer`: - npt_logger.attach( - evaluator, - log_handler=OutputHandler( - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer) - ), - event_name=Events.EPOCH_COMPLETED - ) - # or equivalently - npt_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer) - ) - - Another example, where model is evaluated every 500 iterations: - - .. code-block:: python - - from ignite.contrib.handlers.neptune_logger import * - - @trainer.on(Events.ITERATION_COMPLETED(every=500)) - def evaluate(engine): - evaluator.run(validation_set, max_epochs=1) - - # We are using the api_token for the anonymous user neptuner but you can use your own. - - npt_logger = NeptuneLogger( - api_token="ANONYMOUS", - project_name="shared/pytorch-ignite-integration", - experiment_name="cnn-mnist", # Optional, - params={"max_epochs": 10}, # Optional, - tags=["pytorch-ignite", "minst"] # Optional - ) - - def global_step_transform(*args, **kwargs): - return trainer.state.iteration - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # every 500 iterations. Since evaluator engine does not have access to the training iteration, we - # provide a global_step_transform to return the trainer.state.iteration for the global_step, each time - # evaluator metrics are plotted on NeptuneML. - - npt_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metrics=["nll", "accuracy"], - global_step_transform=global_step_transform - ) - - Another example where the State Attributes ``trainer.state.alpha`` and ``trainer.state.beta`` - are also logged along with the NLL and Accuracy after each iteration: - - .. code-block:: python - - npt_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - metrics=["nll", "accuracy"], - state_attributes=["alpha", "beta"], - ) - - Example of `global_step_transform`: - - .. code-block:: python - - def global_step_transform(engine, event_name): - return engine.state.get_event_attrib_value(event_name) - - .. versionchanged:: 0.4.7 - accepts an optional list of `state_attributes` - """ - - def __init__( - self, - tag: str, - metric_names: Optional[Union[str, List[str]]] = None, - output_transform: Optional[Callable] = None, - global_step_transform: Optional[Callable[[Engine, Union[str, Events]], int]] = None, - state_attributes: Optional[List[str]] = None, - ): - super(OutputHandler, self).__init__( - tag, metric_names, output_transform, global_step_transform, state_attributes - ) - - def __call__(self, engine: Engine, logger: NeptuneLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, NeptuneLogger): - raise TypeError("Handler OutputHandler works only with NeptuneLogger") - - metrics = self._setup_output_metrics_state_attrs(engine, key_tuple=False) - - global_step = self.global_step_transform(engine, event_name) - - if not isinstance(global_step, int): - raise TypeError( - f"global_step must be int, got {type(global_step)}." - " Please check the output of global_step_transform." - ) - - for key, value in metrics.items(): - logger[key].append(value, step=global_step) - - -class OptimizerParamsHandler(BaseOptimizerParamsHandler): - """Helper handler to log optimizer parameters - - Args: - optimizer: torch optimizer or any object with attribute ``param_groups`` - as a sequence. - param_name: parameter name - tag: common title for all produced plots. For example, "generator" - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.neptune_logger import * - - # Create a logger - # We are using the api_token for the anonymous user neptuner but you can use your own. - - npt_logger = NeptuneLogger( - api_token="ANONYMOUS", - project_name="shared/pytorch-ignite-integration", - experiment_name="cnn-mnist", # Optional, - params={"max_epochs": 10}, # Optional, - tags=["pytorch-ignite","minst"] # Optional - ) - - # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration - npt_logger.attach( - trainer, - log_handler=OptimizerParamsHandler(optimizer), - event_name=Events.ITERATION_STARTED - ) - # or equivalently - npt_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer - ) - """ - - def __init__(self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None): - super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) - - def __call__(self, engine: Engine, logger: NeptuneLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, NeptuneLogger): - raise TypeError("Handler OptimizerParamsHandler works only with NeptuneLogger") - - global_step = engine.state.get_event_attrib_value(event_name) - tag_prefix = f"{self.tag}/" if self.tag else "" - params = { - f"{tag_prefix}{self.param_name}/group_{i}": float(param_group[self.param_name]) - for i, param_group in enumerate(self.optimizer.param_groups) - } - - for k, v in params.items(): - logger[k].append(v, step=global_step) - - -class WeightsScalarHandler(BaseWeightsScalarHandler): - """Helper handler to log model's weights as scalars. - Handler, upon construction, iterates over named parameters of the model and keep - reference to ones permitted by `whitelist`. Then at every call, applies - reduction function to each parameter, produces a scalar and logs it. - - Args: - model: model to log weights - reduction: function to reduce parameters into scalar - tag: common title for all produced plots. For example, "generator" - whitelist: specific weights to log. Should be list of model's submodules - or parameters names, or a callable which gets weight along with its name - and determines if it should be logged. Names should be fully-qualified. - For more information please refer to `PyTorch docs - `_. - If not given, all of model's weights are logged. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.neptune_logger import * - - # Create a logger - # We are using the api_token for the anonymous user neptuner but you can use your own. - - npt_logger = NeptuneLogger( - api_token="ANONYMOUS", - project_name="shared/pytorch-ignite-integration", - experiment_name="cnn-mnist", # Optional, - params={"max_epochs": 10}, # Optional, - tags=["pytorch-ignite","minst"] # Optional - ) - - # Attach the logger to the trainer to log model's weights norm after each iteration - npt_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsScalarHandler(model, reduction=torch.norm) - ) - - .. code-block:: python - - from ignite.contrib.handlers.neptune_logger import * - - npt_logger = NeptuneLogger( - api_token="ANONYMOUS", - project_name="shared/pytorch-ignite-integration", - experiment_name="cnn-mnist", # Optional, - params={"max_epochs": 10}, # Optional, - tags=["pytorch-ignite","minst"] # Optional - ) - - # Log only `fc` weights - npt_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsScalarHandler( - model, - whitelist=['fc'] - ) - ) - - .. code-block:: python - - from ignite.contrib.handlers.neptune_logger import * - - npt_logger = NeptuneLogger( - api_token="ANONYMOUS", - project_name="shared/pytorch-ignite-integration", - experiment_name="cnn-mnist", # Optional, - params={"max_epochs": 10}, # Optional, - tags=["pytorch-ignite","minst"] # Optional - ) - - # Log weights which have `bias` in their names - def has_bias_in_name(n, p): - return 'bias' in n - - npt_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsScalarHandler(model, whitelist=has_bias_in_name) - ) - - .. versionchanged:: 0.4.9 - optional argument `whitelist` added. - """ - - def __call__(self, engine: Engine, logger: NeptuneLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, NeptuneLogger): - raise TypeError("Handler WeightsScalarHandler works only with NeptuneLogger") - - global_step = engine.state.get_event_attrib_value(event_name) - tag_prefix = f"{self.tag}/" if self.tag else "" - for name, p in self.weights: - if p.grad is None: - continue - - name = name.replace(".", "/") - key = f"{tag_prefix}weights_{self.reduction.__name__}/{name}" - logger[key].append(self.reduction(p.data), step=global_step) - - -class GradsScalarHandler(BaseWeightsScalarHandler): - """Helper handler to log model's gradients as scalars. - Handler, upon construction, iterates over named parameters of the model and keep - reference to ones permitted by the `whitelist`. Then at every call, applies - reduction function to each parameter's gradient, produces a scalar and logs it. - - Args: - model: model to log weights - reduction: function to reduce parameters into scalar - tag: common title for all produced plots. For example, "generator" - whitelist: specific gradients to log. Should be list of model's submodules - or parameters names, or a callable which gets weight along with its name - and determines if its gradient should be logged. Names should be - fully-qualified. For more information please refer to `PyTorch docs - `_. - If not given, all of model's gradients are logged. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.neptune_logger import * - - # Create a logger - # We are using the api_token for the anonymous user neptuner but you can use your own. - - npt_logger = NeptuneLogger( - api_token="ANONYMOUS", - project_name="shared/pytorch-ignite-integration", - experiment_name="cnn-mnist", # Optional, - params={"max_epochs": 10}, # Optional, - tags=["pytorch-ignite","minst"] # Optional - ) - - # Attach the logger to the trainer to log model's weights norm after each iteration - npt_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=GradsScalarHandler(model, reduction=torch.norm) - ) - - .. code-block:: python - - from ignite.contrib.handlers.neptune_logger import * - - npt_logger = NeptuneLogger( - api_token="ANONYMOUS", - project_name="shared/pytorch-ignite-integration", - experiment_name="cnn-mnist", # Optional, - params={"max_epochs": 10}, # Optional, - tags=["pytorch-ignite","minst"] # Optional - ) - - # Log gradient of `base` - npt_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=GradsScalarHandler( - model, - reduction=torch.norm, - whitelist=['base'] - ) - ) - - .. code-block:: python - - from ignite.contrib.handlers.neptune_logger import * - - npt_logger = NeptuneLogger( - api_token="ANONYMOUS", - project_name="shared/pytorch-ignite-integration", - experiment_name="cnn-mnist", # Optional, - params={"max_epochs": 10}, # Optional, - tags=["pytorch-ignite","minst"] # Optional - ) - - # Log gradient of weights which belong to a `fc` layer - def is_in_fc_layer(n, p): - return 'fc' in n - - npt_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=GradsScalarHandler(model, whitelist=is_in_fc_layer) - ) - - .. versionchanged:: 0.4.9 - optional argument `whitelist` added. - """ - - def __call__(self, engine: Engine, logger: NeptuneLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, NeptuneLogger): - raise TypeError("Handler GradsScalarHandler works only with NeptuneLogger") - - global_step = engine.state.get_event_attrib_value(event_name) - tag_prefix = f"{self.tag}/" if self.tag else "" - for name, p in self.weights: - if p.grad is None: - continue - - name = name.replace(".", "/") - key = f"{tag_prefix}grads_{self.reduction.__name__}/{name}" - logger[key].append(self.reduction(p.grad), step=global_step) - - -class NeptuneSaver(BaseSaveHandler): - """Handler that saves input checkpoint to the Neptune server. - - Args: - neptune_logger: an instance of - NeptuneLogger class. - - .. Note :: - - NeptuneSaver is currently not supported on Windows. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.neptune_logger import * - - # Create a logger - # We are using the api_token for the anonymous user neptuner but you can use your own. - - npt_logger = NeptuneLogger( - api_token="ANONYMOUS", - project_name="shared/pytorch-ignite-integration", - experiment_name="cnn-mnist", # Optional, - params={"max_epochs": 10}, # Optional, - tags=["pytorch-ignite","minst"] # Optional - ) - - ... - evaluator = create_supervised_evaluator(model, metrics=metrics, ...) - ... - - from ignite.handlers import Checkpoint - - def score_function(engine): - return engine.state.metrics["accuracy"] - - to_save = {"model": model} - - # pass neptune logger to NeptuneServer - - handler = Checkpoint( - to_save, - NeptuneSaver(npt_logger), n_saved=2, - filename_prefix="best", score_function=score_function, - score_name="validation_accuracy", - global_step_transform=global_step_from_engine(trainer) - ) - - evaluator.add_event_handler(Events.COMPLETED, handler) - - # We need to close the logger when we are done - npt_logger.close() - - For example, you can access model checkpoints and download them from here: - https://ui.neptune.ai/o/shared/org/pytorch-ignite-integration/e/PYTOR1-18/charts - - """ - - @idist.one_rank_only() - def __init__(self, neptune_logger: NeptuneLogger): - self._logger = neptune_logger - - @idist.one_rank_only() - def __call__(self, checkpoint: Mapping, filename: str, metadata: Optional[Mapping] = None) -> None: - # wont work on XLA - - # Imports for BC compatibility - try: - # neptune-client<1.0.0 package structure - with warnings.catch_warnings(): - # ignore the deprecation warnings - warnings.simplefilter("ignore") - from neptune.new.types import File - except ImportError: - # neptune>=1.0.0 package structure - from neptune.types import File - - with tempfile.NamedTemporaryFile() as tmp: - # we can not use tmp.name to open tmp.file twice on Win32 - # https://docs.python.org/3/library/tempfile.html#tempfile.NamedTemporaryFile - torch.save(checkpoint, tmp.file) - - # rewind the buffer - tmp.file.seek(0) - - # hold onto the file stream for uploading. - # NOTE: This won't load the whole file in memory and upload - # the stream in smaller chunks. - self._logger[filename].upload(File.from_stream(tmp.file)) - - @idist.one_rank_only(with_barrier=True) - def remove(self, filename: str) -> None: - del self._logger.experiment[filename] +NeptuneLogger = NeptuneLogger +NeptuneSaver = NeptuneSaver +OptimizerParamsHandler = OptimizerParamsHandler +OutputHandler = OutputHandler +WeightsScalarHandler = WeightsScalarHandler +GradsScalarHandler = GradsScalarHandler +_INTEGRATION_VERSION_KEY = _INTEGRATION_VERSION_KEY diff --git a/ignite/contrib/handlers/polyaxon_logger.py b/ignite/contrib/handlers/polyaxon_logger.py index 5c85ffd5a30..906510dbf06 100644 --- a/ignite/contrib/handlers/polyaxon_logger.py +++ b/ignite/contrib/handlers/polyaxon_logger.py @@ -1,305 +1,22 @@ -"""Polyaxon logger and its helper handlers.""" - -from typing import Any, Callable, List, Optional, Union - -from torch.optim import Optimizer - -from ignite.contrib.handlers.base_logger import BaseLogger, BaseOptimizerParamsHandler, BaseOutputHandler -from ignite.engine import Engine, Events -from ignite.handlers import global_step_from_engine - -__all__ = ["PolyaxonLogger", "OutputHandler", "OptimizerParamsHandler", "global_step_from_engine"] - - -class PolyaxonLogger(BaseLogger): - """ - `Polyaxon tracking client `_ handler to log parameters and metrics during the training - and validation. - - This class requires `polyaxon `_ package to be installed: - - .. code-block:: bash - - pip install polyaxon - - // If you are using polyaxon v0.x - - pip install polyaxon-client - - Args: - args: Positional arguments accepted from - `Experiment `_. - kwargs: Keyword arguments accepted from - `Experiment `_. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.polyaxon_logger import * - - # Create a logger - plx_logger = PolyaxonLogger() - - # Log experiment parameters: - plx_logger.log_inputs(**{ - "seed": seed, - "batch_size": batch_size, - "model": model.__class__.__name__, - - "pytorch version": torch.__version__, - "ignite version": ignite.__version__, - "cuda version": torch.version.cuda, - "device name": torch.cuda.get_device_name(0) - }) - - # Attach the logger to the trainer to log training loss at each iteration - plx_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - output_transform=lambda loss: {"loss": loss} - ) - - # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch - # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch - # of the `trainer` instead of `train_evaluator`. - plx_logger.attach_output_handler( - train_evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="training", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer), - ) - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the - # `trainer` instead of `evaluator`. - plx_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer)), - ) - - # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration - plx_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer, - param_name='lr' # optional - ) - # to manually end a run - plx_logger.close() - """ - - def __init__(self, *args: Any, **kwargs: Any): - try: - from polyaxon.tracking import Run - - self.experiment = Run(*args, **kwargs) - - except ImportError: - try: - from polyaxon_client.tracking import Experiment - - self.experiment = Experiment(*args, **kwargs) - except ImportError: - raise ModuleNotFoundError( - "This contrib module requires polyaxon to be installed.\n" - "For Polyaxon v1.x please install it with command: \n pip install polyaxon\n" - "For Polyaxon v0.x please install it with command: \n pip install polyaxon-client" - ) - - def close(self) -> None: - try: - self.experiment.end() - except: - pass - - def __getattr__(self, attr: Any) -> Any: - return getattr(self.experiment, attr) - - def _create_output_handler(self, *args: Any, **kwargs: Any) -> "OutputHandler": - return OutputHandler(*args, **kwargs) - - def _create_opt_params_handler(self, *args: Any, **kwargs: Any) -> "OptimizerParamsHandler": - return OptimizerParamsHandler(*args, **kwargs) - - -class OutputHandler(BaseOutputHandler): - """Helper handler to log engine's output and/or metrics. - - Args: - tag: common title for all produced plots. For example, "training" - metric_names: list of metric names to plot or a string "all" to plot all available - metrics. - output_transform: output transform function to prepare `engine.state.output` as a number. - For example, `output_transform = lambda output: output` - This function can also return a dictionary, e.g `{"loss": loss1, "another_loss": loss2}` to label the plot - with corresponding keys. - global_step_transform: global step transform function to output a desired global step. - Input of the function is `(engine, event_name)`. Output of function should be an integer. - Default is None, global_step based on attached engine. If provided, - uses function output as global_step. To setup global step from another engine, please use - :meth:`~ignite.contrib.handlers.polyaxon_logger.global_step_from_engine`. - state_attributes: list of attributes of the ``trainer.state`` to plot. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.polyaxon_logger import * - - # Create a logger - plx_logger = PolyaxonLogger() - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch - # of the `trainer`: - plx_logger.attach( - evaluator, - log_handler=OutputHandler( - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer) - ), - event_name=Events.EPOCH_COMPLETED - ) - # or equivalently - plx_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer) - ) - - Another example, where model is evaluated every 500 iterations: - - .. code-block:: python - - from ignite.contrib.handlers.polyaxon_logger import * - - @trainer.on(Events.ITERATION_COMPLETED(every=500)) - def evaluate(engine): - evaluator.run(validation_set, max_epochs=1) - - plx_logger = PolyaxonLogger() - - def global_step_transform(*args, **kwargs): - return trainer.state.iteration - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # every 500 iterations. Since evaluator engine does not have access to the training iteration, we - # provide a global_step_transform to return the trainer.state.iteration for the global_step, each time - # evaluator metrics are plotted on Polyaxon. - - plx_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metrics=["nll", "accuracy"], - global_step_transform=global_step_transform - ) - - Another example where the State Attributes ``trainer.state.alpha`` and ``trainer.state.beta`` - are also logged along with the NLL and Accuracy after each iteration: - - .. code-block:: python - - plx_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - metrics=["nll", "accuracy"], - state_attributes=["alpha", "beta"], - ) - - Example of `global_step_transform`: - - .. code-block:: python - - def global_step_transform(engine, event_name): - return engine.state.get_event_attrib_value(event_name) - - .. versionchanged:: 0.4.7 - accepts an optional list of `state_attributes` - """ - - def __init__( - self, - tag: str, - metric_names: Optional[List[str]] = None, - output_transform: Optional[Callable] = None, - global_step_transform: Optional[Callable[[Engine, Union[str, Events]], int]] = None, - state_attributes: Optional[List[str]] = None, - ): - super(OutputHandler, self).__init__( - tag, metric_names, output_transform, global_step_transform, state_attributes - ) - - def __call__(self, engine: Engine, logger: PolyaxonLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, PolyaxonLogger): - raise RuntimeError("Handler 'OutputHandler' works only with PolyaxonLogger") - - metrics = self._setup_output_metrics_state_attrs(engine, key_tuple=False) - - global_step = self.global_step_transform(engine, event_name) - - if not isinstance(global_step, int): - raise TypeError( - f"global_step must be int, got {type(global_step)}." - " Please check the output of global_step_transform." - ) - - metrics.update({"step": global_step}) - - logger.log_metrics(**metrics) - - -class OptimizerParamsHandler(BaseOptimizerParamsHandler): - """Helper handler to log optimizer parameters - - Args: - optimizer: torch optimizer or any object with attribute ``param_groups`` - as a sequence. - param_name: parameter name - tag: common title for all produced plots. For example, "generator" - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.polyaxon_logger import * - - # Create a logger - plx_logger = PolyaxonLogger() - - # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration - plx_logger.attach( - trainer, - log_handler=OptimizerParamsHandler(optimizer), - event_name=Events.ITERATION_STARTED - ) - # or equivalently - plx_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer - ) - """ - - def __init__(self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None): - super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) - - def __call__(self, engine: Engine, logger: PolyaxonLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, PolyaxonLogger): - raise RuntimeError("Handler OptimizerParamsHandler works only with PolyaxonLogger") - - global_step = engine.state.get_event_attrib_value(event_name) - tag_prefix = f"{self.tag}/" if self.tag else "" - params = { - f"{tag_prefix}{self.param_name}/group_{i}": float(param_group[self.param_name]) - for i, param_group in enumerate(self.optimizer.param_groups) - } - params["step"] = global_step - logger.log_metrics(**params) +""" ``ignite.contrib.handlers.polyaxon_logger`` was moved to ``ignite.handlers.polyaxon_logger``. +Note: + ``ignite.contrib.handlers.polyaxon_logger`` was moved to ``ignite.handlers.polyaxon_logger``. + Please refer to :mod:`~ignite.handlers.polyaxon_logger`. +""" + +import warnings + +removed_in = "0.6.0" +deprecation_warning = ( + f"{__file__} has been moved to /ignite/handlers/polyaxon_logger.py" + + (f" and will be removed in version {removed_in}" if removed_in else "") + + ".\n Please refer to the documentation for more details." +) +warnings.warn(deprecation_warning, DeprecationWarning, stacklevel=2) +from ignite.handlers.polyaxon_logger import OptimizerParamsHandler, OutputHandler, PolyaxonLogger +from ignite.handlers.utils import global_step_from_engine # noqa + +__all__ = ["PolyaxonLogger", "OutputHandler", "OptimizerParamsHandler"] +PolyaxonLogger = PolyaxonLogger +OutputHandler = OutputHandler +OptimizerParamsHandler = OptimizerParamsHandler diff --git a/ignite/contrib/handlers/tensorboard_logger.py b/ignite/contrib/handlers/tensorboard_logger.py index 94ab98bb36e..72006bb92a0 100644 --- a/ignite/contrib/handlers/tensorboard_logger.py +++ b/ignite/contrib/handlers/tensorboard_logger.py @@ -1,18 +1,29 @@ -"""TensorBoard logger and its helper handlers.""" - -from typing import Any, Callable, List, Optional, Union - -from torch.optim import Optimizer - -from ignite.contrib.handlers.base_logger import ( - BaseLogger, - BaseOptimizerParamsHandler, - BaseOutputHandler, - BaseWeightsHandler, - BaseWeightsScalarHandler, +""" ``ignite.contrib.handlers.tensorboard_logger`` was moved to ``ignite.handlers.tensorboard_logger``. +Note: + ``ignite.contrib.handlers.tensorboard_logger`` was moved to ``ignite.handlers.tensorboard_logger``. + Please refer to :mod:`~ignite.handlers.tensorboard_logger`. +""" + +import warnings + +removed_in = "0.6.0" +deprecation_warning = ( + f"{__file__} has been moved to /ignite/handlers/tensorboard_logger.py" + + (f" and will be removed in version {removed_in}" if removed_in else "") + + ".\n Please refer to the documentation for more details." ) -from ignite.engine import Engine, Events -from ignite.handlers import global_step_from_engine +warnings.warn(deprecation_warning, DeprecationWarning, stacklevel=2) +from ignite.handlers.tensorboard_logger import ( + GradsHistHandler, + GradsScalarHandler, + OptimizerParamsHandler, + OutputHandler, + TensorboardLogger, + WeightsHistHandler, + WeightsScalarHandler, +) +from ignite.handlers.utils import global_step_from_engine # noqa + __all__ = [ "TensorboardLogger", @@ -22,654 +33,11 @@ "WeightsHistHandler", "GradsScalarHandler", "GradsHistHandler", - "global_step_from_engine", ] - - -class TensorboardLogger(BaseLogger): - """ - TensorBoard handler to log metrics, model/optimizer parameters, gradients during the training and validation. - - By default, this class favors `tensorboardX `_ package if installed: - - .. code-block:: bash - - pip install tensorboardX - - otherwise, it falls back to using - `PyTorch's SummaryWriter - `_ - (>=v1.2.0). - - Args: - args: Positional arguments accepted from - `SummaryWriter - `_. - kwargs: Keyword arguments accepted from - `SummaryWriter - `_. - For example, `log_dir` to setup path to the directory where to log. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.tensorboard_logger import * - - # Create a logger - tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") - - # Attach the logger to the trainer to log training loss at each iteration - tb_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - output_transform=lambda loss: {"loss": loss} - ) - - # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch - # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch - # of the `trainer` instead of `train_evaluator`. - tb_logger.attach_output_handler( - train_evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="training", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer), - ) - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the - # `trainer` instead of `evaluator`. - tb_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer)), - ) - - # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration - tb_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer, - param_name='lr' # optional - ) - - # Attach the logger to the trainer to log model's weights norm after each iteration - tb_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsScalarHandler(model) - ) - - # Attach the logger to the trainer to log model's weights as a histogram after each epoch - tb_logger.attach( - trainer, - event_name=Events.EPOCH_COMPLETED, - log_handler=WeightsHistHandler(model) - ) - - # Attach the logger to the trainer to log model's gradients norm after each iteration - tb_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=GradsScalarHandler(model) - ) - - # Attach the logger to the trainer to log model's gradients as a histogram after each epoch - tb_logger.attach( - trainer, - event_name=Events.EPOCH_COMPLETED, - log_handler=GradsHistHandler(model) - ) - - # We need to close the logger when we are done - tb_logger.close() - - It is also possible to use the logger as context manager: - - .. code-block:: python - - from ignite.contrib.handlers.tensorboard_logger import * - - with TensorboardLogger(log_dir="experiments/tb_logs") as tb_logger: - - trainer = Engine(update_fn) - # Attach the logger to the trainer to log training loss at each iteration - tb_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - output_transform=lambda loss: {"loss": loss} - ) - - """ - - def __init__(self, *args: Any, **kwargs: Any): - try: - from tensorboardX import SummaryWriter - except ImportError: - try: - from torch.utils.tensorboard import SummaryWriter - except ImportError: - raise ModuleNotFoundError( - "This contrib module requires either tensorboardX or torch >= 1.2.0. " - "You may install tensorboardX with command: \n pip install tensorboardX \n" - "or upgrade PyTorch using your package manager of choice (pip or conda)." - ) - - self.writer = SummaryWriter(*args, **kwargs) - - def __getattr__(self, attr: Any) -> Any: - return getattr(self.writer, attr) - - def close(self) -> None: - self.writer.close() - - def _create_output_handler(self, *args: Any, **kwargs: Any) -> "OutputHandler": - return OutputHandler(*args, **kwargs) - - def _create_opt_params_handler(self, *args: Any, **kwargs: Any) -> "OptimizerParamsHandler": - return OptimizerParamsHandler(*args, **kwargs) - - -class OutputHandler(BaseOutputHandler): - """Helper handler to log engine's output, engine's state attributes and/or metrics - - Args: - tag: common title for all produced plots. For example, "training" - metric_names: list of metric names to plot or a string "all" to plot all available - metrics. - output_transform: output transform function to prepare `engine.state.output` as a number. - For example, `output_transform = lambda output: output` - This function can also return a dictionary, e.g `{"loss": loss1, "another_loss": loss2}` to label the plot - with corresponding keys. - global_step_transform: global step transform function to output a desired global step. - Input of the function is `(engine, event_name)`. Output of function should be an integer. - Default is None, global_step based on attached engine. If provided, - uses function output as global_step. To setup global step from another engine, please use - :meth:`~ignite.contrib.handlers.tensorboard_logger.global_step_from_engine`. - state_attributes: list of attributes of the ``trainer.state`` to plot. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.tensorboard_logger import * - - # Create a logger - tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch - # of the `trainer`: - tb_logger.attach( - evaluator, - log_handler=OutputHandler( - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer) - ), - event_name=Events.EPOCH_COMPLETED - ) - # or equivalently - tb_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer) - ) - - Another example, where model is evaluated every 500 iterations: - - .. code-block:: python - - from ignite.contrib.handlers.tensorboard_logger import * - - @trainer.on(Events.ITERATION_COMPLETED(every=500)) - def evaluate(engine): - evaluator.run(validation_set, max_epochs=1) - - tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") - - def global_step_transform(*args, **kwargs): - return trainer.state.iteration - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # every 500 iterations. Since evaluator engine does not have access to the training iteration, we - # provide a global_step_transform to return the trainer.state.iteration for the global_step, each time - # evaluator metrics are plotted on Tensorboard. - - tb_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metrics=["nll", "accuracy"], - global_step_transform=global_step_transform - ) - - Another example where the State Attributes ``trainer.state.alpha`` and ``trainer.state.beta`` - are also logged along with the NLL and Accuracy after each iteration: - - .. code-block:: python - - tb_logger.attach( - trainer, - log_handler=OutputHandler( - tag="training", - metric_names=["nll", "accuracy"], - state_attributes=["alpha", "beta"], - ), - event_name=Events.ITERATION_COMPLETED - ) - - Example of `global_step_transform`: - - .. code-block:: python - - def global_step_transform(engine, event_name): - return engine.state.get_event_attrib_value(event_name) - - .. versionchanged:: 0.4.7 - accepts an optional list of `state_attributes` - """ - - def __init__( - self, - tag: str, - metric_names: Optional[List[str]] = None, - output_transform: Optional[Callable] = None, - global_step_transform: Optional[Callable[[Engine, Union[str, Events]], int]] = None, - state_attributes: Optional[List[str]] = None, - ): - super(OutputHandler, self).__init__( - tag, metric_names, output_transform, global_step_transform, state_attributes - ) - - def __call__(self, engine: Engine, logger: TensorboardLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, TensorboardLogger): - raise RuntimeError("Handler 'OutputHandler' works only with TensorboardLogger") - - metrics = self._setup_output_metrics_state_attrs(engine, key_tuple=False) - - global_step = self.global_step_transform(engine, event_name) - if not isinstance(global_step, int): - raise TypeError( - f"global_step must be int, got {type(global_step)}." - " Please check the output of global_step_transform." - ) - - for key, value in metrics.items(): - logger.writer.add_scalar(key, value, global_step) - - -class OptimizerParamsHandler(BaseOptimizerParamsHandler): - """Helper handler to log optimizer parameters - - Args: - optimizer: torch optimizer or any object with attribute ``param_groups`` - as a sequence. - param_name: parameter name - tag: common title for all produced plots. For example, "generator" - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.tensorboard_logger import * - - # Create a logger - tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") - - # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration - tb_logger.attach( - trainer, - log_handler=OptimizerParamsHandler(optimizer), - event_name=Events.ITERATION_STARTED - ) - # or equivalently - tb_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer - ) - """ - - def __init__(self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None): - super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) - - def __call__(self, engine: Engine, logger: TensorboardLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, TensorboardLogger): - raise RuntimeError("Handler OptimizerParamsHandler works only with TensorboardLogger") - - global_step = engine.state.get_event_attrib_value(event_name) - tag_prefix = f"{self.tag}/" if self.tag else "" - params = { - f"{tag_prefix}{self.param_name}/group_{i}": float(param_group[self.param_name]) - for i, param_group in enumerate(self.optimizer.param_groups) - } - - for k, v in params.items(): - logger.writer.add_scalar(k, v, global_step) - - -class WeightsScalarHandler(BaseWeightsScalarHandler): - """Helper handler to log model's weights as scalars. - Handler, upon construction, iterates over named parameters of the model and keep - reference to ones permitted by `whitelist`. Then at every call, applies - reduction function to each parameter, produces a scalar and logs it. - - Args: - model: model to log weights - reduction: function to reduce parameters into scalar - tag: common title for all produced plots. For example, "generator" - whitelist: specific weights to log. Should be list of model's submodules - or parameters names, or a callable which gets weight along with its name - and determines if it should be logged. Names should be fully-qualified. - For more information please refer to `PyTorch docs - `_. - If not given, all of model's weights are logged. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.tensorboard_logger import * - - # Create a logger - tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") - - # Attach the logger to the trainer to log model's weights norm after each iteration - tb_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsScalarHandler(model, reduction=torch.norm) - ) - - .. code-block:: python - - from ignite.contrib.handlers.tensorboard_logger import * - - tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") - - # Log only `fc` weights - tb_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsScalarHandler( - model, - whitelist=['fc'] - ) - ) - - .. code-block:: python - - from ignite.contrib.handlers.tensorboard_logger import * - - tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") - - # Log weights which have `bias` in their names - def has_bias_in_name(n, p): - return 'bias' in n - - tb_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsScalarHandler(model, whitelist=has_bias_in_name) - ) - - .. versionchanged:: 0.4.9 - optional argument `whitelist` added. - """ - - def __call__(self, engine: Engine, logger: TensorboardLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, TensorboardLogger): - raise RuntimeError("Handler 'WeightsScalarHandler' works only with TensorboardLogger") - - global_step = engine.state.get_event_attrib_value(event_name) - tag_prefix = f"{self.tag}/" if self.tag else "" - for name, p in self.weights: - name = name.replace(".", "/") - logger.writer.add_scalar( - f"{tag_prefix}weights_{self.reduction.__name__}/{name}", - self.reduction(p.data), - global_step, - ) - - -class WeightsHistHandler(BaseWeightsHandler): - """Helper handler to log model's weights as histograms. - - Args: - model: model to log weights - tag: common title for all produced plots. For example, "generator" - whitelist: specific weights to log. Should be list of model's submodules - or parameters names, or a callable which gets weight along with its name - and determines if it should be logged. Names should be fully-qualified. - For more information please refer to `PyTorch docs - `_. - If not given, all of model's weights are logged. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.tensorboard_logger import * - - # Create a logger - tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") - - # Attach the logger to the trainer to log model's weights norm after each iteration - tb_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsHistHandler(model) - ) - - .. code-block:: python - - from ignite.contrib.handlers.tensorboard_logger import * - - tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") - - # Log weights of `fc` layer - weights = ['fc'] - - # Attach the logger to the trainer to log weights norm after each iteration - tb_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsHistHandler(model, whitelist=weights) - ) - - .. code-block:: python - - from ignite.contrib.handlers.tensorboard_logger import * - - tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") - - # Log weights which name include 'conv'. - weight_selector = lambda name, p: 'conv' in name - - # Attach the logger to the trainer to log weights norm after each iteration - tb_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsHistHandler(model, whitelist=weight_selector) - ) - - .. versionchanged:: 0.4.9 - optional argument `whitelist` added. - """ - - def __call__(self, engine: Engine, logger: TensorboardLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, TensorboardLogger): - raise RuntimeError("Handler 'WeightsHistHandler' works only with TensorboardLogger") - - global_step = engine.state.get_event_attrib_value(event_name) - tag_prefix = f"{self.tag}/" if self.tag else "" - for name, p in self.weights: - name = name.replace(".", "/") - logger.writer.add_histogram( - tag=f"{tag_prefix}weights/{name}", values=p.data.cpu().numpy(), global_step=global_step - ) - - -class GradsScalarHandler(BaseWeightsScalarHandler): - """Helper handler to log model's gradients as scalars. - Handler, upon construction, iterates over named parameters of the model and keep - reference to ones permitted by the `whitelist`. Then at every call, applies - reduction function to each parameter's gradient, produces a scalar and logs it. - - Args: - model: model to log weights - reduction: function to reduce parameters into scalar - tag: common title for all produced plots. For example, "generator" - whitelist: specific gradients to log. Should be list of model's submodules - or parameters names, or a callable which gets weight along with its name - and determines if its gradient should be logged. Names should be - fully-qualified. For more information please refer to `PyTorch docs - `_. - If not given, all of model's gradients are logged. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.tensorboard_logger import * - - # Create a logger - tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") - - # Attach the logger to the trainer to log model's gradients norm after each iteration - tb_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=GradsScalarHandler(model, reduction=torch.norm) - ) - - .. code-block:: python - - from ignite.contrib.handlers.tensorboard_logger import * - - tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") - - # Log gradient of `base` - tb_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=GradsScalarHandler( - model, - reduction=torch.norm, - whitelist=['base'] - ) - ) - - .. code-block:: python - - from ignite.contrib.handlers.tensorboard_logger import * - - tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") - - # Log gradient of weights which belong to a `fc` layer - def is_in_fc_layer(n, p): - return 'fc' in n - - tb_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=GradsScalarHandler(model, whitelist=is_in_fc_layer) - ) - - .. versionchanged:: 0.4.9 - optional argument `whitelist` added. - """ - - def __call__(self, engine: Engine, logger: TensorboardLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, TensorboardLogger): - raise RuntimeError("Handler 'GradsScalarHandler' works only with TensorboardLogger") - - global_step = engine.state.get_event_attrib_value(event_name) - tag_prefix = f"{self.tag}/" if self.tag else "" - for name, p in self.weights: - if p.grad is None: - continue - - name = name.replace(".", "/") - logger.writer.add_scalar( - f"{tag_prefix}grads_{self.reduction.__name__}/{name}", self.reduction(p.grad), global_step - ) - - -class GradsHistHandler(BaseWeightsHandler): - """Helper handler to log model's gradients as histograms. - - Args: - model: model to log weights - tag: common title for all produced plots. For example, "generator" - whitelist: specific gradients to log. Should be list of model's submodules - or parameters names, or a callable which gets weight along with its name - and determines if its gradient should be logged. Names should be - fully-qualified. For more information please refer to `PyTorch docs - `_. - If not given, all of model's gradients are logged. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.tensorboard_logger import * - - # Create a logger - tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") - - # Attach the logger to the trainer to log model's weights norm after each iteration - tb_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=GradsHistHandler(model) - ) - - .. code-block:: python - - from ignite.contrib.handlers.tensorboard_logger import * - - tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") - - # Log gradient of `fc.bias` - tb_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=GradsHistHandler(model, whitelist=['fc.bias']) - ) - - .. code-block:: python - - from ignite.contrib.handlers.tensorboard_logger import * - - tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") - - # Log gradient of weights which have shape (2, 1) - def has_shape_2_1(n, p): - return p.shape == (2,1) - - tb_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=GradsHistHandler(model, whitelist=has_shape_2_1) - ) - - .. versionchanged:: 0.4.9 - optional argument `whitelist` added. - """ - - def __call__(self, engine: Engine, logger: TensorboardLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, TensorboardLogger): - raise RuntimeError("Handler 'GradsHistHandler' works only with TensorboardLogger") - - global_step = engine.state.get_event_attrib_value(event_name) - tag_prefix = f"{self.tag}/" if self.tag else "" - for name, p in self.weights: - if p.grad is None: - continue - - name = name.replace(".", "/") - logger.writer.add_histogram( - tag=f"{tag_prefix}grads/{name}", values=p.grad.cpu().numpy(), global_step=global_step - ) +TensorboardLogger = TensorboardLogger +OptimizerParamsHandler = OptimizerParamsHandler +OutputHandler = OutputHandler +WeightsScalarHandler = WeightsScalarHandler +WeightsHistHandler = WeightsHistHandler +GradsScalarHandler = GradsScalarHandler +GradsHistHandler = GradsHistHandler diff --git a/ignite/contrib/handlers/tqdm_logger.py b/ignite/contrib/handlers/tqdm_logger.py index 37d79b7c4a0..ad3a04a5ba3 100644 --- a/ignite/contrib/handlers/tqdm_logger.py +++ b/ignite/contrib/handlers/tqdm_logger.py @@ -1,310 +1,21 @@ -# -*- coding: utf-8 -*- -"""TQDM logger.""" -from collections import OrderedDict -from typing import Any, Callable, List, Optional, Union - -from ignite.contrib.handlers.base_logger import BaseLogger, BaseOutputHandler -from ignite.engine import Engine, Events -from ignite.engine.events import CallableEventWithFilter, RemovableEventHandle - - -class ProgressBar(BaseLogger): - """ - TQDM progress bar handler to log training progress and computed metrics. - - Args: - persist: set to ``True`` to persist the progress bar after completion (default = ``False``) - bar_format : Specify a custom bar string formatting. May impact performance. - [default: '{desc}[{n_fmt}/{total_fmt}] {percentage:3.0f}%|{bar}{postfix} [{elapsed}<{remaining}]']. - Set to ``None`` to use ``tqdm`` default bar formatting: '{l_bar}{bar}{r_bar}', where - l_bar='{desc}: {percentage:3.0f}%|' and - r_bar='| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}{postfix}]'. For more details on the - formatting, see `tqdm docs `_. - tqdm_kwargs: kwargs passed to tqdm progress bar. - By default, progress bar description displays "Epoch [5/10]" where 5 is the current epoch and 10 is the - number of epochs; however, if ``max_epochs`` are set to 1, the progress bar instead displays - "Iteration: [5/10]". If tqdm_kwargs defines `desc`, e.g. "Predictions", than the description is - "Predictions [5/10]" if number of epochs is more than one otherwise it is simply "Predictions". - - Examples: - Simple progress bar - - .. code-block:: python - - trainer = create_supervised_trainer(model, optimizer, loss) - - pbar = ProgressBar() - pbar.attach(trainer) - - # Progress bar will looks like - # Epoch [2/50]: [64/128] 50%|█████ [06:17<12:34] - - Log output to a file instead of stderr (tqdm's default output) - - .. code-block:: python - - trainer = create_supervised_trainer(model, optimizer, loss) - - log_file = open("output.log", "w") - pbar = ProgressBar(file=log_file) - pbar.attach(trainer) - - Attach metrics that already have been computed at :attr:`~ignite.engine.events.Events.ITERATION_COMPLETED` - (such as :class:`~ignite.metrics.RunningAverage`) - - .. code-block:: python - - trainer = create_supervised_trainer(model, optimizer, loss) - - RunningAverage(output_transform=lambda x: x).attach(trainer, 'loss') - - pbar = ProgressBar() - pbar.attach(trainer, ['loss']) - - # Progress bar will looks like - # Epoch [2/50]: [64/128] 50%|█████ , loss=0.123 [06:17<12:34] - - Directly attach the engine's output - - .. code-block:: python - - trainer = create_supervised_trainer(model, optimizer, loss) - - pbar = ProgressBar() - pbar.attach(trainer, output_transform=lambda x: {'loss': x}) - - # Progress bar will looks like - # Epoch [2/50]: [64/128] 50%|█████ , loss=0.123 [06:17<12:34] - - - Example where the State Attributes ``trainer.state.alpha`` and ``trainer.state.beta`` - are also logged along with the NLL and Accuracy after each iteration: - - .. code-block:: python - - pbar.attach( - trainer, - metric_names=["nll", "accuracy"], - state_attributes=["alpha", "beta"], - ) - - - Note: - When attaching the progress bar to an engine, it is recommended that you replace - every print operation in the engine's handlers triggered every iteration with - ``pbar.log_message`` to guarantee the correct format of the stdout. - - Note: - When using inside jupyter notebook, `ProgressBar` automatically uses `tqdm_notebook`. For correct rendering, - please install `ipywidgets `_. - Due to `tqdm notebook bugs `_, bar format may be needed to be set - to an empty string value. - - .. versionchanged:: 0.4.7 - `attach` now accepts an optional list of `state_attributes` - - """ - - _events_order: List[Union[Events, CallableEventWithFilter]] = [ - Events.STARTED, - Events.EPOCH_STARTED, - Events.ITERATION_STARTED, - Events.ITERATION_COMPLETED, - Events.EPOCH_COMPLETED, - Events.COMPLETED, - ] - - def __init__( - self, - persist: bool = False, - bar_format: Union[ - str, None - ] = "{desc}[{n_fmt}/{total_fmt}] {percentage:3.0f}%|{bar}{postfix} [{elapsed}<{remaining}]", - **tqdm_kwargs: Any, - ): - try: - from tqdm.autonotebook import tqdm - except ImportError: - raise ModuleNotFoundError( - "This contrib module requires tqdm to be installed. " - "Please install it with command: \n pip install tqdm" - ) - - self.pbar_cls = tqdm - self.pbar = None - self.persist = persist - self.bar_format = bar_format - self.tqdm_kwargs = tqdm_kwargs - - def _reset(self, pbar_total: Optional[int]) -> None: - self.pbar = self.pbar_cls( - total=pbar_total, leave=self.persist, bar_format=self.bar_format, initial=1, **self.tqdm_kwargs - ) - - def _close(self, engine: Engine) -> None: - if self.pbar is not None: - # https://github.com/tqdm/notebook.py#L240-L250 - # issue #1115 : notebook backend of tqdm checks if n < total (error or KeyboardInterrupt) - # and the bar persists in 'danger' mode - if self.pbar.total is not None: - self.pbar.n = self.pbar.total - self.pbar.close() - self.pbar = None - - @staticmethod - def _compare_lt( - event1: Union[Events, CallableEventWithFilter], event2: Union[Events, CallableEventWithFilter] - ) -> bool: - i1 = ProgressBar._events_order.index(event1) - i2 = ProgressBar._events_order.index(event2) - return i1 < i2 - - def log_message(self, message: str) -> None: - """ - Logs a message, preserving the progress bar correct output format. - - Args: - message: string you wish to log. - """ - from tqdm import tqdm - - tqdm.write(message, file=self.tqdm_kwargs.get("file", None)) - - def attach( # type: ignore[override] - self, - engine: Engine, - metric_names: Optional[Union[str, List[str]]] = None, - output_transform: Optional[Callable] = None, - event_name: Union[Events, CallableEventWithFilter] = Events.ITERATION_COMPLETED, - closing_event_name: Union[Events, CallableEventWithFilter] = Events.EPOCH_COMPLETED, - state_attributes: Optional[List[str]] = None, - ) -> None: - """ - Attaches the progress bar to an engine object. - - Args: - engine: engine object. - metric_names: list of metric names to plot or a string "all" to plot all available - metrics. - output_transform: a function to select what you want to print from the engine's - output. This function may return either a dictionary with entries in the format of ``{name: value}``, - or a single scalar, which will be displayed with the default name `output`. - event_name: event's name on which the progress bar advances. Valid events are from - :class:`~ignite.engine.events.Events`. - closing_event_name: event's name on which the progress bar is closed. Valid events are from - :class:`~ignite.engine.events.Events`. - state_attributes: list of attributes of the ``trainer.state`` to plot. - - Note: - Accepted output value types are numbers, 0d and 1d torch tensors and strings. - - """ - desc = self.tqdm_kwargs.get("desc", None) - - if event_name not in engine._allowed_events: - raise ValueError(f"Logging event {event_name.name} is not in allowed events for this engine") - - if isinstance(closing_event_name, CallableEventWithFilter): - if closing_event_name.filter is not None: - raise ValueError("Closing Event should not be a filtered event") - - if not self._compare_lt(event_name, closing_event_name): - raise ValueError(f"Logging event {event_name} should be called before closing event {closing_event_name}") - - log_handler = _OutputHandler( - desc, - metric_names, - output_transform, - closing_event_name=closing_event_name, - state_attributes=state_attributes, - ) - - super(ProgressBar, self).attach(engine, log_handler, event_name) - engine.add_event_handler(closing_event_name, self._close) - - def attach_opt_params_handler( # type: ignore[empty-body] - self, engine: Engine, event_name: Union[str, Events], *args: Any, **kwargs: Any - ) -> RemovableEventHandle: - """Intentionally empty""" - pass - - def _create_output_handler(self, *args: Any, **kwargs: Any) -> "_OutputHandler": - return _OutputHandler(*args, **kwargs) - - def _create_opt_params_handler(self, *args: Any, **kwargs: Any) -> Callable: # type: ignore[empty-body] - """Intentionally empty""" - pass - - -class _OutputHandler(BaseOutputHandler): - """Helper handler to log engine's output and/or metrics - - pbar = ProgressBar() - Args: - description: progress bar description. - metric_names: list of metric names to plot or a string "all" to plot all available - metrics. - output_transform: output transform function to prepare `engine.state.output` as a number. - For example, `output_transform = lambda output: output` - This function can also return a dictionary, e.g `{'loss': loss1, 'another_loss': loss2}` to label the plot - with corresponding keys. - closing_event_name: event's name on which the progress bar is closed. Valid events are from - :class:`~ignite.engine.events.Events` or any `event_name` added by - :meth:`~ignite.engine.engine.Engine.register_events`. - state_attributes: list of attributes of the ``trainer.state`` to plot. - - """ - - def __init__( - self, - description: str, - metric_names: Optional[Union[str, List[str]]] = None, - output_transform: Optional[Callable] = None, - closing_event_name: Union[Events, CallableEventWithFilter] = Events.EPOCH_COMPLETED, - state_attributes: Optional[List[str]] = None, - ): - if metric_names is None and output_transform is None: - # This helps to avoid 'Either metric_names or output_transform should be defined' of BaseOutputHandler - metric_names = [] - super(_OutputHandler, self).__init__( - description, metric_names, output_transform, global_step_transform=None, state_attributes=state_attributes - ) - self.closing_event_name = closing_event_name - - @staticmethod - def get_max_number_events(event_name: Union[str, Events, CallableEventWithFilter], engine: Engine) -> Optional[int]: - if event_name in (Events.ITERATION_STARTED, Events.ITERATION_COMPLETED): - return engine.state.epoch_length - if event_name in (Events.EPOCH_STARTED, Events.EPOCH_COMPLETED): - return engine.state.max_epochs - return 1 - - def __call__(self, engine: Engine, logger: ProgressBar, event_name: Union[str, Events]) -> None: - pbar_total = self.get_max_number_events(event_name, engine) - if logger.pbar is None: - logger._reset(pbar_total=pbar_total) - - max_epochs = engine.state.max_epochs - default_desc = "Iteration" if max_epochs == 1 else "Epoch" - - desc = self.tag or default_desc - max_num_of_closing_events = self.get_max_number_events(self.closing_event_name, engine) - if max_num_of_closing_events and max_num_of_closing_events > 1: - global_step = engine.state.get_event_attrib_value(self.closing_event_name) - desc += f" [{global_step}/{max_num_of_closing_events}]" - logger.pbar.set_description(desc) # type: ignore[attr-defined] - - rendered_metrics = self._setup_output_metrics_state_attrs(engine, log_text=True) - metrics = OrderedDict() - for key, value in rendered_metrics.items(): - key = "_".join(key[1:]) # tqdm has tag as description - - metrics[key] = value - - if metrics: - logger.pbar.set_postfix(metrics) # type: ignore[attr-defined] - - global_step = engine.state.get_event_attrib_value(event_name) - if pbar_total is not None: - global_step = (global_step - 1) % pbar_total + 1 - logger.pbar.update(global_step - logger.pbar.n) # type: ignore[attr-defined] +""" ``ignite.contrib.handlers.tqdm_logger`` was moved to ``ignite.handlers.tqdm_logger``. +Note: + ``ignite.contrib.handlers.tqdm_logger`` was moved to ``ignite.handlers.tqdm_logger``. + Please refer to :mod:`~ignite.handlers.tqdm_logger`. +""" + +import warnings + +removed_in = "0.6.0" +deprecation_warning = ( + f"{__file__} has been moved to /ignite/handlers/tqdm_logger.py" + + (f" and will be removed in version {removed_in}" if removed_in else "") + + ".\n Please refer to the documentation for more details." +) +warnings.warn(deprecation_warning, DeprecationWarning, stacklevel=2) +from ignite.handlers.tqdm_logger import ProgressBar + +__all__ = [ + "ProgressBar", +] +ProgressBar = ProgressBar diff --git a/ignite/contrib/handlers/visdom_logger.py b/ignite/contrib/handlers/visdom_logger.py index cd5264437a8..674aa4fc403 100644 --- a/ignite/contrib/handlers/visdom_logger.py +++ b/ignite/contrib/handlers/visdom_logger.py @@ -1,20 +1,27 @@ -"""Visdom logger and its helper handlers.""" - -import os -from typing import Any, Callable, cast, Dict, List, Optional, Union - -import torch -import torch.nn as nn -from torch.optim import Optimizer - -from ignite.contrib.handlers.base_logger import ( - BaseLogger, - BaseOptimizerParamsHandler, - BaseOutputHandler, - BaseWeightsScalarHandler, +""" ``ignite.contrib.handlers.visdom_logger`` was moved to ``ignite.handlers.visdom_logger``. +Note: + ``ignite.contrib.handlers.visdom_logger`` was moved to ``ignite.handlers.visdom_logger``. + Please refer to :mod:`~ignite.handlers.visdom_logger`. +""" + +import warnings + +removed_in = "0.6.0" +deprecation_warning = ( + f"{__file__} has been moved to /ignite/handlers/visdom_logger.py" + + (f" and will be removed in version {removed_in}" if removed_in else "") + + ".\n Please refer to the documentation for more details." +) +warnings.warn(deprecation_warning, DeprecationWarning, stacklevel=2) +from ignite.handlers.utils import global_step_from_engine # noqa +from ignite.handlers.visdom_logger import ( + _DummyExecutor, + GradsScalarHandler, + OptimizerParamsHandler, + OutputHandler, + VisdomLogger, + WeightsScalarHandler, ) -from ignite.engine import Engine, Events -from ignite.handlers import global_step_from_engine __all__ = [ "VisdomLogger", @@ -22,535 +29,10 @@ "OutputHandler", "WeightsScalarHandler", "GradsScalarHandler", - "global_step_from_engine", ] - - -class VisdomLogger(BaseLogger): - """ - VisdomLogger handler to log metrics, model/optimizer parameters, gradients during the training and validation. - - This class requires `visdom `_ package to be installed: - - .. code-block:: bash - - - pip install git+https://github.com/fossasia/visdom.git - - Args: - server: visdom server URL. It can be also specified by environment variable `VISDOM_SERVER_URL` - port: visdom server's port. It can be also specified by environment variable `VISDOM_PORT` - num_workers: number of workers to use in `concurrent.futures.ThreadPoolExecutor` to post data to - visdom server. Default, `num_workers=1`. If `num_workers=0` and logger uses the main thread. If using - Python 2.7 and `num_workers>0` the package `futures` should be installed: `pip install futures` - kwargs: kwargs to pass into - `visdom.Visdom `_. - - Note: - We can also specify username/password using environment variables: VISDOM_USERNAME, VISDOM_PASSWORD - - - .. warning:: - - Frequent logging, e.g. when logger is attached to `Events.ITERATION_COMPLETED`, can slow down the run if the - main thread is used to send the data to visdom server (`num_workers=0`). To avoid this situation we can either - log less frequently or set `num_workers=1`. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.visdom_logger import * - - # Create a logger - vd_logger = VisdomLogger() - - # Attach the logger to the trainer to log training loss at each iteration - vd_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - output_transform=lambda loss: {"loss": loss} - ) - - # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch - # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch - # of the `trainer` instead of `train_evaluator`. - vd_logger.attach_output_handler( - train_evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="training", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer), - ) - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the - # `trainer` instead of `evaluator`. - vd_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer)), - ) - - # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration - vd_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer, - param_name='lr' # optional - ) - - # Attach the logger to the trainer to log model's weights norm after each iteration - vd_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsScalarHandler(model) - ) - - # Attach the logger to the trainer to log model's gradients norm after each iteration - vd_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=GradsScalarHandler(model) - ) - - # We need to close the logger with we are done - vd_logger.close() - - It is also possible to use the logger as context manager: - - .. code-block:: python - - from ignite.contrib.handlers.visdom_logger import * - - with VisdomLogger() as vd_logger: - - trainer = Engine(update_fn) - # Attach the logger to the trainer to log training loss at each iteration - vd_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - output_transform=lambda loss: {"loss": loss} - ) - - .. versionchanged:: 0.4.7 - accepts an optional list of `state_attributes` - """ - - def __init__( - self, - server: Optional[str] = None, - port: Optional[int] = None, - num_workers: int = 1, - raise_exceptions: bool = True, - **kwargs: Any, - ): - try: - import visdom - except ImportError: - raise ModuleNotFoundError( - "This contrib module requires visdom package. " - "Please install it with command:\n" - "pip install git+https://github.com/fossasia/visdom.git" - ) - - if num_workers > 0: - # If visdom is installed, one of its dependencies `tornado` - # requires also `futures` to be installed. - # Let's check anyway if we can import it. - try: - from concurrent.futures import ThreadPoolExecutor - except ImportError: - raise ModuleNotFoundError( - "This contrib module requires concurrent.futures module" - "Please install it with command:\n" - "pip install futures" - ) - - if server is None: - server = cast(str, os.environ.get("VISDOM_SERVER_URL", "localhost")) - - if port is None: - port = int(os.environ.get("VISDOM_PORT", 8097)) - - if "username" not in kwargs: - username = os.environ.get("VISDOM_USERNAME", None) - kwargs["username"] = username - - if "password" not in kwargs: - password = os.environ.get("VISDOM_PASSWORD", None) - kwargs["password"] = password - - self.vis = visdom.Visdom(server=server, port=port, raise_exceptions=raise_exceptions, **kwargs) - - if not self.vis.offline and not self.vis.check_connection(): # type: ignore[attr-defined] - raise RuntimeError(f"Failed to connect to Visdom server at {server}. Did you run python -m visdom.server ?") - - self.executor: Union[_DummyExecutor, "ThreadPoolExecutor"] = _DummyExecutor() - if num_workers > 0: - self.executor = ThreadPoolExecutor(max_workers=num_workers) - - def _save(self) -> None: - self.vis.save([self.vis.env]) # type: ignore[attr-defined] - - def close(self) -> None: - self.executor.shutdown() - self.vis.close() - - def _create_output_handler(self, *args: Any, **kwargs: Any) -> "OutputHandler": - return OutputHandler(*args, **kwargs) - - def _create_opt_params_handler(self, *args: Any, **kwargs: Any) -> "OptimizerParamsHandler": - return OptimizerParamsHandler(*args, **kwargs) - - -class _BaseVisDrawer: - def __init__(self, show_legend: bool = False): - self.windows: Dict[str, Any] = {} - self.show_legend = show_legend - - def add_scalar( - self, logger: VisdomLogger, k: str, v: Union[str, float, torch.Tensor], event_name: Any, global_step: int - ) -> None: - """ - Helper method to log a scalar with VisdomLogger. - - Args: - logger: visdom logger - k: scalar name which is used to set window title and y-axis label - v: scalar value, y-axis value - event_name: Event name which is used to setup x-axis label. Valid events are from - :class:`~ignite.engine.events.Events` or any `event_name` added by - :meth:`~ignite.engine.engine.Engine.register_events`. - global_step: global step, x-axis value - - """ - if k not in self.windows: - self.windows[k] = { - "win": None, - "opts": {"title": k, "xlabel": str(event_name), "ylabel": k, "showlegend": self.show_legend}, - } - - update = None if self.windows[k]["win"] is None else "append" - - kwargs = { - "X": [global_step], - "Y": [v], - "env": logger.vis.env, # type: ignore[attr-defined] - "win": self.windows[k]["win"], - "update": update, - "opts": self.windows[k]["opts"], - "name": k, - } - - future = logger.executor.submit(logger.vis.line, **kwargs) - if self.windows[k]["win"] is None: - self.windows[k]["win"] = future.result() - - -class OutputHandler(BaseOutputHandler, _BaseVisDrawer): - """Helper handler to log engine's output and/or metrics - - Args: - tag: common title for all produced plots. For example, "training" - metric_names: list of metric names to plot or a string "all" to plot all available - metrics. - output_transform: output transform function to prepare `engine.state.output` as a number. - For example, `output_transform = lambda output: output` - This function can also return a dictionary, e.g `{"loss": loss1, "another_loss": loss2}` to label the plot - with corresponding keys. - global_step_transform: global step transform function to output a desired global step. - Input of the function is `(engine, event_name)`. Output of function should be an integer. - Default is None, global_step based on attached engine. If provided, - uses function output as global_step. To setup global step from another engine, please use - :meth:`~ignite.contrib.handlers.visdom_logger.global_step_from_engine`. - show_legend: flag to show legend in the window - state_attributes: list of attributes of the ``trainer.state`` to plot. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.visdom_logger import * - - # Create a logger - vd_logger = VisdomLogger() - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch - # of the `trainer`: - vd_logger.attach( - evaluator, - log_handler=OutputHandler( - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer) - ), - event_name=Events.EPOCH_COMPLETED - ) - # or equivalently - vd_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer) - ) - - Another example, where model is evaluated every 500 iterations: - - .. code-block:: python - - from ignite.contrib.handlers.visdom_logger import * - - @trainer.on(Events.ITERATION_COMPLETED(every=500)) - def evaluate(engine): - evaluator.run(validation_set, max_epochs=1) - - vd_logger = VisdomLogger() - - def global_step_transform(*args, **kwargs): - return trainer.state.iteration - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # every 500 iterations. Since evaluator engine does not have access to the training iteration, we - # provide a global_step_transform to return the trainer.state.iteration for the global_step, each time - # evaluator metrics are plotted on Visdom. - - vd_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metrics=["nll", "accuracy"], - global_step_transform=global_step_transform - ) - - Another example where the State Attributes ``trainer.state.alpha`` and ``trainer.state.beta`` - are also logged along with the NLL and Accuracy after each iteration: - - .. code-block:: python - - vd_logger.attach( - trainer, - log_handler=OutputHandler( - tag="training", - metric_names=["nll", "accuracy"], - state_attributes=["alpha", "beta"], - ), - event_name=Events.ITERATION_COMPLETED - ) - - Example of `global_step_transform`: - - .. code-block:: python - - def global_step_transform(engine, event_name): - return engine.state.get_event_attrib_value(event_name) - """ - - def __init__( - self, - tag: str, - metric_names: Optional[str] = None, - output_transform: Optional[Callable] = None, - global_step_transform: Optional[Callable[[Engine, Union[str, Events]], int]] = None, - show_legend: bool = False, - state_attributes: Optional[List[str]] = None, - ): - super(OutputHandler, self).__init__( - tag, metric_names, output_transform, global_step_transform, state_attributes - ) - _BaseVisDrawer.__init__(self, show_legend=show_legend) - - def __call__(self, engine: Engine, logger: VisdomLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, VisdomLogger): - raise RuntimeError("Handler 'OutputHandler' works only with VisdomLogger") - - metrics = self._setup_output_metrics_state_attrs(engine, key_tuple=False) - - global_step = self.global_step_transform(engine, event_name) - - if not isinstance(global_step, int): - raise TypeError( - f"global_step must be int, got {type(global_step)}." - " Please check the output of global_step_transform." - ) - - for key, value in metrics.items(): - self.add_scalar(logger, key, value, event_name, global_step) - - logger._save() - - -class OptimizerParamsHandler(BaseOptimizerParamsHandler, _BaseVisDrawer): - """Helper handler to log optimizer parameters - - Args: - optimizer: torch optimizer or any object with attribute ``param_groups`` - as a sequence. - param_name: parameter name - tag: common title for all produced plots. For example, "generator" - show_legend: flag to show legend in the window - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.visdom_logger import * - - # Create a logger - vb_logger = VisdomLogger() - - # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration - vd_logger.attach( - trainer, - log_handler=OptimizerParamsHandler(optimizer), - event_name=Events.ITERATION_STARTED - ) - # or equivalently - vd_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer - ) - """ - - def __init__( - self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None, show_legend: bool = False - ): - super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) - _BaseVisDrawer.__init__(self, show_legend=show_legend) - - def __call__(self, engine: Engine, logger: VisdomLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, VisdomLogger): - raise RuntimeError("Handler OptimizerParamsHandler works only with VisdomLogger") - - global_step = engine.state.get_event_attrib_value(event_name) - tag_prefix = f"{self.tag}/" if self.tag else "" - params = { - f"{tag_prefix}{self.param_name}/group_{i}": float(param_group[self.param_name]) - for i, param_group in enumerate(self.optimizer.param_groups) - } - - for k, v in params.items(): - self.add_scalar(logger, k, v, event_name, global_step) - - logger._save() - - -class WeightsScalarHandler(BaseWeightsScalarHandler, _BaseVisDrawer): - """Helper handler to log model's weights as scalars. - Handler iterates over named parameters of the model, applies reduction function to each parameter - produce a scalar and then logs the scalar. - - Args: - model: model to log weights - reduction: function to reduce parameters into scalar - tag: common title for all produced plots. For example, "generator" - show_legend: flag to show legend in the window - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.visdom_logger import * - - # Create a logger - vd_logger = VisdomLogger() - - # Attach the logger to the trainer to log model's weights norm after each iteration - vd_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsScalarHandler(model, reduction=torch.norm) - ) - """ - - def __init__( - self, model: nn.Module, reduction: Callable = torch.norm, tag: Optional[str] = None, show_legend: bool = False - ): - super(WeightsScalarHandler, self).__init__(model, reduction, tag=tag) - _BaseVisDrawer.__init__(self, show_legend=show_legend) - - def __call__(self, engine: Engine, logger: VisdomLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, VisdomLogger): - raise RuntimeError("Handler 'WeightsScalarHandler' works only with VisdomLogger") - - global_step = engine.state.get_event_attrib_value(event_name) - tag_prefix = f"{self.tag}/" if self.tag else "" - for name, p in self.model.named_parameters(): - name = name.replace(".", "/") - k = f"{tag_prefix}weights_{self.reduction.__name__}/{name}" - v = self.reduction(p.data) - self.add_scalar(logger, k, v, event_name, global_step) - - logger._save() - - -class GradsScalarHandler(BaseWeightsScalarHandler, _BaseVisDrawer): - """Helper handler to log model's gradients as scalars. - Handler iterates over the gradients of named parameters of the model, applies reduction function to each parameter - produce a scalar and then logs the scalar. - - Args: - model: model to log weights - reduction: function to reduce parameters into scalar - tag: common title for all produced plots. For example, "generator" - show_legend: flag to show legend in the window - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.visdom_logger import * - - # Create a logger - vd_logger = VisdomLogger() - - # Attach the logger to the trainer to log model's weights norm after each iteration - vd_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=GradsScalarHandler(model, reduction=torch.norm) - ) - """ - - def __init__( - self, model: nn.Module, reduction: Callable = torch.norm, tag: Optional[str] = None, show_legend: bool = False - ): - super(GradsScalarHandler, self).__init__(model, reduction, tag) - _BaseVisDrawer.__init__(self, show_legend=show_legend) - - def __call__(self, engine: Engine, logger: VisdomLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, VisdomLogger): - raise RuntimeError("Handler 'GradsScalarHandler' works only with VisdomLogger") - - global_step = engine.state.get_event_attrib_value(event_name) - tag_prefix = f"{self.tag}/" if self.tag else "" - for name, p in self.model.named_parameters(): - if p.grad is None: - continue - - name = name.replace(".", "/") - k = f"{tag_prefix}grads_{self.reduction.__name__}/{name}" - v = self.reduction(p.grad) - self.add_scalar(logger, k, v, event_name, global_step) - - logger._save() - - -class _DummyExecutor: - class _DummyFuture: - def __init__(self, result: Any) -> None: - self._output = result - - def result(self) -> Any: - return self._output - - def __init__(self, *args: Any, **kwargs: Any) -> None: - pass - - def submit(self, fn: Callable, **kwargs: Any) -> "_DummyFuture": - return _DummyExecutor._DummyFuture(fn(**kwargs)) - - def shutdown(self, *args: Any, **kwargs: Any) -> None: - pass +VisdomLogger = VisdomLogger +OptimizerParamsHandler = OptimizerParamsHandler +OutputHandler = OutputHandler +WeightsScalarHandler = WeightsScalarHandler +GradsScalarHandler = GradsScalarHandler +_DummyExecutor = _DummyExecutor diff --git a/ignite/contrib/handlers/wandb_logger.py b/ignite/contrib/handlers/wandb_logger.py index 037ee618bf1..beb5397732f 100644 --- a/ignite/contrib/handlers/wandb_logger.py +++ b/ignite/contrib/handlers/wandb_logger.py @@ -1,353 +1,22 @@ -"""WandB logger and its helper handlers.""" - -from typing import Any, Callable, List, Optional, Union - -from torch.optim import Optimizer - -from ignite.contrib.handlers.base_logger import BaseLogger, BaseOptimizerParamsHandler, BaseOutputHandler -from ignite.engine import Engine, Events -from ignite.handlers import global_step_from_engine - -__all__ = ["WandBLogger", "OutputHandler", "OptimizerParamsHandler", "global_step_from_engine"] - - -class WandBLogger(BaseLogger): - """`Weights & Biases `_ handler to log metrics, model/optimizer parameters, gradients - during training and validation. It can also be used to log model checkpoints to the Weights & Biases cloud. - - .. code-block:: bash - - pip install wandb - - This class is also a wrapper for the wandb module. This means that you can call any wandb function using - this wrapper. See examples on how to save model parameters and gradients. - - Args: - args: Positional arguments accepted by `wandb.init`. - kwargs: Keyword arguments accepted by `wandb.init`. - Please see `wandb.init `_ for documentation of possible parameters. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.wandb_logger import * - - # Create a logger. All parameters are optional. See documentation - # on wandb.init for details. - - wandb_logger = WandBLogger( - entity="shared", - project="pytorch-ignite-integration", - name="cnn-mnist", - config={"max_epochs": 10}, - tags=["pytorch-ignite", "minst"] - ) - - # Attach the logger to the trainer to log training loss at each iteration - wandb_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - output_transform=lambda loss: {"loss": loss} - ) - - # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch - # We setup `global_step_transform=lambda *_: trainer.state.iteration` to take iteration value - # of the `trainer`: - wandb_logger.attach_output_handler( - train_evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="training", - metric_names=["nll", "accuracy"], - global_step_transform=lambda *_: trainer.state.iteration, - ) - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # each epoch. We setup `global_step_transform=lambda *_: trainer.state.iteration` to take iteration value - # of the `trainer` instead of `evaluator`. - wandb_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=lambda *_: trainer.state.iteration, - ) - - # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration - wandb_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer, - param_name='lr' # optional - ) - - # We need to close the logger when we are done - wandb_logger.close() - - If you want to log model gradients, the model call graph, etc., use the logger as wrapper of wandb. Refer - to the documentation of wandb.watch for details: - - .. code-block:: python - - wandb_logger = WandBLogger( - entity="shared", - project="pytorch-ignite-integration", - name="cnn-mnist", - config={"max_epochs": 10}, - tags=["pytorch-ignite", "minst"] - ) - - model = torch.nn.Sequential(...) - wandb_logger.watch(model) - - For model checkpointing, Weights & Biases creates a local run dir, and automatically synchronizes all - files saved there at the end of the run. You can just use the `wandb_logger.run.dir` as path for the - `ModelCheckpoint`: - - .. code-block:: python - - from ignite.handlers import ModelCheckpoint - - def score_function(engine): - return engine.state.metrics['accuracy'] - - model_checkpoint = ModelCheckpoint( - wandb_logger.run.dir, n_saved=2, filename_prefix='best', - require_empty=False, score_function=score_function, - score_name="validation_accuracy", - global_step_transform=global_step_from_engine(trainer) - ) - evaluator.add_event_handler(Events.COMPLETED, model_checkpoint, {'model': model}) - - - """ - - def __init__(self, *args: Any, **kwargs: Any): - try: - import wandb - - self._wandb = wandb - except ImportError: - raise ModuleNotFoundError( - "This contrib module requires wandb to be installed. " - "You man install wandb with the command:\n pip install wandb\n" - ) - if kwargs.get("init", True): - wandb.init(*args, **kwargs) - - def __getattr__(self, attr: Any) -> Any: - return getattr(self._wandb, attr) - - def close(self) -> None: - self._wandb.finish() - - def _create_output_handler(self, *args: Any, **kwargs: Any) -> "OutputHandler": - return OutputHandler(*args, **kwargs) - - def _create_opt_params_handler(self, *args: Any, **kwargs: Any) -> "OptimizerParamsHandler": - return OptimizerParamsHandler(*args, **kwargs) - - -class OutputHandler(BaseOutputHandler): - """Helper handler to log engine's output and/or metrics - - Args: - tag: common title for all produced plots. For example, "training" - metric_names: list of metric names to plot or a string "all" to plot all available - metrics. - output_transform: output transform function to prepare `engine.state.output` as a number. - For example, `output_transform = lambda output: output` - This function can also return a dictionary, e.g `{"loss": loss1, "another_loss": loss2}` to label the plot - with corresponding keys. - global_step_transform: global step transform function to output a desired global step. - Input of the function is `(engine, event_name)`. Output of function should be an integer. - Default is None, global_step based on attached engine. If provided, - uses function output as global_step. To setup global step from another engine, please use - :meth:`~ignite.contrib.handlers.wandb_logger.global_step_from_engine`. - sync: If set to False, process calls to log in a seperate thread. Default (None) uses whatever - the default value of wandb.log. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.wandb_logger import * - - # Create a logger. All parameters are optional. See documentation - # on wandb.init for details. - - wandb_logger = WandBLogger( - entity="shared", - project="pytorch-ignite-integration", - name="cnn-mnist", - config={"max_epochs": 10}, - tags=["pytorch-ignite", "minst"] - ) - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # each epoch. We setup `global_step_transform=lambda *_: trainer.state.iteration,` to take iteration value - # of the `trainer`: - wandb_logger.attach( - evaluator, - log_handler=OutputHandler( - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=lambda *_: trainer.state.iteration, - ), - event_name=Events.EPOCH_COMPLETED - ) - # or equivalently - wandb_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=lambda *_: trainer.state.iteration, - ) - - Another example, where model is evaluated every 500 iterations: - - .. code-block:: python - - from ignite.contrib.handlers.wandb_logger import * - - @trainer.on(Events.ITERATION_COMPLETED(every=500)) - def evaluate(engine): - evaluator.run(validation_set, max_epochs=1) - - # Create a logger. All parameters are optional. See documentation - # on wandb.init for details. - - wandb_logger = WandBLogger( - entity="shared", - project="pytorch-ignite-integration", - name="cnn-mnist", - config={"max_epochs": 10}, - tags=["pytorch-ignite", "minst"] - ) - - def global_step_transform(*args, **kwargs): - return trainer.state.iteration - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # every 500 iterations. Since evaluator engine does not have access to the training iteration, we - # provide a global_step_transform to return the trainer.state.iteration for the global_step, each time - # evaluator metrics are plotted on Weights & Biases. - - wandb_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metrics=["nll", "accuracy"], - global_step_transform=global_step_transform - ) - - Another example where the State Attributes ``trainer.state.alpha`` and ``trainer.state.beta`` - are also logged along with the NLL and Accuracy after each iteration: - - .. code-block:: python - - wandb_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - metrics=["nll", "accuracy"], - state_attributes=["alpha", "beta"], - ) - - - Example of `global_step_transform`: - - .. code-block:: python - - def global_step_transform(engine, event_name): - return engine.state.get_event_attrib_value(event_name) - - .. versionchanged:: 0.4.7 - accepts an optional list of `state_attributes` - """ - - def __init__( - self, - tag: str, - metric_names: Optional[List[str]] = None, - output_transform: Optional[Callable] = None, - global_step_transform: Optional[Callable[[Engine, Union[str, Events]], int]] = None, - sync: Optional[bool] = None, - state_attributes: Optional[List[str]] = None, - ): - super().__init__(tag, metric_names, output_transform, global_step_transform, state_attributes) - self.sync = sync - - def __call__(self, engine: Engine, logger: WandBLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, WandBLogger): - raise RuntimeError(f"Handler '{self.__class__.__name__}' works only with WandBLogger.") - - global_step = self.global_step_transform(engine, event_name) - if not isinstance(global_step, int): - raise TypeError( - f"global_step must be int, got {type(global_step)}." - " Please check the output of global_step_transform." - ) - - metrics = self._setup_output_metrics_state_attrs(engine, log_text=True, key_tuple=False) - logger.log(metrics, step=global_step, sync=self.sync) - - -class OptimizerParamsHandler(BaseOptimizerParamsHandler): - """Helper handler to log optimizer parameters - - Args: - optimizer: torch optimizer or any object with attribute ``param_groups`` - as a sequence. - param_name: parameter name - tag: common title for all produced plots. For example, "generator" - sync: If set to False, process calls to log in a seperate thread. Default (None) uses whatever - the default value of wandb.log. - - Examples: - .. code-block:: python - - from ignite.contrib.handlers.wandb_logger import * - - # Create a logger. All parameters are optional. See documentation - # on wandb.init for details. - - wandb_logger = WandBLogger( - entity="shared", - project="pytorch-ignite-integration", - name="cnn-mnist", - config={"max_epochs": 10}, - tags=["pytorch-ignite", "minst"] - ) - - # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration - wandb_logger.attach( - trainer, - log_handler=OptimizerParamsHandler(optimizer), - event_name=Events.ITERATION_STARTED - ) - # or equivalently - wandb_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer - ) - """ - - def __init__( - self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None, sync: Optional[bool] = None - ): - super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) - self.sync = sync - - def __call__(self, engine: Engine, logger: WandBLogger, event_name: Union[str, Events]) -> None: - if not isinstance(logger, WandBLogger): - raise RuntimeError("Handler OptimizerParamsHandler works only with WandBLogger") - - global_step = engine.state.get_event_attrib_value(event_name) - tag_prefix = f"{self.tag}/" if self.tag else "" - params = { - f"{tag_prefix}{self.param_name}/group_{i}": float(param_group[self.param_name]) - for i, param_group in enumerate(self.optimizer.param_groups) - } - logger.log(params, step=global_step, sync=self.sync) +""" ``ignite.contrib.handlers.wandb_logger`` was moved to ``ignite.handlers.wandb_logger``. +Note: + ``ignite.contrib.handlers.wandb_logger`` was moved to ``ignite.handlers.wandb_logger``. + Please refer to :mod:`~ignite.handlers.wandb_logger`. +""" + +import warnings + +removed_in = "0.6.0" +deprecation_warning = ( + f"{__file__} has been moved to /ignite/handlers/wandb_logger.py" + + (f" and will be removed in version {removed_in}" if removed_in else "") + + ".\n Please refer to the documentation for more details." +) +warnings.warn(deprecation_warning, DeprecationWarning, stacklevel=2) +from ignite.handlers.utils import global_step_from_engine # noqa +from ignite.handlers.wandb_logger import OptimizerParamsHandler, OutputHandler, WandBLogger + +__all__ = ["WandBLogger", "OutputHandler", "OptimizerParamsHandler"] +WandBLogger = WandBLogger +OutputHandler = OutputHandler +OptimizerParamsHandler = OptimizerParamsHandler diff --git a/ignite/engine/__init__.py b/ignite/engine/__init__.py index a67dbe08ee1..83e76cf8db6 100644 --- a/ignite/engine/__init__.py +++ b/ignite/engine/__init__.py @@ -480,7 +480,7 @@ def create_supervised_trainer( from ignite.engine import create_supervised_trainer from ignite.utils import convert_tensor - from ignite.contrib.handlers.tqdm_logger import ProgressBar + from ignite.handlers.tqdm_logger import ProgressBar model = ... loss = ... diff --git a/ignite/handlers/__init__.py b/ignite/handlers/__init__.py index d7f3002ced0..f897d7cbe25 100644 --- a/ignite/handlers/__init__.py +++ b/ignite/handlers/__init__.py @@ -3,9 +3,12 @@ from ignite.engine import Engine from ignite.engine.events import Events from ignite.handlers.checkpoint import Checkpoint, DiskSaver, ModelCheckpoint +from ignite.handlers.clearml_logger import ClearMLLogger from ignite.handlers.early_stopping import EarlyStopping from ignite.handlers.ema_handler import EMAHandler from ignite.handlers.lr_finder import FastaiLRFinder +from ignite.handlers.mlflow_logger import MLflowLogger +from ignite.handlers.neptune_logger import NeptuneLogger from ignite.handlers.param_scheduler import ( BaseParamScheduler, ConcatScheduler, @@ -19,6 +22,8 @@ PiecewiseLinear, ReduceLROnPlateauScheduler, ) + +from ignite.handlers.polyaxon_logger import PolyaxonLogger from ignite.handlers.state_param_scheduler import ( ExpStateScheduler, LambdaStateScheduler, @@ -28,10 +33,16 @@ StepStateScheduler, ) from ignite.handlers.stores import EpochOutputStore +from ignite.handlers.tensorboard_logger import TensorboardLogger from ignite.handlers.terminate_on_nan import TerminateOnNan from ignite.handlers.time_limit import TimeLimit from ignite.handlers.time_profilers import BasicTimeProfiler, HandlersTimeProfiler from ignite.handlers.timing import Timer +from ignite.handlers.tqdm_logger import ProgressBar +from ignite.handlers.utils import global_step_from_engine # noqa + +from ignite.handlers.visdom_logger import VisdomLogger +from ignite.handlers.wandb_logger import WandBLogger __all__ = [ "ModelCheckpoint", @@ -64,24 +75,12 @@ "StepStateScheduler", "MultiStepStateScheduler", "ReduceLROnPlateauScheduler", + "ClearMLLogger", + "MLflowLogger", + "NeptuneLogger", + "PolyaxonLogger", + "TensorboardLogger", + "ProgressBar", + "VisdomLogger", + "WandBLogger", ] - - -def global_step_from_engine(engine: Engine, custom_event_name: Optional[Events] = None) -> Callable: - """Helper method to setup `global_step_transform` function using another engine. - This can be helpful for logging trainer epoch/iteration while output handler is attached to an evaluator. - - Args: - engine: engine which state is used to provide the global step - custom_event_name: registered event name. Optional argument, event name to use. - - Returns: - global step based on provided engine - """ - - def wrapper(_: Any, event_name: Events) -> int: - if custom_event_name is not None: - event_name = custom_event_name - return engine.state.get_event_attrib_value(event_name) - - return wrapper diff --git a/ignite/handlers/base_logger.py b/ignite/handlers/base_logger.py new file mode 100644 index 00000000000..e25849554de --- /dev/null +++ b/ignite/handlers/base_logger.py @@ -0,0 +1,294 @@ +"""Base logger and its helper handlers.""" + +import numbers +import warnings +from abc import ABCMeta, abstractmethod +from collections import OrderedDict +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union + +import torch +import torch.nn as nn +from torch.optim import Optimizer + +from ignite.engine import Engine, Events, EventsList, State +from ignite.engine.events import CallableEventWithFilter, RemovableEventHandle + + +class BaseHandler(metaclass=ABCMeta): + """Base handler for defining various useful handlers.""" + + @abstractmethod + def __call__(self, engine: Engine, logger: Any, event_name: Union[str, Events]) -> None: + pass + + +class BaseWeightsHandler(BaseHandler): + """ + Base handler for logging weights or their gradients. + """ + + def __init__( + self, + model: nn.Module, + tag: Optional[str] = None, + whitelist: Optional[Union[List[str], Callable[[str, nn.Parameter], bool]]] = None, + ): + if not isinstance(model, torch.nn.Module): + raise TypeError(f"Argument model should be of type torch.nn.Module, but given {type(model)}") + + self.model = model + self.tag = tag + + weights = {} + if whitelist is None: + weights = dict(model.named_parameters()) + elif callable(whitelist): + for n, p in model.named_parameters(): + if whitelist(n, p): + weights[n] = p + else: + for n, p in model.named_parameters(): + for item in whitelist: + if n.startswith(item): + weights[n] = p + + self.weights = weights.items() + + +class BaseOptimizerParamsHandler(BaseHandler): + """ + Base handler for logging optimizer parameters + """ + + def __init__(self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None): + if not ( + isinstance(optimizer, Optimizer) + or (hasattr(optimizer, "param_groups") and isinstance(optimizer.param_groups, Sequence)) + ): + raise TypeError( + "Argument optimizer should be torch.optim.Optimizer or has attribute 'param_groups' as list/tuple, " + f"but given {type(optimizer)}" + ) + + self.optimizer = optimizer + self.param_name = param_name + self.tag = tag + + +class BaseOutputHandler(BaseHandler): + """ + Helper handler to log engine's output and/or metrics + """ + + def __init__( + self, + tag: str, + metric_names: Optional[Union[str, List[str]]] = None, + output_transform: Optional[Callable] = None, + global_step_transform: Optional[Callable[[Engine, Union[str, Events]], int]] = None, + state_attributes: Optional[List[str]] = None, + ): + if metric_names is not None: + if not (isinstance(metric_names, list) or (isinstance(metric_names, str) and metric_names == "all")): + raise TypeError( + f"metric_names should be either a list or equal 'all', got {type(metric_names)} instead." + ) + + if output_transform is not None and not callable(output_transform): + raise TypeError(f"output_transform should be a function, got {type(output_transform)} instead.") + + if output_transform is None and metric_names is None and state_attributes is None: + raise ValueError("Either metric_names, output_transform or state_attributes should be defined") + + if global_step_transform is not None and not callable(global_step_transform): + raise TypeError(f"global_step_transform should be a function, got {type(global_step_transform)} instead.") + + if global_step_transform is None: + + def global_step_transform(engine: Engine, event_name: Union[str, Events]) -> int: + return engine.state.get_event_attrib_value(event_name) + + self.tag = tag + self.metric_names = metric_names + self.output_transform = output_transform + self.global_step_transform = global_step_transform + self.state_attributes = state_attributes + + def _setup_output_metrics_state_attrs( + self, engine: Engine, log_text: Optional[bool] = False, key_tuple: Optional[bool] = True + ) -> Dict[Any, Any]: + """Helper method to setup metrics and state attributes to log""" + metrics_state_attrs = OrderedDict() + if self.metric_names is not None: + if isinstance(self.metric_names, str) and self.metric_names == "all": + metrics_state_attrs = OrderedDict(engine.state.metrics) + else: + for name in self.metric_names: + if name not in engine.state.metrics: + warnings.warn( + f"Provided metric name '{name}' is missing " + f"in engine's state metrics: {list(engine.state.metrics.keys())}" + ) + continue + metrics_state_attrs[name] = engine.state.metrics[name] + + if self.output_transform is not None: + output_dict = self.output_transform(engine.state.output) + + if not isinstance(output_dict, dict): + output_dict = {"output": output_dict} + + metrics_state_attrs.update(output_dict) + + if self.state_attributes is not None: + metrics_state_attrs.update({name: getattr(engine.state, name, None) for name in self.state_attributes}) + + metrics_state_attrs_dict: Dict[Any, Union[str, float, numbers.Number]] = OrderedDict() + + def key_tuple_tf(tag: str, name: str, *args: str) -> Tuple[str, ...]: + return (tag, name) + args + + def key_str_tf(tag: str, name: str, *args: str) -> str: + return "/".join((tag, name) + args) + + key_tf = key_tuple_tf if key_tuple else key_str_tf + + for name, value in metrics_state_attrs.items(): + if isinstance(value, numbers.Number): + metrics_state_attrs_dict[key_tf(self.tag, name)] = value + elif isinstance(value, torch.Tensor) and value.ndimension() == 0: + metrics_state_attrs_dict[key_tf(self.tag, name)] = value.item() + elif isinstance(value, torch.Tensor) and value.ndimension() == 1: + for i, v in enumerate(value): + metrics_state_attrs_dict[key_tf(self.tag, name, str(i))] = v.item() + else: + if isinstance(value, str) and log_text: + metrics_state_attrs_dict[key_tf(self.tag, name)] = value + else: + warnings.warn(f"Logger output_handler can not log metrics value type {type(value)}") + return metrics_state_attrs_dict + + +class BaseWeightsScalarHandler(BaseWeightsHandler): + """ + Helper handler to log model's weights or gradients as scalars. + """ + + def __init__( + self, + model: nn.Module, + reduction: Callable[[torch.Tensor], Union[float, torch.Tensor]] = torch.norm, + tag: Optional[str] = None, + whitelist: Optional[Union[List[str], Callable[[str, nn.Parameter], bool]]] = None, + ): + super(BaseWeightsScalarHandler, self).__init__(model, tag=tag, whitelist=whitelist) + + if not callable(reduction): + raise TypeError(f"Argument reduction should be callable, but given {type(reduction)}") + + def _is_0D_tensor(t: Any) -> bool: + return isinstance(t, torch.Tensor) and t.ndimension() == 0 + + # Test reduction function on a tensor + o = reduction(torch.ones(4, 2)) + if not (isinstance(o, numbers.Number) or _is_0D_tensor(o)): + raise TypeError(f"Output of the reduction function should be a scalar, but got {type(o)}") + + self.reduction = reduction + + +class BaseLogger(metaclass=ABCMeta): + """ + Base logger handler. See implementations: TensorboardLogger, VisdomLogger, PolyaxonLogger, MLflowLogger, ... + + """ + + def attach( + self, + engine: Engine, + log_handler: Callable, + event_name: Union[str, Events, CallableEventWithFilter, EventsList], + *args: Any, + **kwargs: Any, + ) -> RemovableEventHandle: + """Attach the logger to the engine and execute `log_handler` function at `event_name` events. + + Args: + engine: engine object. + log_handler: a logging handler to execute + event_name: event to attach the logging handler to. Valid events are from + :class:`~ignite.engine.events.Events` or :class:`~ignite.engine.events.EventsList` or any `event_name` + added by :meth:`~ignite.engine.engine.Engine.register_events`. + args: args forwarded to the `log_handler` method + kwargs: kwargs forwarded to the `log_handler` method + + Returns: + :class:`~ignite.engine.events.RemovableEventHandle`, which can be used to remove the handler. + """ + if isinstance(event_name, EventsList): + for name in event_name: + if name not in State.event_to_attr: + raise RuntimeError(f"Unknown event name '{name}'") + engine.add_event_handler(name, log_handler, self, name) + + return RemovableEventHandle(event_name, log_handler, engine) + + else: + if event_name not in State.event_to_attr: + raise RuntimeError(f"Unknown event name '{event_name}'") + + return engine.add_event_handler(event_name, log_handler, self, event_name, *args, **kwargs) + + def attach_output_handler(self, engine: Engine, event_name: Any, *args: Any, **kwargs: Any) -> RemovableEventHandle: + """Shortcut method to attach `OutputHandler` to the logger. + + Args: + engine: engine object. + event_name: event to attach the logging handler to. Valid events are from + :class:`~ignite.engine.events.Events` or any `event_name` added by + :meth:`~ignite.engine.engine.Engine.register_events`. + args: args to initialize `OutputHandler` + kwargs: kwargs to initialize `OutputHandler` + + Returns: + :class:`~ignite.engine.events.RemovableEventHandle`, which can be used to remove the handler. + """ + return self.attach(engine, self._create_output_handler(*args, **kwargs), event_name=event_name) + + def attach_opt_params_handler( + self, engine: Engine, event_name: Any, *args: Any, **kwargs: Any + ) -> RemovableEventHandle: + """Shortcut method to attach `OptimizerParamsHandler` to the logger. + + Args: + engine: engine object. + event_name: event to attach the logging handler to. Valid events are from + :class:`~ignite.engine.events.Events` or any `event_name` added by + :meth:`~ignite.engine.engine.Engine.register_events`. + args: args to initialize `OptimizerParamsHandler` + kwargs: kwargs to initialize `OptimizerParamsHandler` + + Returns: + :class:`~ignite.engine.events.RemovableEventHandle`, which can be used to remove the handler. + + .. versionchanged:: 0.4.3 + Added missing return statement. + """ + return self.attach(engine, self._create_opt_params_handler(*args, **kwargs), event_name=event_name) + + @abstractmethod + def _create_output_handler(self, engine: Engine, *args: Any, **kwargs: Any) -> Callable: + pass + + @abstractmethod + def _create_opt_params_handler(self, *args: Any, **kwargs: Any) -> Callable: + pass + + def __enter__(self) -> "BaseLogger": + return self + + def __exit__(self, type: Any, value: Any, traceback: Any) -> None: + self.close() + + def close(self) -> None: + pass diff --git a/ignite/handlers/clearml_logger.py b/ignite/handlers/clearml_logger.py new file mode 100644 index 00000000000..cf07a6a170a --- /dev/null +++ b/ignite/handlers/clearml_logger.py @@ -0,0 +1,991 @@ +"""ClearML logger and its helper handlers.""" + +import os +import tempfile +import warnings +from collections import defaultdict +from datetime import datetime +from enum import Enum +from typing import Any, Callable, DefaultDict, List, Mapping, Optional, Tuple, Type, Union + +from torch.optim import Optimizer + +import ignite.distributed as idist +from ignite.engine import Engine, Events +from ignite.handlers.base_logger import ( + BaseLogger, + BaseOptimizerParamsHandler, + BaseOutputHandler, + BaseWeightsHandler, + BaseWeightsScalarHandler, +) +from ignite.handlers.checkpoint import DiskSaver +from ignite.handlers.utils import global_step_from_engine # noqa + +__all__ = [ + "ClearMLLogger", + "ClearMLSaver", + "OptimizerParamsHandler", + "OutputHandler", + "WeightsScalarHandler", + "WeightsHistHandler", + "GradsScalarHandler", + "GradsHistHandler", + "global_step_from_engine", +] + + +class ClearMLLogger(BaseLogger): + """ + `ClearML `_ handler to log metrics, text, model/optimizer parameters, + plots during training and validation. + Also supports model checkpoints logging and upload to the storage solution of your choice (i.e. ClearML File server, + S3 bucket etc.) + + .. code-block:: bash + + pip install clearml + clearml-init + + Args: + kwargs: Keyword arguments accepted from ``Task.init`` method. + All arguments are optional. If a ClearML Task has already been created, + kwargs will be ignored and the current ClearML Task will be used. + + Examples: + .. code-block:: python + + from ignite.handlers.clearml_logger import * + + # Create a logger + + clearml_logger = ClearMLLogger( + project_name="pytorch-ignite-integration", + task_name="cnn-mnist" + ) + + # Attach the logger to the trainer to log training loss at each iteration + clearml_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + output_transform=lambda loss: {"loss": loss} + ) + + # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch + # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch + # of the `trainer` instead of `train_evaluator`. + clearml_logger.attach_output_handler( + train_evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="training", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer), + ) + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the + # `trainer` instead of `evaluator`. + clearml_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer)), + ) + + # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration + clearml_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer, + param_name='lr' # optional + ) + + # Attach the logger to the trainer to log model's weights norm after each iteration + clearml_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsScalarHandler(model) + ) + + """ + + def __init__(self, **kwargs: Any): + try: + from clearml import Task + from clearml.binding.frameworks.tensorflow_bind import WeightsGradientHistHelper + except ImportError: + raise ModuleNotFoundError( + "This contrib module requires clearml to be installed. " + "You may install clearml using: \n pip install clearml \n" + ) + + experiment_kwargs = {k: v for k, v in kwargs.items() if k not in ("project_name", "task_name", "task_type")} + + if self.bypass_mode(): + warnings.warn("ClearMLSaver: running in bypass mode") + + # Try to retrieve current the ClearML Task before trying to create a new one + self._task = Task.current_task() + + if self._task is None: + self._task = Task.init( + project_name=kwargs.get("project_name"), + task_name=kwargs.get("task_name"), + task_type=kwargs.get("task_type", Task.TaskTypes.training), + **experiment_kwargs, + ) + + self.clearml_logger = self._task.get_logger() + + self.grad_helper = WeightsGradientHistHelper(logger=self.clearml_logger, report_freq=1) + + @classmethod + def set_bypass_mode(cls, bypass: bool) -> None: + """ + Set ``clearml.Task`` to offline mode. + Will bypass all outside communication, and will save all data and logs to a local session folder. + Should only be used in "standalone mode", when there is no access to the *clearml-server*. + + Args: + bypass: If ``True``, all outside communication is skipped. + Data and logs will be stored in a local session folder. + For more information, please refer to `ClearML docs + `_. + """ + from clearml import Task + + setattr(cls, "_bypass", bypass) + Task.set_offline(offline_mode=bypass) + + @classmethod + def bypass_mode(cls) -> bool: + """ + Returns the bypass mode state. + + Note: + `GITHUB_ACTIONS` env will automatically set bypass_mode to ``True`` + unless overridden specifically with ``ClearMLLogger.set_bypass_mode(False)``. + For more information, please refer to `ClearML docs + `_. + + Return: + If True, ``clearml.Task`` is on offline mode, and all outside communication is skipped. + """ + return getattr(cls, "_bypass", bool(os.environ.get("CI"))) + + def __getattr__(self, attr: Any) -> Any: + """ + Calls the corresponding method of ``clearml.Logger``. + + Args: + attr: methods of the ``clearml.Logger`` class. + """ + return getattr(self.clearml_logger, attr) + + def get_task(self) -> Any: + """ + Returns the task context that the logger is reporting. + + Return: + Returns the current task, equivalent to ``clearml.Task.current_task()``. + """ + return self._task + + def close(self) -> None: + self.clearml_logger.flush() + + def _create_output_handler(self, *args: Any, **kwargs: Any) -> "OutputHandler": + return OutputHandler(*args, **kwargs) + + def _create_opt_params_handler(self, *args: Any, **kwargs: Any) -> "OptimizerParamsHandler": + return OptimizerParamsHandler(*args, **kwargs) + + +class OutputHandler(BaseOutputHandler): + """Helper handler to log engine's output and/or metrics + + Args: + tag: common title for all produced plots. For example, "training" + metric_names: list of metric names to plot or a string "all" to plot all available + metrics. + output_transform: output transform function to prepare `engine.state.output` as a number. + For example, `output_transform = lambda output: output` + This function can also return a dictionary, e.g `{"loss": loss1, "another_loss": loss2}` to label the plot + with corresponding keys. + global_step_transform: global step transform function to output a desired global step. + Input of the function is `(engine, event_name)`. Output of function should be an integer. + Default is None, global_step based on attached engine. If provided, + uses function output as global_step. To setup global step from another engine, please use + :meth:`~ignite.handlers.clearml_logger.global_step_from_engine`. + state_attributes: list of attributes of the ``trainer.state`` to plot. + + Examples: + .. code-block:: python + + from ignite.handlers.clearml_logger import * + + # Create a logger + + clearml_logger = ClearMLLogger( + project_name="pytorch-ignite-integration", + task_name="cnn-mnist" + ) + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch + # of the `trainer`: + clearml_logger.attach( + evaluator, + log_handler=OutputHandler( + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer) + ), + event_name=Events.EPOCH_COMPLETED + ) + # or equivalently + clearml_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer) + ) + + Another example, where model is evaluated every 500 iterations: + + .. code-block:: python + + from ignite.handlers.clearml_logger import * + + @trainer.on(Events.ITERATION_COMPLETED(every=500)) + def evaluate(engine): + evaluator.run(validation_set, max_epochs=1) + + # Create a logger + + clearml_logger = ClearMLLogger( + project_name="pytorch-ignite-integration", + task_name="cnn-mnist" + ) + + def global_step_transform(*args, **kwargs): + return trainer.state.iteration + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # every 500 iterations. Since evaluator engine does not have access to the training iteration, we + # provide a global_step_transform to return the trainer.state.iteration for the global_step, each time + # evaluator metrics are plotted on ClearML. + + clearml_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metrics=["nll", "accuracy"], + global_step_transform=global_step_transform + ) + + Another example where the State Attributes ``trainer.state.alpha`` and ``trainer.state.beta`` + are also logged along with the NLL and Accuracy after each iteration: + + .. code-block:: python + + clearml_logger.attach( + trainer, + log_handler=OutputHandler( + tag="training", + metric_names=["nll", "accuracy"], + state_attributes=["alpha", "beta"], + ), + event_name=Events.ITERATION_COMPLETED + ) + + Example of `global_step_transform` + + .. code-block:: python + + def global_step_transform(engine, event_name): + return engine.state.get_event_attrib_value(event_name) + + .. versionchanged:: 0.4.7 + accepts an optional list of `state_attributes` + """ + + def __init__( + self, + tag: str, + metric_names: Optional[List[str]] = None, + output_transform: Optional[Callable] = None, + global_step_transform: Optional[Callable[[Engine, Union[str, Events]], int]] = None, + state_attributes: Optional[List[str]] = None, + ): + super(OutputHandler, self).__init__( + tag, metric_names, output_transform, global_step_transform, state_attributes + ) + + def __call__(self, engine: Engine, logger: ClearMLLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, ClearMLLogger): + raise RuntimeError("Handler OutputHandler works only with ClearMLLogger") + + metrics = self._setup_output_metrics_state_attrs(engine) + + global_step = self.global_step_transform(engine, event_name) + + if not isinstance(global_step, int): + raise TypeError( + f"global_step must be int, got {type(global_step)}." + " Please check the output of global_step_transform." + ) + + for key, value in metrics.items(): + if len(key) == 2: + logger.clearml_logger.report_scalar(title=key[0], series=key[1], iteration=global_step, value=value) + elif len(key) == 3: + logger.clearml_logger.report_scalar( + title=f"{key[0]}/{key[1]}", series=key[2], iteration=global_step, value=value + ) + + +class OptimizerParamsHandler(BaseOptimizerParamsHandler): + """Helper handler to log optimizer parameters + + Args: + optimizer: torch optimizer or any object with attribute ``param_groups`` + as a sequence. + param_name: parameter name + tag: common title for all produced plots. For example, "generator" + + Examples: + .. code-block:: python + + from ignite.handlers.clearml_logger import * + + # Create a logger + + clearml_logger = ClearMLLogger( + project_name="pytorch-ignite-integration", + task_name="cnn-mnist" + ) + + # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration + clearml_logger.attach( + trainer, + log_handler=OptimizerParamsHandler(optimizer), + event_name=Events.ITERATION_STARTED + ) + # or equivalently + clearml_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer + ) + """ + + def __init__(self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None): + super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) + + def __call__(self, engine: Engine, logger: ClearMLLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, ClearMLLogger): + raise RuntimeError("Handler OptimizerParamsHandler works only with ClearMLLogger") + + global_step = engine.state.get_event_attrib_value(event_name) + tag_prefix = f"{self.tag}/" if self.tag else "" + params = { + str(i): float(param_group[self.param_name]) for i, param_group in enumerate(self.optimizer.param_groups) + } + + for k, v in params.items(): + logger.clearml_logger.report_scalar( + title=f"{tag_prefix}{self.param_name}", series=k, value=v, iteration=global_step + ) + + +class WeightsScalarHandler(BaseWeightsScalarHandler): + """Helper handler to log model's weights as scalars. + Handler, upon construction, iterates over named parameters of the model and keep + reference to ones permitted by `whitelist`. Then at every call, applies + reduction function to each parameter, produces a scalar and logs it. + + Args: + model: model to log weights + reduction: function to reduce parameters into scalar + tag: common title for all produced plots. For example, "generator" + whitelist: specific weights to log. Should be list of model's submodules + or parameters names, or a callable which gets weight along with its name + and determines if it should be logged. Names should be fully-qualified. + For more information please refer to `PyTorch docs + `_. + If not given, all of model's weights are logged. + + Examples: + .. code-block:: python + + from ignite.handlers.clearml_logger import * + + # Create a logger + + clearml_logger = ClearMLLogger( + project_name="pytorch-ignite-integration", + task_name="cnn-mnist" + ) + + # Attach the logger to the trainer to log model's weights norm after each iteration + clearml_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsScalarHandler(model, reduction=torch.norm) + ) + + .. code-block:: python + + from ignite.handlers.clearml_logger import * + + clearml_logger = ClearMLLogger( + project_name="pytorch-ignite-integration", + task_name="cnn-mnist" + ) + + # Log only `fc` weights + clearml_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsScalarHandler( + model, + whitelist=['fc'] + ) + ) + + .. code-block:: python + + from ignite.handlers.clearml_logger import * + + clearml_logger = ClearMLLogger( + project_name="pytorch-ignite-integration", + task_name="cnn-mnist" + ) + + # Log weights which have `bias` in their names + def has_bias_in_name(n, p): + return 'bias' in n + + clearml_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsScalarHandler(model, whitelist=has_bias_in_name) + ) + + .. versionchanged:: 0.4.9 + optional argument `whitelist` added. + """ + + def __call__(self, engine: Engine, logger: ClearMLLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, ClearMLLogger): + raise RuntimeError("Handler WeightsScalarHandler works only with ClearMLLogger") + + global_step = engine.state.get_event_attrib_value(event_name) + tag_prefix = f"{self.tag}/" if self.tag else "" + for name, p in self.weights: + title_name, _, series_name = name.partition(".") + logger.clearml_logger.report_scalar( + title=f"{tag_prefix}weights_{self.reduction.__name__}/{title_name}", + series=series_name, + value=self.reduction(p.data), + iteration=global_step, + ) + + +class WeightsHistHandler(BaseWeightsHandler): + """Helper handler to log model's weights as histograms. + + Args: + model: model to log weights + tag: common title for all produced plots. For example, 'generator' + whitelist: specific weights to log. Should be list of model's submodules + or parameters names, or a callable which gets weight along with its name + and determines if it should be logged. Names should be fully-qualified. + For more information please refer to `PyTorch docs + `_. + If not given, all of model's weights are logged. + + Examples: + .. code-block:: python + + from ignite.handlers.clearml_logger import * + + # Create a logger + + clearml_logger = ClearMLLogger( + project_name="pytorch-ignite-integration", + task_name="cnn-mnist" + ) + + # Attach the logger to the trainer to log model's weights norm after each iteration + clearml_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsHistHandler(model) + ) + + .. code-block:: python + + from ignite.handlers.clearml_logger import * + + clearml_logger = ClearMLLogger( + project_name="pytorch-ignite-integration", + task_name="cnn-mnist" + ) + + # Log weights of `fc` layer + weights = ['fc'] + + # Attach the logger to the trainer to log weights norm after each iteration + clearml_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsHistHandler(model, whitelist=weights) + ) + + .. code-block:: python + + from ignite.handlers.clearml_logger import * + + clearml_logger = ClearMLLogger( + project_name="pytorch-ignite-integration", + task_name="cnn-mnist" + ) + + # Log weights which name include 'conv'. + weight_selector = lambda name, p: 'conv' in name + + # Attach the logger to the trainer to log weights norm after each iteration + clearml_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsHistHandler(model, whitelist=weight_selector) + ) + + .. versionchanged:: 0.4.9 + optional argument `whitelist` added. + """ + + def __call__(self, engine: Engine, logger: ClearMLLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, ClearMLLogger): + raise RuntimeError("Handler 'WeightsHistHandler' works only with ClearMLLogger") + + global_step = engine.state.get_event_attrib_value(event_name) + tag_prefix = f"{self.tag}/" if self.tag else "" + for name, p in self.weights: + title_name, _, series_name = name.partition(".") + + logger.grad_helper.add_histogram( + title=f"{tag_prefix}weights_{title_name}", + series=series_name, + step=global_step, + hist_data=p.data.cpu().numpy(), + ) + + +class GradsScalarHandler(BaseWeightsScalarHandler): + """Helper handler to log model's gradients as scalars. + Handler, upon construction, iterates over named parameters of the model and keep + reference to ones permitted by the `whitelist`. Then at every call, applies + reduction function to each parameter's gradient, produces a scalar and logs it. + + Args: + model: model to log weights + reduction: function to reduce parameters into scalar + tag: common title for all produced plots. For example, "generator" + whitelist: specific gradients to log. Should be list of model's submodules + or parameters names, or a callable which gets weight along with its name + and determines if its gradient should be logged. Names should be + fully-qualified. For more information please refer to `PyTorch docs + `_. + If not given, all of model's gradients are logged. + + Examples: + .. code-block:: python + + from ignite.handlers.clearml_logger import * + + # Create a logger + + clearml_logger = ClearMLLogger( + project_name="pytorch-ignite-integration", + task_name="cnn-mnist" + ) + + # Attach the logger to the trainer to log model's weights norm after each iteration + clearml_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=GradsScalarHandler(model, reduction=torch.norm) + ) + + .. code-block:: python + + from ignite.handlers.clearml_logger import * + + clearml_logger = ClearMLLogger( + project_name="pytorch-ignite-integration", + task_name="cnn-mnist" + ) + + # Log gradient of `base` + clearml_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=GradsScalarHandler( + model, + reduction=torch.norm, + whitelist=['base'] + ) + ) + + .. code-block:: python + + from ignite.handlers.clearml_logger import * + + clearml_logger = ClearMLLogger( + project_name="pytorch-ignite-integration", + task_name="cnn-mnist" + ) + + # Log gradient of weights which belong to a `fc` layer + def is_in_fc_layer(n, p): + return 'fc' in n + + clearml_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=GradsScalarHandler(model, whitelist=is_in_fc_layer) + ) + + .. versionchanged:: 0.4.9 + optional argument `whitelist` added. + """ + + def __call__(self, engine: Engine, logger: ClearMLLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, ClearMLLogger): + raise RuntimeError("Handler GradsScalarHandler works only with ClearMLLogger") + + global_step = engine.state.get_event_attrib_value(event_name) + tag_prefix = f"{self.tag}/" if self.tag else "" + for name, p in self.weights: + if p.grad is None: + continue + + title_name, _, series_name = name.partition(".") + logger.clearml_logger.report_scalar( + title=f"{tag_prefix}grads_{self.reduction.__name__}/{title_name}", + series=series_name, + value=self.reduction(p.grad), + iteration=global_step, + ) + + +class GradsHistHandler(BaseWeightsHandler): + """Helper handler to log model's gradients as histograms. + + Args: + model: model to log weights + tag: common title for all produced plots. For example, 'generator' + whitelist: specific gradients to log. Should be list of model's submodules + or parameters names, or a callable which gets weight along with its name + and determines if its gradient should be logged. Names should be + fully-qualified. For more information please refer to `PyTorch docs + `_. + If not given, all of model's gradients are logged. + + Examples: + .. code-block:: python + + from ignite.handlers.clearml_logger import * + + # Create a logger + + clearml_logger = ClearMLLogger( + project_name="pytorch-ignite-integration", + task_name="cnn-mnist" + ) + + # Attach the logger to the trainer to log model's weights norm after each iteration + clearml_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=GradsHistHandler(model) + ) + + .. code-block:: python + + from ignite.handlers.clearml_logger import * + + clearml_logger = ClearMLLogger( + project_name="pytorch-ignite-integration", + task_name="cnn-mnist" + ) + + # Log gradient of `fc.bias` + clearml_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=GradsHistHandler(model, whitelist=['fc.bias']) + ) + + .. code-block:: python + + from ignite.handlers.clearml_logger import * + + clearml_logger = ClearMLLogger( + project_name="pytorch-ignite-integration", + task_name="cnn-mnist" + ) + + # Log gradient of weights which have shape (2, 1) + def has_shape_2_1(n, p): + return p.shape == (2,1) + + clearml_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=GradsHistHandler(model, whitelist=has_shape_2_1) + ) + + .. versionchanged:: 0.4.9 + optional argument `whitelist` added. + """ + + def __call__(self, engine: Engine, logger: ClearMLLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, ClearMLLogger): + raise RuntimeError("Handler 'GradsHistHandler' works only with ClearMLLogger") + + global_step = engine.state.get_event_attrib_value(event_name) + tag_prefix = f"{self.tag}/" if self.tag else "" + for name, p in self.weights: + if p.grad is None: + continue + + title_name, _, series_name = name.partition(".") + logger.grad_helper.add_histogram( + title=f"{tag_prefix}grads_{title_name}", + series=series_name, + step=global_step, + hist_data=p.grad.cpu().numpy(), + ) + + +class ClearMLSaver(DiskSaver): + """ + Handler that saves input checkpoint as ClearML artifacts + + Args: + logger: An instance of :class:`~ignite.handlers.clearml_logger.ClearMLLogger`, + ensuring a valid ClearML ``Task`` has been initialized. If not provided, and a ClearML Task + has not been manually initialized, a runtime error will be raised. + output_uri: The default location for output models and other artifacts uploaded by ClearML. For + more information, see ``clearml.Task.init``. + dirname: Directory path where the checkpoint will be saved. If not provided, a temporary + directory will be created. + + Examples: + .. code-block:: python + + from ignite.handlers.clearml_logger import * + from ignite.handlers import Checkpoint + + clearml_logger = ClearMLLogger( + project_name="pytorch-ignite-integration", + task_name="cnn-mnist" + ) + + to_save = {"model": model} + + handler = Checkpoint( + to_save, + ClearMLSaver(), + n_saved=1, + score_function=lambda e: 123, + score_name="acc", + filename_prefix="best", + global_step_transform=global_step_from_engine(trainer) + ) + + validation_evaluator.add_event_handler(Events.EVENT_COMPLETED, handler) + + """ + + def __init__( + self, + logger: Optional[ClearMLLogger] = None, + output_uri: Optional[str] = None, + dirname: Optional[str] = None, + *args: Any, + **kwargs: Any, + ): + self._setup_check_clearml(logger, output_uri) + + if not dirname: + dirname = "" + if idist.get_rank() == 0: + dirname = tempfile.mkdtemp(prefix=f"ignite_checkpoints_{datetime.now().strftime('%Y_%m_%d_%H_%M_%S_')}") + if idist.get_world_size() > 1: + dirname = idist.all_gather(dirname)[0] # type: ignore[index, assignment] + + warnings.warn(f"ClearMLSaver created a temporary checkpoints directory: {dirname}") + idist.barrier() + + # Let's set non-atomic tmp dir saving behaviour + if "atomic" not in kwargs: + kwargs["atomic"] = False + + self._checkpoint_slots: DefaultDict[Union[str, Tuple[str, str]], List[Any]] = defaultdict(list) + + super(ClearMLSaver, self).__init__(dirname=dirname, *args, **kwargs) # type: ignore[misc] + + @idist.one_rank_only() + def _setup_check_clearml(self, logger: ClearMLLogger, output_uri: str) -> None: + try: + from clearml import Task + except ImportError: + try: + # Backwards-compatibility for legacy Trains SDK + from trains import Task + except ImportError: + raise ModuleNotFoundError( + "This contrib module requires clearml to be installed. " + "You may install clearml using: \n pip install clearml \n" + ) + + if logger and not isinstance(logger, ClearMLLogger): + raise TypeError("logger must be an instance of ClearMLLogger") + + self._task = Task.current_task() + if not self._task: + raise RuntimeError( + "ClearMLSaver requires a ClearML Task to be initialized. " + "Please use the `logger` argument or call `clearml.Task.init()`." + ) + + if output_uri: + self._task.output_uri = output_uri + + class _CallbacksContext: + def __init__( + self, + callback_type: Type[Enum], + slots: List, + checkpoint_key: str, + filename: str, + basename: str, + metadata: Optional[Mapping] = None, + ) -> None: + self._callback_type = callback_type + self._slots = slots + self._checkpoint_key = str(checkpoint_key) + self._filename = filename + self._basename = basename + self._metadata = metadata + + def pre_callback(self, action: str, model_info: Any) -> Any: + if action != self._callback_type.save: # type: ignore[attr-defined] + return model_info + + try: + slot = self._slots.index(None) + self._slots[slot] = model_info.upload_filename + except ValueError: + self._slots.append(model_info.upload_filename) + slot = len(self._slots) - 1 + + model_info.upload_filename = f"{self._basename}_{slot}{os.path.splitext(self._filename)[1]}" + model_info.local_model_id = f"{self._checkpoint_key}:{model_info.upload_filename}" + return model_info + + def post_callback(self, action: str, model_info: Any) -> Any: + if action != self._callback_type.save: # type: ignore[attr-defined] + return model_info + + model_info.model.name = f"{model_info.task.name}: {self._filename}" + prefix = "Checkpoint Metadata: " + metadata_items = ", ".join(f"{k}={v}" for k, v in self._metadata.items()) if self._metadata else "none" + metadata = f"{prefix}{metadata_items}" + comment = "\n".join( + metadata if line.startswith(prefix) else line for line in (model_info.model.comment or "").split("\n") + ) + if prefix not in comment: + comment += "\n" + metadata + model_info.model.comment = comment + + return model_info + + def __call__(self, checkpoint: Mapping, filename: str, metadata: Optional[Mapping] = None) -> None: + try: + from clearml.binding.frameworks import WeightsFileHandler + except ImportError: + try: + # Backwards-compatibility for legacy Trains SDK + from trains.binding.frameworks import WeightsFileHandler + except ImportError: + raise ModuleNotFoundError( + "This contrib module requires clearml to be installed. " + "You may install clearml using: \n pip install clearml \n" + ) + + try: + basename = metadata["basename"] # type: ignore[index] + except (TypeError, KeyError): + warnings.warn("Checkpoint metadata missing or basename cannot be found") + basename = "checkpoint" + + checkpoint_key = (str(self.dirname), basename) + + cb_context = self._CallbacksContext( + callback_type=WeightsFileHandler.CallbackType, + slots=self._checkpoint_slots[checkpoint_key], + checkpoint_key=str(checkpoint_key), + filename=filename, + basename=basename, + metadata=metadata, + ) + + pre_cb_id = WeightsFileHandler.add_pre_callback(cb_context.pre_callback) + post_cb_id = WeightsFileHandler.add_post_callback(cb_context.post_callback) + + try: + super(ClearMLSaver, self).__call__(checkpoint, filename, metadata) + finally: + WeightsFileHandler.remove_pre_callback(pre_cb_id) + WeightsFileHandler.remove_post_callback(post_cb_id) + + @idist.one_rank_only() + def get_local_copy(self, filename: str) -> Optional[str]: + """Get artifact local copy. + + .. warning:: + + In distributed configuration this method should be called on rank 0 process. + + Args: + filename: artifact name. + + Returns: + a local path to a downloaded copy of the artifact + """ + artifact = self._task.artifacts.get(filename) + if artifact: + return artifact.get_local_copy() + self._task.get_logger().report_text(f"Can not find artifact {filename}") + + return None + + @idist.one_rank_only() + def remove(self, filename: str) -> None: + super(ClearMLSaver, self).remove(filename) + for slots in self._checkpoint_slots.values(): + try: + slots[slots.index(filename)] = None + except ValueError: + pass + else: + break diff --git a/ignite/handlers/mlflow_logger.py b/ignite/handlers/mlflow_logger.py new file mode 100644 index 00000000000..68a996d8fb1 --- /dev/null +++ b/ignite/handlers/mlflow_logger.py @@ -0,0 +1,313 @@ +"""MLflow logger and its helper handlers.""" + +import warnings +from typing import Any, Callable, List, Optional, Union + +from torch.optim import Optimizer + +from ignite.engine import Engine, Events + +from ignite.handlers.base_logger import BaseLogger, BaseOptimizerParamsHandler, BaseOutputHandler +from ignite.handlers.utils import global_step_from_engine # noqa + +__all__ = ["MLflowLogger", "OutputHandler", "OptimizerParamsHandler", "global_step_from_engine"] + + +class MLflowLogger(BaseLogger): + """ + `MLflow `_ tracking client handler to log parameters and metrics during the training + and validation. + + This class requires `mlflow package `_ to be installed: + + .. code-block:: bash + + pip install mlflow + + Args: + tracking_uri: MLflow tracking uri. See MLflow docs for more details + + Examples: + .. code-block:: python + + from ignite.handlers.mlflow_logger import * + + # Create a logger + mlflow_logger = MLflowLogger() + + # Log experiment parameters: + mlflow_logger.log_params({ + "seed": seed, + "batch_size": batch_size, + "model": model.__class__.__name__, + + "pytorch version": torch.__version__, + "ignite version": ignite.__version__, + "cuda version": torch.version.cuda, + "device name": torch.cuda.get_device_name(0) + }) + + # Attach the logger to the trainer to log training loss at each iteration + mlflow_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + output_transform=lambda loss: {'loss': loss} + ) + + # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch + # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch + # of the `trainer` instead of `train_evaluator`. + mlflow_logger.attach_output_handler( + train_evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="training", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer), + ) + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the + # `trainer` instead of `evaluator`. + mlflow_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer)), + ) + + # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration + mlflow_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer, + param_name='lr' # optional + ) + """ + + def __init__(self, tracking_uri: Optional[str] = None): + try: + import mlflow + except ImportError: + raise ModuleNotFoundError( + "This contrib module requires mlflow to be installed. " + "Please install it with command: \n pip install mlflow" + ) + + if tracking_uri is not None: + mlflow.set_tracking_uri(tracking_uri) + + self.active_run = mlflow.active_run() + if self.active_run is None: + self.active_run = mlflow.start_run() + + def __getattr__(self, attr: Any) -> Any: + import mlflow + + return getattr(mlflow, attr) + + def close(self) -> None: + import mlflow + + mlflow.end_run() + + def _create_output_handler(self, *args: Any, **kwargs: Any) -> "OutputHandler": + return OutputHandler(*args, **kwargs) + + def _create_opt_params_handler(self, *args: Any, **kwargs: Any) -> "OptimizerParamsHandler": + return OptimizerParamsHandler(*args, **kwargs) + + +class OutputHandler(BaseOutputHandler): + """Helper handler to log engine's output and/or metrics. + + Args: + tag: common title for all produced plots. For example, 'training' + metric_names: list of metric names to plot or a string "all" to plot all available + metrics. + output_transform: output transform function to prepare `engine.state.output` as a number. + For example, `output_transform = lambda output: output` + This function can also return a dictionary, e.g `{'loss': loss1, 'another_loss': loss2}` to label the plot + with corresponding keys. + global_step_transform: global step transform function to output a desired global step. + Input of the function is `(engine, event_name)`. Output of function should be an integer. + Default is None, global_step based on attached engine. If provided, + uses function output as global_step. To setup global step from another engine, please use + :meth:`~ignite.handlers.mlflow_logger.global_step_from_engine`. + state_attributes: list of attributes of the ``trainer.state`` to plot. + + Examples: + .. code-block:: python + + from ignite.handlers.mlflow_logger import * + + # Create a logger + mlflow_logger = MLflowLogger() + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch + # of the `trainer`: + mlflow_logger.attach( + evaluator, + log_handler=OutputHandler( + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer) + ), + event_name=Events.EPOCH_COMPLETED + ) + # or equivalently + mlflow_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer) + ) + + Another example, where model is evaluated every 500 iterations: + + .. code-block:: python + + from ignite.handlers.mlflow_logger import * + + @trainer.on(Events.ITERATION_COMPLETED(every=500)) + def evaluate(engine): + evaluator.run(validation_set, max_epochs=1) + + mlflow_logger = MLflowLogger() + + def global_step_transform(*args, **kwargs): + return trainer.state.iteration + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # every 500 iterations. Since evaluator engine does not have access to the training iteration, we + # provide a global_step_transform to return the trainer.state.iteration for the global_step, each time + # evaluator metrics are plotted on MLflow. + + mlflow_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metrics=["nll", "accuracy"], + global_step_transform=global_step_transform + ) + + Another example where the State Attributes ``trainer.state.alpha`` and ``trainer.state.beta`` + are also logged along with the NLL and Accuracy after each iteration: + + .. code-block:: python + + mlflow_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + metrics=["nll", "accuracy"], + state_attributes=["alpha", "beta"], + ) + + Example of `global_step_transform`: + + .. code-block:: python + + def global_step_transform(engine, event_name): + return engine.state.get_event_attrib_value(event_name) + + .. versionchanged:: 0.4.7 + accepts an optional list of `state_attributes` + """ + + def __init__( + self, + tag: str, + metric_names: Optional[Union[str, List[str]]] = None, + output_transform: Optional[Callable] = None, + global_step_transform: Optional[Callable[[Engine, Union[str, Events]], int]] = None, + state_attributes: Optional[List[str]] = None, + ) -> None: + super(OutputHandler, self).__init__( + tag, metric_names, output_transform, global_step_transform, state_attributes + ) + + def __call__(self, engine: Engine, logger: MLflowLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, MLflowLogger): + raise TypeError("Handler 'OutputHandler' works only with MLflowLogger") + + rendered_metrics = self._setup_output_metrics_state_attrs(engine) + + global_step = self.global_step_transform(engine, event_name) + + if not isinstance(global_step, int): + raise TypeError( + f"global_step must be int, got {type(global_step)}." + " Please check the output of global_step_transform." + ) + + # Additionally recheck metric names as MLflow rejects non-valid names with MLflowException + from mlflow.utils.validation import _VALID_PARAM_AND_METRIC_NAMES + + metrics = {} + for keys, value in rendered_metrics.items(): + key = " ".join(keys) + metrics[key] = value + + for key in list(metrics.keys()): + if not _VALID_PARAM_AND_METRIC_NAMES.match(key): + warnings.warn( + f"MLflowLogger output_handler encountered an invalid metric name '{key}' that " + "will be ignored and not logged to MLflow" + ) + del metrics[key] + + logger.log_metrics(metrics, step=global_step) + + +class OptimizerParamsHandler(BaseOptimizerParamsHandler): + """Helper handler to log optimizer parameters + + Args: + optimizer: torch optimizer or any object with attribute ``param_groups`` + as a sequence. + param_name: parameter name + tag: common title for all produced plots. For example, 'generator' + + Examples: + .. code-block:: python + + from ignite.handlers.mlflow_logger import * + + # Create a logger + mlflow_logger = MLflowLogger() + # Optionally, user can specify tracking_uri with corresponds to MLFLOW_TRACKING_URI + # mlflow_logger = MLflowLogger(tracking_uri="uri") + + # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration + mlflow_logger.attach( + trainer, + log_handler=OptimizerParamsHandler(optimizer), + event_name=Events.ITERATION_STARTED + ) + # or equivalently + mlflow_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer + ) + """ + + def __init__(self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None): + super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) + + def __call__(self, engine: Engine, logger: MLflowLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, MLflowLogger): + raise TypeError("Handler OptimizerParamsHandler works only with MLflowLogger") + + global_step = engine.state.get_event_attrib_value(event_name) + tag_prefix = f"{self.tag} " if self.tag else "" + params = { + f"{tag_prefix}{self.param_name} group_{i}": float(param_group[self.param_name]) + for i, param_group in enumerate(self.optimizer.param_groups) + } + + logger.log_metrics(params, step=global_step) diff --git a/ignite/handlers/neptune_logger.py b/ignite/handlers/neptune_logger.py new file mode 100644 index 00000000000..092d1f81574 --- /dev/null +++ b/ignite/handlers/neptune_logger.py @@ -0,0 +1,706 @@ +"""Neptune logger and its helper handlers.""" + +import tempfile +import warnings +from typing import Any, Callable, List, Mapping, Optional, Union + +import torch +from torch.optim import Optimizer + +import ignite.distributed as idist + +from ignite.engine import Engine, Events +from ignite.handlers.base_logger import ( + BaseLogger, + BaseOptimizerParamsHandler, + BaseOutputHandler, + BaseWeightsScalarHandler, +) +from ignite.handlers.checkpoint import BaseSaveHandler +from ignite.handlers.utils import global_step_from_engine # noqa + +__all__ = [ + "NeptuneLogger", + "NeptuneSaver", + "OptimizerParamsHandler", + "OutputHandler", + "WeightsScalarHandler", + "GradsScalarHandler", + "global_step_from_engine", +] + +_INTEGRATION_VERSION_KEY = "source_code/integrations/neptune-pytorch-ignite" + + +class NeptuneLogger(BaseLogger): + """ + `Neptune `_ handler to log metrics, model/optimizer parameters and gradients during training + and validation. It can also log model checkpoints to Neptune. + + .. code-block:: bash + + pip install neptune + + Args: + api_token: Neptune API token, found on https://neptune.ai -> User menu -> "Get your API token". + If None, the value of the NEPTUNE_API_TOKEN environment variable is used. To keep your token + secure, you should set it to the environment variable rather than including it in your code. + project: Name of a Neptune project, in the form "workspace-name/project-name". + For example "tom/mnist-classification". + If None, the value of the NEPTUNE_PROJECT environment variable is used. + **kwargs: Other arguments to be passed to the `init_run()` function. + + Examples: + .. code-block:: python + + from ignite.handlers.neptune_logger import * + + # Create a logger + # Note: We are using the API token for anonymous logging. You can pass your own token, or save it as an + # environment variable and leave out the api_token argument. + + npt_logger = NeptuneLogger( + api_token="ANONYMOUS", + project="common/pytorch-ignite-integration", + name="cnn-mnist", # Optional, + tags=["pytorch-ignite", "minst"], # Optional + ) + + # Attach the logger to the trainer to log training loss at each iteration. + npt_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + output_transform=lambda loss: {"loss": loss}, + ) + + # Attach the logger to the evaluator on the training dataset and log NLL + # and accuracy metrics after each epoch. + # We set up `global_step_transform=global_step_from_engine(trainer)` to take the epoch + # of the `trainer` instead of `train_evaluator`. + npt_logger.attach_output_handler( + train_evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="training", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer), + ) + + # Attach the logger to the evaluator on the validation dataset and log NLL and accuracy metrics after + # each epoch. We set up `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the + # `trainer` instead of `evaluator`. + npt_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer), + ) + + # Attach the logger to the trainer to log optimizer parameters, such as learning rate at each iteration. + npt_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer, + param_name="lr", # optional + ) + + # Attach the logger to the trainer to log model's weights norm after each iteration. + npt_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsScalarHandler(model), + ) + + Explore runs with Neptune tracking here: + https://app.neptune.ai/o/common/org/pytorch-ignite-integration/ + + You can also save model checkpoints to a Neptune: + + .. code-block:: python + + from ignite.handlers import Checkpoint + + + def score_function(engine): + return engine.state.metrics["accuracy"] + + + to_save = {"model": model} + handler = Checkpoint( + to_save, + NeptuneSaver(npt_logger), n_saved=2, + filename_prefix="best", + score_function=score_function, + score_name="validation_accuracy", + global_step_transform=global_step_from_engine(trainer), + ) + validation_evaluator.add_event_handler(Events.COMPLETED, handler) + + It is also possible to use the logger as a context manager: + + .. code-block:: python + + from ignite.handlers.neptune_logger import * + + with NeptuneLogger() as npt_logger: + trainer = Engine(update_fn) + # Attach the logger to the trainer to log training loss at each iteration + npt_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + output_transform=lambda loss: {"loss": loss}, + ) + + """ + + def __getattr__(self, attr: Any) -> Any: + return getattr(self.experiment, attr) + + def __getitem__(self, key: str) -> Any: + return self.experiment[key] + + def __setitem__(self, key: str, val: Any) -> Any: + self.experiment[key] = val + + def __init__(self, api_token: Optional[str] = None, project: Optional[str] = None, **kwargs: Any) -> None: + try: + try: + # neptune-client<1.0.0 package structure + with warnings.catch_warnings(): + # ignore the deprecation warnings + warnings.simplefilter("ignore") + import neptune.new as neptune + except ImportError: + # neptune>=1.0.0 package structure + import neptune + except ImportError: + raise ModuleNotFoundError( + "This contrib module requires the Neptune client library to be installed. " + "Install neptune with the command: \n pip install neptune \n" + ) + + run = neptune.init_run( + api_token=api_token, + project=project, + **kwargs, + ) + from ignite import __version__ + + run[_INTEGRATION_VERSION_KEY] = __version__ + + self.experiment = run + + def close(self) -> None: + self.experiment.stop() + + def _create_output_handler(self, *args: Any, **kwargs: Any) -> "OutputHandler": + return OutputHandler(*args, **kwargs) + + def _create_opt_params_handler(self, *args: Any, **kwargs: Any) -> "OptimizerParamsHandler": + return OptimizerParamsHandler(*args, **kwargs) + + +class OutputHandler(BaseOutputHandler): + """Helper handler to log engine's output and/or metrics. + + Args: + tag: common title for all produced plots. For example, "training" + metric_names: list of metric names to plot or a string "all" to plot all available + metrics. + output_transform: output transform function to prepare `engine.state.output` as a number. + For example, `output_transform = lambda output: output` + This function can also return a dictionary, e.g `{"loss": loss1, "another_loss": loss2}` to label the plot + with corresponding keys. + global_step_transform: global step transform function to output a desired global step. + Input of the function is `(engine, event_name)`. Output of function should be an integer. + Default is None, global_step based on attached engine. If provided, + uses function output as global_step. To setup global step from another engine, please use + :meth:`~ignite.handlers.neptune_logger.global_step_from_engine`. + state_attributes: list of attributes of the ``trainer.state`` to plot. + + Examples: + .. code-block:: python + + from ignite.handlers.neptune_logger import * + + # Create a logger + # We are using the api_token for the anonymous user neptuner but you can use your own. + + npt_logger = NeptuneLogger( + api_token="ANONYMOUS", + project_name="shared/pytorch-ignite-integration", + experiment_name="cnn-mnist", # Optional, + params={"max_epochs": 10}, # Optional, + tags=["pytorch-ignite","minst"] # Optional + ) + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch + # of the `trainer`: + npt_logger.attach( + evaluator, + log_handler=OutputHandler( + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer) + ), + event_name=Events.EPOCH_COMPLETED + ) + # or equivalently + npt_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer) + ) + + Another example, where model is evaluated every 500 iterations: + + .. code-block:: python + + from ignite.handlers.neptune_logger import * + + @trainer.on(Events.ITERATION_COMPLETED(every=500)) + def evaluate(engine): + evaluator.run(validation_set, max_epochs=1) + + # We are using the api_token for the anonymous user neptuner but you can use your own. + + npt_logger = NeptuneLogger( + api_token="ANONYMOUS", + project_name="shared/pytorch-ignite-integration", + experiment_name="cnn-mnist", # Optional, + params={"max_epochs": 10}, # Optional, + tags=["pytorch-ignite", "minst"] # Optional + ) + + def global_step_transform(*args, **kwargs): + return trainer.state.iteration + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # every 500 iterations. Since evaluator engine does not have access to the training iteration, we + # provide a global_step_transform to return the trainer.state.iteration for the global_step, each time + # evaluator metrics are plotted on NeptuneML. + + npt_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metrics=["nll", "accuracy"], + global_step_transform=global_step_transform + ) + + Another example where the State Attributes ``trainer.state.alpha`` and ``trainer.state.beta`` + are also logged along with the NLL and Accuracy after each iteration: + + .. code-block:: python + + npt_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + metrics=["nll", "accuracy"], + state_attributes=["alpha", "beta"], + ) + + Example of `global_step_transform`: + + .. code-block:: python + + def global_step_transform(engine, event_name): + return engine.state.get_event_attrib_value(event_name) + + .. versionchanged:: 0.4.7 + accepts an optional list of `state_attributes` + """ + + def __init__( + self, + tag: str, + metric_names: Optional[Union[str, List[str]]] = None, + output_transform: Optional[Callable] = None, + global_step_transform: Optional[Callable[[Engine, Union[str, Events]], int]] = None, + state_attributes: Optional[List[str]] = None, + ): + super(OutputHandler, self).__init__( + tag, metric_names, output_transform, global_step_transform, state_attributes + ) + + def __call__(self, engine: Engine, logger: NeptuneLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, NeptuneLogger): + raise TypeError("Handler OutputHandler works only with NeptuneLogger") + + metrics = self._setup_output_metrics_state_attrs(engine, key_tuple=False) + + global_step = self.global_step_transform(engine, event_name) + + if not isinstance(global_step, int): + raise TypeError( + f"global_step must be int, got {type(global_step)}." + " Please check the output of global_step_transform." + ) + + for key, value in metrics.items(): + logger[key].append(value, step=global_step) + + +class OptimizerParamsHandler(BaseOptimizerParamsHandler): + """Helper handler to log optimizer parameters + + Args: + optimizer: torch optimizer or any object with attribute ``param_groups`` + as a sequence. + param_name: parameter name + tag: common title for all produced plots. For example, "generator" + + Examples: + .. code-block:: python + + from ignite.handlers.neptune_logger import * + + # Create a logger + # We are using the api_token for the anonymous user neptuner but you can use your own. + + npt_logger = NeptuneLogger( + api_token="ANONYMOUS", + project_name="shared/pytorch-ignite-integration", + experiment_name="cnn-mnist", # Optional, + params={"max_epochs": 10}, # Optional, + tags=["pytorch-ignite","minst"] # Optional + ) + + # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration + npt_logger.attach( + trainer, + log_handler=OptimizerParamsHandler(optimizer), + event_name=Events.ITERATION_STARTED + ) + # or equivalently + npt_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer + ) + """ + + def __init__(self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None): + super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) + + def __call__(self, engine: Engine, logger: NeptuneLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, NeptuneLogger): + raise TypeError("Handler OptimizerParamsHandler works only with NeptuneLogger") + + global_step = engine.state.get_event_attrib_value(event_name) + tag_prefix = f"{self.tag}/" if self.tag else "" + params = { + f"{tag_prefix}{self.param_name}/group_{i}": float(param_group[self.param_name]) + for i, param_group in enumerate(self.optimizer.param_groups) + } + + for k, v in params.items(): + logger[k].append(v, step=global_step) + + +class WeightsScalarHandler(BaseWeightsScalarHandler): + """Helper handler to log model's weights as scalars. + Handler, upon construction, iterates over named parameters of the model and keep + reference to ones permitted by `whitelist`. Then at every call, applies + reduction function to each parameter, produces a scalar and logs it. + + Args: + model: model to log weights + reduction: function to reduce parameters into scalar + tag: common title for all produced plots. For example, "generator" + whitelist: specific weights to log. Should be list of model's submodules + or parameters names, or a callable which gets weight along with its name + and determines if it should be logged. Names should be fully-qualified. + For more information please refer to `PyTorch docs + `_. + If not given, all of model's weights are logged. + + Examples: + .. code-block:: python + + from ignite.handlers.neptune_logger import * + + # Create a logger + # We are using the api_token for the anonymous user neptuner but you can use your own. + + npt_logger = NeptuneLogger( + api_token="ANONYMOUS", + project_name="shared/pytorch-ignite-integration", + experiment_name="cnn-mnist", # Optional, + params={"max_epochs": 10}, # Optional, + tags=["pytorch-ignite","minst"] # Optional + ) + + # Attach the logger to the trainer to log model's weights norm after each iteration + npt_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsScalarHandler(model, reduction=torch.norm) + ) + + .. code-block:: python + + from ignite.handlers.neptune_logger import * + + npt_logger = NeptuneLogger( + api_token="ANONYMOUS", + project_name="shared/pytorch-ignite-integration", + experiment_name="cnn-mnist", # Optional, + params={"max_epochs": 10}, # Optional, + tags=["pytorch-ignite","minst"] # Optional + ) + + # Log only `fc` weights + npt_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsScalarHandler( + model, + whitelist=['fc'] + ) + ) + + .. code-block:: python + + from ignite.handlers.neptune_logger import * + + npt_logger = NeptuneLogger( + api_token="ANONYMOUS", + project_name="shared/pytorch-ignite-integration", + experiment_name="cnn-mnist", # Optional, + params={"max_epochs": 10}, # Optional, + tags=["pytorch-ignite","minst"] # Optional + ) + + # Log weights which have `bias` in their names + def has_bias_in_name(n, p): + return 'bias' in n + + npt_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsScalarHandler(model, whitelist=has_bias_in_name) + ) + + .. versionchanged:: 0.4.9 + optional argument `whitelist` added. + """ + + def __call__(self, engine: Engine, logger: NeptuneLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, NeptuneLogger): + raise TypeError("Handler WeightsScalarHandler works only with NeptuneLogger") + + global_step = engine.state.get_event_attrib_value(event_name) + tag_prefix = f"{self.tag}/" if self.tag else "" + for name, p in self.weights: + if p.grad is None: + continue + + name = name.replace(".", "/") + key = f"{tag_prefix}weights_{self.reduction.__name__}/{name}" + logger[key].append(self.reduction(p.data), step=global_step) + + +class GradsScalarHandler(BaseWeightsScalarHandler): + """Helper handler to log model's gradients as scalars. + Handler, upon construction, iterates over named parameters of the model and keep + reference to ones permitted by the `whitelist`. Then at every call, applies + reduction function to each parameter's gradient, produces a scalar and logs it. + + Args: + model: model to log weights + reduction: function to reduce parameters into scalar + tag: common title for all produced plots. For example, "generator" + whitelist: specific gradients to log. Should be list of model's submodules + or parameters names, or a callable which gets weight along with its name + and determines if its gradient should be logged. Names should be + fully-qualified. For more information please refer to `PyTorch docs + `_. + If not given, all of model's gradients are logged. + + Examples: + .. code-block:: python + + from ignite.handlers.neptune_logger import * + + # Create a logger + # We are using the api_token for the anonymous user neptuner but you can use your own. + + npt_logger = NeptuneLogger( + api_token="ANONYMOUS", + project_name="shared/pytorch-ignite-integration", + experiment_name="cnn-mnist", # Optional, + params={"max_epochs": 10}, # Optional, + tags=["pytorch-ignite","minst"] # Optional + ) + + # Attach the logger to the trainer to log model's weights norm after each iteration + npt_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=GradsScalarHandler(model, reduction=torch.norm) + ) + + .. code-block:: python + + from ignite.handlers.neptune_logger import * + + npt_logger = NeptuneLogger( + api_token="ANONYMOUS", + project_name="shared/pytorch-ignite-integration", + experiment_name="cnn-mnist", # Optional, + params={"max_epochs": 10}, # Optional, + tags=["pytorch-ignite","minst"] # Optional + ) + + # Log gradient of `base` + npt_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=GradsScalarHandler( + model, + reduction=torch.norm, + whitelist=['base'] + ) + ) + + .. code-block:: python + + from ignite.handlers.neptune_logger import * + + npt_logger = NeptuneLogger( + api_token="ANONYMOUS", + project_name="shared/pytorch-ignite-integration", + experiment_name="cnn-mnist", # Optional, + params={"max_epochs": 10}, # Optional, + tags=["pytorch-ignite","minst"] # Optional + ) + + # Log gradient of weights which belong to a `fc` layer + def is_in_fc_layer(n, p): + return 'fc' in n + + npt_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=GradsScalarHandler(model, whitelist=is_in_fc_layer) + ) + + .. versionchanged:: 0.4.9 + optional argument `whitelist` added. + """ + + def __call__(self, engine: Engine, logger: NeptuneLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, NeptuneLogger): + raise TypeError("Handler GradsScalarHandler works only with NeptuneLogger") + + global_step = engine.state.get_event_attrib_value(event_name) + tag_prefix = f"{self.tag}/" if self.tag else "" + for name, p in self.weights: + if p.grad is None: + continue + + name = name.replace(".", "/") + key = f"{tag_prefix}grads_{self.reduction.__name__}/{name}" + logger[key].append(self.reduction(p.grad), step=global_step) + + +class NeptuneSaver(BaseSaveHandler): + """Handler that saves input checkpoint to the Neptune server. + + Args: + neptune_logger: an instance of + NeptuneLogger class. + + .. Note :: + + NeptuneSaver is currently not supported on Windows. + + Examples: + .. code-block:: python + + from ignite.handlers.neptune_logger import * + + # Create a logger + # We are using the api_token for the anonymous user neptuner but you can use your own. + + npt_logger = NeptuneLogger( + api_token="ANONYMOUS", + project_name="shared/pytorch-ignite-integration", + experiment_name="cnn-mnist", # Optional, + params={"max_epochs": 10}, # Optional, + tags=["pytorch-ignite","minst"] # Optional + ) + + ... + evaluator = create_supervised_evaluator(model, metrics=metrics, ...) + ... + + from ignite.handlers import Checkpoint + + def score_function(engine): + return engine.state.metrics["accuracy"] + + to_save = {"model": model} + + # pass neptune logger to NeptuneServer + + handler = Checkpoint( + to_save, + NeptuneSaver(npt_logger), n_saved=2, + filename_prefix="best", score_function=score_function, + score_name="validation_accuracy", + global_step_transform=global_step_from_engine(trainer) + ) + + evaluator.add_event_handler(Events.COMPLETED, handler) + + # We need to close the logger when we are done + npt_logger.close() + + For example, you can access model checkpoints and download them from here: + https://ui.neptune.ai/o/shared/org/pytorch-ignite-integration/e/PYTOR1-18/charts + + """ + + @idist.one_rank_only() + def __init__(self, neptune_logger: NeptuneLogger): + self._logger = neptune_logger + + @idist.one_rank_only() + def __call__(self, checkpoint: Mapping, filename: str, metadata: Optional[Mapping] = None) -> None: + # wont work on XLA + + # Imports for BC compatibility + try: + # neptune-client<1.0.0 package structure + with warnings.catch_warnings(): + # ignore the deprecation warnings + warnings.simplefilter("ignore") + from neptune.new.types import File + except ImportError: + # neptune>=1.0.0 package structure + from neptune.types import File + + with tempfile.NamedTemporaryFile() as tmp: + # we can not use tmp.name to open tmp.file twice on Win32 + # https://docs.python.org/3/library/tempfile.html#tempfile.NamedTemporaryFile + torch.save(checkpoint, tmp.file) + + # rewind the buffer + tmp.file.seek(0) + + # hold onto the file stream for uploading. + # NOTE: This won't load the whole file in memory and upload + # the stream in smaller chunks. + self._logger[filename].upload(File.from_stream(tmp.file)) + + @idist.one_rank_only(with_barrier=True) + def remove(self, filename: str) -> None: + del self._logger.experiment[filename] diff --git a/ignite/handlers/polyaxon_logger.py b/ignite/handlers/polyaxon_logger.py new file mode 100644 index 00000000000..e98a5b2b34a --- /dev/null +++ b/ignite/handlers/polyaxon_logger.py @@ -0,0 +1,306 @@ +"""Polyaxon logger and its helper handlers.""" + +from typing import Any, Callable, List, Optional, Union + +from torch.optim import Optimizer + +from ignite.engine import Engine, Events + +from ignite.handlers.base_logger import BaseLogger, BaseOptimizerParamsHandler, BaseOutputHandler +from ignite.handlers.utils import global_step_from_engine # noqa + +__all__ = ["PolyaxonLogger", "OutputHandler", "OptimizerParamsHandler", "global_step_from_engine"] + + +class PolyaxonLogger(BaseLogger): + """ + `Polyaxon tracking client `_ handler to log parameters and metrics during the training + and validation. + + This class requires `polyaxon `_ package to be installed: + + .. code-block:: bash + + pip install polyaxon + + // If you are using polyaxon v0.x + + pip install polyaxon-client + + Args: + args: Positional arguments accepted from + `Experiment `_. + kwargs: Keyword arguments accepted from + `Experiment `_. + + Examples: + .. code-block:: python + + from ignite.handlers.polyaxon_logger import * + + # Create a logger + plx_logger = PolyaxonLogger() + + # Log experiment parameters: + plx_logger.log_inputs(**{ + "seed": seed, + "batch_size": batch_size, + "model": model.__class__.__name__, + + "pytorch version": torch.__version__, + "ignite version": ignite.__version__, + "cuda version": torch.version.cuda, + "device name": torch.cuda.get_device_name(0) + }) + + # Attach the logger to the trainer to log training loss at each iteration + plx_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + output_transform=lambda loss: {"loss": loss} + ) + + # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch + # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch + # of the `trainer` instead of `train_evaluator`. + plx_logger.attach_output_handler( + train_evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="training", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer), + ) + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the + # `trainer` instead of `evaluator`. + plx_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer)), + ) + + # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration + plx_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer, + param_name='lr' # optional + ) + # to manually end a run + plx_logger.close() + """ + + def __init__(self, *args: Any, **kwargs: Any): + try: + from polyaxon.tracking import Run + + self.experiment = Run(*args, **kwargs) + + except ImportError: + try: + from polyaxon_client.tracking import Experiment + + self.experiment = Experiment(*args, **kwargs) + except ImportError: + raise ModuleNotFoundError( + "This contrib module requires polyaxon to be installed.\n" + "For Polyaxon v1.x please install it with command: \n pip install polyaxon\n" + "For Polyaxon v0.x please install it with command: \n pip install polyaxon-client" + ) + + def close(self) -> None: + try: + self.experiment.end() + except: + pass + + def __getattr__(self, attr: Any) -> Any: + return getattr(self.experiment, attr) + + def _create_output_handler(self, *args: Any, **kwargs: Any) -> "OutputHandler": + return OutputHandler(*args, **kwargs) + + def _create_opt_params_handler(self, *args: Any, **kwargs: Any) -> "OptimizerParamsHandler": + return OptimizerParamsHandler(*args, **kwargs) + + +class OutputHandler(BaseOutputHandler): + """Helper handler to log engine's output and/or metrics. + + Args: + tag: common title for all produced plots. For example, "training" + metric_names: list of metric names to plot or a string "all" to plot all available + metrics. + output_transform: output transform function to prepare `engine.state.output` as a number. + For example, `output_transform = lambda output: output` + This function can also return a dictionary, e.g `{"loss": loss1, "another_loss": loss2}` to label the plot + with corresponding keys. + global_step_transform: global step transform function to output a desired global step. + Input of the function is `(engine, event_name)`. Output of function should be an integer. + Default is None, global_step based on attached engine. If provided, + uses function output as global_step. To setup global step from another engine, please use + :meth:`~ignite.handlers.polyaxon_logger.global_step_from_engine`. + state_attributes: list of attributes of the ``trainer.state`` to plot. + + Examples: + .. code-block:: python + + from ignite.handlers.polyaxon_logger import * + + # Create a logger + plx_logger = PolyaxonLogger() + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch + # of the `trainer`: + plx_logger.attach( + evaluator, + log_handler=OutputHandler( + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer) + ), + event_name=Events.EPOCH_COMPLETED + ) + # or equivalently + plx_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer) + ) + + Another example, where model is evaluated every 500 iterations: + + .. code-block:: python + + from ignite.handlers.polyaxon_logger import * + + @trainer.on(Events.ITERATION_COMPLETED(every=500)) + def evaluate(engine): + evaluator.run(validation_set, max_epochs=1) + + plx_logger = PolyaxonLogger() + + def global_step_transform(*args, **kwargs): + return trainer.state.iteration + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # every 500 iterations. Since evaluator engine does not have access to the training iteration, we + # provide a global_step_transform to return the trainer.state.iteration for the global_step, each time + # evaluator metrics are plotted on Polyaxon. + + plx_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metrics=["nll", "accuracy"], + global_step_transform=global_step_transform + ) + + Another example where the State Attributes ``trainer.state.alpha`` and ``trainer.state.beta`` + are also logged along with the NLL and Accuracy after each iteration: + + .. code-block:: python + + plx_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + metrics=["nll", "accuracy"], + state_attributes=["alpha", "beta"], + ) + + Example of `global_step_transform`: + + .. code-block:: python + + def global_step_transform(engine, event_name): + return engine.state.get_event_attrib_value(event_name) + + .. versionchanged:: 0.4.7 + accepts an optional list of `state_attributes` + """ + + def __init__( + self, + tag: str, + metric_names: Optional[List[str]] = None, + output_transform: Optional[Callable] = None, + global_step_transform: Optional[Callable[[Engine, Union[str, Events]], int]] = None, + state_attributes: Optional[List[str]] = None, + ): + super(OutputHandler, self).__init__( + tag, metric_names, output_transform, global_step_transform, state_attributes + ) + + def __call__(self, engine: Engine, logger: PolyaxonLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, PolyaxonLogger): + raise RuntimeError("Handler 'OutputHandler' works only with PolyaxonLogger") + + metrics = self._setup_output_metrics_state_attrs(engine, key_tuple=False) + + global_step = self.global_step_transform(engine, event_name) + + if not isinstance(global_step, int): + raise TypeError( + f"global_step must be int, got {type(global_step)}." + " Please check the output of global_step_transform." + ) + + metrics.update({"step": global_step}) + + logger.log_metrics(**metrics) + + +class OptimizerParamsHandler(BaseOptimizerParamsHandler): + """Helper handler to log optimizer parameters + + Args: + optimizer: torch optimizer or any object with attribute ``param_groups`` + as a sequence. + param_name: parameter name + tag: common title for all produced plots. For example, "generator" + + Examples: + .. code-block:: python + + from ignite.handlers.polyaxon_logger import * + + # Create a logger + plx_logger = PolyaxonLogger() + + # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration + plx_logger.attach( + trainer, + log_handler=OptimizerParamsHandler(optimizer), + event_name=Events.ITERATION_STARTED + ) + # or equivalently + plx_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer + ) + """ + + def __init__(self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None): + super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) + + def __call__(self, engine: Engine, logger: PolyaxonLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, PolyaxonLogger): + raise RuntimeError("Handler OptimizerParamsHandler works only with PolyaxonLogger") + + global_step = engine.state.get_event_attrib_value(event_name) + tag_prefix = f"{self.tag}/" if self.tag else "" + params = { + f"{tag_prefix}{self.param_name}/group_{i}": float(param_group[self.param_name]) + for i, param_group in enumerate(self.optimizer.param_groups) + } + params["step"] = global_step + logger.log_metrics(**params) diff --git a/ignite/handlers/tensorboard_logger.py b/ignite/handlers/tensorboard_logger.py new file mode 100644 index 00000000000..001adf17389 --- /dev/null +++ b/ignite/handlers/tensorboard_logger.py @@ -0,0 +1,676 @@ +"""TensorBoard logger and its helper handlers.""" + +from typing import Any, Callable, List, Optional, Union + +from torch.optim import Optimizer + +from ignite.engine import Engine, Events + +from ignite.handlers.base_logger import ( + BaseLogger, + BaseOptimizerParamsHandler, + BaseOutputHandler, + BaseWeightsHandler, + BaseWeightsScalarHandler, +) +from ignite.handlers.utils import global_step_from_engine # noqa + +__all__ = [ + "TensorboardLogger", + "OptimizerParamsHandler", + "OutputHandler", + "WeightsScalarHandler", + "WeightsHistHandler", + "GradsScalarHandler", + "GradsHistHandler", + "global_step_from_engine", +] + + +class TensorboardLogger(BaseLogger): + """ + TensorBoard handler to log metrics, model/optimizer parameters, gradients during the training and validation. + + By default, this class favors `tensorboardX `_ package if installed: + + .. code-block:: bash + + pip install tensorboardX + + otherwise, it falls back to using + `PyTorch's SummaryWriter + `_ + (>=v1.2.0). + + Args: + args: Positional arguments accepted from + `SummaryWriter + `_. + kwargs: Keyword arguments accepted from + `SummaryWriter + `_. + For example, `log_dir` to setup path to the directory where to log. + + Examples: + .. code-block:: python + + from ignite.handlers.tensorboard_logger import * + + # Create a logger + tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") + + # Attach the logger to the trainer to log training loss at each iteration + tb_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + output_transform=lambda loss: {"loss": loss} + ) + + # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch + # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch + # of the `trainer` instead of `train_evaluator`. + tb_logger.attach_output_handler( + train_evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="training", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer), + ) + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the + # `trainer` instead of `evaluator`. + tb_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer)), + ) + + # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration + tb_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer, + param_name='lr' # optional + ) + + # Attach the logger to the trainer to log model's weights norm after each iteration + tb_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsScalarHandler(model) + ) + + # Attach the logger to the trainer to log model's weights as a histogram after each epoch + tb_logger.attach( + trainer, + event_name=Events.EPOCH_COMPLETED, + log_handler=WeightsHistHandler(model) + ) + + # Attach the logger to the trainer to log model's gradients norm after each iteration + tb_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=GradsScalarHandler(model) + ) + + # Attach the logger to the trainer to log model's gradients as a histogram after each epoch + tb_logger.attach( + trainer, + event_name=Events.EPOCH_COMPLETED, + log_handler=GradsHistHandler(model) + ) + + # We need to close the logger when we are done + tb_logger.close() + + It is also possible to use the logger as context manager: + + .. code-block:: python + + from ignite.handlers.tensorboard_logger import * + + with TensorboardLogger(log_dir="experiments/tb_logs") as tb_logger: + + trainer = Engine(update_fn) + # Attach the logger to the trainer to log training loss at each iteration + tb_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + output_transform=lambda loss: {"loss": loss} + ) + + """ + + def __init__(self, *args: Any, **kwargs: Any): + try: + from tensorboardX import SummaryWriter + except ImportError: + try: + from torch.utils.tensorboard import SummaryWriter + except ImportError: + raise ModuleNotFoundError( + "This contrib module requires either tensorboardX or torch >= 1.2.0. " + "You may install tensorboardX with command: \n pip install tensorboardX \n" + "or upgrade PyTorch using your package manager of choice (pip or conda)." + ) + + self.writer = SummaryWriter(*args, **kwargs) + + def __getattr__(self, attr: Any) -> Any: + return getattr(self.writer, attr) + + def close(self) -> None: + self.writer.close() + + def _create_output_handler(self, *args: Any, **kwargs: Any) -> "OutputHandler": + return OutputHandler(*args, **kwargs) + + def _create_opt_params_handler(self, *args: Any, **kwargs: Any) -> "OptimizerParamsHandler": + return OptimizerParamsHandler(*args, **kwargs) + + +class OutputHandler(BaseOutputHandler): + """Helper handler to log engine's output, engine's state attributes and/or metrics + + Args: + tag: common title for all produced plots. For example, "training" + metric_names: list of metric names to plot or a string "all" to plot all available + metrics. + output_transform: output transform function to prepare `engine.state.output` as a number. + For example, `output_transform = lambda output: output` + This function can also return a dictionary, e.g `{"loss": loss1, "another_loss": loss2}` to label the plot + with corresponding keys. + global_step_transform: global step transform function to output a desired global step. + Input of the function is `(engine, event_name)`. Output of function should be an integer. + Default is None, global_step based on attached engine. If provided, + uses function output as global_step. To setup global step from another engine, please use + :meth:`~ignite.handlers.tensorboard_logger.global_step_from_engine`. + state_attributes: list of attributes of the ``trainer.state`` to plot. + + Examples: + .. code-block:: python + + from ignite.handlers.tensorboard_logger import * + + # Create a logger + tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch + # of the `trainer`: + tb_logger.attach( + evaluator, + log_handler=OutputHandler( + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer) + ), + event_name=Events.EPOCH_COMPLETED + ) + # or equivalently + tb_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer) + ) + + Another example, where model is evaluated every 500 iterations: + + .. code-block:: python + + from ignite.handlers.tensorboard_logger import * + + @trainer.on(Events.ITERATION_COMPLETED(every=500)) + def evaluate(engine): + evaluator.run(validation_set, max_epochs=1) + + tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") + + def global_step_transform(*args, **kwargs): + return trainer.state.iteration + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # every 500 iterations. Since evaluator engine does not have access to the training iteration, we + # provide a global_step_transform to return the trainer.state.iteration for the global_step, each time + # evaluator metrics are plotted on Tensorboard. + + tb_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metrics=["nll", "accuracy"], + global_step_transform=global_step_transform + ) + + Another example where the State Attributes ``trainer.state.alpha`` and ``trainer.state.beta`` + are also logged along with the NLL and Accuracy after each iteration: + + .. code-block:: python + + tb_logger.attach( + trainer, + log_handler=OutputHandler( + tag="training", + metric_names=["nll", "accuracy"], + state_attributes=["alpha", "beta"], + ), + event_name=Events.ITERATION_COMPLETED + ) + + Example of `global_step_transform`: + + .. code-block:: python + + def global_step_transform(engine, event_name): + return engine.state.get_event_attrib_value(event_name) + + .. versionchanged:: 0.4.7 + accepts an optional list of `state_attributes` + """ + + def __init__( + self, + tag: str, + metric_names: Optional[List[str]] = None, + output_transform: Optional[Callable] = None, + global_step_transform: Optional[Callable[[Engine, Union[str, Events]], int]] = None, + state_attributes: Optional[List[str]] = None, + ): + super(OutputHandler, self).__init__( + tag, metric_names, output_transform, global_step_transform, state_attributes + ) + + def __call__(self, engine: Engine, logger: TensorboardLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, TensorboardLogger): + raise RuntimeError("Handler 'OutputHandler' works only with TensorboardLogger") + + metrics = self._setup_output_metrics_state_attrs(engine, key_tuple=False) + + global_step = self.global_step_transform(engine, event_name) + if not isinstance(global_step, int): + raise TypeError( + f"global_step must be int, got {type(global_step)}." + " Please check the output of global_step_transform." + ) + + for key, value in metrics.items(): + logger.writer.add_scalar(key, value, global_step) + + +class OptimizerParamsHandler(BaseOptimizerParamsHandler): + """Helper handler to log optimizer parameters + + Args: + optimizer: torch optimizer or any object with attribute ``param_groups`` + as a sequence. + param_name: parameter name + tag: common title for all produced plots. For example, "generator" + + Examples: + .. code-block:: python + + from ignite.handlers.tensorboard_logger import * + + # Create a logger + tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") + + # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration + tb_logger.attach( + trainer, + log_handler=OptimizerParamsHandler(optimizer), + event_name=Events.ITERATION_STARTED + ) + # or equivalently + tb_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer + ) + """ + + def __init__(self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None): + super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) + + def __call__(self, engine: Engine, logger: TensorboardLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, TensorboardLogger): + raise RuntimeError("Handler OptimizerParamsHandler works only with TensorboardLogger") + + global_step = engine.state.get_event_attrib_value(event_name) + tag_prefix = f"{self.tag}/" if self.tag else "" + params = { + f"{tag_prefix}{self.param_name}/group_{i}": float(param_group[self.param_name]) + for i, param_group in enumerate(self.optimizer.param_groups) + } + + for k, v in params.items(): + logger.writer.add_scalar(k, v, global_step) + + +class WeightsScalarHandler(BaseWeightsScalarHandler): + """Helper handler to log model's weights as scalars. + Handler, upon construction, iterates over named parameters of the model and keep + reference to ones permitted by `whitelist`. Then at every call, applies + reduction function to each parameter, produces a scalar and logs it. + + Args: + model: model to log weights + reduction: function to reduce parameters into scalar + tag: common title for all produced plots. For example, "generator" + whitelist: specific weights to log. Should be list of model's submodules + or parameters names, or a callable which gets weight along with its name + and determines if it should be logged. Names should be fully-qualified. + For more information please refer to `PyTorch docs + `_. + If not given, all of model's weights are logged. + + Examples: + .. code-block:: python + + from ignite.handlers.tensorboard_logger import * + + # Create a logger + tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") + + # Attach the logger to the trainer to log model's weights norm after each iteration + tb_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsScalarHandler(model, reduction=torch.norm) + ) + + .. code-block:: python + + from ignite.handlers.tensorboard_logger import * + + tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") + + # Log only `fc` weights + tb_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsScalarHandler( + model, + whitelist=['fc'] + ) + ) + + .. code-block:: python + + from ignite.handlers.tensorboard_logger import * + + tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") + + # Log weights which have `bias` in their names + def has_bias_in_name(n, p): + return 'bias' in n + + tb_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsScalarHandler(model, whitelist=has_bias_in_name) + ) + + .. versionchanged:: 0.4.9 + optional argument `whitelist` added. + """ + + def __call__(self, engine: Engine, logger: TensorboardLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, TensorboardLogger): + raise RuntimeError("Handler 'WeightsScalarHandler' works only with TensorboardLogger") + + global_step = engine.state.get_event_attrib_value(event_name) + tag_prefix = f"{self.tag}/" if self.tag else "" + for name, p in self.weights: + name = name.replace(".", "/") + logger.writer.add_scalar( + f"{tag_prefix}weights_{self.reduction.__name__}/{name}", + self.reduction(p.data), + global_step, + ) + + +class WeightsHistHandler(BaseWeightsHandler): + """Helper handler to log model's weights as histograms. + + Args: + model: model to log weights + tag: common title for all produced plots. For example, "generator" + whitelist: specific weights to log. Should be list of model's submodules + or parameters names, or a callable which gets weight along with its name + and determines if it should be logged. Names should be fully-qualified. + For more information please refer to `PyTorch docs + `_. + If not given, all of model's weights are logged. + + Examples: + .. code-block:: python + + from ignite.handlers.tensorboard_logger import * + + # Create a logger + tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") + + # Attach the logger to the trainer to log model's weights norm after each iteration + tb_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsHistHandler(model) + ) + + .. code-block:: python + + from ignite.handlers.tensorboard_logger import * + + tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") + + # Log weights of `fc` layer + weights = ['fc'] + + # Attach the logger to the trainer to log weights norm after each iteration + tb_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsHistHandler(model, whitelist=weights) + ) + + .. code-block:: python + + from ignite.handlers.tensorboard_logger import * + + tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") + + # Log weights which name include 'conv'. + weight_selector = lambda name, p: 'conv' in name + + # Attach the logger to the trainer to log weights norm after each iteration + tb_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsHistHandler(model, whitelist=weight_selector) + ) + + .. versionchanged:: 0.4.9 + optional argument `whitelist` added. + """ + + def __call__(self, engine: Engine, logger: TensorboardLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, TensorboardLogger): + raise RuntimeError("Handler 'WeightsHistHandler' works only with TensorboardLogger") + + global_step = engine.state.get_event_attrib_value(event_name) + tag_prefix = f"{self.tag}/" if self.tag else "" + for name, p in self.weights: + name = name.replace(".", "/") + logger.writer.add_histogram( + tag=f"{tag_prefix}weights/{name}", values=p.data.cpu().numpy(), global_step=global_step + ) + + +class GradsScalarHandler(BaseWeightsScalarHandler): + """Helper handler to log model's gradients as scalars. + Handler, upon construction, iterates over named parameters of the model and keep + reference to ones permitted by the `whitelist`. Then at every call, applies + reduction function to each parameter's gradient, produces a scalar and logs it. + + Args: + model: model to log weights + reduction: function to reduce parameters into scalar + tag: common title for all produced plots. For example, "generator" + whitelist: specific gradients to log. Should be list of model's submodules + or parameters names, or a callable which gets weight along with its name + and determines if its gradient should be logged. Names should be + fully-qualified. For more information please refer to `PyTorch docs + `_. + If not given, all of model's gradients are logged. + + Examples: + .. code-block:: python + + from ignite.handlers.tensorboard_logger import * + + # Create a logger + tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") + + # Attach the logger to the trainer to log model's gradients norm after each iteration + tb_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=GradsScalarHandler(model, reduction=torch.norm) + ) + + .. code-block:: python + + from ignite.handlers.tensorboard_logger import * + + tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") + + # Log gradient of `base` + tb_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=GradsScalarHandler( + model, + reduction=torch.norm, + whitelist=['base'] + ) + ) + + .. code-block:: python + + from ignite.handlers.tensorboard_logger import * + + tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") + + # Log gradient of weights which belong to a `fc` layer + def is_in_fc_layer(n, p): + return 'fc' in n + + tb_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=GradsScalarHandler(model, whitelist=is_in_fc_layer) + ) + + .. versionchanged:: 0.4.9 + optional argument `whitelist` added. + """ + + def __call__(self, engine: Engine, logger: TensorboardLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, TensorboardLogger): + raise RuntimeError("Handler 'GradsScalarHandler' works only with TensorboardLogger") + + global_step = engine.state.get_event_attrib_value(event_name) + tag_prefix = f"{self.tag}/" if self.tag else "" + for name, p in self.weights: + if p.grad is None: + continue + + name = name.replace(".", "/") + logger.writer.add_scalar( + f"{tag_prefix}grads_{self.reduction.__name__}/{name}", self.reduction(p.grad), global_step + ) + + +class GradsHistHandler(BaseWeightsHandler): + """Helper handler to log model's gradients as histograms. + + Args: + model: model to log weights + tag: common title for all produced plots. For example, "generator" + whitelist: specific gradients to log. Should be list of model's submodules + or parameters names, or a callable which gets weight along with its name + and determines if its gradient should be logged. Names should be + fully-qualified. For more information please refer to `PyTorch docs + `_. + If not given, all of model's gradients are logged. + + Examples: + .. code-block:: python + + from ignite.handlers.tensorboard_logger import * + + # Create a logger + tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") + + # Attach the logger to the trainer to log model's weights norm after each iteration + tb_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=GradsHistHandler(model) + ) + + .. code-block:: python + + from ignite.handlers.tensorboard_logger import * + + tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") + + # Log gradient of `fc.bias` + tb_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=GradsHistHandler(model, whitelist=['fc.bias']) + ) + + .. code-block:: python + + from ignite.handlers.tensorboard_logger import * + + tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") + + # Log gradient of weights which have shape (2, 1) + def has_shape_2_1(n, p): + return p.shape == (2,1) + + tb_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=GradsHistHandler(model, whitelist=has_shape_2_1) + ) + + .. versionchanged:: 0.4.9 + optional argument `whitelist` added. + """ + + def __call__(self, engine: Engine, logger: TensorboardLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, TensorboardLogger): + raise RuntimeError("Handler 'GradsHistHandler' works only with TensorboardLogger") + + global_step = engine.state.get_event_attrib_value(event_name) + tag_prefix = f"{self.tag}/" if self.tag else "" + for name, p in self.weights: + if p.grad is None: + continue + + name = name.replace(".", "/") + logger.writer.add_histogram( + tag=f"{tag_prefix}grads/{name}", values=p.grad.cpu().numpy(), global_step=global_step + ) diff --git a/ignite/handlers/tqdm_logger.py b/ignite/handlers/tqdm_logger.py new file mode 100644 index 00000000000..98146f3fb43 --- /dev/null +++ b/ignite/handlers/tqdm_logger.py @@ -0,0 +1,311 @@ +# -*- coding: utf-8 -*- +"""TQDM logger.""" +from collections import OrderedDict +from typing import Any, Callable, List, Optional, Union + +from ignite.engine import Engine, Events +from ignite.engine.events import CallableEventWithFilter, RemovableEventHandle + +from ignite.handlers.base_logger import BaseLogger, BaseOutputHandler + + +class ProgressBar(BaseLogger): + """ + TQDM progress bar handler to log training progress and computed metrics. + + Args: + persist: set to ``True`` to persist the progress bar after completion (default = ``False``) + bar_format : Specify a custom bar string formatting. May impact performance. + [default: '{desc}[{n_fmt}/{total_fmt}] {percentage:3.0f}%|{bar}{postfix} [{elapsed}<{remaining}]']. + Set to ``None`` to use ``tqdm`` default bar formatting: '{l_bar}{bar}{r_bar}', where + l_bar='{desc}: {percentage:3.0f}%|' and + r_bar='| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}{postfix}]'. For more details on the + formatting, see `tqdm docs `_. + tqdm_kwargs: kwargs passed to tqdm progress bar. + By default, progress bar description displays "Epoch [5/10]" where 5 is the current epoch and 10 is the + number of epochs; however, if ``max_epochs`` are set to 1, the progress bar instead displays + "Iteration: [5/10]". If tqdm_kwargs defines `desc`, e.g. "Predictions", than the description is + "Predictions [5/10]" if number of epochs is more than one otherwise it is simply "Predictions". + + Examples: + Simple progress bar + + .. code-block:: python + + trainer = create_supervised_trainer(model, optimizer, loss) + + pbar = ProgressBar() + pbar.attach(trainer) + + # Progress bar will looks like + # Epoch [2/50]: [64/128] 50%|█████ [06:17<12:34] + + Log output to a file instead of stderr (tqdm's default output) + + .. code-block:: python + + trainer = create_supervised_trainer(model, optimizer, loss) + + log_file = open("output.log", "w") + pbar = ProgressBar(file=log_file) + pbar.attach(trainer) + + Attach metrics that already have been computed at :attr:`~ignite.engine.events.Events.ITERATION_COMPLETED` + (such as :class:`~ignite.metrics.RunningAverage`) + + .. code-block:: python + + trainer = create_supervised_trainer(model, optimizer, loss) + + RunningAverage(output_transform=lambda x: x).attach(trainer, 'loss') + + pbar = ProgressBar() + pbar.attach(trainer, ['loss']) + + # Progress bar will looks like + # Epoch [2/50]: [64/128] 50%|█████ , loss=0.123 [06:17<12:34] + + Directly attach the engine's output + + .. code-block:: python + + trainer = create_supervised_trainer(model, optimizer, loss) + + pbar = ProgressBar() + pbar.attach(trainer, output_transform=lambda x: {'loss': x}) + + # Progress bar will looks like + # Epoch [2/50]: [64/128] 50%|█████ , loss=0.123 [06:17<12:34] + + + Example where the State Attributes ``trainer.state.alpha`` and ``trainer.state.beta`` + are also logged along with the NLL and Accuracy after each iteration: + + .. code-block:: python + + pbar.attach( + trainer, + metric_names=["nll", "accuracy"], + state_attributes=["alpha", "beta"], + ) + + + Note: + When attaching the progress bar to an engine, it is recommended that you replace + every print operation in the engine's handlers triggered every iteration with + ``pbar.log_message`` to guarantee the correct format of the stdout. + + Note: + When using inside jupyter notebook, `ProgressBar` automatically uses `tqdm_notebook`. For correct rendering, + please install `ipywidgets `_. + Due to `tqdm notebook bugs `_, bar format may be needed to be set + to an empty string value. + + .. versionchanged:: 0.4.7 + `attach` now accepts an optional list of `state_attributes` + + """ + + _events_order: List[Union[Events, CallableEventWithFilter]] = [ + Events.STARTED, + Events.EPOCH_STARTED, + Events.ITERATION_STARTED, + Events.ITERATION_COMPLETED, + Events.EPOCH_COMPLETED, + Events.COMPLETED, + ] + + def __init__( + self, + persist: bool = False, + bar_format: Union[ + str, None + ] = "{desc}[{n_fmt}/{total_fmt}] {percentage:3.0f}%|{bar}{postfix} [{elapsed}<{remaining}]", + **tqdm_kwargs: Any, + ): + try: + from tqdm.autonotebook import tqdm + except ImportError: + raise ModuleNotFoundError( + "This contrib module requires tqdm to be installed. " + "Please install it with command: \n pip install tqdm" + ) + + self.pbar_cls = tqdm + self.pbar = None + self.persist = persist + self.bar_format = bar_format + self.tqdm_kwargs = tqdm_kwargs + + def _reset(self, pbar_total: Optional[int]) -> None: + self.pbar = self.pbar_cls( + total=pbar_total, leave=self.persist, bar_format=self.bar_format, initial=1, **self.tqdm_kwargs + ) + + def _close(self, engine: Engine) -> None: + if self.pbar is not None: + # https://github.com/tqdm/notebook.py#L240-L250 + # issue #1115 : notebook backend of tqdm checks if n < total (error or KeyboardInterrupt) + # and the bar persists in 'danger' mode + if self.pbar.total is not None: + self.pbar.n = self.pbar.total + self.pbar.close() + self.pbar = None + + @staticmethod + def _compare_lt( + event1: Union[Events, CallableEventWithFilter], event2: Union[Events, CallableEventWithFilter] + ) -> bool: + i1 = ProgressBar._events_order.index(event1) + i2 = ProgressBar._events_order.index(event2) + return i1 < i2 + + def log_message(self, message: str) -> None: + """ + Logs a message, preserving the progress bar correct output format. + + Args: + message: string you wish to log. + """ + from tqdm import tqdm + + tqdm.write(message, file=self.tqdm_kwargs.get("file", None)) + + def attach( # type: ignore[override] + self, + engine: Engine, + metric_names: Optional[Union[str, List[str]]] = None, + output_transform: Optional[Callable] = None, + event_name: Union[Events, CallableEventWithFilter] = Events.ITERATION_COMPLETED, + closing_event_name: Union[Events, CallableEventWithFilter] = Events.EPOCH_COMPLETED, + state_attributes: Optional[List[str]] = None, + ) -> None: + """ + Attaches the progress bar to an engine object. + + Args: + engine: engine object. + metric_names: list of metric names to plot or a string "all" to plot all available + metrics. + output_transform: a function to select what you want to print from the engine's + output. This function may return either a dictionary with entries in the format of ``{name: value}``, + or a single scalar, which will be displayed with the default name `output`. + event_name: event's name on which the progress bar advances. Valid events are from + :class:`~ignite.engine.events.Events`. + closing_event_name: event's name on which the progress bar is closed. Valid events are from + :class:`~ignite.engine.events.Events`. + state_attributes: list of attributes of the ``trainer.state`` to plot. + + Note: + Accepted output value types are numbers, 0d and 1d torch tensors and strings. + + """ + desc = self.tqdm_kwargs.get("desc", None) + + if event_name not in engine._allowed_events: + raise ValueError(f"Logging event {event_name.name} is not in allowed events for this engine") + + if isinstance(closing_event_name, CallableEventWithFilter): + if closing_event_name.filter is not None: + raise ValueError("Closing Event should not be a filtered event") + + if not self._compare_lt(event_name, closing_event_name): + raise ValueError(f"Logging event {event_name} should be called before closing event {closing_event_name}") + + log_handler = _OutputHandler( + desc, + metric_names, + output_transform, + closing_event_name=closing_event_name, + state_attributes=state_attributes, + ) + + super(ProgressBar, self).attach(engine, log_handler, event_name) + engine.add_event_handler(closing_event_name, self._close) + + def attach_opt_params_handler( # type: ignore[empty-body] + self, engine: Engine, event_name: Union[str, Events], *args: Any, **kwargs: Any + ) -> RemovableEventHandle: + """Intentionally empty""" + pass + + def _create_output_handler(self, *args: Any, **kwargs: Any) -> "_OutputHandler": + return _OutputHandler(*args, **kwargs) + + def _create_opt_params_handler(self, *args: Any, **kwargs: Any) -> Callable: # type: ignore[empty-body] + """Intentionally empty""" + pass + + +class _OutputHandler(BaseOutputHandler): + """Helper handler to log engine's output and/or metrics + + pbar = ProgressBar() + Args: + description: progress bar description. + metric_names: list of metric names to plot or a string "all" to plot all available + metrics. + output_transform: output transform function to prepare `engine.state.output` as a number. + For example, `output_transform = lambda output: output` + This function can also return a dictionary, e.g `{'loss': loss1, 'another_loss': loss2}` to label the plot + with corresponding keys. + closing_event_name: event's name on which the progress bar is closed. Valid events are from + :class:`~ignite.engine.events.Events` or any `event_name` added by + :meth:`~ignite.engine.engine.Engine.register_events`. + state_attributes: list of attributes of the ``trainer.state`` to plot. + + """ + + def __init__( + self, + description: str, + metric_names: Optional[Union[str, List[str]]] = None, + output_transform: Optional[Callable] = None, + closing_event_name: Union[Events, CallableEventWithFilter] = Events.EPOCH_COMPLETED, + state_attributes: Optional[List[str]] = None, + ): + if metric_names is None and output_transform is None: + # This helps to avoid 'Either metric_names or output_transform should be defined' of BaseOutputHandler + metric_names = [] + super(_OutputHandler, self).__init__( + description, metric_names, output_transform, global_step_transform=None, state_attributes=state_attributes + ) + self.closing_event_name = closing_event_name + + @staticmethod + def get_max_number_events(event_name: Union[str, Events, CallableEventWithFilter], engine: Engine) -> Optional[int]: + if event_name in (Events.ITERATION_STARTED, Events.ITERATION_COMPLETED): + return engine.state.epoch_length + if event_name in (Events.EPOCH_STARTED, Events.EPOCH_COMPLETED): + return engine.state.max_epochs + return 1 + + def __call__(self, engine: Engine, logger: ProgressBar, event_name: Union[str, Events]) -> None: + pbar_total = self.get_max_number_events(event_name, engine) + if logger.pbar is None: + logger._reset(pbar_total=pbar_total) + + max_epochs = engine.state.max_epochs + default_desc = "Iteration" if max_epochs == 1 else "Epoch" + + desc = self.tag or default_desc + max_num_of_closing_events = self.get_max_number_events(self.closing_event_name, engine) + if max_num_of_closing_events and max_num_of_closing_events > 1: + global_step = engine.state.get_event_attrib_value(self.closing_event_name) + desc += f" [{global_step}/{max_num_of_closing_events}]" + logger.pbar.set_description(desc) # type: ignore[attr-defined] + + rendered_metrics = self._setup_output_metrics_state_attrs(engine, log_text=True) + metrics = OrderedDict() + for key, value in rendered_metrics.items(): + key = "_".join(key[1:]) # tqdm has tag as description + + metrics[key] = value + + if metrics: + logger.pbar.set_postfix(metrics) # type: ignore[attr-defined] + + global_step = engine.state.get_event_attrib_value(event_name) + if pbar_total is not None: + global_step = (global_step - 1) % pbar_total + 1 + logger.pbar.update(global_step - logger.pbar.n) # type: ignore[attr-defined] diff --git a/ignite/handlers/utils.py b/ignite/handlers/utils.py new file mode 100644 index 00000000000..35f3e8c8d33 --- /dev/null +++ b/ignite/handlers/utils.py @@ -0,0 +1,24 @@ +from typing import Any, Callable, Optional + +from ignite.engine import Engine +from ignite.engine.events import Events + + +def global_step_from_engine(engine: Engine, custom_event_name: Optional[Events] = None) -> Callable: + """Helper method to setup `global_step_transform` function using another engine. + This can be helpful for logging trainer epoch/iteration while output handler is attached to an evaluator. + + Args: + engine: engine which state is used to provide the global step + custom_event_name: registered event name. Optional argument, event name to use. + + Returns: + global step based on provided engine + """ + + def wrapper(_: Any, event_name: Events) -> int: + if custom_event_name is not None: + event_name = custom_event_name + return engine.state.get_event_attrib_value(event_name) + + return wrapper diff --git a/ignite/handlers/visdom_logger.py b/ignite/handlers/visdom_logger.py new file mode 100644 index 00000000000..315191c3984 --- /dev/null +++ b/ignite/handlers/visdom_logger.py @@ -0,0 +1,557 @@ +"""Visdom logger and its helper handlers.""" + +import os +from typing import Any, Callable, cast, Dict, List, Optional, Union + +import torch +import torch.nn as nn +from torch.optim import Optimizer + +from ignite.engine import Engine, Events +from ignite.handlers.base_logger import ( + BaseLogger, + BaseOptimizerParamsHandler, + BaseOutputHandler, + BaseWeightsScalarHandler, +) + +from ignite.handlers.utils import global_step_from_engine # noqa + +__all__ = [ + "VisdomLogger", + "OptimizerParamsHandler", + "OutputHandler", + "WeightsScalarHandler", + "GradsScalarHandler", + "global_step_from_engine", +] + + +class VisdomLogger(BaseLogger): + """ + VisdomLogger handler to log metrics, model/optimizer parameters, gradients during the training and validation. + + This class requires `visdom `_ package to be installed: + + .. code-block:: bash + + + pip install git+https://github.com/fossasia/visdom.git + + Args: + server: visdom server URL. It can be also specified by environment variable `VISDOM_SERVER_URL` + port: visdom server's port. It can be also specified by environment variable `VISDOM_PORT` + num_workers: number of workers to use in `concurrent.futures.ThreadPoolExecutor` to post data to + visdom server. Default, `num_workers=1`. If `num_workers=0` and logger uses the main thread. If using + Python 2.7 and `num_workers>0` the package `futures` should be installed: `pip install futures` + kwargs: kwargs to pass into + `visdom.Visdom `_. + + Note: + We can also specify username/password using environment variables: VISDOM_USERNAME, VISDOM_PASSWORD + + + .. warning:: + + Frequent logging, e.g. when logger is attached to `Events.ITERATION_COMPLETED`, can slow down the run if the + main thread is used to send the data to visdom server (`num_workers=0`). To avoid this situation we can either + log less frequently or set `num_workers=1`. + + Examples: + .. code-block:: python + + from ignite.handlers.visdom_logger import * + + # Create a logger + vd_logger = VisdomLogger() + + # Attach the logger to the trainer to log training loss at each iteration + vd_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + output_transform=lambda loss: {"loss": loss} + ) + + # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch + # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch + # of the `trainer` instead of `train_evaluator`. + vd_logger.attach_output_handler( + train_evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="training", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer), + ) + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the + # `trainer` instead of `evaluator`. + vd_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer)), + ) + + # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration + vd_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer, + param_name='lr' # optional + ) + + # Attach the logger to the trainer to log model's weights norm after each iteration + vd_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsScalarHandler(model) + ) + + # Attach the logger to the trainer to log model's gradients norm after each iteration + vd_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=GradsScalarHandler(model) + ) + + # We need to close the logger with we are done + vd_logger.close() + + It is also possible to use the logger as context manager: + + .. code-block:: python + + from ignite.handlers.visdom_logger import * + + with VisdomLogger() as vd_logger: + + trainer = Engine(update_fn) + # Attach the logger to the trainer to log training loss at each iteration + vd_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + output_transform=lambda loss: {"loss": loss} + ) + + .. versionchanged:: 0.4.7 + accepts an optional list of `state_attributes` + """ + + def __init__( + self, + server: Optional[str] = None, + port: Optional[int] = None, + num_workers: int = 1, + raise_exceptions: bool = True, + **kwargs: Any, + ): + try: + import visdom + except ImportError: + raise ModuleNotFoundError( + "This contrib module requires visdom package. " + "Please install it with command:\n" + "pip install git+https://github.com/fossasia/visdom.git" + ) + + if num_workers > 0: + # If visdom is installed, one of its dependencies `tornado` + # requires also `futures` to be installed. + # Let's check anyway if we can import it. + try: + from concurrent.futures import ThreadPoolExecutor + except ImportError: + raise ModuleNotFoundError( + "This contrib module requires concurrent.futures module" + "Please install it with command:\n" + "pip install futures" + ) + + if server is None: + server = cast(str, os.environ.get("VISDOM_SERVER_URL", "localhost")) + + if port is None: + port = int(os.environ.get("VISDOM_PORT", 8097)) + + if "username" not in kwargs: + username = os.environ.get("VISDOM_USERNAME", None) + kwargs["username"] = username + + if "password" not in kwargs: + password = os.environ.get("VISDOM_PASSWORD", None) + kwargs["password"] = password + + self.vis = visdom.Visdom(server=server, port=port, raise_exceptions=raise_exceptions, **kwargs) + + if not self.vis.offline and not self.vis.check_connection(): # type: ignore[attr-defined] + raise RuntimeError(f"Failed to connect to Visdom server at {server}. Did you run python -m visdom.server ?") + + self.executor: Union[_DummyExecutor, "ThreadPoolExecutor"] = _DummyExecutor() + if num_workers > 0: + self.executor = ThreadPoolExecutor(max_workers=num_workers) + + def _save(self) -> None: + self.vis.save([self.vis.env]) # type: ignore[attr-defined] + + def close(self) -> None: + self.executor.shutdown() + self.vis.close() + + def _create_output_handler(self, *args: Any, **kwargs: Any) -> "OutputHandler": + return OutputHandler(*args, **kwargs) + + def _create_opt_params_handler(self, *args: Any, **kwargs: Any) -> "OptimizerParamsHandler": + return OptimizerParamsHandler(*args, **kwargs) + + +class _BaseVisDrawer: + def __init__(self, show_legend: bool = False): + self.windows: Dict[str, Any] = {} + self.show_legend = show_legend + + def add_scalar( + self, logger: VisdomLogger, k: str, v: Union[str, float, torch.Tensor], event_name: Any, global_step: int + ) -> None: + """ + Helper method to log a scalar with VisdomLogger. + + Args: + logger: visdom logger + k: scalar name which is used to set window title and y-axis label + v: scalar value, y-axis value + event_name: Event name which is used to setup x-axis label. Valid events are from + :class:`~ignite.engine.events.Events` or any `event_name` added by + :meth:`~ignite.engine.engine.Engine.register_events`. + global_step: global step, x-axis value + + """ + if k not in self.windows: + self.windows[k] = { + "win": None, + "opts": {"title": k, "xlabel": str(event_name), "ylabel": k, "showlegend": self.show_legend}, + } + + update = None if self.windows[k]["win"] is None else "append" + + kwargs = { + "X": [global_step], + "Y": [v], + "env": logger.vis.env, # type: ignore[attr-defined] + "win": self.windows[k]["win"], + "update": update, + "opts": self.windows[k]["opts"], + "name": k, + } + + future = logger.executor.submit(logger.vis.line, **kwargs) + if self.windows[k]["win"] is None: + self.windows[k]["win"] = future.result() + + +class OutputHandler(BaseOutputHandler, _BaseVisDrawer): + """Helper handler to log engine's output and/or metrics + + Args: + tag: common title for all produced plots. For example, "training" + metric_names: list of metric names to plot or a string "all" to plot all available + metrics. + output_transform: output transform function to prepare `engine.state.output` as a number. + For example, `output_transform = lambda output: output` + This function can also return a dictionary, e.g `{"loss": loss1, "another_loss": loss2}` to label the plot + with corresponding keys. + global_step_transform: global step transform function to output a desired global step. + Input of the function is `(engine, event_name)`. Output of function should be an integer. + Default is None, global_step based on attached engine. If provided, + uses function output as global_step. To setup global step from another engine, please use + :meth:`~ignite.handlers.visdom_logger.global_step_from_engine`. + show_legend: flag to show legend in the window + state_attributes: list of attributes of the ``trainer.state`` to plot. + + Examples: + .. code-block:: python + + from ignite.handlers.visdom_logger import * + + # Create a logger + vd_logger = VisdomLogger() + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch + # of the `trainer`: + vd_logger.attach( + evaluator, + log_handler=OutputHandler( + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer) + ), + event_name=Events.EPOCH_COMPLETED + ) + # or equivalently + vd_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer) + ) + + Another example, where model is evaluated every 500 iterations: + + .. code-block:: python + + from ignite.handlers.visdom_logger import * + + @trainer.on(Events.ITERATION_COMPLETED(every=500)) + def evaluate(engine): + evaluator.run(validation_set, max_epochs=1) + + vd_logger = VisdomLogger() + + def global_step_transform(*args, **kwargs): + return trainer.state.iteration + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # every 500 iterations. Since evaluator engine does not have access to the training iteration, we + # provide a global_step_transform to return the trainer.state.iteration for the global_step, each time + # evaluator metrics are plotted on Visdom. + + vd_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metrics=["nll", "accuracy"], + global_step_transform=global_step_transform + ) + + Another example where the State Attributes ``trainer.state.alpha`` and ``trainer.state.beta`` + are also logged along with the NLL and Accuracy after each iteration: + + .. code-block:: python + + vd_logger.attach( + trainer, + log_handler=OutputHandler( + tag="training", + metric_names=["nll", "accuracy"], + state_attributes=["alpha", "beta"], + ), + event_name=Events.ITERATION_COMPLETED + ) + + Example of `global_step_transform`: + + .. code-block:: python + + def global_step_transform(engine, event_name): + return engine.state.get_event_attrib_value(event_name) + """ + + def __init__( + self, + tag: str, + metric_names: Optional[str] = None, + output_transform: Optional[Callable] = None, + global_step_transform: Optional[Callable[[Engine, Union[str, Events]], int]] = None, + show_legend: bool = False, + state_attributes: Optional[List[str]] = None, + ): + super(OutputHandler, self).__init__( + tag, metric_names, output_transform, global_step_transform, state_attributes + ) + _BaseVisDrawer.__init__(self, show_legend=show_legend) + + def __call__(self, engine: Engine, logger: VisdomLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, VisdomLogger): + raise RuntimeError("Handler 'OutputHandler' works only with VisdomLogger") + + metrics = self._setup_output_metrics_state_attrs(engine, key_tuple=False) + + global_step = self.global_step_transform(engine, event_name) + + if not isinstance(global_step, int): + raise TypeError( + f"global_step must be int, got {type(global_step)}." + " Please check the output of global_step_transform." + ) + + for key, value in metrics.items(): + self.add_scalar(logger, key, value, event_name, global_step) + + logger._save() + + +class OptimizerParamsHandler(BaseOptimizerParamsHandler, _BaseVisDrawer): + """Helper handler to log optimizer parameters + + Args: + optimizer: torch optimizer or any object with attribute ``param_groups`` + as a sequence. + param_name: parameter name + tag: common title for all produced plots. For example, "generator" + show_legend: flag to show legend in the window + + Examples: + .. code-block:: python + + from ignite.handlers.visdom_logger import * + + # Create a logger + vb_logger = VisdomLogger() + + # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration + vd_logger.attach( + trainer, + log_handler=OptimizerParamsHandler(optimizer), + event_name=Events.ITERATION_STARTED + ) + # or equivalently + vd_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer + ) + """ + + def __init__( + self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None, show_legend: bool = False + ): + super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) + _BaseVisDrawer.__init__(self, show_legend=show_legend) + + def __call__(self, engine: Engine, logger: VisdomLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, VisdomLogger): + raise RuntimeError("Handler OptimizerParamsHandler works only with VisdomLogger") + + global_step = engine.state.get_event_attrib_value(event_name) + tag_prefix = f"{self.tag}/" if self.tag else "" + params = { + f"{tag_prefix}{self.param_name}/group_{i}": float(param_group[self.param_name]) + for i, param_group in enumerate(self.optimizer.param_groups) + } + + for k, v in params.items(): + self.add_scalar(logger, k, v, event_name, global_step) + + logger._save() + + +class WeightsScalarHandler(BaseWeightsScalarHandler, _BaseVisDrawer): + """Helper handler to log model's weights as scalars. + Handler iterates over named parameters of the model, applies reduction function to each parameter + produce a scalar and then logs the scalar. + + Args: + model: model to log weights + reduction: function to reduce parameters into scalar + tag: common title for all produced plots. For example, "generator" + show_legend: flag to show legend in the window + + Examples: + .. code-block:: python + + from ignite.handlers.visdom_logger import * + + # Create a logger + vd_logger = VisdomLogger() + + # Attach the logger to the trainer to log model's weights norm after each iteration + vd_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsScalarHandler(model, reduction=torch.norm) + ) + """ + + def __init__( + self, model: nn.Module, reduction: Callable = torch.norm, tag: Optional[str] = None, show_legend: bool = False + ): + super(WeightsScalarHandler, self).__init__(model, reduction, tag=tag) + _BaseVisDrawer.__init__(self, show_legend=show_legend) + + def __call__(self, engine: Engine, logger: VisdomLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, VisdomLogger): + raise RuntimeError("Handler 'WeightsScalarHandler' works only with VisdomLogger") + + global_step = engine.state.get_event_attrib_value(event_name) + tag_prefix = f"{self.tag}/" if self.tag else "" + for name, p in self.model.named_parameters(): + name = name.replace(".", "/") + k = f"{tag_prefix}weights_{self.reduction.__name__}/{name}" + v = self.reduction(p.data) + self.add_scalar(logger, k, v, event_name, global_step) + + logger._save() + + +class GradsScalarHandler(BaseWeightsScalarHandler, _BaseVisDrawer): + """Helper handler to log model's gradients as scalars. + Handler iterates over the gradients of named parameters of the model, applies reduction function to each parameter + produce a scalar and then logs the scalar. + + Args: + model: model to log weights + reduction: function to reduce parameters into scalar + tag: common title for all produced plots. For example, "generator" + show_legend: flag to show legend in the window + + Examples: + .. code-block:: python + + from ignite.handlers.visdom_logger import * + + # Create a logger + vd_logger = VisdomLogger() + + # Attach the logger to the trainer to log model's weights norm after each iteration + vd_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=GradsScalarHandler(model, reduction=torch.norm) + ) + """ + + def __init__( + self, model: nn.Module, reduction: Callable = torch.norm, tag: Optional[str] = None, show_legend: bool = False + ): + super(GradsScalarHandler, self).__init__(model, reduction, tag) + _BaseVisDrawer.__init__(self, show_legend=show_legend) + + def __call__(self, engine: Engine, logger: VisdomLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, VisdomLogger): + raise RuntimeError("Handler 'GradsScalarHandler' works only with VisdomLogger") + + global_step = engine.state.get_event_attrib_value(event_name) + tag_prefix = f"{self.tag}/" if self.tag else "" + for name, p in self.model.named_parameters(): + if p.grad is None: + continue + + name = name.replace(".", "/") + k = f"{tag_prefix}grads_{self.reduction.__name__}/{name}" + v = self.reduction(p.grad) + self.add_scalar(logger, k, v, event_name, global_step) + + logger._save() + + +class _DummyExecutor: + class _DummyFuture: + def __init__(self, result: Any) -> None: + self._output = result + + def result(self) -> Any: + return self._output + + def __init__(self, *args: Any, **kwargs: Any) -> None: + pass + + def submit(self, fn: Callable, **kwargs: Any) -> "_DummyFuture": + return _DummyExecutor._DummyFuture(fn(**kwargs)) + + def shutdown(self, *args: Any, **kwargs: Any) -> None: + pass diff --git a/ignite/handlers/wandb_logger.py b/ignite/handlers/wandb_logger.py new file mode 100644 index 00000000000..3f8e44840c7 --- /dev/null +++ b/ignite/handlers/wandb_logger.py @@ -0,0 +1,354 @@ +"""WandB logger and its helper handlers.""" + +from typing import Any, Callable, List, Optional, Union + +from torch.optim import Optimizer + +from ignite.engine import Engine, Events + +from ignite.handlers.base_logger import BaseLogger, BaseOptimizerParamsHandler, BaseOutputHandler +from ignite.handlers.utils import global_step_from_engine # noqa + +__all__ = ["WandBLogger", "OutputHandler", "OptimizerParamsHandler", "global_step_from_engine"] + + +class WandBLogger(BaseLogger): + """`Weights & Biases `_ handler to log metrics, model/optimizer parameters, gradients + during training and validation. It can also be used to log model checkpoints to the Weights & Biases cloud. + + .. code-block:: bash + + pip install wandb + + This class is also a wrapper for the wandb module. This means that you can call any wandb function using + this wrapper. See examples on how to save model parameters and gradients. + + Args: + args: Positional arguments accepted by `wandb.init`. + kwargs: Keyword arguments accepted by `wandb.init`. + Please see `wandb.init `_ for documentation of possible parameters. + + Examples: + .. code-block:: python + + from ignite.handlers.wandb_logger import * + + # Create a logger. All parameters are optional. See documentation + # on wandb.init for details. + + wandb_logger = WandBLogger( + entity="shared", + project="pytorch-ignite-integration", + name="cnn-mnist", + config={"max_epochs": 10}, + tags=["pytorch-ignite", "minst"] + ) + + # Attach the logger to the trainer to log training loss at each iteration + wandb_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + output_transform=lambda loss: {"loss": loss} + ) + + # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch + # We setup `global_step_transform=lambda *_: trainer.state.iteration` to take iteration value + # of the `trainer`: + wandb_logger.attach_output_handler( + train_evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="training", + metric_names=["nll", "accuracy"], + global_step_transform=lambda *_: trainer.state.iteration, + ) + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # each epoch. We setup `global_step_transform=lambda *_: trainer.state.iteration` to take iteration value + # of the `trainer` instead of `evaluator`. + wandb_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=lambda *_: trainer.state.iteration, + ) + + # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration + wandb_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer, + param_name='lr' # optional + ) + + # We need to close the logger when we are done + wandb_logger.close() + + If you want to log model gradients, the model call graph, etc., use the logger as wrapper of wandb. Refer + to the documentation of wandb.watch for details: + + .. code-block:: python + + wandb_logger = WandBLogger( + entity="shared", + project="pytorch-ignite-integration", + name="cnn-mnist", + config={"max_epochs": 10}, + tags=["pytorch-ignite", "minst"] + ) + + model = torch.nn.Sequential(...) + wandb_logger.watch(model) + + For model checkpointing, Weights & Biases creates a local run dir, and automatically synchronizes all + files saved there at the end of the run. You can just use the `wandb_logger.run.dir` as path for the + `ModelCheckpoint`: + + .. code-block:: python + + from ignite.handlers import ModelCheckpoint + + def score_function(engine): + return engine.state.metrics['accuracy'] + + model_checkpoint = ModelCheckpoint( + wandb_logger.run.dir, n_saved=2, filename_prefix='best', + require_empty=False, score_function=score_function, + score_name="validation_accuracy", + global_step_transform=global_step_from_engine(trainer) + ) + evaluator.add_event_handler(Events.COMPLETED, model_checkpoint, {'model': model}) + + + """ + + def __init__(self, *args: Any, **kwargs: Any): + try: + import wandb + + self._wandb = wandb + except ImportError: + raise ModuleNotFoundError( + "This contrib module requires wandb to be installed. " + "You man install wandb with the command:\n pip install wandb\n" + ) + if kwargs.get("init", True): + wandb.init(*args, **kwargs) + + def __getattr__(self, attr: Any) -> Any: + return getattr(self._wandb, attr) + + def close(self) -> None: + self._wandb.finish() + + def _create_output_handler(self, *args: Any, **kwargs: Any) -> "OutputHandler": + return OutputHandler(*args, **kwargs) + + def _create_opt_params_handler(self, *args: Any, **kwargs: Any) -> "OptimizerParamsHandler": + return OptimizerParamsHandler(*args, **kwargs) + + +class OutputHandler(BaseOutputHandler): + """Helper handler to log engine's output and/or metrics + + Args: + tag: common title for all produced plots. For example, "training" + metric_names: list of metric names to plot or a string "all" to plot all available + metrics. + output_transform: output transform function to prepare `engine.state.output` as a number. + For example, `output_transform = lambda output: output` + This function can also return a dictionary, e.g `{"loss": loss1, "another_loss": loss2}` to label the plot + with corresponding keys. + global_step_transform: global step transform function to output a desired global step. + Input of the function is `(engine, event_name)`. Output of function should be an integer. + Default is None, global_step based on attached engine. If provided, + uses function output as global_step. To setup global step from another engine, please use + :meth:`~ignite.handlers.wandb_logger.global_step_from_engine`. + sync: If set to False, process calls to log in a seperate thread. Default (None) uses whatever + the default value of wandb.log. + + Examples: + .. code-block:: python + + from ignite.handlers.wandb_logger import * + + # Create a logger. All parameters are optional. See documentation + # on wandb.init for details. + + wandb_logger = WandBLogger( + entity="shared", + project="pytorch-ignite-integration", + name="cnn-mnist", + config={"max_epochs": 10}, + tags=["pytorch-ignite", "minst"] + ) + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # each epoch. We setup `global_step_transform=lambda *_: trainer.state.iteration,` to take iteration value + # of the `trainer`: + wandb_logger.attach( + evaluator, + log_handler=OutputHandler( + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=lambda *_: trainer.state.iteration, + ), + event_name=Events.EPOCH_COMPLETED + ) + # or equivalently + wandb_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=lambda *_: trainer.state.iteration, + ) + + Another example, where model is evaluated every 500 iterations: + + .. code-block:: python + + from ignite.handlers.wandb_logger import * + + @trainer.on(Events.ITERATION_COMPLETED(every=500)) + def evaluate(engine): + evaluator.run(validation_set, max_epochs=1) + + # Create a logger. All parameters are optional. See documentation + # on wandb.init for details. + + wandb_logger = WandBLogger( + entity="shared", + project="pytorch-ignite-integration", + name="cnn-mnist", + config={"max_epochs": 10}, + tags=["pytorch-ignite", "minst"] + ) + + def global_step_transform(*args, **kwargs): + return trainer.state.iteration + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # every 500 iterations. Since evaluator engine does not have access to the training iteration, we + # provide a global_step_transform to return the trainer.state.iteration for the global_step, each time + # evaluator metrics are plotted on Weights & Biases. + + wandb_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metrics=["nll", "accuracy"], + global_step_transform=global_step_transform + ) + + Another example where the State Attributes ``trainer.state.alpha`` and ``trainer.state.beta`` + are also logged along with the NLL and Accuracy after each iteration: + + .. code-block:: python + + wandb_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + metrics=["nll", "accuracy"], + state_attributes=["alpha", "beta"], + ) + + + Example of `global_step_transform`: + + .. code-block:: python + + def global_step_transform(engine, event_name): + return engine.state.get_event_attrib_value(event_name) + + .. versionchanged:: 0.4.7 + accepts an optional list of `state_attributes` + """ + + def __init__( + self, + tag: str, + metric_names: Optional[List[str]] = None, + output_transform: Optional[Callable] = None, + global_step_transform: Optional[Callable[[Engine, Union[str, Events]], int]] = None, + sync: Optional[bool] = None, + state_attributes: Optional[List[str]] = None, + ): + super().__init__(tag, metric_names, output_transform, global_step_transform, state_attributes) + self.sync = sync + + def __call__(self, engine: Engine, logger: WandBLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, WandBLogger): + raise RuntimeError(f"Handler '{self.__class__.__name__}' works only with WandBLogger.") + + global_step = self.global_step_transform(engine, event_name) + if not isinstance(global_step, int): + raise TypeError( + f"global_step must be int, got {type(global_step)}." + " Please check the output of global_step_transform." + ) + + metrics = self._setup_output_metrics_state_attrs(engine, log_text=True, key_tuple=False) + logger.log(metrics, step=global_step, sync=self.sync) + + +class OptimizerParamsHandler(BaseOptimizerParamsHandler): + """Helper handler to log optimizer parameters + + Args: + optimizer: torch optimizer or any object with attribute ``param_groups`` + as a sequence. + param_name: parameter name + tag: common title for all produced plots. For example, "generator" + sync: If set to False, process calls to log in a seperate thread. Default (None) uses whatever + the default value of wandb.log. + + Examples: + .. code-block:: python + + from ignite.handlers.wandb_logger import * + + # Create a logger. All parameters are optional. See documentation + # on wandb.init for details. + + wandb_logger = WandBLogger( + entity="shared", + project="pytorch-ignite-integration", + name="cnn-mnist", + config={"max_epochs": 10}, + tags=["pytorch-ignite", "minst"] + ) + + # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration + wandb_logger.attach( + trainer, + log_handler=OptimizerParamsHandler(optimizer), + event_name=Events.ITERATION_STARTED + ) + # or equivalently + wandb_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer + ) + """ + + def __init__( + self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None, sync: Optional[bool] = None + ): + super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) + self.sync = sync + + def __call__(self, engine: Engine, logger: WandBLogger, event_name: Union[str, Events]) -> None: + if not isinstance(logger, WandBLogger): + raise RuntimeError("Handler OptimizerParamsHandler works only with WandBLogger") + + global_step = engine.state.get_event_attrib_value(event_name) + tag_prefix = f"{self.tag}/" if self.tag else "" + params = { + f"{tag_prefix}{self.param_name}/group_{i}": float(param_group[self.param_name]) + for i, param_group in enumerate(self.optimizer.param_groups) + } + logger.log(params, step=global_step, sync=self.sync) diff --git a/tests/ignite/contrib/conftest.py b/tests/ignite/contrib/conftest.py index 9c9b15d8699..4b4081f8c69 100644 --- a/tests/ignite/contrib/conftest.py +++ b/tests/ignite/contrib/conftest.py @@ -1,79 +1,7 @@ -import random -from pathlib import Path - import pytest -@pytest.fixture -def no_site_packages(request): - import sys - - modules = {} - for k in sys.modules: - if request.param in k: - modules[k] = sys.modules[k] - for k in modules: - del sys.modules[k] - - prev_path = list(sys.path) - sys.path = [p for p in sys.path if "site-packages" not in p] - yield "no_site_packages" - sys.path = prev_path - for k in modules: - sys.modules[k] = modules[k] - - @pytest.fixture() def visdom_offline_logfile(dirname): log_file = dirname / "logs.visdom" yield log_file - - -vd_hostname = None -vd_port = None -vd_server_process = None - - -@pytest.fixture() -def visdom_server(): - # Start Visdom server once and stop it with visdom_server_stop - global vd_hostname, vd_port, vd_server_process - - if vd_server_process is None: - import subprocess - import time - - from visdom import Visdom - from visdom.server.build import download_scripts - - (Path.home() / ".visdom").mkdir(exist_ok=True) - download_scripts() - - vd_hostname = "localhost" - vd_port = random.randint(8089, 8887) - - try: - vis = Visdom(server=vd_hostname, port=vd_port, raise_exceptions=True) - except ConnectionError: - pass - - vd_server_process = subprocess.Popen( - ["python", "-m", "visdom.server", "--hostname", vd_hostname, "-port", str(vd_port)] - ) - time.sleep(5) - - vis = Visdom(server=vd_hostname, port=vd_port) - assert vis.check_connection() - vis.close() - - yield (vd_hostname, vd_port) - - -@pytest.fixture() -def visdom_server_stop(): - yield None - - import time - - vd_server_process.kill() - time.sleep(2) diff --git a/tests/ignite/contrib/engines/test_common.py b/tests/ignite/contrib/engines/test_common.py index 4749d5db108..97389523004 100644 --- a/tests/ignite/contrib/engines/test_common.py +++ b/tests/ignite/contrib/engines/test_common.py @@ -62,11 +62,11 @@ def _test_setup_common_training_handlers( if lr_scheduler is None: lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=gamma) elif isinstance(lr_scheduler, str) and lr_scheduler == "ignite|LRScheduler": - from ignite.contrib.handlers import LRScheduler + from ignite.handlers import LRScheduler lr_scheduler = LRScheduler(torch.optim.lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=gamma)) elif isinstance(lr_scheduler, str) and lr_scheduler == "ignite": - from ignite.contrib.handlers import PiecewiseLinear + from ignite.handlers import PiecewiseLinear milestones_values = [(0, 0.0), (step_size, lr), (num_iters * (num_epochs - 1), 0.0)] lr_scheduler = PiecewiseLinear(optimizer, param_name="lr", milestones_values=milestones_values) diff --git a/tests/ignite/contrib/handlers/__init__.py b/tests/ignite/contrib/handlers/__init__.py deleted file mode 100644 index f9cd4aebf6a..00000000000 --- a/tests/ignite/contrib/handlers/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -class MockFP16DeepSpeedZeroOptimizer: - def __init__(self, optimizer): - self.optimizer = optimizer - - def step(self, closure=None): - self.optimizer.step() - - def _get_param_groups(self): - return self.optimizer.param_groups - - def _set_param_groups(self, value): - self.optimizer.param_groups = value - - param_groups = property(_get_param_groups, _set_param_groups) diff --git a/tests/ignite/contrib/handlers/conftest.py b/tests/ignite/contrib/handlers/conftest.py deleted file mode 100644 index 578e7d50fc2..00000000000 --- a/tests/ignite/contrib/handlers/conftest.py +++ /dev/null @@ -1,48 +0,0 @@ -from unittest.mock import Mock - -import pytest -import torch - - -@pytest.fixture() -def norm_mock(): - def norm(x: torch.Tensor): - return x.norm() - - norm_mock = Mock(side_effect=norm, spec=norm) - norm_mock.configure_mock(**{"__name__": "norm"}) - norm_mock.reset_mock() - return norm_mock - - -@pytest.fixture() -def dummy_model_factory(): - class DummyModel(torch.nn.Module): - def __init__(self): - super(DummyModel, self).__init__() - self.fc1 = torch.nn.Linear(10, 10) - self.fc2 = torch.nn.Linear(12, 12) - self.fc1.weight.data.zero_() - self.fc1.bias.data.zero_() - self.fc2.weight.data.fill_(1.0) - self.fc2.bias.data.fill_(1.0) - - def get_dummy_model(with_grads=True, with_frozen_layer=False, with_buffer=False): - model = DummyModel() - if with_grads: - model.fc2.weight.grad = torch.zeros_like(model.fc2.weight) - model.fc2.bias.grad = torch.zeros_like(model.fc2.bias) - - if not with_frozen_layer: - model.fc1.weight.grad = torch.zeros_like(model.fc1.weight) - model.fc1.bias.grad = torch.zeros_like(model.fc1.bias) - - if with_frozen_layer: - for param in model.fc1.parameters(): - param.requires_grad = False - - if with_buffer: - model.register_buffer("buffer1", torch.ones(1)) - return model - - return get_dummy_model diff --git a/tests/ignite/contrib/handlers/test_warnings_of_deprecation.py b/tests/ignite/contrib/handlers/test_warnings_of_deprecation.py new file mode 100644 index 00000000000..356ee4ab1c2 --- /dev/null +++ b/tests/ignite/contrib/handlers/test_warnings_of_deprecation.py @@ -0,0 +1,98 @@ +from importlib import __import__ + +import pytest + + +@pytest.mark.parametrize( + "log_module,fromlist", + [ + ("mlflow_logger", ["MLflowLogger", "OptimizerParamsHandler", "OutputHandler"]), + ("polyaxon_logger", ["PolyaxonLogger", "OutputHandler", "OptimizerParamsHandler"]), + ("wandb_logger", ["WandBLogger", "OutputHandler", "OptimizerParamsHandler"]), + ("lr_finder", ["FastaiLRFinder"]), + ("tqdm_logger", ["ProgressBar"]), + ( + "clearml_logger", + [ + "ClearMLLogger", + "ClearMLSaver", + "OptimizerParamsHandler", + "OutputHandler", + "WeightsScalarHandler", + "WeightsHistHandler", + "GradsScalarHandler", + "GradsHistHandler", + ], + ), + ( + "tensorboard_logger", + [ + "TensorboardLogger", + "OptimizerParamsHandler", + "OutputHandler", + "WeightsScalarHandler", + "WeightsHistHandler", + "GradsScalarHandler", + "GradsHistHandler", + ], + ), + ( + "visdom_logger", + [ + "VisdomLogger", + "OptimizerParamsHandler", + "OutputHandler", + "WeightsScalarHandler", + "GradsScalarHandler", + ], + ), + ( + "neptune_logger", + [ + "NeptuneLogger", + "NeptuneSaver", + "OptimizerParamsHandler", + "OutputHandler", + "WeightsScalarHandler", + "GradsScalarHandler", + ], + ), + ( + "base_logger", + [ + "BaseHandler", + "BaseWeightsHandler", + "BaseOptimizerParamsHandler", + "BaseOutputHandler", + "BaseWeightsScalarHandler", + "BaseLogger", + ], + ), + ( + "time_profilers", + [ + "BasicTimeProfiler", + "HandlersTimeProfiler", + ], + ), + ( + "param_scheduler", + [ + "ConcatScheduler", + "CosineAnnealingScheduler", + "LinearCyclicalScheduler", + "LRScheduler", + "ParamGroupScheduler", + "ParamScheduler", + "PiecewiseLinear", + "CyclicalScheduler", + "create_lr_scheduler_with_warmup", + ], + ), + ], +) +def test_imports(log_module, fromlist): + with pytest.warns(DeprecationWarning, match="will be removed in version 0.6.0"): + imported = __import__(f"ignite.contrib.handlers.{log_module}", globals(), locals(), fromlist) + for attr in fromlist: + getattr(imported, attr) diff --git a/tests/ignite/handlers/__init__.py b/tests/ignite/handlers/__init__.py index c5318473e0e..6ceb673d57b 100644 --- a/tests/ignite/handlers/__init__.py +++ b/tests/ignite/handlers/__init__.py @@ -1 +1,15 @@ # Needed to collect coverage data +class MockFP16DeepSpeedZeroOptimizer: + def __init__(self, optimizer): + self.optimizer = optimizer + + def step(self, closure=None): + self.optimizer.step() + + def _get_param_groups(self): + return self.optimizer.param_groups + + def _set_param_groups(self, value): + self.optimizer.param_groups = value + + param_groups = property(_get_param_groups, _set_param_groups) diff --git a/tests/ignite/handlers/conftest.py b/tests/ignite/handlers/conftest.py index 1dcdafb32cb..9d7bb999463 100644 --- a/tests/ignite/handlers/conftest.py +++ b/tests/ignite/handlers/conftest.py @@ -1,6 +1,89 @@ +import random +from pathlib import Path +from unittest.mock import Mock + import pytest import torch +vd_hostname = None +vd_port = None +vd_server_process = None + + +@pytest.fixture() +def visdom_server(): + # Start Visdom server once and stop it with visdom_server_stop + global vd_hostname, vd_port, vd_server_process + + if vd_server_process is None: + import subprocess + import time + + from visdom import Visdom + from visdom.server.build import download_scripts + + (Path.home() / ".visdom").mkdir(exist_ok=True) + download_scripts() + + vd_hostname = "localhost" + vd_port = random.randint(8089, 8887) + + try: + vis = Visdom(server=vd_hostname, port=vd_port, raise_exceptions=True) + except ConnectionError: + pass + + vd_server_process = subprocess.Popen( + ["python", "-m", "visdom.server", "--hostname", vd_hostname, "-port", str(vd_port)] + ) + time.sleep(5) + + vis = Visdom(server=vd_hostname, port=vd_port) + assert vis.check_connection() + vis.close() + + yield (vd_hostname, vd_port) + + +@pytest.fixture() +def visdom_server_stop(): + yield None + + import time + + vd_server_process.kill() + time.sleep(2) + + +@pytest.fixture +def no_site_packages(request): + import sys + + modules = {} + for k in sys.modules: + if request.param in k: + modules[k] = sys.modules[k] + for k in modules: + del sys.modules[k] + + prev_path = list(sys.path) + sys.path = [p for p in sys.path if "site-packages" not in p] + yield "no_site_packages" + sys.path = prev_path + for k in modules: + sys.modules[k] = modules[k] + + +@pytest.fixture() +def norm_mock(): + def norm(x: torch.Tensor): + return x.norm() + + norm_mock = Mock(side_effect=norm, spec=norm) + norm_mock.configure_mock(**{"__name__": "norm"}) + norm_mock.reset_mock() + return norm_mock + @pytest.fixture() def dummy_model_factory(): @@ -14,7 +97,7 @@ def __init__(self): self.fc2.weight.data.fill_(1.0) self.fc2.bias.data.fill_(1.0) - def get_dummy_model(with_grads=True, with_frozen_layer=False): + def get_dummy_model(with_grads=True, with_frozen_layer=False, with_buffer=False): model = DummyModel() if with_grads: model.fc2.weight.grad = torch.zeros_like(model.fc2.weight) @@ -27,6 +110,9 @@ def get_dummy_model(with_grads=True, with_frozen_layer=False): if with_frozen_layer: for param in model.fc1.parameters(): param.requires_grad = False + + if with_buffer: + model.register_buffer("buffer1", torch.ones(1)) return model return get_dummy_model diff --git a/tests/ignite/contrib/handlers/test_base_logger.py b/tests/ignite/handlers/test_base_logger.py similarity index 98% rename from tests/ignite/contrib/handlers/test_base_logger.py rename to tests/ignite/handlers/test_base_logger.py index a8684f78120..a1e4960dcba 100644 --- a/tests/ignite/contrib/handlers/test_base_logger.py +++ b/tests/ignite/handlers/test_base_logger.py @@ -4,15 +4,16 @@ import pytest import torch -from ignite.contrib.handlers.base_logger import ( +from ignite.engine import Engine, Events, EventsList, State + +from ignite.handlers.base_logger import ( BaseLogger, BaseOptimizerParamsHandler, BaseOutputHandler, BaseWeightsHandler, BaseWeightsScalarHandler, ) -from ignite.engine import Engine, Events, EventsList, State -from tests.ignite.contrib.handlers import MockFP16DeepSpeedZeroOptimizer +from tests.ignite.handlers import MockFP16DeepSpeedZeroOptimizer class DummyOutputHandler(BaseOutputHandler): diff --git a/tests/ignite/contrib/handlers/test_clearml_logger.py b/tests/ignite/handlers/test_clearml_logger.py similarity index 99% rename from tests/ignite/contrib/handlers/test_clearml_logger.py rename to tests/ignite/handlers/test_clearml_logger.py index 9f29d2ba8eb..832362a852c 100644 --- a/tests/ignite/contrib/handlers/test_clearml_logger.py +++ b/tests/ignite/handlers/test_clearml_logger.py @@ -10,7 +10,10 @@ from clearml.model import Framework import ignite.distributed as idist -from ignite.contrib.handlers.clearml_logger import ( + +from ignite.engine import Engine, Events, State +from ignite.handlers import Checkpoint +from ignite.handlers.clearml_logger import ( ClearMLLogger, ClearMLSaver, global_step_from_engine, @@ -22,9 +25,6 @@ WeightsScalarHandler, ) -from ignite.engine import Engine, Events, State -from ignite.handlers import Checkpoint - def test_no_clearml(): with patch.dict("sys.modules", {"clearml": None, "trains": None}): diff --git a/tests/ignite/handlers/test_lr_finder.py b/tests/ignite/handlers/test_lr_finder.py index 61726a91930..e12d951dfbf 100644 --- a/tests/ignite/handlers/test_lr_finder.py +++ b/tests/ignite/handlers/test_lr_finder.py @@ -11,8 +11,8 @@ from torch.optim import SGD import ignite.distributed as idist -from ignite.contrib.handlers import FastaiLRFinder from ignite.engine import create_supervised_trainer, Engine, Events +from ignite.handlers import FastaiLRFinder matplotlib.use("agg") diff --git a/tests/ignite/contrib/handlers/test_mlflow_logger.py b/tests/ignite/handlers/test_mlflow_logger.py similarity index 98% rename from tests/ignite/contrib/handlers/test_mlflow_logger.py rename to tests/ignite/handlers/test_mlflow_logger.py index 04bed3e7b91..8930879de71 100644 --- a/tests/ignite/contrib/handlers/test_mlflow_logger.py +++ b/tests/ignite/handlers/test_mlflow_logger.py @@ -4,14 +4,10 @@ import pytest import torch -from ignite.contrib.handlers.mlflow_logger import ( - global_step_from_engine, - MLflowLogger, - OptimizerParamsHandler, - OutputHandler, -) from ignite.engine import Engine, Events, State +from ignite.handlers.mlflow_logger import global_step_from_engine, MLflowLogger, OptimizerParamsHandler, OutputHandler + def test_output_handler_with_wrong_logger_type(): wrapper = OutputHandler("tag", output_transform=lambda x: x) diff --git a/tests/ignite/contrib/handlers/test_neptune_logger.py b/tests/ignite/handlers/test_neptune_logger.py similarity index 99% rename from tests/ignite/contrib/handlers/test_neptune_logger.py rename to tests/ignite/handlers/test_neptune_logger.py index 84d91c75577..3dcda2376bb 100644 --- a/tests/ignite/contrib/handlers/test_neptune_logger.py +++ b/tests/ignite/handlers/test_neptune_logger.py @@ -5,7 +5,9 @@ import pytest import torch -from ignite.contrib.handlers.neptune_logger import ( +from ignite.engine import Engine, Events, State + +from ignite.handlers.neptune_logger import ( global_step_from_engine, GradsScalarHandler, NeptuneLogger, @@ -14,7 +16,6 @@ OutputHandler, WeightsScalarHandler, ) -from ignite.engine import Engine, Events, State def assert_logger_called_once_with(logger, key, value): @@ -523,7 +524,7 @@ def test_neptune_saver(model, serializable): def test_logs_version(): from ignite import __version__ - from ignite.contrib.handlers.neptune_logger import _INTEGRATION_VERSION_KEY + from ignite.handlers.neptune_logger import _INTEGRATION_VERSION_KEY logger = NeptuneLogger( project="tests/dry-run", diff --git a/tests/ignite/handlers/test_param_scheduler.py b/tests/ignite/handlers/test_param_scheduler.py index af5f5cae497..cb096d7fdf4 100644 --- a/tests/ignite/handlers/test_param_scheduler.py +++ b/tests/ignite/handlers/test_param_scheduler.py @@ -17,7 +17,7 @@ PiecewiseLinear, ReduceLROnPlateauScheduler, ) -from tests.ignite.contrib.handlers import MockFP16DeepSpeedZeroOptimizer +from tests.ignite.handlers import MockFP16DeepSpeedZeroOptimizer try: from torch.optim.lr_scheduler import MultiplicativeLR diff --git a/tests/ignite/contrib/handlers/test_polyaxon_logger.py b/tests/ignite/handlers/test_polyaxon_logger.py similarity index 99% rename from tests/ignite/contrib/handlers/test_polyaxon_logger.py rename to tests/ignite/handlers/test_polyaxon_logger.py index 1d025da036c..69f68e9a3bc 100644 --- a/tests/ignite/contrib/handlers/test_polyaxon_logger.py +++ b/tests/ignite/handlers/test_polyaxon_logger.py @@ -4,13 +4,14 @@ import pytest import torch -from ignite.contrib.handlers.polyaxon_logger import ( +from ignite.engine import Engine, Events, State + +from ignite.handlers.polyaxon_logger import ( global_step_from_engine, OptimizerParamsHandler, OutputHandler, PolyaxonLogger, ) -from ignite.engine import Engine, Events, State os.environ["POLYAXON_NO_OP"] = "1" diff --git a/tests/ignite/contrib/handlers/test_tensorboard_logger.py b/tests/ignite/handlers/test_tensorboard_logger.py similarity index 99% rename from tests/ignite/contrib/handlers/test_tensorboard_logger.py rename to tests/ignite/handlers/test_tensorboard_logger.py index 7effd41f046..7b92c345fe2 100644 --- a/tests/ignite/contrib/handlers/test_tensorboard_logger.py +++ b/tests/ignite/handlers/test_tensorboard_logger.py @@ -5,7 +5,9 @@ import pytest import torch -from ignite.contrib.handlers.tensorboard_logger import ( +from ignite.engine import Engine, Events, State + +from ignite.handlers.tensorboard_logger import ( global_step_from_engine, GradsHistHandler, GradsScalarHandler, @@ -15,7 +17,6 @@ WeightsHistHandler, WeightsScalarHandler, ) -from ignite.engine import Engine, Events, State def test_optimizer_params_handler_wrong_setup(): diff --git a/tests/ignite/contrib/handlers/test_tqdm_logger.py b/tests/ignite/handlers/test_tqdm_logger.py similarity index 99% rename from tests/ignite/contrib/handlers/test_tqdm_logger.py rename to tests/ignite/handlers/test_tqdm_logger.py index c7b5acbd2c2..0f9a501ebf8 100644 --- a/tests/ignite/contrib/handlers/test_tqdm_logger.py +++ b/tests/ignite/handlers/test_tqdm_logger.py @@ -9,9 +9,9 @@ import torch from packaging.version import Version -from ignite.contrib.handlers import ProgressBar from ignite.engine import Engine, Events -from ignite.handlers import TerminateOnNan + +from ignite.handlers import ProgressBar, TerminateOnNan from ignite.metrics import RunningAverage if sys.platform.startswith("win"): diff --git a/tests/ignite/contrib/handlers/test_visdom_logger.py b/tests/ignite/handlers/test_visdom_logger.py similarity index 99% rename from tests/ignite/contrib/handlers/test_visdom_logger.py rename to tests/ignite/handlers/test_visdom_logger.py index 39db6558bd4..40657d180cf 100644 --- a/tests/ignite/contrib/handlers/test_visdom_logger.py +++ b/tests/ignite/handlers/test_visdom_logger.py @@ -4,7 +4,9 @@ import pytest import torch -from ignite.contrib.handlers.visdom_logger import ( +from ignite.engine import Engine, Events, State + +from ignite.handlers.visdom_logger import ( _DummyExecutor, global_step_from_engine, GradsScalarHandler, @@ -13,7 +15,6 @@ VisdomLogger, WeightsScalarHandler, ) -from ignite.engine import Engine, Events, State def test_optimizer_params_handler_wrong_setup(): diff --git a/tests/ignite/contrib/handlers/test_wandb_logger.py b/tests/ignite/handlers/test_wandb_logger.py similarity index 98% rename from tests/ignite/contrib/handlers/test_wandb_logger.py rename to tests/ignite/handlers/test_wandb_logger.py index 82103556838..27742befa10 100644 --- a/tests/ignite/contrib/handlers/test_wandb_logger.py +++ b/tests/ignite/handlers/test_wandb_logger.py @@ -3,14 +3,10 @@ import pytest import torch -from ignite.contrib.handlers.wandb_logger import ( - global_step_from_engine, - OptimizerParamsHandler, - OutputHandler, - WandBLogger, -) from ignite.engine import Events, State +from ignite.handlers.wandb_logger import global_step_from_engine, OptimizerParamsHandler, OutputHandler, WandBLogger + def test_optimizer_params_handler_wrong_setup(): with pytest.raises(TypeError):