# Copyright 2021 MosaicML. All Rights Reserved.
"""Core AugMix classes and functions."""
import functools
import textwrap
import weakref
from typing import List, TypeVar
import numpy as np
import torch
from PIL import Image
from PIL.Image import Image as PillowImage
from torchvision.datasets import VisionDataset
from composer.algorithms.utils import augmentation_sets
from composer.algorithms.utils.augmentation_common import map_pillow_function
from composer.core import Algorithm, Event, State
from composer.datasets.utils import add_vision_dataset_transform
from composer.loggers import Logger
__all__ = ["AugMix", "AugmentAndMixTransform", "augmix_image"]
ImgT = TypeVar("ImgT", torch.Tensor, PillowImage)
[docs]def augmix_image(img: ImgT,
severity: int = 3,
depth: int = -1,
width: int = 3,
alpha: float = 1.0,
augmentation_set: List = augmentation_sets["all"]) -> ImgT:
"""Applies AugMix (`Hendrycks et al, 2020 <http://arxiv.org/abs/1912.02781>`_) data
augmentation to a single image or batch of images. See
:class:`.AugMix` and the
:doc:`Method Card </method_cards/augmix>` for details. This function only acts on a
single image (or batch) per call and is unlikely to be used in a training loop. Use
:class:`~composer.algorithms.augmix.augmix.AugmentAndMixTransform` to use AugMix as
part of a :class:`torchvision.datasets.VisionDataset`\\'s ``transform``.
Example:
.. testcode::
import composer.functional as cf
from composer.algorithms.utils import augmentation_sets
augmixed_image = cf.augmix_image(
img=image,
severity=3,
width=3,
depth=-1,
alpha=1.0,
augmentation_set=augmentation_sets["all"]
)
Args:
img (PIL.Image.Image or torch.Tensor): Image or batch of images to be AugMix'd.
severity (int, optional): See :class:`.AugMix`.
depth (int, optional): See :class:`.AugMix`.
width (int, optional): See :class:`.AugMix`.
alpha (float, optional): See :class:`.AugMix`.
augmentation_set (str, optional): See
:class:`.AugMix`.
Returns:
PIL.Image: AugMix'd image.
"""
def _augmix_pil_image(img_pil: PillowImage, severity: int, depth: int, width: int, alpha: float,
augmentation_set: List) -> PillowImage:
chain_weights = np.random.dirichlet([alpha] * width).astype(np.float32)
mixing_weight = np.float32(np.random.beta(alpha, alpha))
augmented_combination = np.zeros_like(img_pil, dtype=np.float32)
# Iterate over image chains
for chain_i in range(width):
augmented_image = img_pil.copy()
# Determine depth of current augmentation chain
if depth > 0:
d = depth
else:
d = np.random.randint(1, 4)
# Iterate through chain depth
for _ in range(d):
aug = np.random.choice(augmentation_set)
augmented_image = aug(augmented_image, severity)
augmented_combination += chain_weights[chain_i] * np.asarray(augmented_image)
mixed = (1 - mixing_weight) * np.asarray(img_pil) + mixing_weight * augmented_combination
mixed = Image.fromarray(np.uint8(mixed))
return mixed
f_pil = functools.partial(_augmix_pil_image,
severity=severity,
depth=depth,
width=width,
alpha=alpha,
augmentation_set=augmentation_set)
return map_pillow_function(f_pil, img)
[docs]class AugMix(Algorithm):
"""AugMix (`Hendrycks et al, 2020 <http://arxiv.org/abs/1912.02781>`_) creates ``width`` sequences of ``depth``
image augmentations, applies each sequence with random intensity, and returns a convex combination of the ``width``
augmented images and the original image. The coefficients for mixing the augmented images are drawn from a uniform
``Dirichlet(alpha, alpha, ...)`` distribution. The coefficient for mixing the combined augmented image and the
original image is drawn from a ``Beta(alpha, alpha)`` distribution, using the same ``alpha``.
This algorithm runs on on :attr:`~composer.core.event.Event.FIT_START` to insert a dataset transformation.
It is a no-op if this algorithm already applied itself on the :attr:`State.train_dataloader.dataset`.
See the :doc:`Method Card </method_cards/augmix>` for more details.
Example:
.. testcode::
from composer.algorithms import AugMix
from composer.trainer import Trainer
augmix_algorithm = AugMix(
severity=3,
width=3,
depth=-1,
alpha=1.0,
augmentation_set="all"
)
trainer = Trainer(
model=model,
train_dataloader=train_dataloader,
eval_dataloader=eval_dataloader,
max_duration="1ep",
algorithms=[augmix_algorithm],
optimizers=[optimizer]
)
Args:
severity (int, optional): Severity of augmentations; ranges from 0
(no augmentation) to 10 (most severe). Default: ``3``.
depth (int, optional): Number of augmentations per sequence. -1 enables stochastic
depth sampled uniformly from [1, 3]. Default: ``-1``.
width (int, optional): Number of augmentation sequences. Default: ``3``.
alpha (float, optional): Pseudocount for Beta and Dirichlet distributions. Must be
> 0. Higher values yield mixing coefficients closer to uniform weighting. As
the value approaches 0, the mixing coefficients approach using only one
version of each image. Default: ``1.0``.
augmentation_set (str, optional): Must be one of the following options:
* ``"augmentations_all"``
Uses all augmentations from the paper.
* ``"augmentations_corruption_safe"``
Like ``"augmentations_all"``, but excludes transforms that are part of
the ImageNet-C/CIFAR10-C test sets
* ``"augmentations_original"``
Like ``"augmentations_all"``, but some of the implementations
are identical to the original Github repository, which contains
implementation specificities for the augmentations
``"color"``, ``"contrast"``, ``"sharpness"``, and ``"brightness"``. The
original implementations have an intensity sampling scheme that samples a
value bounded by 0.118 at a minimum, and a maximum value of
:math:`intensity \\times 0.18 + .1`, which ranges from 0.28 (intensity = 1)
to 1.9 (intensity 10). These augmentations have different effects
depending on whether they are < 0 or > 0 (or < 1 or > 1).
"augmentations_all" uses implementations of "color", "contrast",
"sharpness", and "brightness" that account for diverging effects around 0
(or 1).
Default: ``"all"``.
"""
# TODO document each value of augmentation_set in more detail; i.e.,
# which augmentations are actually used
def __init__(self,
severity: int = 3,
depth: int = -1,
width: int = 3,
alpha: float = 1.0,
augmentation_set: str = "all"):
if severity < 0 or severity > 10:
raise ValueError("AugMix severity value must satisfy 0 โค severity โค 10")
if width < 1:
raise ValueError("AugMix width must be โฅ 1")
if augmentation_set not in augmentation_sets.keys():
raise KeyError(f"AugMix augmentation_set is not one of {augmentation_sets.keys()}")
self.severity = severity
self.depth = depth
self.width = width
self.alpha = alpha
self.augmentation_set = augmentation_set
self._transformed_datasets = weakref.WeakSet()
def match(self, event: Event, state: State) -> bool:
return event == Event.FIT_START and state.train_dataloader.dataset not in self._transformed_datasets
def apply(self, event: Event, state: State, logger: Logger) -> None:
am = AugmentAndMixTransform(severity=self.severity,
depth=self.depth,
width=self.width,
alpha=self.alpha,
augmentation_set=self.augmentation_set)
dataset = state.train_dataloader.dataset
if not isinstance(dataset, VisionDataset):
raise TypeError(
textwrap.dedent(f"""\
To use {type(self).__name__}, the dataset must be a
{VisionDataset.__qualname__}, not {type(dataset).__name__}"""))
add_vision_dataset_transform(dataset, am, is_tensor_transform=False)
self._transformed_datasets.add(dataset)