Optimizers

Gradient descent and the adaptive methods built on top of it.

Each optimizer is written to be read alongside the explanation that derives it, so the update rule appears in the code in the same form it appears in the mathematics.

dlhub.optimizers

Optimizers

Gradient descent and the adaptive methods built on top of it.

Each optimizer is written to be read alongside the explanation that derives it, so the update rule appears in the code in the same form it appears in the mathematics.

Author

Deep Learning Reference Hub

License

MIT

AdamOptimizer

Adam (Adaptive Moment Estimation) Optimizer

Adam combines the advantages of AdaGrad and RMSProp by computing adaptive learning rates for each parameter using estimates of first and second moments of the gradients.

The algorithm maintains exponentially decaying averages of past gradients and past squared gradients, which act as estimates of the first moment (mean) and second moment (uncentered variance) of the gradients.

Parameters:

Name	Type	Description	Default
learning_rate	float	Learning rate (alpha in the paper)	0.001
beta1	float	Exponential decay rate for first moment estimates	0.9
beta2	float	Exponential decay rate for second moment estimates	0.999
epsilon	float	Small constant for numerical stability	1e-8
weight_decay	float	Weight decay coefficient (L2 regularization)	0.0
amsgrad	bool	Whether to use AMSGrad variant which maintains maximum of squared gradients	False
gradient_clip_norm	float	Maximum norm for gradient clipping	None
gradient_clip_value	float	Maximum absolute value for gradient clipping	None

Attributes:

Name	Type	Description
m	dict	First moment estimates (exponentially decaying average of gradients)
v	dict	Second moment estimates (exponentially decaying average of squared gradients)
v_hat_max	dict	Maximum of v_hat values (used in AMSGrad)
t	int	Time step (number of updates performed)

Source code in src/dlhub/optimizers/adam.py

class AdamOptimizer:
    """
    Adam (Adaptive Moment Estimation) Optimizer

    Adam combines the advantages of AdaGrad and RMSProp by computing adaptive
    learning rates for each parameter using estimates of first and second
    moments of the gradients.

    The algorithm maintains exponentially decaying averages of past gradients
    and past squared gradients, which act as estimates of the first moment
    (mean) and second moment (uncentered variance) of the gradients.

    Parameters
    ----------
    learning_rate : float, default=0.001
        Learning rate (alpha in the paper)
    beta1 : float, default=0.9
        Exponential decay rate for first moment estimates
    beta2 : float, default=0.999
        Exponential decay rate for second moment estimates
    epsilon : float, default=1e-8
        Small constant for numerical stability
    weight_decay : float, default=0.0
        Weight decay coefficient (L2 regularization)
    amsgrad : bool, default=False
        Whether to use AMSGrad variant which maintains maximum of squared gradients
    gradient_clip_norm : float, optional
        Maximum norm for gradient clipping
    gradient_clip_value : float, optional
        Maximum absolute value for gradient clipping

    Attributes
    ----------
    m : dict
        First moment estimates (exponentially decaying average of gradients)
    v : dict
        Second moment estimates (exponentially decaying average of squared gradients)
    v_hat_max : dict
        Maximum of v_hat values (used in AMSGrad)
    t : int
        Time step (number of updates performed)
    """

    def __init__(
        self,
        learning_rate: float = 0.001,
        beta1: float = 0.9,
        beta2: float = 0.999,
        epsilon: float = 1e-8,
        weight_decay: float = 0.0,
        amsgrad: bool = False,
        gradient_clip_norm: float | None = None,
        gradient_clip_value: float | None = None,
    ):
        if not 0.0 < learning_rate <= 1.0:
            raise ValueError(f"Invalid learning rate: {learning_rate}")
        if not 0.0 <= beta1 < 1.0:
            raise ValueError(f"Invalid beta1 parameter: {beta1}")
        if not 0.0 <= beta2 < 1.0:
            raise ValueError(f"Invalid beta2 parameter: {beta2}")
        if epsilon <= 0.0:
            raise ValueError(f"Invalid epsilon value: {epsilon}")
        if weight_decay < 0.0:
            raise ValueError(f"Invalid weight_decay value: {weight_decay}")

        self.learning_rate = learning_rate
        self.beta1 = beta1
        self.beta2 = beta2
        self.epsilon = epsilon
        self.weight_decay = weight_decay
        self.amsgrad = amsgrad
        self.gradient_clip_norm = gradient_clip_norm
        self.gradient_clip_value = gradient_clip_value

        self.m: dict[str, np.ndarray] = {}
        self.v: dict[str, np.ndarray] = {}
        self.v_hat_max: dict[str, np.ndarray] = {}
        self.t = 0

        self.history = {
            "loss": [],
            "gradient_norm": [],
            "parameter_norm": [],
            "learning_rate": [],
        }

    def _initialize_moments(
        self, param_name: str, param_shape: tuple[int, ...]
    ) -> None:
        """Initialize moment estimates for a parameter."""
        if param_name not in self.m:
            self.m[param_name] = np.zeros(param_shape, dtype=np.float64)
            self.v[param_name] = np.zeros(param_shape, dtype=np.float64)
            if self.amsgrad:
                self.v_hat_max[param_name] = np.zeros(param_shape, dtype=np.float64)

    def _clip_gradients(
        self, gradients: dict[str, np.ndarray]
    ) -> dict[str, np.ndarray]:
        """
        Apply gradient clipping if specified.

        Parameters
        ----------
        gradients : dict
            Dictionary of gradients for each parameter

        Returns
        -------
        dict
            Clipped gradients
        """
        if self.gradient_clip_norm is not None:
            total_norm = 0.0
            for grad in gradients.values():
                total_norm += np.sum(grad**2)
            total_norm = np.sqrt(total_norm)

            if total_norm > self.gradient_clip_norm:
                clip_coeff = self.gradient_clip_norm / (total_norm + 1e-8)
                gradients = {
                    name: grad * clip_coeff for name, grad in gradients.items()
                }

        if self.gradient_clip_value is not None:
            gradients = {
                name: np.clip(grad, -self.gradient_clip_value, self.gradient_clip_value)
                for name, grad in gradients.items()
            }

        return gradients

    def update(
        self, parameters: dict[str, np.ndarray], gradients: dict[str, np.ndarray]
    ) -> dict[str, np.ndarray]:
        """
        Perform a single optimization step.

        Parameters
        ----------
        parameters : dict
            Dictionary of parameters to optimize
        gradients : dict
            Dictionary of gradients for each parameter

        Returns
        -------
        dict
            Updated parameters
        """
        self.t += 1
        gradients = self._clip_gradients(gradients)

        grad_norm = np.sqrt(sum(np.sum(grad**2) for grad in gradients.values()))
        param_norm = np.sqrt(sum(np.sum(param**2) for param in parameters.values()))

        updated_parameters = {}
        for param_name, param in parameters.items():
            if param_name not in gradients:
                updated_parameters[param_name] = param.copy()
                continue

            grad = gradients[param_name]

            self._initialize_moments(param_name, param.shape)

            if self.weight_decay > 0:
                grad = grad + self.weight_decay * param

            self.m[param_name] = (
                self.beta1 * self.m[param_name] + (1 - self.beta1) * grad
            )
            self.v[param_name] = self.beta2 * self.v[param_name] + (1 - self.beta2) * (
                grad**2
            )
            m_hat = self.m[param_name] / (1 - self.beta1**self.t)
            v_hat = self.v[param_name] / (1 - self.beta2**self.t)

            # AMSGrad modification
            if self.amsgrad:
                self.v_hat_max[param_name] = np.maximum(
                    self.v_hat_max[param_name], v_hat
                )
                v_hat = self.v_hat_max[param_name]

            denominator = np.sqrt(v_hat) + self.epsilon
            step = self.learning_rate * m_hat / denominator

            if np.any(np.isnan(step)) or np.any(np.isinf(step)):
                warnings.warn(
                    f"Numerical instability detected in parameter {param_name}"
                )
                step = np.nan_to_num(step, nan=0.0, posinf=1e-6, neginf=-1e-6)

            updated_parameters[param_name] = param - step

        self.history["gradient_norm"].append(grad_norm)
        self.history["parameter_norm"].append(param_norm)
        self.history["learning_rate"].append(self.learning_rate)

        return updated_parameters

    def get_config(self) -> dict:
        """
        Get optimizer configuration.

        Returns
        -------
        dict
            Configuration dictionary
        """
        return {
            "learning_rate": self.learning_rate,
            "beta1": self.beta1,
            "beta2": self.beta2,
            "epsilon": self.epsilon,
            "weight_decay": self.weight_decay,
            "amsgrad": self.amsgrad,
            "gradient_clip_norm": self.gradient_clip_norm,
            "gradient_clip_value": self.gradient_clip_value,
            "time_step": self.t,
        }

    def reset_state(self) -> None:
        """Reset optimizer state (moments and time step)."""
        self.m.clear()
        self.v.clear()
        self.v_hat_max.clear()
        self.t = 0
        self.history = {
            "loss": [],
            "gradient_norm": [],
            "parameter_norm": [],
            "learning_rate": [],
        }

    def get_state(self) -> dict:
        """
        Get complete optimizer state.

        Returns
        -------
        dict
            Complete state dictionary
        """
        return {
            "config": self.get_config(),
            "moments": {
                "m": self.m.copy(),
                "v": self.v.copy(),
                "v_hat_max": self.v_hat_max.copy() if self.amsgrad else {},
            },
            "history": self.history.copy(),
        }

    def load_state(self, state: dict) -> None:
        """
        Load optimizer state.

        Parameters
        ----------
        state : dict
            State dictionary from get_state()
        """
        config = state["config"]
        self.learning_rate = config["learning_rate"]
        self.beta1 = config["beta1"]
        self.beta2 = config["beta2"]
        self.epsilon = config["epsilon"]
        self.weight_decay = config["weight_decay"]
        self.amsgrad = config["amsgrad"]
        self.gradient_clip_norm = config["gradient_clip_norm"]
        self.gradient_clip_value = config["gradient_clip_value"]
        self.t = config["time_step"]

        moments = state["moments"]
        self.m = moments["m"].copy()
        self.v = moments["v"].copy()
        self.v_hat_max = moments["v_hat_max"].copy()

        self.history = state["history"].copy()

update

update(parameters: dict[str, ndarray], gradients: dict[str, ndarray]) -> dict[str, np.ndarray]

Perform a single optimization step.

Parameters:

Name	Type	Description	Default
parameters	dict	Dictionary of parameters to optimize	required
gradients	dict	Dictionary of gradients for each parameter	required

Returns:

Type	Description
dict	Updated parameters

Source code in src/dlhub/optimizers/adam.py

def update(
    self, parameters: dict[str, np.ndarray], gradients: dict[str, np.ndarray]
) -> dict[str, np.ndarray]:
    """
    Perform a single optimization step.

    Parameters
    ----------
    parameters : dict
        Dictionary of parameters to optimize
    gradients : dict
        Dictionary of gradients for each parameter

    Returns
    -------
    dict
        Updated parameters
    """
    self.t += 1
    gradients = self._clip_gradients(gradients)

    grad_norm = np.sqrt(sum(np.sum(grad**2) for grad in gradients.values()))
    param_norm = np.sqrt(sum(np.sum(param**2) for param in parameters.values()))

    updated_parameters = {}
    for param_name, param in parameters.items():
        if param_name not in gradients:
            updated_parameters[param_name] = param.copy()
            continue

        grad = gradients[param_name]

        self._initialize_moments(param_name, param.shape)

        if self.weight_decay > 0:
            grad = grad + self.weight_decay * param

        self.m[param_name] = (
            self.beta1 * self.m[param_name] + (1 - self.beta1) * grad
        )
        self.v[param_name] = self.beta2 * self.v[param_name] + (1 - self.beta2) * (
            grad**2
        )
        m_hat = self.m[param_name] / (1 - self.beta1**self.t)
        v_hat = self.v[param_name] / (1 - self.beta2**self.t)

        # AMSGrad modification
        if self.amsgrad:
            self.v_hat_max[param_name] = np.maximum(
                self.v_hat_max[param_name], v_hat
            )
            v_hat = self.v_hat_max[param_name]

        denominator = np.sqrt(v_hat) + self.epsilon
        step = self.learning_rate * m_hat / denominator

        if np.any(np.isnan(step)) or np.any(np.isinf(step)):
            warnings.warn(
                f"Numerical instability detected in parameter {param_name}"
            )
            step = np.nan_to_num(step, nan=0.0, posinf=1e-6, neginf=-1e-6)

        updated_parameters[param_name] = param - step

    self.history["gradient_norm"].append(grad_norm)
    self.history["parameter_norm"].append(param_norm)
    self.history["learning_rate"].append(self.learning_rate)

    return updated_parameters

get_config

get_config() -> dict

Get optimizer configuration.

Returns:

Type	Description
dict	Configuration dictionary

Source code in src/dlhub/optimizers/adam.py

def get_config(self) -> dict:
    """
    Get optimizer configuration.

    Returns
    -------
    dict
        Configuration dictionary
    """
    return {
        "learning_rate": self.learning_rate,
        "beta1": self.beta1,
        "beta2": self.beta2,
        "epsilon": self.epsilon,
        "weight_decay": self.weight_decay,
        "amsgrad": self.amsgrad,
        "gradient_clip_norm": self.gradient_clip_norm,
        "gradient_clip_value": self.gradient_clip_value,
        "time_step": self.t,
    }

reset_state

reset_state() -> None

Reset optimizer state (moments and time step).

Source code in src/dlhub/optimizers/adam.py

def reset_state(self) -> None:
    """Reset optimizer state (moments and time step)."""
    self.m.clear()
    self.v.clear()
    self.v_hat_max.clear()
    self.t = 0
    self.history = {
        "loss": [],
        "gradient_norm": [],
        "parameter_norm": [],
        "learning_rate": [],
    }

get_state

get_state() -> dict

Get complete optimizer state.

Returns:

Type	Description
dict	Complete state dictionary

Source code in src/dlhub/optimizers/adam.py

def get_state(self) -> dict:
    """
    Get complete optimizer state.

    Returns
    -------
    dict
        Complete state dictionary
    """
    return {
        "config": self.get_config(),
        "moments": {
            "m": self.m.copy(),
            "v": self.v.copy(),
            "v_hat_max": self.v_hat_max.copy() if self.amsgrad else {},
        },
        "history": self.history.copy(),
    }

load_state

load_state(state: dict) -> None

Load optimizer state.

Parameters:

Name	Type	Description	Default
state	dict	State dictionary from get_state()	required

Source code in src/dlhub/optimizers/adam.py

def load_state(self, state: dict) -> None:
    """
    Load optimizer state.

    Parameters
    ----------
    state : dict
        State dictionary from get_state()
    """
    config = state["config"]
    self.learning_rate = config["learning_rate"]
    self.beta1 = config["beta1"]
    self.beta2 = config["beta2"]
    self.epsilon = config["epsilon"]
    self.weight_decay = config["weight_decay"]
    self.amsgrad = config["amsgrad"]
    self.gradient_clip_norm = config["gradient_clip_norm"]
    self.gradient_clip_value = config["gradient_clip_value"]
    self.t = config["time_step"]

    moments = state["moments"]
    self.m = moments["m"].copy()
    self.v = moments["v"].copy()
    self.v_hat_max = moments["v_hat_max"].copy()

    self.history = state["history"].copy()

BaseOptimizer

Base class for all optimizers with common functionality.

Parameters:

Name	Type	Description	Default
learning_rate	float	Learning rate for parameter updates.	0.01

Attributes:

Name	Type	Description
learning_rate	float	Step size applied to each update.
name	str	Human-readable label, used to key and plot results. Subclasses set it to the name of the method they implement.

Source code in src/dlhub/optimizers/base.py

class BaseOptimizer:
    """
    Base class for all optimizers with common functionality.

    Parameters
    ----------
    learning_rate : float, default=0.01
        Learning rate for parameter updates.

    Attributes
    ----------
    learning_rate : float
        Step size applied to each update.
    name : str
        Human-readable label, used to key and plot results. Subclasses set it to
        the name of the method they implement.
    """

    def __init__(self, learning_rate: float = 0.01):
        self.learning_rate = learning_rate
        self.name = "BaseOptimizer"

    def update_parameters(
        self, params: dict[str, np.ndarray], grads: dict[str, np.ndarray], t: int
    ) -> dict[str, np.ndarray]:
        """
        Update parameters using optimization algorithm.

        Implementations return a new dictionary rather than mutating the one they
        were given, so that a caller can keep the parameter trajectory of a run.

        Parameters
        ----------
        params : dict
            Current parameter values.
        grads : dict
            Gradients for each parameter.
        t : int
            Current iteration, counted from one. Optimizers applying bias
            correction divide by ``1 - beta ** t``, which is why the count starts
            at one rather than zero.

        Returns
        -------
        dict
            Updated parameters.

        Raises
        ------
        NotImplementedError
            Always, on the base class. An optimizer is defined by its update
            rule, so there is no meaningful default to inherit.
        """
        raise NotImplementedError("Subclasses must implement update_parameters")

    def reset(self) -> None:
        """
        Reset optimizer state for new optimization run.

        The base implementation does nothing, which is correct for a stateless
        optimizer such as plain gradient descent. Any optimizer accumulating
        state across steps -- a velocity, a second moment -- must override this
        and clear it, or a second run starts from wherever the first one ended
        and its trajectory is not reproducible.
        """
        pass

update_parameters

update_parameters(params: dict[str, ndarray], grads: dict[str, ndarray], t: int) -> dict[str, np.ndarray]

Update parameters using optimization algorithm.

Implementations return a new dictionary rather than mutating the one they were given, so that a caller can keep the parameter trajectory of a run.

Parameters:

Name	Type	Description	Default
params	dict	Current parameter values.	required
grads	dict	Gradients for each parameter.	required
t	int	Current iteration, counted from one. Optimizers applying bias correction divide by 1 - beta ** t, which is why the count starts at one rather than zero.	required

Returns:

Type	Description
dict	Updated parameters.

Raises:

Type	Description
NotImplementedError	Always, on the base class. An optimizer is defined by its update rule, so there is no meaningful default to inherit.

Source code in src/dlhub/optimizers/base.py

def update_parameters(
    self, params: dict[str, np.ndarray], grads: dict[str, np.ndarray], t: int
) -> dict[str, np.ndarray]:
    """
    Update parameters using optimization algorithm.

    Implementations return a new dictionary rather than mutating the one they
    were given, so that a caller can keep the parameter trajectory of a run.

    Parameters
    ----------
    params : dict
        Current parameter values.
    grads : dict
        Gradients for each parameter.
    t : int
        Current iteration, counted from one. Optimizers applying bias
        correction divide by ``1 - beta ** t``, which is why the count starts
        at one rather than zero.

    Returns
    -------
    dict
        Updated parameters.

    Raises
    ------
    NotImplementedError
        Always, on the base class. An optimizer is defined by its update
        rule, so there is no meaningful default to inherit.
    """
    raise NotImplementedError("Subclasses must implement update_parameters")

reset

reset() -> None

Reset optimizer state for new optimization run.

The base implementation does nothing, which is correct for a stateless optimizer such as plain gradient descent. Any optimizer accumulating state across steps -- a velocity, a second moment -- must override this and clear it, or a second run starts from wherever the first one ended and its trajectory is not reproducible.

Source code in src/dlhub/optimizers/base.py

def reset(self) -> None:
    """
    Reset optimizer state for new optimization run.

    The base implementation does nothing, which is correct for a stateless
    optimizer such as plain gradient descent. Any optimizer accumulating
    state across steps -- a velocity, a second moment -- must override this
    and clear it, or a second run starts from wherever the first one ended
    and its trajectory is not reproducible.
    """
    pass

BealeFunction

Bases: OptimizationProblem

Beale function: f(x,y) = (1.5 - x + xy)² + (2.25 - x + xy²)² + (2.625 - x + xy³)².

Multimodal function with global minimum and several local minima. Tests optimizer robustness to local minima and saddle points.

Source code in src/dlhub/optimizers/comparison.py

class BealeFunction(OptimizationProblem):
    """
    Beale function: f(x,y) = (1.5 - x + xy)² + (2.25 - x + x*y²)² + (2.625 - x + x*y³)².

    Multimodal function with global minimum and several local minima.
    Tests optimizer robustness to local minima and saddle points.
    """

    def __init__(self):
        super().__init__("Beale Function")

    def loss_function(self, params: dict) -> float:
        """Compute Beale function value."""
        x, y = params["x"], params["y"]
        term1 = (1.5 - x + x * y) ** 2
        term2 = (2.25 - x + x * y**2) ** 2
        term3 = (2.625 - x + x * y**3) ** 2
        return term1 + term2 + term3

    def gradients(self, params: dict) -> dict:
        """Compute Beale function gradients analytically."""
        x, y = params["x"], params["y"]

        # Partial derivatives computed analytically
        dx = (
            2 * (1.5 - x + x * y) * (-1 + y)
            + 2 * (2.25 - x + x * y**2) * (-1 + y**2)
            + 2 * (2.625 - x + x * y**3) * (-1 + y**3)
        )

        dy = (
            2 * (1.5 - x + x * y) * x
            + 2 * (2.25 - x + x * y**2) * (2 * x * y)
            + 2 * (2.625 - x + x * y**3) * (3 * x * y**2)
        )

        return {"x": dx, "y": dy}

    def initial_parameters(self) -> dict:
        """Initialize at challenging starting point."""
        return {"x": 4.0, "y": 4.0}

loss_function

loss_function(params: dict) -> float

Compute Beale function value.

Source code in src/dlhub/optimizers/comparison.py

def loss_function(self, params: dict) -> float:
    """Compute Beale function value."""
    x, y = params["x"], params["y"]
    term1 = (1.5 - x + x * y) ** 2
    term2 = (2.25 - x + x * y**2) ** 2
    term3 = (2.625 - x + x * y**3) ** 2
    return term1 + term2 + term3

gradients

gradients(params: dict) -> dict

Compute Beale function gradients analytically.

Source code in src/dlhub/optimizers/comparison.py

def gradients(self, params: dict) -> dict:
    """Compute Beale function gradients analytically."""
    x, y = params["x"], params["y"]

    # Partial derivatives computed analytically
    dx = (
        2 * (1.5 - x + x * y) * (-1 + y)
        + 2 * (2.25 - x + x * y**2) * (-1 + y**2)
        + 2 * (2.625 - x + x * y**3) * (-1 + y**3)
    )

    dy = (
        2 * (1.5 - x + x * y) * x
        + 2 * (2.25 - x + x * y**2) * (2 * x * y)
        + 2 * (2.625 - x + x * y**3) * (3 * x * y**2)
    )

    return {"x": dx, "y": dy}

initial_parameters

initial_parameters() -> dict

Initialize at challenging starting point.

Source code in src/dlhub/optimizers/comparison.py

def initial_parameters(self) -> dict:
    """Initialize at challenging starting point."""
    return {"x": 4.0, "y": 4.0}

OptimizationAnalytics

Advanced analytics utilities for optimization comparison results.

Provides statistical analysis, performance ranking, and detailed insights into optimizer behavior across different problem types.

Source code in src/dlhub/optimizers/comparison.py

class OptimizationAnalytics:
    """
    Advanced analytics utilities for optimization comparison results.

    Provides statistical analysis, performance ranking, and detailed
    insights into optimizer behavior across different problem types.
    """

    @staticmethod
    def compute_convergence_metrics(result: OptimizationRun) -> dict[str, float]:
        """
        Compute detailed convergence metrics for a single optimization run.

        Parameters
        ----------
        result : OptimizationRun
            Single optimization result to analyze.

        Returns
        -------
        dict
            Dictionary of computed metrics.
        """
        losses = np.array(result.losses)

        metrics = {
            "final_loss": result.final_loss,
            "initial_loss": losses[0] if len(losses) > 0 else float("inf"),
            "loss_reduction": losses[0] - result.final_loss if len(losses) > 0 else 0.0,
            "relative_improvement": ((losses[0] - result.final_loss) / losses[0]) * 100
            if len(losses) > 0 and losses[0] != 0
            else 0.0,
            "iterations_to_converge": result.iterations_to_converge,
            "convergence_time": result.convergence_time,
            "convergence_rate": 0.0,
            "stability_score": 0.0,
        }

        # Compute convergence rate (loss decrease per iteration)
        if len(losses) > 1:
            total_improvement = losses[0] - losses[-1]
            metrics["convergence_rate"] = total_improvement / len(losses)

            # Compute stability score (1 - coefficient of variation of loss changes)
            loss_changes = np.diff(losses)
            if len(loss_changes) > 0 and np.mean(loss_changes) != 0:
                cv = np.std(loss_changes) / abs(np.mean(loss_changes))
                metrics["stability_score"] = max(0, 1 - cv)

        return metrics

    @staticmethod
    def rank_optimizers(
        all_results: dict[str, dict[str, OptimizationRun]],
    ) -> dict[str, dict[str, int]]:
        """
        Rank optimizers across different problems and metrics.

        Parameters
        ----------
        all_results : dict
            Complete results from optimization comparison.

        Returns
        -------
        dict
            Rankings for each optimizer on each problem.
        """
        rankings = {}

        for problem_name, results in all_results.items():
            if not results:
                continue

            optimizer_metrics = {}
            for opt_name, result in results.items():
                optimizer_metrics[opt_name] = (
                    OptimizationAnalytics.compute_convergence_metrics(result)
                )

            rankings[problem_name] = {}
            sorted_by_loss = sorted(
                optimizer_metrics.items(), key=lambda x: x[1]["final_loss"]
            )
            for rank, (opt_name, _) in enumerate(sorted_by_loss, 1):
                rankings[problem_name][f"{opt_name}_loss_rank"] = rank

            sorted_by_speed = sorted(
                optimizer_metrics.items(), key=lambda x: x[1]["iterations_to_converge"]
            )
            for rank, (opt_name, _) in enumerate(sorted_by_speed, 1):
                rankings[problem_name][f"{opt_name}_speed_rank"] = rank

            sorted_by_stability = sorted(
                optimizer_metrics.items(),
                key=lambda x: x[1]["stability_score"],
                reverse=True,
            )
            for rank, (opt_name, _) in enumerate(sorted_by_stability, 1):
                rankings[problem_name][f"{opt_name}_stability_rank"] = rank

        return rankings

    @staticmethod
    def generate_performance_heatmap(
        all_results: dict[str, dict[str, OptimizationRun]],
    ):
        """
        Generate performance heatmap comparing optimizers across problems.

        Parameters
        ----------
        all_results : dict
            Complete results from optimization comparison.
        """
        import matplotlib.pyplot as plt
        import numpy as np

        optimizers = []
        problems = list(all_results.keys())

        for problem_results in all_results.values():
            for opt_name in problem_results.keys():
                if opt_name not in optimizers:
                    optimizers.append(opt_name)

        performance_matrix = np.zeros((len(optimizers), len(problems)))

        for j, problem_name in enumerate(problems):
            results = all_results[problem_name]
            losses = [results[opt].final_loss for opt in optimizers if opt in results]

            if losses:
                # Normalize losses (0 = best, 1 = worst)
                min_loss, max_loss = min(losses), max(losses)
                loss_range = max_loss - min_loss if max_loss != min_loss else 1

                for i, opt_name in enumerate(optimizers):
                    if opt_name in results:
                        normalized_loss = (
                            results[opt_name].final_loss - min_loss
                        ) / loss_range
                        performance_matrix[i, j] = normalized_loss
                    else:
                        performance_matrix[i, j] = (
                            1.0  # Worst performance if not available
                        )

        fig, ax = plt.subplots(figsize=(12, 8))
        im = ax.imshow(
            performance_matrix, cmap="RdYlGn_r", aspect="auto", vmin=0, vmax=1
        )

        ax.set_xticks(range(len(problems)))
        ax.set_yticks(range(len(optimizers)))
        ax.set_xticklabels(
            [p.split("(")[0].strip() for p in problems], rotation=45, ha="right"
        )
        ax.set_yticklabels(optimizers)

        cbar = plt.colorbar(im, ax=ax)
        cbar.set_label(
            "Normalized Performance (0=Best, 1=Worst)", rotation=270, labelpad=20
        )

        # Add text annotations
        for i in range(len(optimizers)):
            for j in range(len(problems)):
                text = ax.text(
                    j,
                    i,
                    f"{performance_matrix[i, j]:.2f}",
                    ha="center",
                    va="center",
                    color="black",
                    fontweight="bold",
                )

        ax.set_title(
            "Optimizer Performance Heatmap Across Problems",
            fontsize=14,
            fontweight="bold",
            pad=20,
        )
        plt.tight_layout()
        plt.show()

compute_convergence_metrics staticmethod

compute_convergence_metrics(result: OptimizationRun) -> dict[str, float]

Compute detailed convergence metrics for a single optimization run.

Parameters:

Name	Type	Description	Default
result	OptimizationRun	Single optimization result to analyze.	required

Returns:

Type	Description
dict	Dictionary of computed metrics.

Source code in src/dlhub/optimizers/comparison.py

@staticmethod
def compute_convergence_metrics(result: OptimizationRun) -> dict[str, float]:
    """
    Compute detailed convergence metrics for a single optimization run.

    Parameters
    ----------
    result : OptimizationRun
        Single optimization result to analyze.

    Returns
    -------
    dict
        Dictionary of computed metrics.
    """
    losses = np.array(result.losses)

    metrics = {
        "final_loss": result.final_loss,
        "initial_loss": losses[0] if len(losses) > 0 else float("inf"),
        "loss_reduction": losses[0] - result.final_loss if len(losses) > 0 else 0.0,
        "relative_improvement": ((losses[0] - result.final_loss) / losses[0]) * 100
        if len(losses) > 0 and losses[0] != 0
        else 0.0,
        "iterations_to_converge": result.iterations_to_converge,
        "convergence_time": result.convergence_time,
        "convergence_rate": 0.0,
        "stability_score": 0.0,
    }

    # Compute convergence rate (loss decrease per iteration)
    if len(losses) > 1:
        total_improvement = losses[0] - losses[-1]
        metrics["convergence_rate"] = total_improvement / len(losses)

        # Compute stability score (1 - coefficient of variation of loss changes)
        loss_changes = np.diff(losses)
        if len(loss_changes) > 0 and np.mean(loss_changes) != 0:
            cv = np.std(loss_changes) / abs(np.mean(loss_changes))
            metrics["stability_score"] = max(0, 1 - cv)

    return metrics

rank_optimizers staticmethod

rank_optimizers(all_results: dict[str, dict[str, OptimizationRun]]) -> dict[str, dict[str, int]]

Rank optimizers across different problems and metrics.

Parameters:

Name	Type	Description	Default
all_results	dict	Complete results from optimization comparison.	required

Returns:

Type	Description
dict	Rankings for each optimizer on each problem.

Source code in src/dlhub/optimizers/comparison.py

@staticmethod
def rank_optimizers(
    all_results: dict[str, dict[str, OptimizationRun]],
) -> dict[str, dict[str, int]]:
    """
    Rank optimizers across different problems and metrics.

    Parameters
    ----------
    all_results : dict
        Complete results from optimization comparison.

    Returns
    -------
    dict
        Rankings for each optimizer on each problem.
    """
    rankings = {}

    for problem_name, results in all_results.items():
        if not results:
            continue

        optimizer_metrics = {}
        for opt_name, result in results.items():
            optimizer_metrics[opt_name] = (
                OptimizationAnalytics.compute_convergence_metrics(result)
            )

        rankings[problem_name] = {}
        sorted_by_loss = sorted(
            optimizer_metrics.items(), key=lambda x: x[1]["final_loss"]
        )
        for rank, (opt_name, _) in enumerate(sorted_by_loss, 1):
            rankings[problem_name][f"{opt_name}_loss_rank"] = rank

        sorted_by_speed = sorted(
            optimizer_metrics.items(), key=lambda x: x[1]["iterations_to_converge"]
        )
        for rank, (opt_name, _) in enumerate(sorted_by_speed, 1):
            rankings[problem_name][f"{opt_name}_speed_rank"] = rank

        sorted_by_stability = sorted(
            optimizer_metrics.items(),
            key=lambda x: x[1]["stability_score"],
            reverse=True,
        )
        for rank, (opt_name, _) in enumerate(sorted_by_stability, 1):
            rankings[problem_name][f"{opt_name}_stability_rank"] = rank

    return rankings

generate_performance_heatmap staticmethod

generate_performance_heatmap(all_results: dict[str, dict[str, OptimizationRun]])

Generate performance heatmap comparing optimizers across problems.

Parameters:

Name	Type	Description	Default
all_results	dict	Complete results from optimization comparison.	required

Source code in src/dlhub/optimizers/comparison.py

@staticmethod
def generate_performance_heatmap(
    all_results: dict[str, dict[str, OptimizationRun]],
):
    """
    Generate performance heatmap comparing optimizers across problems.

    Parameters
    ----------
    all_results : dict
        Complete results from optimization comparison.
    """
    import matplotlib.pyplot as plt
    import numpy as np

    optimizers = []
    problems = list(all_results.keys())

    for problem_results in all_results.values():
        for opt_name in problem_results.keys():
            if opt_name not in optimizers:
                optimizers.append(opt_name)

    performance_matrix = np.zeros((len(optimizers), len(problems)))

    for j, problem_name in enumerate(problems):
        results = all_results[problem_name]
        losses = [results[opt].final_loss for opt in optimizers if opt in results]

        if losses:
            # Normalize losses (0 = best, 1 = worst)
            min_loss, max_loss = min(losses), max(losses)
            loss_range = max_loss - min_loss if max_loss != min_loss else 1

            for i, opt_name in enumerate(optimizers):
                if opt_name in results:
                    normalized_loss = (
                        results[opt_name].final_loss - min_loss
                    ) / loss_range
                    performance_matrix[i, j] = normalized_loss
                else:
                    performance_matrix[i, j] = (
                        1.0  # Worst performance if not available
                    )

    fig, ax = plt.subplots(figsize=(12, 8))
    im = ax.imshow(
        performance_matrix, cmap="RdYlGn_r", aspect="auto", vmin=0, vmax=1
    )

    ax.set_xticks(range(len(problems)))
    ax.set_yticks(range(len(optimizers)))
    ax.set_xticklabels(
        [p.split("(")[0].strip() for p in problems], rotation=45, ha="right"
    )
    ax.set_yticklabels(optimizers)

    cbar = plt.colorbar(im, ax=ax)
    cbar.set_label(
        "Normalized Performance (0=Best, 1=Worst)", rotation=270, labelpad=20
    )

    # Add text annotations
    for i in range(len(optimizers)):
        for j in range(len(problems)):
            text = ax.text(
                j,
                i,
                f"{performance_matrix[i, j]:.2f}",
                ha="center",
                va="center",
                color="black",
                fontweight="bold",
            )

    ax.set_title(
        "Optimizer Performance Heatmap Across Problems",
        fontsize=14,
        fontweight="bold",
        pad=20,
    )
    plt.tight_layout()
    plt.show()

OptimizationComparison

Comprehensive framework for comparing optimization algorithms.

Provides utilities to run multiple optimizers on various problems, collect performance metrics, and generate comparative visualizations.

Parameters:

Name	Type	Description	Default
max_iterations	int	Maximum number of optimization iterations.	1000
tolerance	float	Convergence tolerance for loss change.	1e-6
verbose	bool	Whether to print progress information.	True

Source code in src/dlhub/optimizers/comparison.py

class OptimizationComparison:
    """
    Comprehensive framework for comparing optimization algorithms.

    Provides utilities to run multiple optimizers on various problems,
    collect performance metrics, and generate comparative visualizations.

    Parameters
    ----------
    max_iterations : int, default=1000
        Maximum number of optimization iterations.
    tolerance : float, default=1e-6
        Convergence tolerance for loss change.
    verbose : bool, default=True
        Whether to print progress information.
    """

    def __init__(
        self, max_iterations: int = 1000, tolerance: float = 1e-6, verbose: bool = True
    ):
        self.max_iterations = max_iterations
        self.tolerance = tolerance
        self.verbose = verbose
        self.results = {}

    def create_optimizer(
        self, optimizer_type: OptimizerType, **kwargs
    ) -> BaseOptimizer:
        """
        Factory method to create optimizer instances.

        Parameters
        ----------
        optimizer_type : OptimizerType
            Type of optimizer to create.
        **kwargs
            Additional parameters for optimizer initialization.

        Returns
        -------
        BaseOptimizer
            Configured optimizer instance.
        """
        if optimizer_type == OptimizerType.SGD:
            return SGDOptimizer(**kwargs)
        elif optimizer_type == OptimizerType.MOMENTUM:
            return MomentumOptimizer(**kwargs)
        elif optimizer_type == OptimizerType.RMSPROP:
            return RMSpropOptimizer(**kwargs)
        elif optimizer_type == OptimizerType.ADAM:
            return AdamOptimizer(**kwargs)
        else:
            raise ValueError(f"Unknown optimizer type: {optimizer_type}")

    def run_optimization(
        self, problem: OptimizationProblem, optimizer: BaseOptimizer
    ) -> OptimizationRun:
        """
        Run single optimization experiment.

        Parameters
        ----------
        problem : OptimizationProblem
            Problem to optimize.
        optimizer : BaseOptimizer
            Optimizer to use.

        Returns
        -------
        OptimizationRun
            Results of optimization including metrics and trajectory.
        """
        optimizer.reset()

        # The benchmark problems below are two-dimensional surfaces, so they state
        # their starting points as plain floats, which reads naturally for a point
        # on a contour plot. Optimizers are written against arrays -- the contract
        # says so, and a neural network's parameters are never scalars -- so the
        # conversion happens once, here, where the problem's world meets the
        # optimizer's. Doing it at the boundary means every optimizer receives what
        # the contract promises, rather than each one having to tolerate floats.
        params = {
            name: np.asarray(value, dtype=float)
            for name, value in problem.initial_parameters().items()
        }
        losses = []
        parameter_history = []

        start_time = time.time()
        converged = False

        for iteration in range(1, self.max_iterations + 1):
            current_loss = problem.loss_function(params)
            grads = problem.gradients(params)

            losses.append(current_loss)
            parameter_history.append(params.copy())

            if len(losses) > 1:
                loss_change = abs(losses[-2] - losses[-1])
                if loss_change < self.tolerance:
                    converged = True
                    break

            params = optimizer.update_parameters(params, grads, iteration)

            # Prevent divergence
            if current_loss > 1e10 or np.any(
                [np.isnan(v) or np.isinf(v) for v in params.values()]
            ):
                if self.verbose:
                    print(f"{optimizer.name} diverged at iteration {iteration}")
                break

        end_time = time.time()

        result = OptimizationRun(
            optimizer_name=optimizer.name,
            losses=losses,
            parameters=parameter_history,
            convergence_time=end_time - start_time,
            final_loss=losses[-1] if losses else float("inf"),
            iterations_to_converge=len(losses) if converged else self.max_iterations,
        )

        if self.verbose:
            status = "converged" if converged else "max iterations reached"
            print(
                f"{optimizer.name}: {status} in {len(losses)} iterations, "
                f"final loss: {result.final_loss:.6f}, time: {result.convergence_time:.3f}s"
            )

        return result

    def compare_optimizers(
        self, problem: OptimizationProblem, optimizer_configs: dict[OptimizerType, dict]
    ) -> dict[str, OptimizationRun]:
        """
        Compare multiple optimizers on a single problem.

        Parameters
        ----------
        problem : OptimizationProblem
            Problem to optimize.
        optimizer_configs : dict
            Dictionary mapping optimizer types to their configuration parameters.

        Returns
        -------
        dict
            Results for each optimizer.
        """
        results = {}

        if self.verbose:
            print(f"\nOptimizing: {problem.name}")
            print("=" * 50)

        for opt_type, config in optimizer_configs.items():
            optimizer = self.create_optimizer(opt_type, **config)
            result = self.run_optimization(problem, optimizer)
            results[optimizer.name] = result

        return results

    def run_comprehensive_comparison(self) -> dict[str, dict[str, OptimizationRun]]:
        """
        Run comprehensive comparison across multiple problems and optimizers.

        Returns
        -------
        dict
            Nested dictionary: {problem_name: {optimizer_name: result}}
        """
        problems = [
            QuadraticBowl(a=1.0, b=1.0),  # Well-conditioned
            QuadraticBowl(a=1.0, b=100.0),  # Ill-conditioned
            RosenbrockFunction(),  # Non-convex valley
            BealeFunction(),  # Multimodal
        ]

        optimizer_configs = {
            OptimizerType.SGD: {"learning_rate": 0.01},
            OptimizerType.MOMENTUM: {"learning_rate": 0.01, "beta": 0.9},
            OptimizerType.RMSPROP: {"learning_rate": 0.01, "beta": 0.9},
            OptimizerType.ADAM: {"learning_rate": 0.05, "beta1": 0.9, "beta2": 0.999},
        }

        all_results = {}

        for problem in problems:
            results = self.compare_optimizers(problem, optimizer_configs)
            all_results[problem.name] = results

        self.results = all_results
        return all_results

    def plot_convergence_comparison(
        self,
        results: dict[str, OptimizationRun],
        problem_name: str,
        log_scale: bool = True,
    ):
        """
        Plot convergence curves for optimizer comparison.

        Parameters
        ----------
        results : dict
            Results from optimizer comparison.
        problem_name : str
            Name of the problem for plot title.
        log_scale : bool, default=True
            Whether to use logarithmic scale for loss.
        """
        plt.figure(figsize=(12, 8))

        colors = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728"]
        styles = ["-", "--", "-.", ":"]

        for i, (opt_name, result) in enumerate(results.items()):
            if result.losses:
                iterations = range(1, len(result.losses) + 1)
                plt.plot(
                    iterations,
                    result.losses,
                    color=colors[i % len(colors)],
                    linestyle=styles[i % len(styles)],
                    linewidth=2,
                    label=opt_name,
                    alpha=0.8,
                )

        plt.xlabel("Iterations", fontsize=12)
        plt.ylabel("Loss", fontsize=12)
        plt.title(
            f"Convergence Comparison: {problem_name}", fontsize=14, fontweight="bold"
        )
        plt.legend(fontsize=11)
        plt.grid(True, alpha=0.3)

        if log_scale:
            plt.yscale("log")
            plt.ylabel("Loss (log scale)", fontsize=12)

        plt.tight_layout()
        plt.show()

    def plot_optimization_paths(
        self,
        results: dict[str, OptimizationRun],
        problem: OptimizationProblem,
        contour_levels: int = 20,
    ):
        """
        Plot optimization trajectories on loss landscape contours.

        Parameters
        ----------
        results : dict
            Results from optimizer comparison.
        problem : OptimizationProblem
            Problem instance for computing loss landscape.
        contour_levels : int, default=20
            Number of contour levels to display.
        """
        plt.figure(figsize=(14, 10))

        # Create meshgrid for contour plot
        x_range = np.linspace(-6, 6, 100)
        y_range = np.linspace(-6, 6, 100)
        X, Y = np.meshgrid(x_range, y_range)
        Z = np.zeros_like(X)

        for i in range(X.shape[0]):
            for j in range(X.shape[1]):
                params = {"x": X[i, j], "y": Y[i, j]}
                Z[i, j] = problem.loss_function(params)

        contours = plt.contour(
            X, Y, Z, levels=contour_levels, alpha=0.6, cmap="viridis"
        )
        plt.clabel(contours, inline=True, fontsize=8, fmt="%.1f")

        colors = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728"]
        markers = ["o", "s", "^", "D"]

        for i, (opt_name, result) in enumerate(results.items()):
            if result.parameters:
                x_path = [p["x"] for p in result.parameters]
                y_path = [p["y"] for p in result.parameters]

                plt.plot(
                    x_path,
                    y_path,
                    color=colors[i % len(colors)],
                    marker=markers[i % len(markers)],
                    markersize=4,
                    linewidth=2,
                    label=f"{opt_name} ({len(x_path)} steps)",
                    alpha=0.8,
                )

                # Mark start and end points
                plt.plot(
                    x_path[0],
                    y_path[0],
                    marker="*",
                    color=colors[i % len(colors)],
                    markersize=12,
                    markeredgecolor="black",
                    markeredgewidth=1,
                )
                plt.plot(
                    x_path[-1],
                    y_path[-1],
                    marker="x",
                    color=colors[i % len(colors)],
                    markersize=10,
                    markeredgewidth=3,
                )

        plt.xlabel("x", fontsize=12)
        plt.ylabel("y", fontsize=12)
        plt.title(f"Optimization Paths: {problem.name}", fontsize=14, fontweight="bold")
        plt.legend(fontsize=11)
        plt.grid(True, alpha=0.3)
        plt.axis("equal")
        plt.tight_layout()
        plt.show()

    def generate_summary_table(
        self, all_results: dict[str, dict[str, OptimizationRun]]
    ) -> None:
        """
        Generate formatted summary table of optimization results.

        Parameters
        ----------
        all_results : dict
            Complete results from comprehensive comparison.
        """
        print("\n" + "=" * 80)
        print("OPTIMIZATION COMPARISON SUMMARY")
        print("=" * 80)

        for problem_name, results in all_results.items():
            print(f"\n{problem_name}")
            print("-" * len(problem_name))

            # Create formatted table
            headers = [
                "Optimizer",
                "Final Loss",
                "Iterations",
                "Conv. Time (s)",
                "Status",
            ]
            print(
                f"{headers[0]:<12} {headers[1]:<20} {headers[2]:<12} {headers[3]:<15} {headers[4]:<10}"
            )
            print("-" * 80)

            for opt_name, result in results.items():
                status = (
                    "✓"
                    if result.iterations_to_converge < self.max_iterations
                    else "Max iter"
                )
                print(
                    f"{opt_name:<12} {result.final_loss:<14.6f} "
                    f"{result.iterations_to_converge:<12} "
                    f"{result.convergence_time:<15.3f} {status:<10}"
                )

        print("\n" + "=" * 90)

create_optimizer

create_optimizer(optimizer_type: OptimizerType, **kwargs) -> BaseOptimizer

Factory method to create optimizer instances.

Parameters:

Name	Type	Description	Default
optimizer_type	OptimizerType	Type of optimizer to create.	required
**kwargs		Additional parameters for optimizer initialization.	{}

Returns:

Type	Description
BaseOptimizer	Configured optimizer instance.

Source code in src/dlhub/optimizers/comparison.py

def create_optimizer(
    self, optimizer_type: OptimizerType, **kwargs
) -> BaseOptimizer:
    """
    Factory method to create optimizer instances.

    Parameters
    ----------
    optimizer_type : OptimizerType
        Type of optimizer to create.
    **kwargs
        Additional parameters for optimizer initialization.

    Returns
    -------
    BaseOptimizer
        Configured optimizer instance.
    """
    if optimizer_type == OptimizerType.SGD:
        return SGDOptimizer(**kwargs)
    elif optimizer_type == OptimizerType.MOMENTUM:
        return MomentumOptimizer(**kwargs)
    elif optimizer_type == OptimizerType.RMSPROP:
        return RMSpropOptimizer(**kwargs)
    elif optimizer_type == OptimizerType.ADAM:
        return AdamOptimizer(**kwargs)
    else:
        raise ValueError(f"Unknown optimizer type: {optimizer_type}")

run_optimization

run_optimization(problem: OptimizationProblem, optimizer: BaseOptimizer) -> OptimizationRun

Run single optimization experiment.

Parameters:

Name	Type	Description	Default
problem	OptimizationProblem	Problem to optimize.	required
optimizer	BaseOptimizer	Optimizer to use.	required

Returns:

Type	Description
OptimizationRun	Results of optimization including metrics and trajectory.

Source code in src/dlhub/optimizers/comparison.py

def run_optimization(
    self, problem: OptimizationProblem, optimizer: BaseOptimizer
) -> OptimizationRun:
    """
    Run single optimization experiment.

    Parameters
    ----------
    problem : OptimizationProblem
        Problem to optimize.
    optimizer : BaseOptimizer
        Optimizer to use.

    Returns
    -------
    OptimizationRun
        Results of optimization including metrics and trajectory.
    """
    optimizer.reset()

    # The benchmark problems below are two-dimensional surfaces, so they state
    # their starting points as plain floats, which reads naturally for a point
    # on a contour plot. Optimizers are written against arrays -- the contract
    # says so, and a neural network's parameters are never scalars -- so the
    # conversion happens once, here, where the problem's world meets the
    # optimizer's. Doing it at the boundary means every optimizer receives what
    # the contract promises, rather than each one having to tolerate floats.
    params = {
        name: np.asarray(value, dtype=float)
        for name, value in problem.initial_parameters().items()
    }
    losses = []
    parameter_history = []

    start_time = time.time()
    converged = False

    for iteration in range(1, self.max_iterations + 1):
        current_loss = problem.loss_function(params)
        grads = problem.gradients(params)

        losses.append(current_loss)
        parameter_history.append(params.copy())

        if len(losses) > 1:
            loss_change = abs(losses[-2] - losses[-1])
            if loss_change < self.tolerance:
                converged = True
                break

        params = optimizer.update_parameters(params, grads, iteration)

        # Prevent divergence
        if current_loss > 1e10 or np.any(
            [np.isnan(v) or np.isinf(v) for v in params.values()]
        ):
            if self.verbose:
                print(f"{optimizer.name} diverged at iteration {iteration}")
            break

    end_time = time.time()

    result = OptimizationRun(
        optimizer_name=optimizer.name,
        losses=losses,
        parameters=parameter_history,
        convergence_time=end_time - start_time,
        final_loss=losses[-1] if losses else float("inf"),
        iterations_to_converge=len(losses) if converged else self.max_iterations,
    )

    if self.verbose:
        status = "converged" if converged else "max iterations reached"
        print(
            f"{optimizer.name}: {status} in {len(losses)} iterations, "
            f"final loss: {result.final_loss:.6f}, time: {result.convergence_time:.3f}s"
        )

    return result

compare_optimizers

compare_optimizers(problem: OptimizationProblem, optimizer_configs: dict[OptimizerType, dict]) -> dict[str, OptimizationRun]

Compare multiple optimizers on a single problem.

Parameters:

Name	Type	Description	Default
problem	OptimizationProblem	Problem to optimize.	required
optimizer_configs	dict	Dictionary mapping optimizer types to their configuration parameters.	required

Returns:

Type	Description
dict	Results for each optimizer.

Source code in src/dlhub/optimizers/comparison.py

def compare_optimizers(
    self, problem: OptimizationProblem, optimizer_configs: dict[OptimizerType, dict]
) -> dict[str, OptimizationRun]:
    """
    Compare multiple optimizers on a single problem.

    Parameters
    ----------
    problem : OptimizationProblem
        Problem to optimize.
    optimizer_configs : dict
        Dictionary mapping optimizer types to their configuration parameters.

    Returns
    -------
    dict
        Results for each optimizer.
    """
    results = {}

    if self.verbose:
        print(f"\nOptimizing: {problem.name}")
        print("=" * 50)

    for opt_type, config in optimizer_configs.items():
        optimizer = self.create_optimizer(opt_type, **config)
        result = self.run_optimization(problem, optimizer)
        results[optimizer.name] = result

    return results

run_comprehensive_comparison

run_comprehensive_comparison() -> dict[str, dict[str, OptimizationRun]]

Run comprehensive comparison across multiple problems and optimizers.

Returns:

Type	Description
dict	Nested dictionary: {problem_name: {optimizer_name: result}}

Source code in src/dlhub/optimizers/comparison.py

def run_comprehensive_comparison(self) -> dict[str, dict[str, OptimizationRun]]:
    """
    Run comprehensive comparison across multiple problems and optimizers.

    Returns
    -------
    dict
        Nested dictionary: {problem_name: {optimizer_name: result}}
    """
    problems = [
        QuadraticBowl(a=1.0, b=1.0),  # Well-conditioned
        QuadraticBowl(a=1.0, b=100.0),  # Ill-conditioned
        RosenbrockFunction(),  # Non-convex valley
        BealeFunction(),  # Multimodal
    ]

    optimizer_configs = {
        OptimizerType.SGD: {"learning_rate": 0.01},
        OptimizerType.MOMENTUM: {"learning_rate": 0.01, "beta": 0.9},
        OptimizerType.RMSPROP: {"learning_rate": 0.01, "beta": 0.9},
        OptimizerType.ADAM: {"learning_rate": 0.05, "beta1": 0.9, "beta2": 0.999},
    }

    all_results = {}

    for problem in problems:
        results = self.compare_optimizers(problem, optimizer_configs)
        all_results[problem.name] = results

    self.results = all_results
    return all_results

plot_convergence_comparison

plot_convergence_comparison(results: dict[str, OptimizationRun], problem_name: str, log_scale: bool = True)

Plot convergence curves for optimizer comparison.

Parameters:

Name	Type	Description	Default
results	dict	Results from optimizer comparison.	required
problem_name	str	Name of the problem for plot title.	required
log_scale	bool	Whether to use logarithmic scale for loss.	True

Source code in src/dlhub/optimizers/comparison.py

def plot_convergence_comparison(
    self,
    results: dict[str, OptimizationRun],
    problem_name: str,
    log_scale: bool = True,
):
    """
    Plot convergence curves for optimizer comparison.

    Parameters
    ----------
    results : dict
        Results from optimizer comparison.
    problem_name : str
        Name of the problem for plot title.
    log_scale : bool, default=True
        Whether to use logarithmic scale for loss.
    """
    plt.figure(figsize=(12, 8))

    colors = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728"]
    styles = ["-", "--", "-.", ":"]

    for i, (opt_name, result) in enumerate(results.items()):
        if result.losses:
            iterations = range(1, len(result.losses) + 1)
            plt.plot(
                iterations,
                result.losses,
                color=colors[i % len(colors)],
                linestyle=styles[i % len(styles)],
                linewidth=2,
                label=opt_name,
                alpha=0.8,
            )

    plt.xlabel("Iterations", fontsize=12)
    plt.ylabel("Loss", fontsize=12)
    plt.title(
        f"Convergence Comparison: {problem_name}", fontsize=14, fontweight="bold"
    )
    plt.legend(fontsize=11)
    plt.grid(True, alpha=0.3)

    if log_scale:
        plt.yscale("log")
        plt.ylabel("Loss (log scale)", fontsize=12)

    plt.tight_layout()
    plt.show()

plot_optimization_paths

plot_optimization_paths(results: dict[str, OptimizationRun], problem: OptimizationProblem, contour_levels: int = 20)

Plot optimization trajectories on loss landscape contours.

Parameters:

Name	Type	Description	Default
results	dict	Results from optimizer comparison.	required
problem	OptimizationProblem	Problem instance for computing loss landscape.	required
contour_levels	int	Number of contour levels to display.	20

Source code in src/dlhub/optimizers/comparison.py

def plot_optimization_paths(
    self,
    results: dict[str, OptimizationRun],
    problem: OptimizationProblem,
    contour_levels: int = 20,
):
    """
    Plot optimization trajectories on loss landscape contours.

    Parameters
    ----------
    results : dict
        Results from optimizer comparison.
    problem : OptimizationProblem
        Problem instance for computing loss landscape.
    contour_levels : int, default=20
        Number of contour levels to display.
    """
    plt.figure(figsize=(14, 10))

    # Create meshgrid for contour plot
    x_range = np.linspace(-6, 6, 100)
    y_range = np.linspace(-6, 6, 100)
    X, Y = np.meshgrid(x_range, y_range)
    Z = np.zeros_like(X)

    for i in range(X.shape[0]):
        for j in range(X.shape[1]):
            params = {"x": X[i, j], "y": Y[i, j]}
            Z[i, j] = problem.loss_function(params)

    contours = plt.contour(
        X, Y, Z, levels=contour_levels, alpha=0.6, cmap="viridis"
    )
    plt.clabel(contours, inline=True, fontsize=8, fmt="%.1f")

    colors = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728"]
    markers = ["o", "s", "^", "D"]

    for i, (opt_name, result) in enumerate(results.items()):
        if result.parameters:
            x_path = [p["x"] for p in result.parameters]
            y_path = [p["y"] for p in result.parameters]

            plt.plot(
                x_path,
                y_path,
                color=colors[i % len(colors)],
                marker=markers[i % len(markers)],
                markersize=4,
                linewidth=2,
                label=f"{opt_name} ({len(x_path)} steps)",
                alpha=0.8,
            )

            # Mark start and end points
            plt.plot(
                x_path[0],
                y_path[0],
                marker="*",
                color=colors[i % len(colors)],
                markersize=12,
                markeredgecolor="black",
                markeredgewidth=1,
            )
            plt.plot(
                x_path[-1],
                y_path[-1],
                marker="x",
                color=colors[i % len(colors)],
                markersize=10,
                markeredgewidth=3,
            )

    plt.xlabel("x", fontsize=12)
    plt.ylabel("y", fontsize=12)
    plt.title(f"Optimization Paths: {problem.name}", fontsize=14, fontweight="bold")
    plt.legend(fontsize=11)
    plt.grid(True, alpha=0.3)
    plt.axis("equal")
    plt.tight_layout()
    plt.show()

generate_summary_table

generate_summary_table(all_results: dict[str, dict[str, OptimizationRun]]) -> None

Generate formatted summary table of optimization results.

Parameters:

Name	Type	Description	Default
all_results	dict	Complete results from comprehensive comparison.	required

Source code in src/dlhub/optimizers/comparison.py

def generate_summary_table(
    self, all_results: dict[str, dict[str, OptimizationRun]]
) -> None:
    """
    Generate formatted summary table of optimization results.

    Parameters
    ----------
    all_results : dict
        Complete results from comprehensive comparison.
    """
    print("\n" + "=" * 80)
    print("OPTIMIZATION COMPARISON SUMMARY")
    print("=" * 80)

    for problem_name, results in all_results.items():
        print(f"\n{problem_name}")
        print("-" * len(problem_name))

        # Create formatted table
        headers = [
            "Optimizer",
            "Final Loss",
            "Iterations",
            "Conv. Time (s)",
            "Status",
        ]
        print(
            f"{headers[0]:<12} {headers[1]:<20} {headers[2]:<12} {headers[3]:<15} {headers[4]:<10}"
        )
        print("-" * 80)

        for opt_name, result in results.items():
            status = (
                "✓"
                if result.iterations_to_converge < self.max_iterations
                else "Max iter"
            )
            print(
                f"{opt_name:<12} {result.final_loss:<14.6f} "
                f"{result.iterations_to_converge:<12} "
                f"{result.convergence_time:<15.3f} {status:<10}"
            )

    print("\n" + "=" * 90)

OptimizationProblem

Base class for defining optimization problems with loss functions and gradients.

Parameters:

Name	Type	Description	Default
name	str	Name of the optimization problem.	required

Source code in src/dlhub/optimizers/comparison.py

class OptimizationProblem:
    """
    Base class for defining optimization problems with loss functions and gradients.

    Parameters
    ----------
    name : str
        Name of the optimization problem.
    """

    def __init__(self, name: str):
        self.name = name

    def loss_function(self, params: dict) -> float:
        """
        Compute loss for given parameters.

        Parameters
        ----------
        params : dict
            Parameter values.

        Returns
        -------
        float
            Loss value.
        """
        raise NotImplementedError("Subclasses must implement loss_function")

    def gradients(self, params: dict) -> dict:
        """
        Compute gradients for given parameters.

        Parameters
        ----------
        params : dict
            Parameter values.

        Returns
        -------
        dict
            Gradients for each parameter.
        """
        raise NotImplementedError("Subclasses must implement gradients")

    def initial_parameters(self) -> dict:
        """
        Get initial parameter values for optimization.

        Returns
        -------
        dict
            Initial parameter values.
        """
        raise NotImplementedError("Subclasses must implement initial_parameters")

loss_function

loss_function(params: dict) -> float

Compute loss for given parameters.

Parameters:

Name	Type	Description	Default
params	dict	Parameter values.	required

Returns:

Type	Description
float	Loss value.

Source code in src/dlhub/optimizers/comparison.py

def loss_function(self, params: dict) -> float:
    """
    Compute loss for given parameters.

    Parameters
    ----------
    params : dict
        Parameter values.

    Returns
    -------
    float
        Loss value.
    """
    raise NotImplementedError("Subclasses must implement loss_function")

gradients

gradients(params: dict) -> dict

Compute gradients for given parameters.

Parameters:

Name	Type	Description	Default
params	dict	Parameter values.	required

Returns:

Type	Description
dict	Gradients for each parameter.

Source code in src/dlhub/optimizers/comparison.py

def gradients(self, params: dict) -> dict:
    """
    Compute gradients for given parameters.

    Parameters
    ----------
    params : dict
        Parameter values.

    Returns
    -------
    dict
        Gradients for each parameter.
    """
    raise NotImplementedError("Subclasses must implement gradients")

initial_parameters

initial_parameters() -> dict

Get initial parameter values for optimization.

Returns:

Type	Description
dict	Initial parameter values.

Source code in src/dlhub/optimizers/comparison.py

def initial_parameters(self) -> dict:
    """
    Get initial parameter values for optimization.

    Returns
    -------
    dict
        Initial parameter values.
    """
    raise NotImplementedError("Subclasses must implement initial_parameters")

OptimizationRun dataclass

The trace of one optimizer descending one problem, and its summary.

Not to be confused with :class:dlhub.tuning.ExperimentResult, which records the outcome of a hyperparameter search rather than a single descent.

Attributes:

Name	Type	Description
optimizer_name	str	Name of the optimizer used.
losses	List[float]	Loss values recorded during training.
parameters	List[Dict]	Parameter values at each iteration.
convergence_time	float	Time taken for convergence (in seconds).
final_loss	float	Final loss value achieved.
iterations_to_converge	int	Number of iterations required for convergence.

Source code in src/dlhub/optimizers/comparison.py

@dataclass
class OptimizationRun:
    """
    The trace of one optimizer descending one problem, and its summary.

    Not to be confused with :class:`dlhub.tuning.ExperimentResult`, which records
    the outcome of a hyperparameter search rather than a single descent.

    Attributes
    ----------
    optimizer_name : str
        Name of the optimizer used.
    losses : List[float]
        Loss values recorded during training.
    parameters : List[Dict]
        Parameter values at each iteration.
    convergence_time : float
        Time taken for convergence (in seconds).
    final_loss : float
        Final loss value achieved.
    iterations_to_converge : int
        Number of iterations required for convergence.
    """

    optimizer_name: str
    losses: list[float]
    parameters: list[dict]
    convergence_time: float
    final_loss: float
    iterations_to_converge: int

OptimizerType

Bases: Enum

Enumeration of available optimizer types.

Source code in src/dlhub/optimizers/comparison.py

class OptimizerType(Enum):
    """Enumeration of available optimizer types."""

    SGD = "sgd"
    MOMENTUM = "momentum"
    RMSPROP = "rmsprop"
    ADAM = "adam"

QuadraticBowl

Bases: OptimizationProblem

Simple quadratic bowl optimization problem: f(x,y) = ax² + by².

Well-conditioned convex problem useful for demonstrating basic optimizer behavior.

Parameters:

Name	Type	Description	Default
a	float	Coefficient for x² term.	1.0
b	float	Coefficient for y² term.	1.0

Source code in src/dlhub/optimizers/comparison.py

class QuadraticBowl(OptimizationProblem):
    """
    Simple quadratic bowl optimization problem: f(x,y) = ax² + by².

    Well-conditioned convex problem useful for demonstrating basic optimizer behavior.

    Parameters
    ----------
    a : float, default=1.0
        Coefficient for x² term.
    b : float, default=1.0
        Coefficient for y² term.
    """

    def __init__(self, a: float = 1.0, b: float = 1.0):
        super().__init__(f"Quadratic Bowl (a={a}, b={b})")
        self.a = a
        self.b = b

    def loss_function(self, params: dict) -> float:
        """Compute quadratic loss: ax² + by²."""
        x, y = params["x"], params["y"]
        return self.a * x**2 + self.b * y**2

    def gradients(self, params: dict) -> dict:
        """Compute gradients: [2ax, 2by]."""
        x, y = params["x"], params["y"]
        return {"x": 2 * self.a * x, "y": 2 * self.b * y}

    def initial_parameters(self) -> dict:
        """Initialize at (5, 5) for clear visualization."""
        return {"x": 5.0, "y": 5.0}

loss_function

loss_function(params: dict) -> float

Compute quadratic loss: ax² + by².

Source code in src/dlhub/optimizers/comparison.py

def loss_function(self, params: dict) -> float:
    """Compute quadratic loss: ax² + by²."""
    x, y = params["x"], params["y"]
    return self.a * x**2 + self.b * y**2

gradients

gradients(params: dict) -> dict

Compute gradients: [2ax, 2by].

Source code in src/dlhub/optimizers/comparison.py

def gradients(self, params: dict) -> dict:
    """Compute gradients: [2ax, 2by]."""
    x, y = params["x"], params["y"]
    return {"x": 2 * self.a * x, "y": 2 * self.b * y}

initial_parameters

initial_parameters() -> dict

Initialize at (5, 5) for clear visualization.

Source code in src/dlhub/optimizers/comparison.py

def initial_parameters(self) -> dict:
    """Initialize at (5, 5) for clear visualization."""
    return {"x": 5.0, "y": 5.0}

RosenbrockFunction

Bases: OptimizationProblem

Rosenbrock function: f(x,y) = (a-x)² + b(y-x²)².

Classic non-convex optimization benchmark with narrow curved valley. Challenging for optimizers due to ill-conditioning and plateau regions.

Parameters:

Name	Type	Description	Default
a	float	Parameter controlling x-offset of minimum.	1.0
b	float	Parameter controlling valley curvature (higher = more challenging).	100.0

Source code in src/dlhub/optimizers/comparison.py

class RosenbrockFunction(OptimizationProblem):
    """
    Rosenbrock function: f(x,y) = (a-x)² + b(y-x²)².

    Classic non-convex optimization benchmark with narrow curved valley.
    Challenging for optimizers due to ill-conditioning and plateau regions.

    Parameters
    ----------
    a : float, default=1.0
        Parameter controlling x-offset of minimum.
    b : float, default=100.0
        Parameter controlling valley curvature (higher = more challenging).
    """

    def __init__(self, a: float = 1.0, b: float = 100.0):
        super().__init__(f"Rosenbrock Function (a={a}, b={b})")
        self.a = a
        self.b = b

    def loss_function(self, params: dict) -> float:
        """Compute Rosenbrock function value."""
        x, y = params["x"], params["y"]
        return (self.a - x) ** 2 + self.b * (y - x**2) ** 2

    def gradients(self, params: dict) -> dict:
        """Compute Rosenbrock gradients analytically."""
        x, y = params["x"], params["y"]
        dx = -2 * (self.a - x) - 4 * self.b * x * (y - x**2)
        dy = 2 * self.b * (y - x**2)
        return {"x": dx, "y": dy}

    def initial_parameters(self) -> dict:
        """Initialize away from minimum for interesting optimization path."""
        return {"x": -2.0, "y": 2.0}

loss_function

loss_function(params: dict) -> float

Compute Rosenbrock function value.

Source code in src/dlhub/optimizers/comparison.py

def loss_function(self, params: dict) -> float:
    """Compute Rosenbrock function value."""
    x, y = params["x"], params["y"]
    return (self.a - x) ** 2 + self.b * (y - x**2) ** 2

gradients

gradients(params: dict) -> dict

Compute Rosenbrock gradients analytically.

Source code in src/dlhub/optimizers/comparison.py

def gradients(self, params: dict) -> dict:
    """Compute Rosenbrock gradients analytically."""
    x, y = params["x"], params["y"]
    dx = -2 * (self.a - x) - 4 * self.b * x * (y - x**2)
    dy = 2 * self.b * (y - x**2)
    return {"x": dx, "y": dy}

initial_parameters

initial_parameters() -> dict

Initialize away from minimum for interesting optimization path.

Source code in src/dlhub/optimizers/comparison.py

def initial_parameters(self) -> dict:
    """Initialize away from minimum for interesting optimization path."""
    return {"x": -2.0, "y": 2.0}

AveragingStrategy

Bases: Enum

Enumeration of different averaging strategies.

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

class AveragingStrategy(Enum):
    """Enumeration of different averaging strategies."""

    SIMPLE = "simple"
    BIAS_CORRECTED = "bias_corrected"
    VARIANCE_CORRECTED = "variance_corrected"
    EXPONENTIAL_DECAY = "exponential_decay"

ExponentialWeightedAverage

Exponential Weighted Average with bias correction and multiple strategies.

This class implements exponential weighted averages (also known as exponentially weighted moving averages) with various correction techniques commonly used in deep learning optimization algorithms.

The basic formula is: v_t = beta * v_{t-1} + (1 - beta) * theta_t

Where: - v_t is the average at time t - beta is the decay parameter - theta_t is the current value - v_0 = 0 (initial value)

Parameters:

Name	Type	Description	Default
beta	float	Decay parameter (0 < beta < 1). Higher values give more weight to past values. Common values: 0.9 (momentum), 0.999 (second moments in Adam)	0.9
bias_correction	bool	Whether to apply bias correction to account for initialization bias	True
strategy	AveragingStrategy	Averaging strategy to use	AveragingStrategy.BIAS_CORRECTED
epsilon	float	Small constant for numerical stability	1e-8
warmup_steps	int	Number of warmup steps before applying full averaging	0

Attributes:

Name	Type	Description
v	float or ndarray	Current average value
t	int	Time step (number of updates)
history	list	History of average values

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

class ExponentialWeightedAverage:
    """
    Exponential Weighted Average with bias correction and multiple strategies.

    This class implements exponential weighted averages (also known as exponentially
    weighted moving averages) with various correction techniques commonly used in
    deep learning optimization algorithms.

    The basic formula is:
        v_t = beta * v_{t-1} + (1 - beta) * theta_t

    Where:
        - v_t is the average at time t
        - beta is the decay parameter
        - theta_t is the current value
        - v_0 = 0 (initial value)

    Parameters
    ----------
    beta : float, default=0.9
        Decay parameter (0 < beta < 1). Higher values give more weight to past values.
        Common values: 0.9 (momentum), 0.999 (second moments in Adam)
    bias_correction : bool, default=True
        Whether to apply bias correction to account for initialization bias
    strategy : AveragingStrategy, default=AveragingStrategy.BIAS_CORRECTED
        Averaging strategy to use
    epsilon : float, default=1e-8
        Small constant for numerical stability
    warmup_steps : int, default=0
        Number of warmup steps before applying full averaging

    Attributes
    ----------
    v : float or np.ndarray
        Current average value
    t : int
        Time step (number of updates)
    history : list
        History of average values
    """

    def __init__(
        self,
        beta: float = 0.9,
        bias_correction: bool = True,
        strategy: AveragingStrategy = AveragingStrategy.BIAS_CORRECTED,
        epsilon: float = 1e-8,
        warmup_steps: int = 0,
    ):
        if not 0.0 < beta < 1.0:
            raise ValueError(f"Beta must be in (0, 1), got {beta}")
        if epsilon <= 0:
            raise ValueError(f"Epsilon must be positive, got {epsilon}")
        if warmup_steps < 0:
            raise ValueError(f"Warmup steps must be non-negative, got {warmup_steps}")

        self.beta = beta
        self.bias_correction = bias_correction
        self.strategy = strategy
        self.epsilon = epsilon
        self.warmup_steps = warmup_steps

        self.v = None
        self.t = 0
        self.history = []

        # Cumulative weight the accumulator has actually applied to its samples.
        # The textbook correction factor 1 - beta**t is this quantity's closed form
        # for a *constant* beta. Warm-up varies beta with t, which invalidates that
        # closed form, so the weight is tracked directly instead. See Notes.
        self.weight = 0.0

        self.squared_avg = None
        self.variance_history = []

    def update(self, value: float | np.ndarray) -> float | np.ndarray:
        """
        Update the exponential weighted average with a new value.

        Parameters
        ----------
        value : float or np.ndarray
            New value to incorporate into the average

        Returns
        -------
        float or np.ndarray
            Updated average value
        """
        self.t += 1

        if isinstance(value, np.ndarray):
            if np.any(np.isnan(value)) or np.any(np.isinf(value)):
                warnings.warn("NaN or Inf detected in input value")
                value = np.nan_to_num(value, nan=0.0, posinf=1e6, neginf=-1e6)
        else:
            if np.isnan(value) or np.isinf(value):
                warnings.warn("NaN or Inf detected in input value")
                value = 0.0 if np.isnan(value) else (1e6 if value > 0 else -1e6)

        if self.v is None:
            if isinstance(value, np.ndarray):
                self.v = np.zeros_like(value, dtype=np.float64)
                if self.strategy == AveragingStrategy.VARIANCE_CORRECTED:
                    self.squared_avg = np.zeros_like(value, dtype=np.float64)
            else:
                self.v = 0.0
                if self.strategy == AveragingStrategy.VARIANCE_CORRECTED:
                    self.squared_avg = 0.0

        if self.t <= self.warmup_steps:
            effective_beta = min(self.beta, (self.t - 1) / self.t)
        else:
            effective_beta = self.beta

        if self.strategy == AveragingStrategy.EXPONENTIAL_DECAY:
            effective_beta = effective_beta ** (self.t / 1000)  # Decay over time

        # Track the weight with whichever coefficient this step actually applied,
        # so the two stay consistent for every strategy.
        self.weight = effective_beta * self.weight + (1 - effective_beta)

        if self.strategy == AveragingStrategy.SIMPLE:
            self.v = effective_beta * self.v + (1 - effective_beta) * value
            result = self.v

        elif self.strategy == AveragingStrategy.BIAS_CORRECTED:
            self.v = effective_beta * self.v + (1 - effective_beta) * value
            if self.bias_correction:
                result = self.v / (self.weight + self.epsilon)
            else:
                result = self.v

        elif self.strategy == AveragingStrategy.VARIANCE_CORRECTED:
            self.v = effective_beta * self.v + (1 - effective_beta) * value
            self.squared_avg = effective_beta * self.squared_avg + (
                1 - effective_beta
            ) * (value**2)

            if self.bias_correction:
                mean_corrected = self.v / (self.weight + self.epsilon)
                variance_corrected = self.squared_avg / (self.weight + self.epsilon)

                variance = variance_corrected - mean_corrected**2
                self.variance_history.append(variance)
                result = mean_corrected
            else:
                result = self.v

        elif self.strategy == AveragingStrategy.EXPONENTIAL_DECAY:
            self.v = effective_beta * self.v + (1 - effective_beta) * value
            result = self.v

        else:
            raise ValueError(f"Unknown averaging strategy: {self.strategy}")

        self.history.append(result.copy() if isinstance(result, np.ndarray) else result)
        return result

    def get_current_average(self) -> float | np.ndarray | None:
        """
        Get the current average value.

        Returns
        -------
        float, np.ndarray, or None
            Current average value, None if no updates have been made
        """
        if self.v is None:
            return None

        if self.strategy == AveragingStrategy.BIAS_CORRECTED and self.bias_correction:
            return self.v / (self.weight + self.epsilon)
        else:
            return self.v

    def get_variance(self) -> float | np.ndarray | None:
        """
        Get the current variance estimate (only available with VARIANCE_CORRECTED strategy).

        Returns
        -------
        float, np.ndarray, or None
            Current variance estimate, None if not available
        """
        if (
            self.strategy != AveragingStrategy.VARIANCE_CORRECTED
            or self.squared_avg is None
        ):
            return None

        if self.bias_correction:
            mean_corrected = self.v / (self.weight + self.epsilon)
            variance_corrected = self.squared_avg / (self.weight + self.epsilon)
            return variance_corrected - mean_corrected**2
        else:
            return self.squared_avg - self.v**2

    def reset(self) -> None:
        """Reset the exponential weighted average to initial state."""
        self.v = None
        self.squared_avg = None
        self.t = 0
        self.weight = 0.0
        self.history.clear()
        self.variance_history.clear()

    def get_effective_window_size(self) -> float:
        """
        Get the effective window size of the exponential weighted average.

        The effective window size is approximately 1/(1-beta).

        Returns
        -------
        float
            Effective window size
        """
        return 1.0 / (1.0 - self.beta)

    def get_config(self) -> dict[str, Any]:
        """
        Get configuration dictionary.

        Returns
        -------
        dict
            Configuration dictionary
        """
        return {
            "beta": self.beta,
            "bias_correction": self.bias_correction,
            "strategy": self.strategy.value,
            "epsilon": self.epsilon,
            "warmup_steps": self.warmup_steps,
            "time_step": self.t,
        }

    def get_state(self) -> dict[str, Any]:
        """
        Get complete state dictionary.

        Returns
        -------
        dict
            Complete state dictionary
        """
        return {
            "config": self.get_config(),
            "v": self.v,
            "weight": self.weight,
            "squared_avg": self.squared_avg,
            "history": self.history.copy(),
            "variance_history": self.variance_history.copy(),
        }

    def load_state(self, state: dict[str, Any]) -> None:
        """
        Load state from dictionary.

        Parameters
        ----------
        state : dict
            State dictionary from get_state()
        """
        config = state["config"]
        self.beta = config["beta"]
        self.bias_correction = config["bias_correction"]
        self.strategy = AveragingStrategy(config["strategy"])
        self.epsilon = config["epsilon"]
        self.warmup_steps = config["warmup_steps"]
        self.t = config["time_step"]

        self.v = state["v"]
        # Older states predate the tracked weight. Fall back to the constant-beta
        # closed form, which is exact whenever warmup_steps is 0.
        self.weight = state.get("weight", 1 - self.beta**self.t if self.t else 0.0)
        self.squared_avg = state["squared_avg"]
        self.history = state["history"].copy()
        self.variance_history = state["variance_history"].copy()

update

update(value: float | ndarray) -> float | np.ndarray

Update the exponential weighted average with a new value.

Parameters:

Name	Type	Description	Default
value	float or ndarray	New value to incorporate into the average	required

Returns:

Type	Description
float or ndarray	Updated average value

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def update(self, value: float | np.ndarray) -> float | np.ndarray:
    """
    Update the exponential weighted average with a new value.

    Parameters
    ----------
    value : float or np.ndarray
        New value to incorporate into the average

    Returns
    -------
    float or np.ndarray
        Updated average value
    """
    self.t += 1

    if isinstance(value, np.ndarray):
        if np.any(np.isnan(value)) or np.any(np.isinf(value)):
            warnings.warn("NaN or Inf detected in input value")
            value = np.nan_to_num(value, nan=0.0, posinf=1e6, neginf=-1e6)
    else:
        if np.isnan(value) or np.isinf(value):
            warnings.warn("NaN or Inf detected in input value")
            value = 0.0 if np.isnan(value) else (1e6 if value > 0 else -1e6)

    if self.v is None:
        if isinstance(value, np.ndarray):
            self.v = np.zeros_like(value, dtype=np.float64)
            if self.strategy == AveragingStrategy.VARIANCE_CORRECTED:
                self.squared_avg = np.zeros_like(value, dtype=np.float64)
        else:
            self.v = 0.0
            if self.strategy == AveragingStrategy.VARIANCE_CORRECTED:
                self.squared_avg = 0.0

    if self.t <= self.warmup_steps:
        effective_beta = min(self.beta, (self.t - 1) / self.t)
    else:
        effective_beta = self.beta

    if self.strategy == AveragingStrategy.EXPONENTIAL_DECAY:
        effective_beta = effective_beta ** (self.t / 1000)  # Decay over time

    # Track the weight with whichever coefficient this step actually applied,
    # so the two stay consistent for every strategy.
    self.weight = effective_beta * self.weight + (1 - effective_beta)

    if self.strategy == AveragingStrategy.SIMPLE:
        self.v = effective_beta * self.v + (1 - effective_beta) * value
        result = self.v

    elif self.strategy == AveragingStrategy.BIAS_CORRECTED:
        self.v = effective_beta * self.v + (1 - effective_beta) * value
        if self.bias_correction:
            result = self.v / (self.weight + self.epsilon)
        else:
            result = self.v

    elif self.strategy == AveragingStrategy.VARIANCE_CORRECTED:
        self.v = effective_beta * self.v + (1 - effective_beta) * value
        self.squared_avg = effective_beta * self.squared_avg + (
            1 - effective_beta
        ) * (value**2)

        if self.bias_correction:
            mean_corrected = self.v / (self.weight + self.epsilon)
            variance_corrected = self.squared_avg / (self.weight + self.epsilon)

            variance = variance_corrected - mean_corrected**2
            self.variance_history.append(variance)
            result = mean_corrected
        else:
            result = self.v

    elif self.strategy == AveragingStrategy.EXPONENTIAL_DECAY:
        self.v = effective_beta * self.v + (1 - effective_beta) * value
        result = self.v

    else:
        raise ValueError(f"Unknown averaging strategy: {self.strategy}")

    self.history.append(result.copy() if isinstance(result, np.ndarray) else result)
    return result

get_current_average

get_current_average() -> float | np.ndarray | None

Get the current average value.

Returns:

Type	Description
float, np.ndarray, or None	Current average value, None if no updates have been made

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def get_current_average(self) -> float | np.ndarray | None:
    """
    Get the current average value.

    Returns
    -------
    float, np.ndarray, or None
        Current average value, None if no updates have been made
    """
    if self.v is None:
        return None

    if self.strategy == AveragingStrategy.BIAS_CORRECTED and self.bias_correction:
        return self.v / (self.weight + self.epsilon)
    else:
        return self.v

get_variance

get_variance() -> float | np.ndarray | None

Get the current variance estimate (only available with VARIANCE_CORRECTED strategy).

Returns:

Type	Description
float, np.ndarray, or None	Current variance estimate, None if not available

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def get_variance(self) -> float | np.ndarray | None:
    """
    Get the current variance estimate (only available with VARIANCE_CORRECTED strategy).

    Returns
    -------
    float, np.ndarray, or None
        Current variance estimate, None if not available
    """
    if (
        self.strategy != AveragingStrategy.VARIANCE_CORRECTED
        or self.squared_avg is None
    ):
        return None

    if self.bias_correction:
        mean_corrected = self.v / (self.weight + self.epsilon)
        variance_corrected = self.squared_avg / (self.weight + self.epsilon)
        return variance_corrected - mean_corrected**2
    else:
        return self.squared_avg - self.v**2

reset

reset() -> None

Reset the exponential weighted average to initial state.

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def reset(self) -> None:
    """Reset the exponential weighted average to initial state."""
    self.v = None
    self.squared_avg = None
    self.t = 0
    self.weight = 0.0
    self.history.clear()
    self.variance_history.clear()

get_effective_window_size

get_effective_window_size() -> float

Get the effective window size of the exponential weighted average.

The effective window size is approximately 1/(1-beta).

Returns:

Type	Description
float	Effective window size

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def get_effective_window_size(self) -> float:
    """
    Get the effective window size of the exponential weighted average.

    The effective window size is approximately 1/(1-beta).

    Returns
    -------
    float
        Effective window size
    """
    return 1.0 / (1.0 - self.beta)

get_config

get_config() -> dict[str, Any]

Get configuration dictionary.

Returns:

Type	Description
dict	Configuration dictionary

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def get_config(self) -> dict[str, Any]:
    """
    Get configuration dictionary.

    Returns
    -------
    dict
        Configuration dictionary
    """
    return {
        "beta": self.beta,
        "bias_correction": self.bias_correction,
        "strategy": self.strategy.value,
        "epsilon": self.epsilon,
        "warmup_steps": self.warmup_steps,
        "time_step": self.t,
    }

get_state

get_state() -> dict[str, Any]

Get complete state dictionary.

Returns:

Type	Description
dict	Complete state dictionary

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def get_state(self) -> dict[str, Any]:
    """
    Get complete state dictionary.

    Returns
    -------
    dict
        Complete state dictionary
    """
    return {
        "config": self.get_config(),
        "v": self.v,
        "weight": self.weight,
        "squared_avg": self.squared_avg,
        "history": self.history.copy(),
        "variance_history": self.variance_history.copy(),
    }

load_state

load_state(state: dict[str, Any]) -> None

Load state from dictionary.

Parameters:

Name	Type	Description	Default
state	dict	State dictionary from get_state()	required

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def load_state(self, state: dict[str, Any]) -> None:
    """
    Load state from dictionary.

    Parameters
    ----------
    state : dict
        State dictionary from get_state()
    """
    config = state["config"]
    self.beta = config["beta"]
    self.bias_correction = config["bias_correction"]
    self.strategy = AveragingStrategy(config["strategy"])
    self.epsilon = config["epsilon"]
    self.warmup_steps = config["warmup_steps"]
    self.t = config["time_step"]

    self.v = state["v"]
    # Older states predate the tracked weight. Fall back to the constant-beta
    # closed form, which is exact whenever warmup_steps is 0.
    self.weight = state.get("weight", 1 - self.beta**self.t if self.t else 0.0)
    self.squared_avg = state["squared_avg"]
    self.history = state["history"].copy()
    self.variance_history = state["variance_history"].copy()

MultiVariateEWA

Multi-variate Exponential Weighted Average for handling multiple variables simultaneously.

This class manages multiple exponential weighted averages, commonly used in optimization algorithms where different parameters need separate averages.

Parameters:

Name	Type	Description	Default
beta	float	Common decay parameter for all variables	0.9
bias_correction	bool	Whether to apply bias correction	True
strategy	AveragingStrategy	Averaging strategy	AveragingStrategy.BIAS_CORRECTED
**kwargs		Additional parameters passed to individual EWA instances	{}

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

class MultiVariateEWA:
    """
    Multi-variate Exponential Weighted Average for handling multiple variables simultaneously.

    This class manages multiple exponential weighted averages, commonly used in
    optimization algorithms where different parameters need separate averages.

    Parameters
    ----------
    beta : float, default=0.9
        Common decay parameter for all variables
    bias_correction : bool, default=True
        Whether to apply bias correction
    strategy : AveragingStrategy, default=AveragingStrategy.BIAS_CORRECTED
        Averaging strategy
    **kwargs
        Additional parameters passed to individual EWA instances
    """

    def __init__(
        self,
        beta: float = 0.9,
        bias_correction: bool = True,
        strategy: AveragingStrategy = AveragingStrategy.BIAS_CORRECTED,
        **kwargs,
    ):
        self.beta = beta
        self.bias_correction = bias_correction
        self.strategy = strategy
        self.kwargs = kwargs

        self.averages: dict[str, ExponentialWeightedAverage] = {}

    def update(
        self, values: dict[str, float | np.ndarray]
    ) -> dict[str, float | np.ndarray]:
        """
        Update all averages with new values.

        Parameters
        ----------
        values : dict
            Dictionary of new values for each variable

        Returns
        -------
        dict
            Dictionary of updated averages
        """
        results = {}

        for name, value in values.items():
            if name not in self.averages:
                self.averages[name] = ExponentialWeightedAverage(
                    beta=self.beta,
                    bias_correction=self.bias_correction,
                    strategy=self.strategy,
                    **self.kwargs,
                )

            results[name] = self.averages[name].update(value)

        return results

    def get_averages(self) -> dict[str, float | np.ndarray]:
        """Get current averages for all variables."""
        return {name: ewa.get_current_average() for name, ewa in self.averages.items()}

    def reset(self) -> None:
        """Reset all averages."""
        for ewa in self.averages.values():
            ewa.reset()

    def get_state(self) -> dict[str, Any]:
        """Get complete state for all averages."""
        return {
            "config": {
                "beta": self.beta,
                "bias_correction": self.bias_correction,
                "strategy": self.strategy.value,
                "kwargs": self.kwargs,
            },
            "averages": {name: ewa.get_state() for name, ewa in self.averages.items()},
        }

update

update(values: dict[str, float | ndarray]) -> dict[str, float | np.ndarray]

Update all averages with new values.

Parameters:

Name	Type	Description	Default
values	dict	Dictionary of new values for each variable	required

Returns:

Type	Description
dict	Dictionary of updated averages

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def update(
    self, values: dict[str, float | np.ndarray]
) -> dict[str, float | np.ndarray]:
    """
    Update all averages with new values.

    Parameters
    ----------
    values : dict
        Dictionary of new values for each variable

    Returns
    -------
    dict
        Dictionary of updated averages
    """
    results = {}

    for name, value in values.items():
        if name not in self.averages:
            self.averages[name] = ExponentialWeightedAverage(
                beta=self.beta,
                bias_correction=self.bias_correction,
                strategy=self.strategy,
                **self.kwargs,
            )

        results[name] = self.averages[name].update(value)

    return results

get_averages

get_averages() -> dict[str, float | np.ndarray]

Get current averages for all variables.

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def get_averages(self) -> dict[str, float | np.ndarray]:
    """Get current averages for all variables."""
    return {name: ewa.get_current_average() for name, ewa in self.averages.items()}

reset

reset() -> None

Reset all averages.

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def reset(self) -> None:
    """Reset all averages."""
    for ewa in self.averages.values():
        ewa.reset()

get_state

get_state() -> dict[str, Any]

Get complete state for all averages.

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def get_state(self) -> dict[str, Any]:
    """Get complete state for all averages."""
    return {
        "config": {
            "beta": self.beta,
            "bias_correction": self.bias_correction,
            "strategy": self.strategy.value,
            "kwargs": self.kwargs,
        },
        "averages": {name: ewa.get_state() for name, ewa in self.averages.items()},
    }

MiniBatchGradientDescent

Mini-batch Gradient Descent optimizer with configurable batch size and shuffling.

This implementation provides efficient mini-batch processing with proper data shuffling and batch creation for neural network training.

Parameters:

Name	Type	Description	Default
learning_rate	float	Learning rate for gradient descent updates	0.001
batch_size	int	Size of mini-batches for training	64
shuffle	bool	Whether to shuffle data at each epoch	True
random_seed	int	Random seed for reproducibility	None

Attributes:

Name	Type	Description
learning_rate	float	Current learning rate
batch_size	int	Mini-batch size
shuffle	bool	Shuffling flag
history	dict	Training history including losses and metrics

Source code in src/dlhub/optimizers/mini_batch.py

class MiniBatchGradientDescent:
    """
    Mini-batch Gradient Descent optimizer with configurable batch size and shuffling.

    This implementation provides efficient mini-batch processing with proper data
    shuffling and batch creation for neural network training.

    Parameters
    ----------
    learning_rate : float, default=0.001
        Learning rate for gradient descent updates
    batch_size : int, default=64
        Size of mini-batches for training
    shuffle : bool, default=True
        Whether to shuffle data at each epoch
    random_seed : int, optional
        Random seed for reproducibility

    Attributes
    ----------
    learning_rate : float
        Current learning rate
    batch_size : int
        Mini-batch size
    shuffle : bool
        Shuffling flag
    history : dict
        Training history including losses and metrics
    """

    def __init__(
        self,
        learning_rate: float = 0.001,
        batch_size: int = 64,
        shuffle: bool = True,
        random_seed: int | None = None,
    ):
        self.learning_rate = learning_rate
        self.batch_size = batch_size
        self.shuffle = shuffle
        self.history = {"loss": [], "accuracy": []}

        if random_seed is not None:
            np.random.seed(random_seed)
            random.seed(random_seed)

    def create_mini_batches(
        self, X: np.ndarray, Y: np.ndarray
    ) -> list[tuple[np.ndarray, np.ndarray]]:
        """
        Create mini-batches from training data with optional shuffling.

        Parameters
        ----------
        X : np.ndarray
            Input features of shape (n_features, m_examples)
        Y : np.ndarray
            Target labels of shape (n_classes, m_examples)

        Returns
        -------
        List[Tuple[np.ndarray, np.ndarray]]
            List of (X_batch, Y_batch) tuples

        Notes
        -----
        If shuffle is True, data is randomly permuted before creating batches.
        The last batch may be smaller if the dataset size is not divisible by batch_size.
        """
        m = X.shape[1]
        mini_batchs = []

        if self.shuffle:
            permutation = np.random.permutation(m)
            X_shuffled = X[:, permutation]
            Y_shuffled = Y[:, permutation]
        else:
            X_shuffled = X
            Y_shuffled = Y

        full_batches = m // self.batch_size
        for k in range(full_batches):
            start = k * self.batch_size
            end = start + self.batch_size

            X_batch = X_shuffled[:, start:end]
            Y_batch = Y_shuffled[:, start:end]

            mini_batchs.append((X_batch, Y_batch))

        if m % self.batch_size:
            start = full_batches * self.batch_size
            X_batch = X_shuffled[:, start:]
            Y_batch = Y_shuffled[:, start:]
            mini_batchs.append((X_batch, Y_batch))

        return mini_batchs

    def update_parameters(
        self, parameters: dict[str, np.ndarray], gradients: dict[str, np.ndarray]
    ) -> dict[str, np.ndarray]:
        """
        Update model parameters using gradient descent.

        Parameters
        ----------
        parameters : Dict[str, np.ndarray]
            Current model parameters
        gradients : Dict[str, np.ndarray]
            Computed gradients for each parameter

        Returns
        -------
        Dict[str, np.ndarray]
            Updated parameters

        Notes
        -----
        Updates parameters using the standard gradient descent rule:
        θ = θ - α * ∇J(θ)
        """
        updated_parameters = {}

        for key in parameters:
            updated_parameters[key] = (
                parameters[key] - self.learning_rate * gradients[key]
            )

        return updated_parameters

    def train_epoch(
        self,
        X: np.ndarray,
        Y: np.ndarray,
        parameters: dict[str, np.ndarray],
        forward_propagation_fn: callable,
        backward_propagation_fn: callable,
        compute_cost_fn: callable,
    ) -> tuple[dict[str, np.ndarray], float]:
        """
        Train for one epoch using mini-batch gradient descent.

        Parameters
        ----------
        X : np.ndarray
            Input features of shape (n_features, m_examples)
        Y : np.ndarray
            Target labels of shape (n_classes, m_examples)
        parameters : Dict[str, np.ndarray]
            Current model parameters
        forward_propagation_fn : callable
            Function to compute forward propagation
        backward_propagation_fn : callable
            Function to compute backward propagation
        compute_cost_fn : callable
            Function to compute cost/loss

        Returns
        -------
        Tuple[Dict[str, np.ndarray], float]
            Updated parameters and epoch loss

        Notes
        -----
        Performs one complete epoch of mini-batch gradient descent training.
        """
        epoch_cost = 0.0
        mini_batches = self.create_mini_batches(X, Y)

        for X_batch, Y_batch in mini_batches:
            AL, caches = forward_propagation_fn(X_batch, parameters)
            epoch_cost += compute_cost_fn(AL, Y_batch)
            gradients = backward_propagation_fn(AL, Y_batch, caches)
            parameters = self.update_parameters(parameters, gradients)

        epoch_cost /= len(mini_batches)
        return parameters, epoch_cost

    def fit(
        self,
        X: np.ndarray,
        Y: np.ndarray,
        parameters: dict[str, np.ndarray],
        forward_propagation_fn: callable,
        backward_propagation_fn: callable,
        compute_cost_fn: callable,
        epochs: int = 1000,
        print_cost: bool = True,
        print_every: int = 100,
    ) -> dict[str, np.ndarray]:
        """
        Train the model using mini-batch gradient descent.

        Parameters
        ----------
        X : np.ndarray
            Input features of shape (n_features, m_examples)
        Y : np.ndarray
            Target labels of shape (n_classes, m_examples)
        parameters : Dict[str, np.ndarray]
            Initial model parameters
        forward_propagation_fn : callable
            Function to compute forward propagation
        backward_propagation_fn : callable
            Function to compute backward propagation
        compute_cost_fn : callable
            Function to compute cost/loss
        epochs : int, default=1000
            Number of training epochs
        print_cost : bool, default=True
            Whether to print cost during training
        print_every : int, default=100
            Print cost every N epochs

        Returns
        -------
        Dict[str, np.ndarray]
            Trained parameters

        Notes
        -----
        Trains the model for the specified number of epochs using mini-batch
        gradient descent. Training history is stored in self.history.
        """
        for epoch in range(epochs):
            parameters, epoch_cost = self.train_epoch(
                X,
                Y,
                parameters,
                forward_propagation_fn,
                backward_propagation_fn,
                compute_cost_fn,
            )

            self.history["loss"].append(epoch_cost)

            if print_cost and epoch % print_every == 0:
                print(f"Epoch {epoch}: Cost = {epoch_cost:.6f}")

        return parameters

    def get_config(self) -> dict[str, Any]:
        """
        Get optimizer configuration.

        Returns
        -------
        Dict[str, Any]
            Configuration dictionary
        """
        return {
            "learning_rate": self.learning_rate,
            "batch_size": self.batch_size,
            "shuffle": self.shuffle,
            "optimizer": "MiniBatchGradientDescent",
        }

create_mini_batches

create_mini_batches(X: ndarray, Y: ndarray) -> list[tuple[np.ndarray, np.ndarray]]

Create mini-batches from training data with optional shuffling.

Parameters:

Name	Type	Description	Default
X	ndarray	Input features of shape (n_features, m_examples)	required
Y	ndarray	Target labels of shape (n_classes, m_examples)	required

Returns:

Type	Description
List[Tuple[ndarray, ndarray]]	List of (X_batch, Y_batch) tuples

Notes

If shuffle is True, data is randomly permuted before creating batches. The last batch may be smaller if the dataset size is not divisible by batch_size.

Source code in src/dlhub/optimizers/mini_batch.py

def create_mini_batches(
    self, X: np.ndarray, Y: np.ndarray
) -> list[tuple[np.ndarray, np.ndarray]]:
    """
    Create mini-batches from training data with optional shuffling.

    Parameters
    ----------
    X : np.ndarray
        Input features of shape (n_features, m_examples)
    Y : np.ndarray
        Target labels of shape (n_classes, m_examples)

    Returns
    -------
    List[Tuple[np.ndarray, np.ndarray]]
        List of (X_batch, Y_batch) tuples

    Notes
    -----
    If shuffle is True, data is randomly permuted before creating batches.
    The last batch may be smaller if the dataset size is not divisible by batch_size.
    """
    m = X.shape[1]
    mini_batchs = []

    if self.shuffle:
        permutation = np.random.permutation(m)
        X_shuffled = X[:, permutation]
        Y_shuffled = Y[:, permutation]
    else:
        X_shuffled = X
        Y_shuffled = Y

    full_batches = m // self.batch_size
    for k in range(full_batches):
        start = k * self.batch_size
        end = start + self.batch_size

        X_batch = X_shuffled[:, start:end]
        Y_batch = Y_shuffled[:, start:end]

        mini_batchs.append((X_batch, Y_batch))

    if m % self.batch_size:
        start = full_batches * self.batch_size
        X_batch = X_shuffled[:, start:]
        Y_batch = Y_shuffled[:, start:]
        mini_batchs.append((X_batch, Y_batch))

    return mini_batchs

update_parameters

update_parameters(parameters: dict[str, ndarray], gradients: dict[str, ndarray]) -> dict[str, np.ndarray]

Update model parameters using gradient descent.

Parameters:

Name	Type	Description	Default
parameters	Dict[str, ndarray]	Current model parameters	required
gradients	Dict[str, ndarray]	Computed gradients for each parameter	required

Returns:

Type	Description
Dict[str, ndarray]	Updated parameters

Notes

Updates parameters using the standard gradient descent rule: θ = θ - α * ∇J(θ)

Source code in src/dlhub/optimizers/mini_batch.py

def update_parameters(
    self, parameters: dict[str, np.ndarray], gradients: dict[str, np.ndarray]
) -> dict[str, np.ndarray]:
    """
    Update model parameters using gradient descent.

    Parameters
    ----------
    parameters : Dict[str, np.ndarray]
        Current model parameters
    gradients : Dict[str, np.ndarray]
        Computed gradients for each parameter

    Returns
    -------
    Dict[str, np.ndarray]
        Updated parameters

    Notes
    -----
    Updates parameters using the standard gradient descent rule:
    θ = θ - α * ∇J(θ)
    """
    updated_parameters = {}

    for key in parameters:
        updated_parameters[key] = (
            parameters[key] - self.learning_rate * gradients[key]
        )

    return updated_parameters

train_epoch

train_epoch(X: ndarray, Y: ndarray, parameters: dict[str, ndarray], forward_propagation_fn: callable, backward_propagation_fn: callable, compute_cost_fn: callable) -> tuple[dict[str, np.ndarray], float]

Train for one epoch using mini-batch gradient descent.

Parameters:

Name	Type	Description	Default
X	ndarray	Input features of shape (n_features, m_examples)	required
Y	ndarray	Target labels of shape (n_classes, m_examples)	required
parameters	Dict[str, ndarray]	Current model parameters	required
forward_propagation_fn	callable	Function to compute forward propagation	required
backward_propagation_fn	callable	Function to compute backward propagation	required
compute_cost_fn	callable	Function to compute cost/loss	required

Returns:

Type	Description
Tuple[Dict[str, ndarray], float]	Updated parameters and epoch loss

Notes

Performs one complete epoch of mini-batch gradient descent training.

Source code in src/dlhub/optimizers/mini_batch.py

def train_epoch(
    self,
    X: np.ndarray,
    Y: np.ndarray,
    parameters: dict[str, np.ndarray],
    forward_propagation_fn: callable,
    backward_propagation_fn: callable,
    compute_cost_fn: callable,
) -> tuple[dict[str, np.ndarray], float]:
    """
    Train for one epoch using mini-batch gradient descent.

    Parameters
    ----------
    X : np.ndarray
        Input features of shape (n_features, m_examples)
    Y : np.ndarray
        Target labels of shape (n_classes, m_examples)
    parameters : Dict[str, np.ndarray]
        Current model parameters
    forward_propagation_fn : callable
        Function to compute forward propagation
    backward_propagation_fn : callable
        Function to compute backward propagation
    compute_cost_fn : callable
        Function to compute cost/loss

    Returns
    -------
    Tuple[Dict[str, np.ndarray], float]
        Updated parameters and epoch loss

    Notes
    -----
    Performs one complete epoch of mini-batch gradient descent training.
    """
    epoch_cost = 0.0
    mini_batches = self.create_mini_batches(X, Y)

    for X_batch, Y_batch in mini_batches:
        AL, caches = forward_propagation_fn(X_batch, parameters)
        epoch_cost += compute_cost_fn(AL, Y_batch)
        gradients = backward_propagation_fn(AL, Y_batch, caches)
        parameters = self.update_parameters(parameters, gradients)

    epoch_cost /= len(mini_batches)
    return parameters, epoch_cost

fit

fit(X: ndarray, Y: ndarray, parameters: dict[str, ndarray], forward_propagation_fn: callable, backward_propagation_fn: callable, compute_cost_fn: callable, epochs: int = 1000, print_cost: bool = True, print_every: int = 100) -> dict[str, np.ndarray]

Train the model using mini-batch gradient descent.

Parameters:

Name	Type	Description	Default
X	ndarray	Input features of shape (n_features, m_examples)	required
Y	ndarray	Target labels of shape (n_classes, m_examples)	required
parameters	Dict[str, ndarray]	Initial model parameters	required
forward_propagation_fn	callable	Function to compute forward propagation	required
backward_propagation_fn	callable	Function to compute backward propagation	required
compute_cost_fn	callable	Function to compute cost/loss	required
epochs	int	Number of training epochs	1000
print_cost	bool	Whether to print cost during training	True
print_every	int	Print cost every N epochs	100

Returns:

Type	Description
Dict[str, ndarray]	Trained parameters

Notes

Trains the model for the specified number of epochs using mini-batch gradient descent. Training history is stored in self.history.

Source code in src/dlhub/optimizers/mini_batch.py

def fit(
    self,
    X: np.ndarray,
    Y: np.ndarray,
    parameters: dict[str, np.ndarray],
    forward_propagation_fn: callable,
    backward_propagation_fn: callable,
    compute_cost_fn: callable,
    epochs: int = 1000,
    print_cost: bool = True,
    print_every: int = 100,
) -> dict[str, np.ndarray]:
    """
    Train the model using mini-batch gradient descent.

    Parameters
    ----------
    X : np.ndarray
        Input features of shape (n_features, m_examples)
    Y : np.ndarray
        Target labels of shape (n_classes, m_examples)
    parameters : Dict[str, np.ndarray]
        Initial model parameters
    forward_propagation_fn : callable
        Function to compute forward propagation
    backward_propagation_fn : callable
        Function to compute backward propagation
    compute_cost_fn : callable
        Function to compute cost/loss
    epochs : int, default=1000
        Number of training epochs
    print_cost : bool, default=True
        Whether to print cost during training
    print_every : int, default=100
        Print cost every N epochs

    Returns
    -------
    Dict[str, np.ndarray]
        Trained parameters

    Notes
    -----
    Trains the model for the specified number of epochs using mini-batch
    gradient descent. Training history is stored in self.history.
    """
    for epoch in range(epochs):
        parameters, epoch_cost = self.train_epoch(
            X,
            Y,
            parameters,
            forward_propagation_fn,
            backward_propagation_fn,
            compute_cost_fn,
        )

        self.history["loss"].append(epoch_cost)

        if print_cost and epoch % print_every == 0:
            print(f"Epoch {epoch}: Cost = {epoch_cost:.6f}")

    return parameters

get_config

get_config() -> dict[str, Any]

Get optimizer configuration.

Returns:

Type	Description
Dict[str, Any]	Configuration dictionary

Source code in src/dlhub/optimizers/mini_batch.py

def get_config(self) -> dict[str, Any]:
    """
    Get optimizer configuration.

    Returns
    -------
    Dict[str, Any]
        Configuration dictionary
    """
    return {
        "learning_rate": self.learning_rate,
        "batch_size": self.batch_size,
        "shuffle": self.shuffle,
        "optimizer": "MiniBatchGradientDescent",
    }

MomentumOptimizer

Gradient Descent with Momentum optimizer.

This implementation uses exponential weighted averages to accumulate gradients and includes bias correction for better convergence, especially in early training.

Parameters:

Name	Type	Description	Default
learning_rate	float	Learning rate for parameter updates	0.001
beta	float	Momentum parameter (exponential decay rate)	0.9
bias_correction	bool	Whether to apply bias correction to momentum estimates	True
epsilon	float	Small constant for numerical stability	1e-8

Attributes:

Name	Type	Description
learning_rate	float	Current learning rate
beta	float	Momentum parameter
bias_correction	bool	Bias correction flag
epsilon	float	Numerical stability constant
v	Dict[str, ndarray]	Momentum (velocity) estimates for each parameter
t	int	Time step counter for bias correction
history	Dict[str, List[float]]	Training history

Source code in src/dlhub/optimizers/momentum.py

class MomentumOptimizer:
    """
    Gradient Descent with Momentum optimizer.

    This implementation uses exponential weighted averages to accumulate gradients
    and includes bias correction for better convergence, especially in early training.

    Parameters
    ----------
    learning_rate : float, default=0.001
        Learning rate for parameter updates
    beta : float, default=0.9
        Momentum parameter (exponential decay rate)
    bias_correction : bool, default=True
        Whether to apply bias correction to momentum estimates
    epsilon : float, default=1e-8
        Small constant for numerical stability

    Attributes
    ----------
    learning_rate : float
        Current learning rate
    beta : float
        Momentum parameter
    bias_correction : bool
        Bias correction flag
    epsilon : float
        Numerical stability constant
    v : Dict[str, np.ndarray]
        Momentum (velocity) estimates for each parameter
    t : int
        Time step counter for bias correction
    history : Dict[str, List[float]]
        Training history
    """

    def __init__(
        self,
        learning_rate: float = 0.001,
        beta: float = 0.9,
        bias_correction: bool = True,
        epsilon: float = 1e-8,
    ):
        self.learning_rate = learning_rate
        self.beta = beta
        self.bias_correction = bias_correction
        self.epsilon = epsilon

        self.v = {}  # Momentum estimates
        self.t = 0  # Time step
        self.history = {"loss": [], "gradient_norm": []}

    def initialize_velocity(self, parameters: dict[str, np.ndarray]) -> None:
        """
        Initialize velocity (momentum) estimates for all parameters.

        Parameters
        ----------
        parameters : Dict[str, np.ndarray]
            Model parameters to initialize velocity for

        Notes
        -----
        Velocities are initialized to zero arrays with the same shape as parameters.
        """
        for key in parameters:
            self.v[key] = np.zeros_like(parameters[key])

    def update_parameters(
        self, parameters: dict[str, np.ndarray], gradients: dict[str, np.ndarray]
    ) -> dict[str, np.ndarray]:
        """
        Update parameters using momentum-based gradient descent.

        Parameters
        ----------
        parameters : Dict[str, np.ndarray]
            Current model parameters
        gradients : Dict[str, np.ndarray]
            Computed gradients for each parameter

        Returns
        -------
        Dict[str, np.ndarray]
            Updated parameters

        Notes
        -----
        Updates parameters using momentum:
        v_t = β * v_{t-1} + (1-β) * g_t
        θ_t = θ_{t-1} - α * v_t_corrected

        Where v_t_corrected includes bias correction if enabled.
        """
        if not self.v:
            self.initialize_velocity(parameters)

        self.t += 1
        updated_parameters = {}

        for key in parameters:
            self.v[key] = self.beta * self.v[key] + (1 - self.beta) * gradients[key]

            if self.bias_correction:
                v_corrected = self.v[key] / (1 - self.beta**self.t)
            else:
                v_corrected = self.v[key]

            updated_parameters[key] = parameters[key] - self.learning_rate * v_corrected

        return updated_parameters

    def compute_gradient_norm(self, gradients: dict[str, np.ndarray]) -> float:
        """
        Compute the L2 norm of gradients for monitoring convergence.

        Parameters
        ----------
        gradients : Dict[str, np.ndarray]
            Gradients for each parameter

        Returns
        -------
        float
            L2 norm of all gradients
        """
        total_norm = 0.0
        for grad in gradients.values():
            total_norm += np.sum(grad**2)
        return np.sqrt(total_norm)

    def train_step(
        self,
        X: np.ndarray,
        Y: np.ndarray,
        parameters: dict[str, np.ndarray],
        forward_propagation_fn: callable,
        backward_propagation_fn: callable,
        compute_cost_fn: callable,
    ) -> tuple[dict[str, np.ndarray]]:
        """
        Perform one training step with momentum optimizer.

        Parameters
        ----------
        X : np.ndarray
            Input features
        Y : np.ndarray
            Target labels
        parameters : Dict[str, np.ndarray]
            Current model parameters
        forward_propagation_fn : callable
            Function to compute forward propagation
        backward_propagation_fn : callable
            Function to compute backward propagation
        compute_cost_fn : callable
            Function to compute cost/loss

        Returns
        -------
        Tuple[Dict[str, np.ndarray], float]
            Updated parameters and current loss
        """
        AL, caches = forward_propagation_fn(X, parameters)
        cost = compute_cost_fn(AL, Y)
        gradients = backward_propagation_fn(AL, Y, caches)
        parameters = self.update_parameters(parameters, gradients)

        grad_norm = self.compute_gradient_norm(gradients)
        self.history["gradient_norm"].append(grad_norm)

        return parameters, cost

    def fit(
        self,
        X: np.ndarray,
        Y: np.ndarray,
        parameters: dict[str, np.ndarray],
        forward_propagation_fn: callable,
        backward_propagation_fn: callable,
        compute_cost_fn: callable,
        epochs: int = 1000,
        print_cost: bool = True,
        print_every: int = 100,
    ) -> dict[str, np.ndarray]:
        """
        Train the model using momentum optimizer.

        Parameters
        ----------
        X : np.ndarray
            Input features
        Y : np.ndarray
            Target labels
        parameters : Dict[str, np.ndarray]
            Initial model parameters
        forward_propagation_fn : callable
            Function to compute forward propagation
        backward_propagation_fn : callable
            Function to compute backward propagation
        compute_cost_fn : callable
            Function to compute cost/loss
        epochs : int, default=1000
            Number of training epochs
        print_cost : bool, default=True
            Whether to print cost during training
        print_every : int, default=100
            Print cost every N epochs

        Returns
        -------
        Dict[str, np.ndarray]
            Trained parameters
        """
        for epoch in range(epochs):
            parameters, cost = self.train_step(
                X,
                Y,
                parameters,
                forward_propagation_fn,
                backward_propagation_fn,
                compute_cost_fn,
            )

            self.history["loss"].append(cost)

            if print_cost and epoch % print_every == 0:
                grad_norm = self.history["gradient_norm"][-1]
                print(
                    f"Epoch {epoch}: Cost = {cost:.6f}, Gradient Norm = {grad_norm:.6f}"
                )

        return parameters

    def get_momentum_statistics(self) -> dict[str, dict[str, float]]:
        """
        Get statistics about momentum estimates.

        Returns
        -------
        Dict[str, Dict[str, float]]
            Statistics for each parameter's momentum
        """
        stats = {}
        for key, v in self.v.items():
            stats[key] = {
                "mean": np.mean(v),
                "std": np.std(v),
                "max": np.max(v),
                "min": np.min(v),
                "norm": np.linalg.norm(v),
            }
        return stats

    def reset_optimizer_state(self) -> None:
        """Reset optimizer state including velocity estimates and time step."""
        self.v = {}
        self.t = 0
        self.history = {"loss": [], "gradient_norm": []}

    def get_config(self) -> dict[str, Any]:
        """
        Get optimizer configuration.

        Returns
        -------
        Dict[str, Any]
            Configuration dictionary
        """
        return {
            "learning_rate": self.learning_rate,
            "beta": self.beta,
            "bias_correction": self.bias_correction,
            "epsilon": self.epsilon,
            "optimizer": "MomentumOptimizer",
        }

initialize_velocity

initialize_velocity(parameters: dict[str, ndarray]) -> None

Initialize velocity (momentum) estimates for all parameters.

Parameters:

Name	Type	Description	Default
parameters	Dict[str, ndarray]	Model parameters to initialize velocity for	required

Notes

Velocities are initialized to zero arrays with the same shape as parameters.

Source code in src/dlhub/optimizers/momentum.py

def initialize_velocity(self, parameters: dict[str, np.ndarray]) -> None:
    """
    Initialize velocity (momentum) estimates for all parameters.

    Parameters
    ----------
    parameters : Dict[str, np.ndarray]
        Model parameters to initialize velocity for

    Notes
    -----
    Velocities are initialized to zero arrays with the same shape as parameters.
    """
    for key in parameters:
        self.v[key] = np.zeros_like(parameters[key])

update_parameters

update_parameters(parameters: dict[str, ndarray], gradients: dict[str, ndarray]) -> dict[str, np.ndarray]

Update parameters using momentum-based gradient descent.

Parameters:

Name	Type	Description	Default
parameters	Dict[str, ndarray]	Current model parameters	required
gradients	Dict[str, ndarray]	Computed gradients for each parameter	required

Returns:

Type	Description
Dict[str, ndarray]	Updated parameters

Notes

Updates parameters using momentum: v_t = β * v_{t-1} + (1-β) * g_t θ_t = θ_{t-1} - α * v_t_corrected

Where v_t_corrected includes bias correction if enabled.

Source code in src/dlhub/optimizers/momentum.py

def update_parameters(
    self, parameters: dict[str, np.ndarray], gradients: dict[str, np.ndarray]
) -> dict[str, np.ndarray]:
    """
    Update parameters using momentum-based gradient descent.

    Parameters
    ----------
    parameters : Dict[str, np.ndarray]
        Current model parameters
    gradients : Dict[str, np.ndarray]
        Computed gradients for each parameter

    Returns
    -------
    Dict[str, np.ndarray]
        Updated parameters

    Notes
    -----
    Updates parameters using momentum:
    v_t = β * v_{t-1} + (1-β) * g_t
    θ_t = θ_{t-1} - α * v_t_corrected

    Where v_t_corrected includes bias correction if enabled.
    """
    if not self.v:
        self.initialize_velocity(parameters)

    self.t += 1
    updated_parameters = {}

    for key in parameters:
        self.v[key] = self.beta * self.v[key] + (1 - self.beta) * gradients[key]

        if self.bias_correction:
            v_corrected = self.v[key] / (1 - self.beta**self.t)
        else:
            v_corrected = self.v[key]

        updated_parameters[key] = parameters[key] - self.learning_rate * v_corrected

    return updated_parameters

compute_gradient_norm

compute_gradient_norm(gradients: dict[str, ndarray]) -> float

Compute the L2 norm of gradients for monitoring convergence.

Parameters:

Name	Type	Description	Default
gradients	Dict[str, ndarray]	Gradients for each parameter	required

Returns:

Type	Description
float	L2 norm of all gradients

Source code in src/dlhub/optimizers/momentum.py

def compute_gradient_norm(self, gradients: dict[str, np.ndarray]) -> float:
    """
    Compute the L2 norm of gradients for monitoring convergence.

    Parameters
    ----------
    gradients : Dict[str, np.ndarray]
        Gradients for each parameter

    Returns
    -------
    float
        L2 norm of all gradients
    """
    total_norm = 0.0
    for grad in gradients.values():
        total_norm += np.sum(grad**2)
    return np.sqrt(total_norm)

train_step

train_step(X: ndarray, Y: ndarray, parameters: dict[str, ndarray], forward_propagation_fn: callable, backward_propagation_fn: callable, compute_cost_fn: callable) -> tuple[dict[str, np.ndarray]]

Perform one training step with momentum optimizer.

Parameters:

Name	Type	Description	Default
X	ndarray	Input features	required
Y	ndarray	Target labels	required
parameters	Dict[str, ndarray]	Current model parameters	required
forward_propagation_fn	callable	Function to compute forward propagation	required
backward_propagation_fn	callable	Function to compute backward propagation	required
compute_cost_fn	callable	Function to compute cost/loss	required

Returns:

Type	Description
Tuple[Dict[str, ndarray], float]	Updated parameters and current loss

Source code in src/dlhub/optimizers/momentum.py

def train_step(
    self,
    X: np.ndarray,
    Y: np.ndarray,
    parameters: dict[str, np.ndarray],
    forward_propagation_fn: callable,
    backward_propagation_fn: callable,
    compute_cost_fn: callable,
) -> tuple[dict[str, np.ndarray]]:
    """
    Perform one training step with momentum optimizer.

    Parameters
    ----------
    X : np.ndarray
        Input features
    Y : np.ndarray
        Target labels
    parameters : Dict[str, np.ndarray]
        Current model parameters
    forward_propagation_fn : callable
        Function to compute forward propagation
    backward_propagation_fn : callable
        Function to compute backward propagation
    compute_cost_fn : callable
        Function to compute cost/loss

    Returns
    -------
    Tuple[Dict[str, np.ndarray], float]
        Updated parameters and current loss
    """
    AL, caches = forward_propagation_fn(X, parameters)
    cost = compute_cost_fn(AL, Y)
    gradients = backward_propagation_fn(AL, Y, caches)
    parameters = self.update_parameters(parameters, gradients)

    grad_norm = self.compute_gradient_norm(gradients)
    self.history["gradient_norm"].append(grad_norm)

    return parameters, cost

fit

fit(X: ndarray, Y: ndarray, parameters: dict[str, ndarray], forward_propagation_fn: callable, backward_propagation_fn: callable, compute_cost_fn: callable, epochs: int = 1000, print_cost: bool = True, print_every: int = 100) -> dict[str, np.ndarray]

Train the model using momentum optimizer.

Parameters:

Name	Type	Description	Default
X	ndarray	Input features	required
Y	ndarray	Target labels	required
parameters	Dict[str, ndarray]	Initial model parameters	required
forward_propagation_fn	callable	Function to compute forward propagation	required
backward_propagation_fn	callable	Function to compute backward propagation	required
compute_cost_fn	callable	Function to compute cost/loss	required
epochs	int	Number of training epochs	1000
print_cost	bool	Whether to print cost during training	True
print_every	int	Print cost every N epochs	100

Returns:

Type	Description
Dict[str, ndarray]	Trained parameters

Source code in src/dlhub/optimizers/momentum.py

def fit(
    self,
    X: np.ndarray,
    Y: np.ndarray,
    parameters: dict[str, np.ndarray],
    forward_propagation_fn: callable,
    backward_propagation_fn: callable,
    compute_cost_fn: callable,
    epochs: int = 1000,
    print_cost: bool = True,
    print_every: int = 100,
) -> dict[str, np.ndarray]:
    """
    Train the model using momentum optimizer.

    Parameters
    ----------
    X : np.ndarray
        Input features
    Y : np.ndarray
        Target labels
    parameters : Dict[str, np.ndarray]
        Initial model parameters
    forward_propagation_fn : callable
        Function to compute forward propagation
    backward_propagation_fn : callable
        Function to compute backward propagation
    compute_cost_fn : callable
        Function to compute cost/loss
    epochs : int, default=1000
        Number of training epochs
    print_cost : bool, default=True
        Whether to print cost during training
    print_every : int, default=100
        Print cost every N epochs

    Returns
    -------
    Dict[str, np.ndarray]
        Trained parameters
    """
    for epoch in range(epochs):
        parameters, cost = self.train_step(
            X,
            Y,
            parameters,
            forward_propagation_fn,
            backward_propagation_fn,
            compute_cost_fn,
        )

        self.history["loss"].append(cost)

        if print_cost and epoch % print_every == 0:
            grad_norm = self.history["gradient_norm"][-1]
            print(
                f"Epoch {epoch}: Cost = {cost:.6f}, Gradient Norm = {grad_norm:.6f}"
            )

    return parameters

get_momentum_statistics

get_momentum_statistics() -> dict[str, dict[str, float]]

Get statistics about momentum estimates.

Returns:

Type	Description
Dict[str, Dict[str, float]]	Statistics for each parameter's momentum

Source code in src/dlhub/optimizers/momentum.py

def get_momentum_statistics(self) -> dict[str, dict[str, float]]:
    """
    Get statistics about momentum estimates.

    Returns
    -------
    Dict[str, Dict[str, float]]
        Statistics for each parameter's momentum
    """
    stats = {}
    for key, v in self.v.items():
        stats[key] = {
            "mean": np.mean(v),
            "std": np.std(v),
            "max": np.max(v),
            "min": np.min(v),
            "norm": np.linalg.norm(v),
        }
    return stats

reset_optimizer_state

reset_optimizer_state() -> None

Reset optimizer state including velocity estimates and time step.

Source code in src/dlhub/optimizers/momentum.py

def reset_optimizer_state(self) -> None:
    """Reset optimizer state including velocity estimates and time step."""
    self.v = {}
    self.t = 0
    self.history = {"loss": [], "gradient_norm": []}

get_config

get_config() -> dict[str, Any]

Get optimizer configuration.

Returns:

Type	Description
Dict[str, Any]	Configuration dictionary

Source code in src/dlhub/optimizers/momentum.py

def get_config(self) -> dict[str, Any]:
    """
    Get optimizer configuration.

    Returns
    -------
    Dict[str, Any]
        Configuration dictionary
    """
    return {
        "learning_rate": self.learning_rate,
        "beta": self.beta,
        "bias_correction": self.bias_correction,
        "epsilon": self.epsilon,
        "optimizer": "MomentumOptimizer",
    }

RMSpropOptimizer

RMSprop (Root Mean Square Propagation) optimizer.

RMSprop adapts the learning rate for each parameter by dividing by a running average of the magnitudes of recent gradients. This helps with convergence on non-convex functions and handles different scaling of parameters. It also applies a learning rate decay technique based on current step.

Parameters:

Name	Type	Description	Default
learning_rate	float	Learning rate for parameter updates	0.001
beta	float	Exponential decay rate for the second moment estimates	0.9
epsilon	float	Small constant for numerical stability	1e-8
bias_correction	bool	Whether to apply bias correction (not standard in RMSprop)	False
decay	float	Learning rate decay factor	0.0

Attributes:

Name	Type	Description
learning_rate	float	Current learning rate
beta	float	Decay rate for second moment estimates
epsilon	float	Numerical stability constant
s	Dict[str, ndarray]	Second moment estimates (squared gradients) for each parameter
t	int	Time step counter
history	Dict[str, List[float]]	Training history including losses and learning rates

Source code in src/dlhub/optimizers/rmsprop.py

class RMSpropOptimizer:
    """
    RMSprop (Root Mean Square Propagation) optimizer.

    RMSprop adapts the learning rate for each parameter by dividing by a running
    average of the magnitudes of recent gradients. This helps with convergence
    on non-convex functions and handles different scaling of parameters. It also
    applies a learning rate decay technique based on current step.

    Parameters
    ----------
    learning_rate : float, default=0.001
        Learning rate for parameter updates
    beta : float, default=0.9
        Exponential decay rate for the second moment estimates
    epsilon : float, default=1e-8
        Small constant for numerical stability
    bias_correction : bool, default=False
        Whether to apply bias correction (not standard in RMSprop)
    decay : float, default=0.0
        Learning rate decay factor

    Attributes
    ----------
    learning_rate : float
        Current learning rate
    beta : float
        Decay rate for second moment estimates
    epsilon : float
        Numerical stability constant
    s : Dict[str, np.ndarray]
        Second moment estimates (squared gradients) for each parameter
    t : int
        Time step counter
    history : Dict[str, List[float]]
        Training history including losses and learning rates
    """

    def __init__(
        self,
        learning_rate: float = 0.001,
        beta: float = 0.9,
        epsilon: float = 1e-8,
        bias_correction: bool = False,
        decay: float = 0.0,
    ):
        self.learning_rate = learning_rate
        self.initial_learning_rate = learning_rate
        self.beta = beta
        self.epsilon = epsilon
        self.bias_correction = bias_correction
        self.decay = decay

        self.s = {}  # Second moment estimates
        self.t = 0  # Time step
        self.history = {
            "loss": [],
            "gradient_norm": [],
            "learning_rate": [],
            "rms_grad": [],
        }

    def initialize_second_moments(self, parameters: dict[str, np.ndarray]) -> None:
        """
        Initialize second moment estimates for all parameters.

        Parameters
        ----------
        parameters : Dict[str, np.ndarray]
            Model parameters to initialize second moments for

        Notes
        -----
        Second moments are initialized to zero arrays with the same shape as parameters.
        """
        for key in parameters:
            self.s[key] = np.zeros_like(parameters[key])

    def update_learning_rate(self) -> None:
        """
        Update learning rate with decay if specified.

        Notes
        -----
        Applies learning rate decay: lr = lr_initial / (1 + decay * t)
        """
        if self.decay > 0:
            self.learning_rate = self.initial_learning_rate / (1 + self.decay * self.t)

    def update_parameters(
        self, parameters: dict[str, np.ndarray], gradients: dict[str, np.ndarray]
    ) -> dict[str, np.ndarray]:
        """
        Update parameters using RMSprop optimization.

        Parameters
        ----------
        parameters : Dict[str, np.ndarray]
            Current model parameters
        gradients : Dict[str, np.ndarray]
            Computed gradients for each parameter

        Returns
        -------
        Dict[str, np.ndarray]
            Updated parameters

        Notes
        -----
        Updates parameters using RMSprop:
        s_t = β * s_{t-1} + (1-β) * g_t²
        θ_t = θ_{t-1} - α * g_t / (√s_t + ε)

        Where s_t is the exponential weighted average of squared gradients.
        Stores average RMS gradient for monitoring
        """
        if not self.s:
            self.initialize_second_moments(parameters)

        self.t += 1
        self.update_learning_rate()

        updated_parameters = {}
        rms_gradients = {}

        for key in parameters:
            self.s[key] = self.beta * self.s[key] + (1 - self.beta) * (
                gradients[key] ** 2
            )

            if self.bias_correction:
                s_corrected = self.s[key] / (1 - self.beta**self.t)
            else:
                s_corrected = self.s[key]

            rms_grad = np.sqrt(np.mean(s_corrected))
            rms_gradients[key] = rms_grad

            updated_parameters[key] = parameters[key] - self.learning_rate * gradients[
                key
            ] / (np.sqrt(s_corrected) + self.epsilon)

        avg_rms_grad = np.mean(list(rms_gradients.values()))
        self.history["rms_grad"].append(avg_rms_grad)
        self.history["learning_rate"].append(self.learning_rate)

        return updated_parameters

    def compute_gradient_norm(self, gradients: dict[str, np.ndarray]) -> float:
        """
        Compute the L2 norm of gradients for monitoring convergence.

        Parameters
        ----------
        gradients : Dict[str, np.ndarray]
            Gradients for each parameter

        Returns
        -------
        float
            L2 norm of all gradients
        """
        total_norm = 0.0
        for grad in gradients.values():
            total_norm += np.sum(grad**2)
        return np.sqrt(total_norm)

    def get_effective_learning_rates(
        self, parameters: dict[str, np.ndarray]
    ) -> dict[str, np.ndarray]:
        """
        Compute effective learning rates for each parameter.

        Parameters
        ----------
        parameters : Dict[str, np.ndarray]
            Current model parameters

        Returns
        -------
        Dict[str, np.ndarray]
            Effective learning rates for each parameter
        """
        effective_lrs = {}

        if not self.s:
            for key in parameters:  # If not initialized, return base learning rate
                effective_lrs[key] = np.full_like(parameters[key], self.learning_rate)
        else:
            for key in parameters:
                if self.bias_correction:
                    s_corrected = self.s[key] / (1 - self.beta**self.t)
                else:
                    s_corrected = self.s[key]

                effective_lrs[key] = self.learning_rate / (
                    np.sqrt(s_corrected) + self.epsilon
                )

        return effective_lrs

    def train_step(
        self,
        X: np.ndarray,
        Y: np.ndarray,
        parameters: dict[str, np.ndarray],
        forward_propagation_fn: callable,
        backward_propagation_fn: callable,
        compute_cost_fn: callable,
    ) -> tuple[dict[str, np.ndarray], float]:
        """
        Perform one training step with RMSprop optimizer.

        Parameters
        ----------
        X : np.ndarray
            Input features
        Y : np.ndarray
            Target labels
        parameters : Dict[str, np.ndarray]
            Current model parameters
        forward_propagation_fn : callable
            Function to compute forward propagation
        backward_propagation_fn : callable
            Function to compute backward propagation
        compute_cost_fn : callable
            Function to compute cost/loss

        Returns
        -------
        Tuple[Dict[str, np.ndarray], float]
            Updated parameters and current loss
        """
        AL, caches = forward_propagation_fn(X, parameters)
        cost = compute_cost_fn(AL, Y)
        gradients = backward_propagation_fn(AL, Y, caches)
        parameters = self.update_parameters(parameters, gradients)

        grad_norm = self.compute_gradient_norm(gradients)
        self.history["gradient_norm"].append(grad_norm)

        return parameters, cost

    def fit(
        self,
        X: np.ndarray,
        Y: np.ndarray,
        parameters: dict[str, np.ndarray],
        forward_propagation_fn: callable,
        backward_propagation_fn: callable,
        compute_cost_fn: callable,
        epochs: int = 1000,
        print_cost: bool = True,
        print_every: int = 100,
    ) -> dict[str, np.ndarray]:
        """
        Train the model using RMSprop optimizer.

        Parameters
        ----------
        X : np.ndarray
            Input features
        Y : np.ndarray
            Target labels
        parameters : Dict[str, np.ndarray]
            Initial model parameters
        forward_propagation_fn : callable
            Function to compute forward propagation
        backward_propagation_fn : callable
            Function to compute backward propagation
        compute_cost_fn : callable
            Function to compute cost/loss
        epochs : int, default=1000
            Number of training epochs
        print_cost : bool, default=True
            Whether to print cost during training
        print_every : int, default=100
            Print cost every N epochs

        Returns
        -------
        Dict[str, np.ndarray]
            Trained parameters
        """
        for epoch in range(epochs):
            parameters, cost = self.train_step(
                X,
                Y,
                parameters,
                forward_propagation_fn,
                backward_propagation_fn,
                compute_cost_fn,
            )

            self.history["loss"].append(cost)

            if print_cost and epoch % print_every == 0:
                grad_norm = self.history["gradient_norm"][-1]
                rms_grad = self.history["rms_grad"][-1]
                lr = self.history["learning_rate"][-1]
                print(
                    f"Epoch {epoch}: Cost = {cost:.6f}, "
                    f"Gradient Norm = {grad_norm:.6f}, "
                    f"RMS Grad = {rms_grad:.6f}, "
                    f"LR = {lr:.6f}"
                )

        return parameters

    def get_second_moment_statistics(self) -> dict[str, dict[str, float]]:
        """
        Get statistics about second moment estimates.

        Returns
        -------
        Dict[str, Dict[str, float]]
            Statistics for each parameter's second moments
        """
        stats = {}
        for key, s in self.s.items():
            stats[key] = {
                "mean": np.mean(s),
                "std": np.std(s),
                "max": np.max(s),
                "min": np.min(s),
                "norm": np.linalg.norm(s),
            }
        return stats

    def reset_optimizer_state(self) -> None:
        """Reset optimizer state including second moment estimates and time step."""
        self.s = {}
        self.t = 0
        self.learning_rate = self.initial_learning_rate
        self.history = {
            "loss": [],
            "gradient_norm": [],
            "learning_rate": [],
            "rms_grad": [],
        }

    def get_config(self) -> dict[str, Any]:
        """
        Get optimizer configuration.

        Returns
        -------
        Dict[str, Any]
            Configuration dictionary
        """
        return {
            "learning_rate": self.learning_rate,
            "initial_learning_rate": self.initial_learning_rate,
            "beta": self.beta,
            "epsilon": self.epsilon,
            "bias_correction": self.bias_correction,
            "decay": self.decay,
            "optimizer": "RMSpropOptimizer",
        }

initialize_second_moments

initialize_second_moments(parameters: dict[str, ndarray]) -> None

Initialize second moment estimates for all parameters.

Parameters:

Name	Type	Description	Default
parameters	Dict[str, ndarray]	Model parameters to initialize second moments for	required

Notes

Second moments are initialized to zero arrays with the same shape as parameters.

Source code in src/dlhub/optimizers/rmsprop.py

def initialize_second_moments(self, parameters: dict[str, np.ndarray]) -> None:
    """
    Initialize second moment estimates for all parameters.

    Parameters
    ----------
    parameters : Dict[str, np.ndarray]
        Model parameters to initialize second moments for

    Notes
    -----
    Second moments are initialized to zero arrays with the same shape as parameters.
    """
    for key in parameters:
        self.s[key] = np.zeros_like(parameters[key])

update_learning_rate

update_learning_rate() -> None

Update learning rate with decay if specified.

Notes

Applies learning rate decay: lr = lr_initial / (1 + decay * t)

Source code in src/dlhub/optimizers/rmsprop.py

def update_learning_rate(self) -> None:
    """
    Update learning rate with decay if specified.

    Notes
    -----
    Applies learning rate decay: lr = lr_initial / (1 + decay * t)
    """
    if self.decay > 0:
        self.learning_rate = self.initial_learning_rate / (1 + self.decay * self.t)

update_parameters

update_parameters(parameters: dict[str, ndarray], gradients: dict[str, ndarray]) -> dict[str, np.ndarray]

Update parameters using RMSprop optimization.

Parameters:

Name	Type	Description	Default
parameters	Dict[str, ndarray]	Current model parameters	required
gradients	Dict[str, ndarray]	Computed gradients for each parameter	required

Returns:

Type	Description
Dict[str, ndarray]	Updated parameters

Notes

Updates parameters using RMSprop: s_t = β * s_{t-1} + (1-β) * g_t² θ_t = θ_{t-1} - α * g_t / (√s_t + ε)

Where s_t is the exponential weighted average of squared gradients. Stores average RMS gradient for monitoring

Source code in src/dlhub/optimizers/rmsprop.py

def update_parameters(
    self, parameters: dict[str, np.ndarray], gradients: dict[str, np.ndarray]
) -> dict[str, np.ndarray]:
    """
    Update parameters using RMSprop optimization.

    Parameters
    ----------
    parameters : Dict[str, np.ndarray]
        Current model parameters
    gradients : Dict[str, np.ndarray]
        Computed gradients for each parameter

    Returns
    -------
    Dict[str, np.ndarray]
        Updated parameters

    Notes
    -----
    Updates parameters using RMSprop:
    s_t = β * s_{t-1} + (1-β) * g_t²
    θ_t = θ_{t-1} - α * g_t / (√s_t + ε)

    Where s_t is the exponential weighted average of squared gradients.
    Stores average RMS gradient for monitoring
    """
    if not self.s:
        self.initialize_second_moments(parameters)

    self.t += 1
    self.update_learning_rate()

    updated_parameters = {}
    rms_gradients = {}

    for key in parameters:
        self.s[key] = self.beta * self.s[key] + (1 - self.beta) * (
            gradients[key] ** 2
        )

        if self.bias_correction:
            s_corrected = self.s[key] / (1 - self.beta**self.t)
        else:
            s_corrected = self.s[key]

        rms_grad = np.sqrt(np.mean(s_corrected))
        rms_gradients[key] = rms_grad

        updated_parameters[key] = parameters[key] - self.learning_rate * gradients[
            key
        ] / (np.sqrt(s_corrected) + self.epsilon)

    avg_rms_grad = np.mean(list(rms_gradients.values()))
    self.history["rms_grad"].append(avg_rms_grad)
    self.history["learning_rate"].append(self.learning_rate)

    return updated_parameters

compute_gradient_norm

compute_gradient_norm(gradients: dict[str, ndarray]) -> float

Compute the L2 norm of gradients for monitoring convergence.

Parameters:

Name	Type	Description	Default
gradients	Dict[str, ndarray]	Gradients for each parameter	required

Returns:

Type	Description
float	L2 norm of all gradients

Source code in src/dlhub/optimizers/rmsprop.py

def compute_gradient_norm(self, gradients: dict[str, np.ndarray]) -> float:
    """
    Compute the L2 norm of gradients for monitoring convergence.

    Parameters
    ----------
    gradients : Dict[str, np.ndarray]
        Gradients for each parameter

    Returns
    -------
    float
        L2 norm of all gradients
    """
    total_norm = 0.0
    for grad in gradients.values():
        total_norm += np.sum(grad**2)
    return np.sqrt(total_norm)

get_effective_learning_rates

get_effective_learning_rates(parameters: dict[str, ndarray]) -> dict[str, np.ndarray]

Compute effective learning rates for each parameter.

Parameters:

Name	Type	Description	Default
parameters	Dict[str, ndarray]	Current model parameters	required

Returns:

Type	Description
Dict[str, ndarray]	Effective learning rates for each parameter

Source code in src/dlhub/optimizers/rmsprop.py

def get_effective_learning_rates(
    self, parameters: dict[str, np.ndarray]
) -> dict[str, np.ndarray]:
    """
    Compute effective learning rates for each parameter.

    Parameters
    ----------
    parameters : Dict[str, np.ndarray]
        Current model parameters

    Returns
    -------
    Dict[str, np.ndarray]
        Effective learning rates for each parameter
    """
    effective_lrs = {}

    if not self.s:
        for key in parameters:  # If not initialized, return base learning rate
            effective_lrs[key] = np.full_like(parameters[key], self.learning_rate)
    else:
        for key in parameters:
            if self.bias_correction:
                s_corrected = self.s[key] / (1 - self.beta**self.t)
            else:
                s_corrected = self.s[key]

            effective_lrs[key] = self.learning_rate / (
                np.sqrt(s_corrected) + self.epsilon
            )

    return effective_lrs

train_step

train_step(X: ndarray, Y: ndarray, parameters: dict[str, ndarray], forward_propagation_fn: callable, backward_propagation_fn: callable, compute_cost_fn: callable) -> tuple[dict[str, np.ndarray], float]

Perform one training step with RMSprop optimizer.

Parameters:

Name	Type	Description	Default
X	ndarray	Input features	required
Y	ndarray	Target labels	required
parameters	Dict[str, ndarray]	Current model parameters	required
forward_propagation_fn	callable	Function to compute forward propagation	required
backward_propagation_fn	callable	Function to compute backward propagation	required
compute_cost_fn	callable	Function to compute cost/loss	required

Returns:

Type	Description
Tuple[Dict[str, ndarray], float]	Updated parameters and current loss

Source code in src/dlhub/optimizers/rmsprop.py

def train_step(
    self,
    X: np.ndarray,
    Y: np.ndarray,
    parameters: dict[str, np.ndarray],
    forward_propagation_fn: callable,
    backward_propagation_fn: callable,
    compute_cost_fn: callable,
) -> tuple[dict[str, np.ndarray], float]:
    """
    Perform one training step with RMSprop optimizer.

    Parameters
    ----------
    X : np.ndarray
        Input features
    Y : np.ndarray
        Target labels
    parameters : Dict[str, np.ndarray]
        Current model parameters
    forward_propagation_fn : callable
        Function to compute forward propagation
    backward_propagation_fn : callable
        Function to compute backward propagation
    compute_cost_fn : callable
        Function to compute cost/loss

    Returns
    -------
    Tuple[Dict[str, np.ndarray], float]
        Updated parameters and current loss
    """
    AL, caches = forward_propagation_fn(X, parameters)
    cost = compute_cost_fn(AL, Y)
    gradients = backward_propagation_fn(AL, Y, caches)
    parameters = self.update_parameters(parameters, gradients)

    grad_norm = self.compute_gradient_norm(gradients)
    self.history["gradient_norm"].append(grad_norm)

    return parameters, cost

fit

fit(X: ndarray, Y: ndarray, parameters: dict[str, ndarray], forward_propagation_fn: callable, backward_propagation_fn: callable, compute_cost_fn: callable, epochs: int = 1000, print_cost: bool = True, print_every: int = 100) -> dict[str, np.ndarray]

Train the model using RMSprop optimizer.

Parameters:

Name	Type	Description	Default
X	ndarray	Input features	required
Y	ndarray	Target labels	required
parameters	Dict[str, ndarray]	Initial model parameters	required
forward_propagation_fn	callable	Function to compute forward propagation	required
backward_propagation_fn	callable	Function to compute backward propagation	required
compute_cost_fn	callable	Function to compute cost/loss	required
epochs	int	Number of training epochs	1000
print_cost	bool	Whether to print cost during training	True
print_every	int	Print cost every N epochs	100

Returns:

Type	Description
Dict[str, ndarray]	Trained parameters

Source code in src/dlhub/optimizers/rmsprop.py

def fit(
    self,
    X: np.ndarray,
    Y: np.ndarray,
    parameters: dict[str, np.ndarray],
    forward_propagation_fn: callable,
    backward_propagation_fn: callable,
    compute_cost_fn: callable,
    epochs: int = 1000,
    print_cost: bool = True,
    print_every: int = 100,
) -> dict[str, np.ndarray]:
    """
    Train the model using RMSprop optimizer.

    Parameters
    ----------
    X : np.ndarray
        Input features
    Y : np.ndarray
        Target labels
    parameters : Dict[str, np.ndarray]
        Initial model parameters
    forward_propagation_fn : callable
        Function to compute forward propagation
    backward_propagation_fn : callable
        Function to compute backward propagation
    compute_cost_fn : callable
        Function to compute cost/loss
    epochs : int, default=1000
        Number of training epochs
    print_cost : bool, default=True
        Whether to print cost during training
    print_every : int, default=100
        Print cost every N epochs

    Returns
    -------
    Dict[str, np.ndarray]
        Trained parameters
    """
    for epoch in range(epochs):
        parameters, cost = self.train_step(
            X,
            Y,
            parameters,
            forward_propagation_fn,
            backward_propagation_fn,
            compute_cost_fn,
        )

        self.history["loss"].append(cost)

        if print_cost and epoch % print_every == 0:
            grad_norm = self.history["gradient_norm"][-1]
            rms_grad = self.history["rms_grad"][-1]
            lr = self.history["learning_rate"][-1]
            print(
                f"Epoch {epoch}: Cost = {cost:.6f}, "
                f"Gradient Norm = {grad_norm:.6f}, "
                f"RMS Grad = {rms_grad:.6f}, "
                f"LR = {lr:.6f}"
            )

    return parameters

get_second_moment_statistics

get_second_moment_statistics() -> dict[str, dict[str, float]]

Get statistics about second moment estimates.

Returns:

Type	Description
Dict[str, Dict[str, float]]	Statistics for each parameter's second moments

Source code in src/dlhub/optimizers/rmsprop.py

def get_second_moment_statistics(self) -> dict[str, dict[str, float]]:
    """
    Get statistics about second moment estimates.

    Returns
    -------
    Dict[str, Dict[str, float]]
        Statistics for each parameter's second moments
    """
    stats = {}
    for key, s in self.s.items():
        stats[key] = {
            "mean": np.mean(s),
            "std": np.std(s),
            "max": np.max(s),
            "min": np.min(s),
            "norm": np.linalg.norm(s),
        }
    return stats

reset_optimizer_state

reset_optimizer_state() -> None

Reset optimizer state including second moment estimates and time step.

Source code in src/dlhub/optimizers/rmsprop.py

def reset_optimizer_state(self) -> None:
    """Reset optimizer state including second moment estimates and time step."""
    self.s = {}
    self.t = 0
    self.learning_rate = self.initial_learning_rate
    self.history = {
        "loss": [],
        "gradient_norm": [],
        "learning_rate": [],
        "rms_grad": [],
    }

get_config

get_config() -> dict[str, Any]

Get optimizer configuration.

Returns:

Type	Description
Dict[str, Any]	Configuration dictionary

Source code in src/dlhub/optimizers/rmsprop.py

def get_config(self) -> dict[str, Any]:
    """
    Get optimizer configuration.

    Returns
    -------
    Dict[str, Any]
        Configuration dictionary
    """
    return {
        "learning_rate": self.learning_rate,
        "initial_learning_rate": self.initial_learning_rate,
        "beta": self.beta,
        "epsilon": self.epsilon,
        "bias_correction": self.bias_correction,
        "decay": self.decay,
        "optimizer": "RMSpropOptimizer",
    }

LearningRateScheduler

Comprehensive Learning Rate Scheduler with multiple scheduling strategies.

This class provides various learning rate scheduling strategies commonly used in deep learning training, including step decay, cosine annealing, cyclical learning rates, and warm restarts.

Parameters:

Name	Type	Description	Default
initial_lr	float	Initial learning rate	required
scheduler_type	SchedulerType	Type of scheduling strategy to use	CONSTANT
total_steps	int	Total number of training steps (required for some schedulers)	None
**kwargs		Additional parameters specific to each scheduler type	{}

Attributes:

Name	Type	Description
current_lr	float	Current learning rate
step_count	int	Number of steps taken
history	list	History of learning rates

Source code in src/dlhub/optimizers/schedules.py

class LearningRateScheduler:
    """
    Comprehensive Learning Rate Scheduler with multiple scheduling strategies.

    This class provides various learning rate scheduling strategies commonly used
    in deep learning training, including step decay, cosine annealing, cyclical
    learning rates, and warm restarts.

    Parameters
    ----------
    initial_lr : float
        Initial learning rate
    scheduler_type : SchedulerType
        Type of scheduling strategy to use
    total_steps : int, optional
        Total number of training steps (required for some schedulers)
    **kwargs
        Additional parameters specific to each scheduler type

    Attributes
    ----------
    current_lr : float
        Current learning rate
    step_count : int
        Number of steps taken
    history : list
        History of learning rates
    """

    def __init__(
        self,
        initial_lr: float,
        scheduler_type: SchedulerType = SchedulerType.CONSTANT,
        total_steps: int | None = None,
        **kwargs,
    ):
        if initial_lr <= 0:
            raise ValueError(
                f"Initial learning rate must be positive, got {initial_lr}"
            )

        self.initial_lr = initial_lr
        self.scheduler_type = scheduler_type
        self.total_steps = total_steps
        self.kwargs = kwargs

        self.current_lr = initial_lr
        self.step_count = 0
        self.history = [initial_lr]

        self._plateau_count = 0
        self._best_metric = None
        self._cycle_count = 0
        self._restart_count = 0
        self._cooldown_counter = 0

        self._validate_parameters()

    def _validate_parameters(self) -> None:
        """Validate scheduler-specific parameters."""
        if self.scheduler_type == SchedulerType.STEP_DECAY:
            if "step_size" not in self.kwargs:
                raise ValueError("step_size required for STEP_DECAY scheduler")
            if "gamma" not in self.kwargs:
                self.kwargs["gamma"] = 0.1

        elif self.scheduler_type == SchedulerType.EXPONENTIAL_DECAY:
            if "gamma" not in self.kwargs:
                raise ValueError("gamma required for EXPONENTIAL_DECAY scheduler")

        elif self.scheduler_type == SchedulerType.POLYNOMIAL_DECAY:
            if "power" not in self.kwargs:
                self.kwargs["power"] = 1.0
            if self.total_steps is None:
                raise ValueError("total_steps required for POLYNOMIAL_DECAY scheduler")

        elif self.scheduler_type == SchedulerType.COSINE_ANNEALING:
            if "T_max" not in self.kwargs and self.total_steps is None:
                raise ValueError(
                    "Either T_max or total_steps required for COSINE_ANNEALING"
                )
            if "eta_min" not in self.kwargs:
                self.kwargs["eta_min"] = 0.0

        elif self.scheduler_type == SchedulerType.COSINE_ANNEALING_WARM_RESTARTS:
            if "T_0" not in self.kwargs:
                self.kwargs["T_0"] = 10
            if "T_mult" not in self.kwargs:
                self.kwargs["T_mult"] = 2
            if "eta_min" not in self.kwargs:
                self.kwargs["eta_min"] = 0.0

        elif self.scheduler_type == SchedulerType.CYCLICAL:
            if "base_lr" not in self.kwargs:
                self.kwargs["base_lr"] = self.initial_lr * 0.1
            if "max_lr" not in self.kwargs:
                self.kwargs["max_lr"] = self.initial_lr
            if "step_size_up" not in self.kwargs:
                self.kwargs["step_size_up"] = 2000
            if "mode" not in self.kwargs:
                self.kwargs["mode"] = "triangular"

        elif self.scheduler_type == SchedulerType.ONE_CYCLE:
            if "max_lr" not in self.kwargs:
                self.kwargs["max_lr"] = self.initial_lr * 10
            if self.total_steps is None:
                raise ValueError("total_steps required for ONE_CYCLE scheduler")
            if "pct_start" not in self.kwargs:
                self.kwargs["pct_start"] = 0.3
            if "anneal_strategy" not in self.kwargs:
                self.kwargs["anneal_strategy"] = "cos"

        elif self.scheduler_type == SchedulerType.REDUCE_ON_PLATEAU:
            if "factor" not in self.kwargs:
                self.kwargs["factor"] = 0.1
            if "patience" not in self.kwargs:
                self.kwargs["patience"] = 10
            if "threshold" not in self.kwargs:
                self.kwargs["threshold"] = 1e-4
            if "cooldown" not in self.kwargs:
                self.kwargs["cooldown"] = 0
            if "min_lr" not in self.kwargs:
                self.kwargs["min_lr"] = 0.0

        elif self.scheduler_type == SchedulerType.WARMUP_COSINE:
            if "warmup_steps" not in self.kwargs:
                self.kwargs["warmup_steps"] = 1000
            if self.total_steps is None:
                raise ValueError("total_steps required for WARMUP_COSINE scheduler")

        elif self.scheduler_type == SchedulerType.LINEAR_WARMUP:
            if "warmup_steps" not in self.kwargs:
                raise ValueError("warmup_steps required for LINEAR_WARMUP scheduler")

        elif self.scheduler_type == SchedulerType.CUSTOM:
            if "custom_func" not in self.kwargs:
                raise ValueError("custom_func required for CUSTOM scheduler")

    def step(self, metric: float | None = None) -> float:
        """
        Update the learning rate for one step.

        Parameters
        ----------
        metric : float, optional
            Current metric value (required for REDUCE_ON_PLATEAU)

        Returns
        -------
        float
            Updated learning rate
        """
        self.step_count += 1

        if self.scheduler_type == SchedulerType.CONSTANT:
            self.current_lr = self.initial_lr

        elif self.scheduler_type == SchedulerType.STEP_DECAY:
            self.current_lr = self._step_decay()

        elif self.scheduler_type == SchedulerType.EXPONENTIAL_DECAY:
            self.current_lr = self._exponential_decay()

        elif self.scheduler_type == SchedulerType.POLYNOMIAL_DECAY:
            self.current_lr = self._polynomial_decay()

        elif self.scheduler_type == SchedulerType.COSINE_ANNEALING:
            self.current_lr = self._cosine_annealing()

        elif self.scheduler_type == SchedulerType.COSINE_ANNEALING_WARM_RESTARTS:
            self.current_lr = self._cosine_annealing_warm_restarts()

        elif self.scheduler_type == SchedulerType.CYCLICAL:
            self.current_lr = self._cyclical()

        elif self.scheduler_type == SchedulerType.ONE_CYCLE:
            self.current_lr = self._one_cycle()

        elif self.scheduler_type == SchedulerType.REDUCE_ON_PLATEAU:
            self.current_lr = self._reduce_on_plateau(metric)

        elif self.scheduler_type == SchedulerType.WARMUP_COSINE:
            self.current_lr = self._warmup_cosine()

        elif self.scheduler_type == SchedulerType.LINEAR_WARMUP:
            self.current_lr = self._linear_warmup()

        elif self.scheduler_type == SchedulerType.CUSTOM:
            self.current_lr = self._custom()

        else:
            raise ValueError(f"Unknown scheduler type: {self.scheduler_type}")

        self.current_lr = max(self.current_lr, 0.0)
        self.history.append(self.current_lr)
        return self.current_lr

    def _step_decay(self) -> float:
        """Step decay scheduler."""
        step_size = self.kwargs["step_size"]
        gamma = self.kwargs["gamma"]
        return self.initial_lr * (gamma ** (self.step_count // step_size))

    def _exponential_decay(self) -> float:
        """Exponential decay scheduler."""
        gamma = self.kwargs["gamma"]
        return self.initial_lr * (gamma**self.step_count)

    def _polynomial_decay(self) -> float:
        """Polynomial decay scheduler."""
        power = self.kwargs["power"]
        if self.step_count >= self.total_steps:
            return 0.0
        return self.initial_lr * ((1 - self.step_count / self.total_steps) ** power)

    def _cosine_annealing(self) -> float:
        """Cosine annealing scheduler."""
        T_max = self.kwargs.get("T_max", self.total_steps)
        eta_min = self.kwargs["eta_min"]

        if T_max is None:
            T_max = self.total_steps

        return (
            eta_min
            + (self.initial_lr - eta_min)
            * (1 + math.cos(math.pi * self.step_count / T_max))
            / 2
        )

    def _cosine_annealing_warm_restarts(self) -> float:
        """Cosine annealing with warm restarts (SGDR)."""
        T_0 = self.kwargs["T_0"]
        T_mult = self.kwargs["T_mult"]
        eta_min = self.kwargs["eta_min"]

        # The cycle is recomputed from step_count on every call, so the restart
        # count has to be assigned, not accumulated: incrementing inside this
        # loop adds the whole cycle history again at every step.
        T_cur = self.step_count
        T_i = T_0
        restarts = 0

        while T_cur >= T_i:
            T_cur -= T_i
            T_i *= T_mult
            restarts += 1

        self._restart_count = restarts

        return (
            eta_min
            + (self.initial_lr - eta_min) * (1 + math.cos(math.pi * T_cur / T_i)) / 2
        )

    def _cyclical(self) -> float:
        """Cyclical learning rate scheduler."""
        base_lr = self.kwargs["base_lr"]
        max_lr = self.kwargs["max_lr"]
        step_size_up = self.kwargs["step_size_up"]
        mode = self.kwargs["mode"]

        cycle = math.floor(1 + self.step_count / (2 * step_size_up))
        x = abs(self.step_count / step_size_up - 2 * cycle + 1)
        self._cycle_count = cycle

        if mode == "triangular":
            scale_fn = lambda x: 1.0
            scale_mode = "cycle"
        elif mode == "triangular2":
            scale_fn = lambda x: 1 / (2.0 ** (cycle - 1))
            scale_mode = "cycle"
        elif mode == "exp_range":
            gamma = self.kwargs.get("gamma", 1.0)
            scale_fn = lambda x: gamma**self.step_count
            scale_mode = "iterations"
        else:
            raise ValueError(f"Unknown cyclical mode: {mode}")

        if scale_mode == "cycle":
            scale_factor = scale_fn(cycle)
        else:
            scale_factor = scale_fn(self.step_count)

        return base_lr + (max_lr - base_lr) * max(0, (1 - x)) * scale_factor

    def _one_cycle(self) -> float:
        """One cycle learning rate scheduler."""
        max_lr = self.kwargs["max_lr"]
        pct_start = self.kwargs["pct_start"]
        anneal_strategy = self.kwargs["anneal_strategy"]

        step_ratio = self.step_count / self.total_steps

        if step_ratio <= pct_start:
            # Warmup phase
            if anneal_strategy == "linear":
                return (
                    self.initial_lr
                    + (max_lr - self.initial_lr) * step_ratio / pct_start
                )
            else:  # cosine
                return (
                    self.initial_lr
                    + (max_lr - self.initial_lr)
                    * (1 - math.cos(math.pi * step_ratio / pct_start))
                    / 2
                )
        else:
            # Annealing phase
            remaining_ratio = (step_ratio - pct_start) / (1 - pct_start)
            if anneal_strategy == "linear":
                return max_lr - (max_lr - self.initial_lr) * remaining_ratio
            else:  # cosine
                return (
                    self.initial_lr
                    + (max_lr - self.initial_lr)
                    * (1 + math.cos(math.pi * remaining_ratio))
                    / 2
                )

    def _reduce_on_plateau(self, metric: float | None) -> float:
        """Reduce on plateau scheduler."""
        if metric is None:
            warnings.warn("Metric required for REDUCE_ON_PLATEAU scheduler")
            return self.current_lr

        factor = self.kwargs["factor"]
        patience = self.kwargs["patience"]
        threshold = self.kwargs["threshold"]
        cooldown = self.kwargs["cooldown"]
        min_lr = self.kwargs["min_lr"]
        mode = self.kwargs.get("mode", "min")

        if self._best_metric is None:
            self._best_metric = metric
            return self.current_lr

        # Check if metric improved
        if mode == "min":
            improved = metric < self._best_metric - threshold
        else:  # mode == 'max'
            improved = metric > self._best_metric + threshold

        if improved:
            self._best_metric = metric
            self._plateau_count = 0
            return self.current_lr

        # A reduction takes time to show up in the metric, so `cooldown` steps
        # after one are not counted against patience. Without this the next
        # reduction can fire before the previous one has had any effect.
        if self._cooldown_counter > 0:
            self._cooldown_counter -= 1
            self._plateau_count = 0
            return self.current_lr

        self._plateau_count += 1

        # Reduce learning rate if patience exceeded
        if self._plateau_count > patience:
            new_lr = max(self.current_lr * factor, min_lr)
            if new_lr < self.current_lr:
                self._plateau_count = 0
                self._cooldown_counter = cooldown
            return new_lr

        return self.current_lr

    def _warmup_cosine(self) -> float:
        """Warmup followed by cosine annealing."""
        warmup_steps = self.kwargs["warmup_steps"]

        if self.step_count <= warmup_steps:
            # Linear warmup
            return self.initial_lr * self.step_count / warmup_steps
        else:
            # Cosine annealing
            progress = (self.step_count - warmup_steps) / (
                self.total_steps - warmup_steps
            )
            return self.initial_lr * (1 + math.cos(math.pi * progress)) / 2

    def _linear_warmup(self) -> float:
        """Linear warmup scheduler."""
        warmup_steps = self.kwargs["warmup_steps"]

        if self.step_count <= warmup_steps:
            return self.initial_lr * self.step_count / warmup_steps
        else:
            return self.initial_lr

    def _custom(self) -> float:
        """Create custom scheduler using user-provided function."""
        custom_func = self.kwargs["custom_func"]
        return custom_func(self.step_count, self.initial_lr, **self.kwargs)

    def get_lr(self) -> float:
        """Get current learning rate."""
        return self.current_lr

    def reset(self) -> None:
        """Reset scheduler to initial state."""
        self.current_lr = self.initial_lr
        self.step_count = 0
        self.history = [self.initial_lr]
        self._plateau_count = 0
        self._best_metric = None
        self._cycle_count = 0
        self._restart_count = 0
        self._cooldown_counter = 0

    def get_config(self) -> dict[str, Any]:
        """Get scheduler configuration."""
        return {
            "initial_lr": self.initial_lr,
            "scheduler_type": self.scheduler_type.value,
            "total_steps": self.total_steps,
            "step_count": self.step_count,
            "kwargs": self.kwargs.copy(),
        }

    def get_state(self) -> dict[str, Any]:
        """Get complete scheduler state."""
        return {
            "config": self.get_config(),
            "current_lr": self.current_lr,
            "history": self.history.copy(),
            "plateau_count": self._plateau_count,
            "best_metric": self._best_metric,
            "cycle_count": self._cycle_count,
            "restart_count": self._restart_count,
            "cooldown_counter": self._cooldown_counter,
        }

    def load_state(self, state: dict[str, Any]) -> None:
        """Load scheduler state."""
        config = state["config"]
        self.initial_lr = config["initial_lr"]
        self.scheduler_type = SchedulerType(config["scheduler_type"])
        self.total_steps = config["total_steps"]
        self.step_count = config["step_count"]
        self.kwargs = config["kwargs"]

        self.current_lr = state["current_lr"]
        self.history = state["history"]
        self._plateau_count = state["plateau_count"]
        self._best_metric = state["best_metric"]
        self._cycle_count = state["cycle_count"]
        self._restart_count = state["restart_count"]
        self._cooldown_counter = state.get("cooldown_counter", 0)

step

step(metric: float | None = None) -> float

Update the learning rate for one step.

Parameters:

Name	Type	Description	Default
metric	float	Current metric value (required for REDUCE_ON_PLATEAU)	None

Returns:

Type	Description
float	Updated learning rate

Source code in src/dlhub/optimizers/schedules.py

def step(self, metric: float | None = None) -> float:
    """
    Update the learning rate for one step.

    Parameters
    ----------
    metric : float, optional
        Current metric value (required for REDUCE_ON_PLATEAU)

    Returns
    -------
    float
        Updated learning rate
    """
    self.step_count += 1

    if self.scheduler_type == SchedulerType.CONSTANT:
        self.current_lr = self.initial_lr

    elif self.scheduler_type == SchedulerType.STEP_DECAY:
        self.current_lr = self._step_decay()

    elif self.scheduler_type == SchedulerType.EXPONENTIAL_DECAY:
        self.current_lr = self._exponential_decay()

    elif self.scheduler_type == SchedulerType.POLYNOMIAL_DECAY:
        self.current_lr = self._polynomial_decay()

    elif self.scheduler_type == SchedulerType.COSINE_ANNEALING:
        self.current_lr = self._cosine_annealing()

    elif self.scheduler_type == SchedulerType.COSINE_ANNEALING_WARM_RESTARTS:
        self.current_lr = self._cosine_annealing_warm_restarts()

    elif self.scheduler_type == SchedulerType.CYCLICAL:
        self.current_lr = self._cyclical()

    elif self.scheduler_type == SchedulerType.ONE_CYCLE:
        self.current_lr = self._one_cycle()

    elif self.scheduler_type == SchedulerType.REDUCE_ON_PLATEAU:
        self.current_lr = self._reduce_on_plateau(metric)

    elif self.scheduler_type == SchedulerType.WARMUP_COSINE:
        self.current_lr = self._warmup_cosine()

    elif self.scheduler_type == SchedulerType.LINEAR_WARMUP:
        self.current_lr = self._linear_warmup()

    elif self.scheduler_type == SchedulerType.CUSTOM:
        self.current_lr = self._custom()

    else:
        raise ValueError(f"Unknown scheduler type: {self.scheduler_type}")

    self.current_lr = max(self.current_lr, 0.0)
    self.history.append(self.current_lr)
    return self.current_lr

get_lr

get_lr() -> float

Get current learning rate.

Source code in src/dlhub/optimizers/schedules.py

def get_lr(self) -> float:
    """Get current learning rate."""
    return self.current_lr

reset

reset() -> None

Reset scheduler to initial state.

Source code in src/dlhub/optimizers/schedules.py

def reset(self) -> None:
    """Reset scheduler to initial state."""
    self.current_lr = self.initial_lr
    self.step_count = 0
    self.history = [self.initial_lr]
    self._plateau_count = 0
    self._best_metric = None
    self._cycle_count = 0
    self._restart_count = 0
    self._cooldown_counter = 0

get_config

get_config() -> dict[str, Any]

Get scheduler configuration.

Source code in src/dlhub/optimizers/schedules.py

def get_config(self) -> dict[str, Any]:
    """Get scheduler configuration."""
    return {
        "initial_lr": self.initial_lr,
        "scheduler_type": self.scheduler_type.value,
        "total_steps": self.total_steps,
        "step_count": self.step_count,
        "kwargs": self.kwargs.copy(),
    }

get_state

get_state() -> dict[str, Any]

Get complete scheduler state.

Source code in src/dlhub/optimizers/schedules.py

def get_state(self) -> dict[str, Any]:
    """Get complete scheduler state."""
    return {
        "config": self.get_config(),
        "current_lr": self.current_lr,
        "history": self.history.copy(),
        "plateau_count": self._plateau_count,
        "best_metric": self._best_metric,
        "cycle_count": self._cycle_count,
        "restart_count": self._restart_count,
        "cooldown_counter": self._cooldown_counter,
    }

load_state

load_state(state: dict[str, Any]) -> None

Load scheduler state.

Source code in src/dlhub/optimizers/schedules.py

def load_state(self, state: dict[str, Any]) -> None:
    """Load scheduler state."""
    config = state["config"]
    self.initial_lr = config["initial_lr"]
    self.scheduler_type = SchedulerType(config["scheduler_type"])
    self.total_steps = config["total_steps"]
    self.step_count = config["step_count"]
    self.kwargs = config["kwargs"]

    self.current_lr = state["current_lr"]
    self.history = state["history"]
    self._plateau_count = state["plateau_count"]
    self._best_metric = state["best_metric"]
    self._cycle_count = state["cycle_count"]
    self._restart_count = state["restart_count"]
    self._cooldown_counter = state.get("cooldown_counter", 0)

SchedulerType

Bases: Enum

Enumeration of different scheduling strategies.

Source code in src/dlhub/optimizers/schedules.py

class SchedulerType(Enum):
    """Enumeration of different scheduling strategies."""

    CONSTANT = "constant"
    STEP_DECAY = "step_decay"
    EXPONENTIAL_DECAY = "exponential_decay"
    POLYNOMIAL_DECAY = "polynomial_decay"
    COSINE_ANNEALING = "cosine_annealing"
    COSINE_ANNEALING_WARM_RESTARTS = "cosine_annealing_warm_restarts"
    CYCLICAL = "cyclical"
    ONE_CYCLE = "one_cycle"
    REDUCE_ON_PLATEAU = "reduce_on_plateau"
    WARMUP_COSINE = "warmup_cosine"
    LINEAR_WARMUP = "linear_warmup"
    CUSTOM = "custom"

create_adam_optimizer

create_adam_optimizer(learning_rate: float = 0.001, beta1: float = 0.9, beta2: float = 0.999, epsilon: float = 1e-08, **kwargs) -> AdamOptimizer

Factory function to create Adam optimizer with common configurations.

Parameters:

Name	Type	Description	Default
learning_rate	float	Learning rate	0.001
beta1	float	First moment decay rate	0.9
beta2	float	Second moment decay rate	0.999
epsilon	float	Numerical stability constant	1e-08
**kwargs		Additional optimizer parameters	{}

Returns:

Type	Description
AdamOptimizer	Configured Adam optimizer

Source code in src/dlhub/optimizers/adam.py

def create_adam_optimizer(
    learning_rate: float = 0.001,
    beta1: float = 0.9,
    beta2: float = 0.999,
    epsilon: float = 1e-8,
    **kwargs,
) -> AdamOptimizer:
    """
    Factory function to create Adam optimizer with common configurations.

    Parameters
    ----------
    learning_rate : float
        Learning rate
    beta1 : float
        First moment decay rate
    beta2 : float
        Second moment decay rate
    epsilon : float
        Numerical stability constant
    **kwargs
        Additional optimizer parameters

    Returns
    -------
    AdamOptimizer
        Configured Adam optimizer
    """
    return AdamOptimizer(
        learning_rate=learning_rate, beta1=beta1, beta2=beta2, epsilon=epsilon, **kwargs
    )

create_adam_ewa_pair

create_adam_ewa_pair(beta1: float = 0.9, beta2: float = 0.999) -> tuple[ExponentialWeightedAverage, ExponentialWeightedAverage]

Create EWA pair for Adam optimizer (first and second moments).

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def create_adam_ewa_pair(
    beta1: float = 0.9, beta2: float = 0.999
) -> tuple[ExponentialWeightedAverage, ExponentialWeightedAverage]:
    """Create EWA pair for Adam optimizer (first and second moments)."""
    first_moment = ExponentialWeightedAverage(
        beta=beta1, bias_correction=True, strategy=AveragingStrategy.BIAS_CORRECTED
    )

    second_moment = ExponentialWeightedAverage(
        beta=beta2, bias_correction=True, strategy=AveragingStrategy.BIAS_CORRECTED
    )

    return first_moment, second_moment

create_momentum_ewa

create_momentum_ewa(beta: float = 0.9) -> ExponentialWeightedAverage

Create EWA for momentum optimization.

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def create_momentum_ewa(beta: float = 0.9) -> ExponentialWeightedAverage:
    """Create EWA for momentum optimization."""
    return ExponentialWeightedAverage(
        beta=beta, bias_correction=True, strategy=AveragingStrategy.BIAS_CORRECTED
    )

create_rmsprop_ewa

create_rmsprop_ewa(beta: float = 0.999) -> ExponentialWeightedAverage

Create EWA for RMSprop (second moments).

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def create_rmsprop_ewa(beta: float = 0.999) -> ExponentialWeightedAverage:
    """Create EWA for RMSprop (second moments)."""
    return ExponentialWeightedAverage(
        beta=beta, bias_correction=True, strategy=AveragingStrategy.BIAS_CORRECTED
    )

create_cosine_scheduler

create_cosine_scheduler(initial_lr: float, total_steps: int, eta_min: float = 0.0) -> LearningRateScheduler

Create cosine annealing scheduler.

Source code in src/dlhub/optimizers/schedules.py

def create_cosine_scheduler(
    initial_lr: float, total_steps: int, eta_min: float = 0.0
) -> LearningRateScheduler:
    """Create cosine annealing scheduler."""
    return LearningRateScheduler(
        initial_lr=initial_lr,
        scheduler_type=SchedulerType.COSINE_ANNEALING,
        total_steps=total_steps,
        eta_min=eta_min,
    )

create_one_cycle_scheduler

create_one_cycle_scheduler(initial_lr: float, max_lr: float, total_steps: int, pct_start: float = 0.3) -> LearningRateScheduler

Create one cycle scheduler.

Source code in src/dlhub/optimizers/schedules.py

def create_one_cycle_scheduler(
    initial_lr: float, max_lr: float, total_steps: int, pct_start: float = 0.3
) -> LearningRateScheduler:
    """Create one cycle scheduler."""
    return LearningRateScheduler(
        initial_lr=initial_lr,
        scheduler_type=SchedulerType.ONE_CYCLE,
        total_steps=total_steps,
        max_lr=max_lr,
        pct_start=pct_start,
    )

create_step_scheduler

create_step_scheduler(initial_lr: float, step_size: int, gamma: float = 0.1) -> LearningRateScheduler

Create step decay scheduler.

Source code in src/dlhub/optimizers/schedules.py

def create_step_scheduler(
    initial_lr: float, step_size: int, gamma: float = 0.1
) -> LearningRateScheduler:
    """Create step decay scheduler."""
    return LearningRateScheduler(
        initial_lr=initial_lr,
        scheduler_type=SchedulerType.STEP_DECAY,
        step_size=step_size,
        gamma=gamma,
    )

create_warmup_cosine_scheduler

create_warmup_cosine_scheduler(initial_lr: float, total_steps: int, warmup_steps: int) -> LearningRateScheduler

Create warmup + cosine scheduler.

Source code in src/dlhub/optimizers/schedules.py

def create_warmup_cosine_scheduler(
    initial_lr: float, total_steps: int, warmup_steps: int
) -> LearningRateScheduler:
    """Create warmup + cosine scheduler."""
    return LearningRateScheduler(
        initial_lr=initial_lr,
        scheduler_type=SchedulerType.WARMUP_COSINE,
        total_steps=total_steps,
        warmup_steps=warmup_steps,
    )

adam

Adam Optimizer Implementation

A comprehensive implementation of the Adam (Adaptive Moment Estimation) optimizer with bias correction, gradient clipping, and numerical stability features.

References

• Kingma, D. P., & Ba, J. (2014). Adam: A method for stochastic optimization. https://arxiv.org/abs/1412.6980

Author

Deep Learning Reference Hub

License

MIT

AdamOptimizer

Adam (Adaptive Moment Estimation) Optimizer

Adam combines the advantages of AdaGrad and RMSProp by computing adaptive learning rates for each parameter using estimates of first and second moments of the gradients.

The algorithm maintains exponentially decaying averages of past gradients and past squared gradients, which act as estimates of the first moment (mean) and second moment (uncentered variance) of the gradients.

Parameters:

Name	Type	Description	Default
learning_rate	float	Learning rate (alpha in the paper)	0.001
beta1	float	Exponential decay rate for first moment estimates	0.9
beta2	float	Exponential decay rate for second moment estimates	0.999
epsilon	float	Small constant for numerical stability	1e-8
weight_decay	float	Weight decay coefficient (L2 regularization)	0.0
amsgrad	bool	Whether to use AMSGrad variant which maintains maximum of squared gradients	False
gradient_clip_norm	float	Maximum norm for gradient clipping	None
gradient_clip_value	float	Maximum absolute value for gradient clipping	None

Attributes:

Name	Type	Description
m	dict	First moment estimates (exponentially decaying average of gradients)
v	dict	Second moment estimates (exponentially decaying average of squared gradients)
v_hat_max	dict	Maximum of v_hat values (used in AMSGrad)
t	int	Time step (number of updates performed)

Source code in src/dlhub/optimizers/adam.py

class AdamOptimizer:
    """
    Adam (Adaptive Moment Estimation) Optimizer

    Adam combines the advantages of AdaGrad and RMSProp by computing adaptive
    learning rates for each parameter using estimates of first and second
    moments of the gradients.

    The algorithm maintains exponentially decaying averages of past gradients
    and past squared gradients, which act as estimates of the first moment
    (mean) and second moment (uncentered variance) of the gradients.

    Parameters
    ----------
    learning_rate : float, default=0.001
        Learning rate (alpha in the paper)
    beta1 : float, default=0.9
        Exponential decay rate for first moment estimates
    beta2 : float, default=0.999
        Exponential decay rate for second moment estimates
    epsilon : float, default=1e-8
        Small constant for numerical stability
    weight_decay : float, default=0.0
        Weight decay coefficient (L2 regularization)
    amsgrad : bool, default=False
        Whether to use AMSGrad variant which maintains maximum of squared gradients
    gradient_clip_norm : float, optional
        Maximum norm for gradient clipping
    gradient_clip_value : float, optional
        Maximum absolute value for gradient clipping

    Attributes
    ----------
    m : dict
        First moment estimates (exponentially decaying average of gradients)
    v : dict
        Second moment estimates (exponentially decaying average of squared gradients)
    v_hat_max : dict
        Maximum of v_hat values (used in AMSGrad)
    t : int
        Time step (number of updates performed)
    """

    def __init__(
        self,
        learning_rate: float = 0.001,
        beta1: float = 0.9,
        beta2: float = 0.999,
        epsilon: float = 1e-8,
        weight_decay: float = 0.0,
        amsgrad: bool = False,
        gradient_clip_norm: float | None = None,
        gradient_clip_value: float | None = None,
    ):
        if not 0.0 < learning_rate <= 1.0:
            raise ValueError(f"Invalid learning rate: {learning_rate}")
        if not 0.0 <= beta1 < 1.0:
            raise ValueError(f"Invalid beta1 parameter: {beta1}")
        if not 0.0 <= beta2 < 1.0:
            raise ValueError(f"Invalid beta2 parameter: {beta2}")
        if epsilon <= 0.0:
            raise ValueError(f"Invalid epsilon value: {epsilon}")
        if weight_decay < 0.0:
            raise ValueError(f"Invalid weight_decay value: {weight_decay}")

        self.learning_rate = learning_rate
        self.beta1 = beta1
        self.beta2 = beta2
        self.epsilon = epsilon
        self.weight_decay = weight_decay
        self.amsgrad = amsgrad
        self.gradient_clip_norm = gradient_clip_norm
        self.gradient_clip_value = gradient_clip_value

        self.m: dict[str, np.ndarray] = {}
        self.v: dict[str, np.ndarray] = {}
        self.v_hat_max: dict[str, np.ndarray] = {}
        self.t = 0

        self.history = {
            "loss": [],
            "gradient_norm": [],
            "parameter_norm": [],
            "learning_rate": [],
        }

    def _initialize_moments(
        self, param_name: str, param_shape: tuple[int, ...]
    ) -> None:
        """Initialize moment estimates for a parameter."""
        if param_name not in self.m:
            self.m[param_name] = np.zeros(param_shape, dtype=np.float64)
            self.v[param_name] = np.zeros(param_shape, dtype=np.float64)
            if self.amsgrad:
                self.v_hat_max[param_name] = np.zeros(param_shape, dtype=np.float64)

    def _clip_gradients(
        self, gradients: dict[str, np.ndarray]
    ) -> dict[str, np.ndarray]:
        """
        Apply gradient clipping if specified.

        Parameters
        ----------
        gradients : dict
            Dictionary of gradients for each parameter

        Returns
        -------
        dict
            Clipped gradients
        """
        if self.gradient_clip_norm is not None:
            total_norm = 0.0
            for grad in gradients.values():
                total_norm += np.sum(grad**2)
            total_norm = np.sqrt(total_norm)

            if total_norm > self.gradient_clip_norm:
                clip_coeff = self.gradient_clip_norm / (total_norm + 1e-8)
                gradients = {
                    name: grad * clip_coeff for name, grad in gradients.items()
                }

        if self.gradient_clip_value is not None:
            gradients = {
                name: np.clip(grad, -self.gradient_clip_value, self.gradient_clip_value)
                for name, grad in gradients.items()
            }

        return gradients

    def update(
        self, parameters: dict[str, np.ndarray], gradients: dict[str, np.ndarray]
    ) -> dict[str, np.ndarray]:
        """
        Perform a single optimization step.

        Parameters
        ----------
        parameters : dict
            Dictionary of parameters to optimize
        gradients : dict
            Dictionary of gradients for each parameter

        Returns
        -------
        dict
            Updated parameters
        """
        self.t += 1
        gradients = self._clip_gradients(gradients)

        grad_norm = np.sqrt(sum(np.sum(grad**2) for grad in gradients.values()))
        param_norm = np.sqrt(sum(np.sum(param**2) for param in parameters.values()))

        updated_parameters = {}
        for param_name, param in parameters.items():
            if param_name not in gradients:
                updated_parameters[param_name] = param.copy()
                continue

            grad = gradients[param_name]

            self._initialize_moments(param_name, param.shape)

            if self.weight_decay > 0:
                grad = grad + self.weight_decay * param

            self.m[param_name] = (
                self.beta1 * self.m[param_name] + (1 - self.beta1) * grad
            )
            self.v[param_name] = self.beta2 * self.v[param_name] + (1 - self.beta2) * (
                grad**2
            )
            m_hat = self.m[param_name] / (1 - self.beta1**self.t)
            v_hat = self.v[param_name] / (1 - self.beta2**self.t)

            # AMSGrad modification
            if self.amsgrad:
                self.v_hat_max[param_name] = np.maximum(
                    self.v_hat_max[param_name], v_hat
                )
                v_hat = self.v_hat_max[param_name]

            denominator = np.sqrt(v_hat) + self.epsilon
            step = self.learning_rate * m_hat / denominator

            if np.any(np.isnan(step)) or np.any(np.isinf(step)):
                warnings.warn(
                    f"Numerical instability detected in parameter {param_name}"
                )
                step = np.nan_to_num(step, nan=0.0, posinf=1e-6, neginf=-1e-6)

            updated_parameters[param_name] = param - step

        self.history["gradient_norm"].append(grad_norm)
        self.history["parameter_norm"].append(param_norm)
        self.history["learning_rate"].append(self.learning_rate)

        return updated_parameters

    def get_config(self) -> dict:
        """
        Get optimizer configuration.

        Returns
        -------
        dict
            Configuration dictionary
        """
        return {
            "learning_rate": self.learning_rate,
            "beta1": self.beta1,
            "beta2": self.beta2,
            "epsilon": self.epsilon,
            "weight_decay": self.weight_decay,
            "amsgrad": self.amsgrad,
            "gradient_clip_norm": self.gradient_clip_norm,
            "gradient_clip_value": self.gradient_clip_value,
            "time_step": self.t,
        }

    def reset_state(self) -> None:
        """Reset optimizer state (moments and time step)."""
        self.m.clear()
        self.v.clear()
        self.v_hat_max.clear()
        self.t = 0
        self.history = {
            "loss": [],
            "gradient_norm": [],
            "parameter_norm": [],
            "learning_rate": [],
        }

    def get_state(self) -> dict:
        """
        Get complete optimizer state.

        Returns
        -------
        dict
            Complete state dictionary
        """
        return {
            "config": self.get_config(),
            "moments": {
                "m": self.m.copy(),
                "v": self.v.copy(),
                "v_hat_max": self.v_hat_max.copy() if self.amsgrad else {},
            },
            "history": self.history.copy(),
        }

    def load_state(self, state: dict) -> None:
        """
        Load optimizer state.

        Parameters
        ----------
        state : dict
            State dictionary from get_state()
        """
        config = state["config"]
        self.learning_rate = config["learning_rate"]
        self.beta1 = config["beta1"]
        self.beta2 = config["beta2"]
        self.epsilon = config["epsilon"]
        self.weight_decay = config["weight_decay"]
        self.amsgrad = config["amsgrad"]
        self.gradient_clip_norm = config["gradient_clip_norm"]
        self.gradient_clip_value = config["gradient_clip_value"]
        self.t = config["time_step"]

        moments = state["moments"]
        self.m = moments["m"].copy()
        self.v = moments["v"].copy()
        self.v_hat_max = moments["v_hat_max"].copy()

        self.history = state["history"].copy()

update

update(parameters: dict[str, ndarray], gradients: dict[str, ndarray]) -> dict[str, np.ndarray]

Perform a single optimization step.

Parameters:

Name	Type	Description	Default
parameters	dict	Dictionary of parameters to optimize	required
gradients	dict	Dictionary of gradients for each parameter	required

Returns:

Type	Description
dict	Updated parameters

Source code in src/dlhub/optimizers/adam.py

def update(
    self, parameters: dict[str, np.ndarray], gradients: dict[str, np.ndarray]
) -> dict[str, np.ndarray]:
    """
    Perform a single optimization step.

    Parameters
    ----------
    parameters : dict
        Dictionary of parameters to optimize
    gradients : dict
        Dictionary of gradients for each parameter

    Returns
    -------
    dict
        Updated parameters
    """
    self.t += 1
    gradients = self._clip_gradients(gradients)

    grad_norm = np.sqrt(sum(np.sum(grad**2) for grad in gradients.values()))
    param_norm = np.sqrt(sum(np.sum(param**2) for param in parameters.values()))

    updated_parameters = {}
    for param_name, param in parameters.items():
        if param_name not in gradients:
            updated_parameters[param_name] = param.copy()
            continue

        grad = gradients[param_name]

        self._initialize_moments(param_name, param.shape)

        if self.weight_decay > 0:
            grad = grad + self.weight_decay * param

        self.m[param_name] = (
            self.beta1 * self.m[param_name] + (1 - self.beta1) * grad
        )
        self.v[param_name] = self.beta2 * self.v[param_name] + (1 - self.beta2) * (
            grad**2
        )
        m_hat = self.m[param_name] / (1 - self.beta1**self.t)
        v_hat = self.v[param_name] / (1 - self.beta2**self.t)

        # AMSGrad modification
        if self.amsgrad:
            self.v_hat_max[param_name] = np.maximum(
                self.v_hat_max[param_name], v_hat
            )
            v_hat = self.v_hat_max[param_name]

        denominator = np.sqrt(v_hat) + self.epsilon
        step = self.learning_rate * m_hat / denominator

        if np.any(np.isnan(step)) or np.any(np.isinf(step)):
            warnings.warn(
                f"Numerical instability detected in parameter {param_name}"
            )
            step = np.nan_to_num(step, nan=0.0, posinf=1e-6, neginf=-1e-6)

        updated_parameters[param_name] = param - step

    self.history["gradient_norm"].append(grad_norm)
    self.history["parameter_norm"].append(param_norm)
    self.history["learning_rate"].append(self.learning_rate)

    return updated_parameters

get_config

get_config() -> dict

Get optimizer configuration.

Returns:

Type	Description
dict	Configuration dictionary

Source code in src/dlhub/optimizers/adam.py

def get_config(self) -> dict:
    """
    Get optimizer configuration.

    Returns
    -------
    dict
        Configuration dictionary
    """
    return {
        "learning_rate": self.learning_rate,
        "beta1": self.beta1,
        "beta2": self.beta2,
        "epsilon": self.epsilon,
        "weight_decay": self.weight_decay,
        "amsgrad": self.amsgrad,
        "gradient_clip_norm": self.gradient_clip_norm,
        "gradient_clip_value": self.gradient_clip_value,
        "time_step": self.t,
    }

reset_state

reset_state() -> None

Reset optimizer state (moments and time step).

Source code in src/dlhub/optimizers/adam.py

def reset_state(self) -> None:
    """Reset optimizer state (moments and time step)."""
    self.m.clear()
    self.v.clear()
    self.v_hat_max.clear()
    self.t = 0
    self.history = {
        "loss": [],
        "gradient_norm": [],
        "parameter_norm": [],
        "learning_rate": [],
    }

get_state

get_state() -> dict

Get complete optimizer state.

Returns:

Type	Description
dict	Complete state dictionary

Source code in src/dlhub/optimizers/adam.py

def get_state(self) -> dict:
    """
    Get complete optimizer state.

    Returns
    -------
    dict
        Complete state dictionary
    """
    return {
        "config": self.get_config(),
        "moments": {
            "m": self.m.copy(),
            "v": self.v.copy(),
            "v_hat_max": self.v_hat_max.copy() if self.amsgrad else {},
        },
        "history": self.history.copy(),
    }

load_state

load_state(state: dict) -> None

Load optimizer state.

Parameters:

Name	Type	Description	Default
state	dict	State dictionary from get_state()	required

Source code in src/dlhub/optimizers/adam.py

def load_state(self, state: dict) -> None:
    """
    Load optimizer state.

    Parameters
    ----------
    state : dict
        State dictionary from get_state()
    """
    config = state["config"]
    self.learning_rate = config["learning_rate"]
    self.beta1 = config["beta1"]
    self.beta2 = config["beta2"]
    self.epsilon = config["epsilon"]
    self.weight_decay = config["weight_decay"]
    self.amsgrad = config["amsgrad"]
    self.gradient_clip_norm = config["gradient_clip_norm"]
    self.gradient_clip_value = config["gradient_clip_value"]
    self.t = config["time_step"]

    moments = state["moments"]
    self.m = moments["m"].copy()
    self.v = moments["v"].copy()
    self.v_hat_max = moments["v_hat_max"].copy()

    self.history = state["history"].copy()

create_adam_optimizer

create_adam_optimizer(learning_rate: float = 0.001, beta1: float = 0.9, beta2: float = 0.999, epsilon: float = 1e-08, **kwargs) -> AdamOptimizer

Factory function to create Adam optimizer with common configurations.

Parameters:

Name	Type	Description	Default
learning_rate	float	Learning rate	0.001
beta1	float	First moment decay rate	0.9
beta2	float	Second moment decay rate	0.999
epsilon	float	Numerical stability constant	1e-08
**kwargs		Additional optimizer parameters	{}

Returns:

Type	Description
AdamOptimizer	Configured Adam optimizer

Source code in src/dlhub/optimizers/adam.py

def create_adam_optimizer(
    learning_rate: float = 0.001,
    beta1: float = 0.9,
    beta2: float = 0.999,
    epsilon: float = 1e-8,
    **kwargs,
) -> AdamOptimizer:
    """
    Factory function to create Adam optimizer with common configurations.

    Parameters
    ----------
    learning_rate : float
        Learning rate
    beta1 : float
        First moment decay rate
    beta2 : float
        Second moment decay rate
    epsilon : float
        Numerical stability constant
    **kwargs
        Additional optimizer parameters

    Returns
    -------
    AdamOptimizer
        Configured Adam optimizer
    """
    return AdamOptimizer(
        learning_rate=learning_rate, beta1=beta1, beta2=beta2, epsilon=epsilon, **kwargs
    )

base

Optimizer Contract

The interface every optimizer in this subpackage presents to code that drives a training loop: given the current parameters and their gradients, return the updated parameters.

The base class is deliberately thin. An optimizer is the artifact a reader came to read, so the update rule stays written out in full in its own module rather than being assembled from hooks defined here. What this class supplies is the uniform signature that lets a driver hold a collection of optimizers without knowing which one it has -- the comparison harness being the case that motivated extracting it.

Two conventions worth stating, because they are the ones a new optimizer gets wrong:

Bias correction is the optimizer's own decision, not the contract's. Momentum and RMSprop maintain running averages that start at zero and are therefore biased toward zero for the first few steps; whether to divide that bias out is a property of the method, and the modules that implement those methods expose it as a constructor flag. A driver that wants a like-for-like race across optimizers has to set that flag deliberately rather than inherit whatever each default happens to be.

The step index is passed in. Optimizers whose update depends on how many steps have been taken -- anything applying bias correction -- read it from the argument rather than counting internally, so that a driver resetting an optimizer between runs does not have to trust it to reset its own counter. The canonical optimizer modules in this subpackage predate this contract and count internally instead; the adapters in comparison bridge the two by clearing that counter in reset, which is the property the contract actually cares about and the one its tests check.

Author

Deep Learning Reference Hub

License

MIT

BaseOptimizer

Base class for all optimizers with common functionality.

Parameters:

Name	Type	Description	Default
learning_rate	float	Learning rate for parameter updates.	0.01

Attributes:

Name	Type	Description
learning_rate	float	Step size applied to each update.
name	str	Human-readable label, used to key and plot results. Subclasses set it to the name of the method they implement.

Source code in src/dlhub/optimizers/base.py

class BaseOptimizer:
    """
    Base class for all optimizers with common functionality.

    Parameters
    ----------
    learning_rate : float, default=0.01
        Learning rate for parameter updates.

    Attributes
    ----------
    learning_rate : float
        Step size applied to each update.
    name : str
        Human-readable label, used to key and plot results. Subclasses set it to
        the name of the method they implement.
    """

    def __init__(self, learning_rate: float = 0.01):
        self.learning_rate = learning_rate
        self.name = "BaseOptimizer"

    def update_parameters(
        self, params: dict[str, np.ndarray], grads: dict[str, np.ndarray], t: int
    ) -> dict[str, np.ndarray]:
        """
        Update parameters using optimization algorithm.

        Implementations return a new dictionary rather than mutating the one they
        were given, so that a caller can keep the parameter trajectory of a run.

        Parameters
        ----------
        params : dict
            Current parameter values.
        grads : dict
            Gradients for each parameter.
        t : int
            Current iteration, counted from one. Optimizers applying bias
            correction divide by ``1 - beta ** t``, which is why the count starts
            at one rather than zero.

        Returns
        -------
        dict
            Updated parameters.

        Raises
        ------
        NotImplementedError
            Always, on the base class. An optimizer is defined by its update
            rule, so there is no meaningful default to inherit.
        """
        raise NotImplementedError("Subclasses must implement update_parameters")

    def reset(self) -> None:
        """
        Reset optimizer state for new optimization run.

        The base implementation does nothing, which is correct for a stateless
        optimizer such as plain gradient descent. Any optimizer accumulating
        state across steps -- a velocity, a second moment -- must override this
        and clear it, or a second run starts from wherever the first one ended
        and its trajectory is not reproducible.
        """
        pass

update_parameters

update_parameters(params: dict[str, ndarray], grads: dict[str, ndarray], t: int) -> dict[str, np.ndarray]

Update parameters using optimization algorithm.

Implementations return a new dictionary rather than mutating the one they were given, so that a caller can keep the parameter trajectory of a run.

Parameters:

Name	Type	Description	Default
params	dict	Current parameter values.	required
grads	dict	Gradients for each parameter.	required
t	int	Current iteration, counted from one. Optimizers applying bias correction divide by 1 - beta ** t, which is why the count starts at one rather than zero.	required

Returns:

Type	Description
dict	Updated parameters.

Raises:

Type	Description
NotImplementedError	Always, on the base class. An optimizer is defined by its update rule, so there is no meaningful default to inherit.

Source code in src/dlhub/optimizers/base.py

def update_parameters(
    self, params: dict[str, np.ndarray], grads: dict[str, np.ndarray], t: int
) -> dict[str, np.ndarray]:
    """
    Update parameters using optimization algorithm.

    Implementations return a new dictionary rather than mutating the one they
    were given, so that a caller can keep the parameter trajectory of a run.

    Parameters
    ----------
    params : dict
        Current parameter values.
    grads : dict
        Gradients for each parameter.
    t : int
        Current iteration, counted from one. Optimizers applying bias
        correction divide by ``1 - beta ** t``, which is why the count starts
        at one rather than zero.

    Returns
    -------
    dict
        Updated parameters.

    Raises
    ------
    NotImplementedError
        Always, on the base class. An optimizer is defined by its update
        rule, so there is no meaningful default to inherit.
    """
    raise NotImplementedError("Subclasses must implement update_parameters")

reset

reset() -> None

Reset optimizer state for new optimization run.

The base implementation does nothing, which is correct for a stateless optimizer such as plain gradient descent. Any optimizer accumulating state across steps -- a velocity, a second moment -- must override this and clear it, or a second run starts from wherever the first one ended and its trajectory is not reproducible.

Source code in src/dlhub/optimizers/base.py

def reset(self) -> None:
    """
    Reset optimizer state for new optimization run.

    The base implementation does nothing, which is correct for a stateless
    optimizer such as plain gradient descent. Any optimizer accumulating
    state across steps -- a velocity, a second moment -- must override this
    and clear it, or a second run starts from wherever the first one ended
    and its trajectory is not reproducible.
    """
    pass

comparison

Optimization Algorithms Comparison

A comprehensive comparison framework for evaluating different optimization algorithms in deep learning contexts. This module provides implementations and utilities to compare gradient descent variants (SGD, Momentum, RMSprop, Adam) on various loss landscapes and datasets, demonstrating their convergence properties and performance characteristics.

This implementation serves as both a practical tool for optimizer selection and an educational resource for understanding optimization dynamics in neural networks.

References

• Kingma, D. P., & Ba, J. (2014). Adam: A Method for Stochastic Optimization. arXiv preprint arXiv:1412.6980.
• Ruder, S. (2016). An overview of gradient descent optimization algorithms. arXiv preprint arXiv:1609.04747.
• Duchi, J., Hazan, E., & Singer, Y. (2011). Adaptive subgradient methods for online learning and stochastic optimization. JMLR, 12, 2121-2159.

Author

Deep Learning Reference Hub

License

MIT License

Notes

This implementation focuses on numerical stability and educational clarity. The optimizers raced here are the canonical implementations from this subpackage, presented through a uniform driver interface rather than reimplemented; see :data:RACE_BIAS_CORRECTION for the one configuration decision the race makes on their behalf. Visualization utilities require matplotlib and are designed for Jupyter notebooks.

OptimizerType

Bases: Enum

Enumeration of available optimizer types.

Source code in src/dlhub/optimizers/comparison.py

class OptimizerType(Enum):
    """Enumeration of available optimizer types."""

    SGD = "sgd"
    MOMENTUM = "momentum"
    RMSPROP = "rmsprop"
    ADAM = "adam"

OptimizationRun dataclass

The trace of one optimizer descending one problem, and its summary.

Not to be confused with :class:dlhub.tuning.ExperimentResult, which records the outcome of a hyperparameter search rather than a single descent.

Attributes:

Name	Type	Description
optimizer_name	str	Name of the optimizer used.
losses	List[float]	Loss values recorded during training.
parameters	List[Dict]	Parameter values at each iteration.
convergence_time	float	Time taken for convergence (in seconds).
final_loss	float	Final loss value achieved.
iterations_to_converge	int	Number of iterations required for convergence.

Source code in src/dlhub/optimizers/comparison.py

@dataclass
class OptimizationRun:
    """
    The trace of one optimizer descending one problem, and its summary.

    Not to be confused with :class:`dlhub.tuning.ExperimentResult`, which records
    the outcome of a hyperparameter search rather than a single descent.

    Attributes
    ----------
    optimizer_name : str
        Name of the optimizer used.
    losses : List[float]
        Loss values recorded during training.
    parameters : List[Dict]
        Parameter values at each iteration.
    convergence_time : float
        Time taken for convergence (in seconds).
    final_loss : float
        Final loss value achieved.
    iterations_to_converge : int
        Number of iterations required for convergence.
    """

    optimizer_name: str
    losses: list[float]
    parameters: list[dict]
    convergence_time: float
    final_loss: float
    iterations_to_converge: int

SGDOptimizer

Bases: BaseOptimizer

Stochastic Gradient Descent optimizer.

Basic gradient descent with fixed learning rate. Simple but often effective baseline for comparison with more sophisticated optimizers.

The update rule itself lives in :mod:dlhub.optimizers.mini_batch, which is where the hub teaches plain gradient descent; this class exists to present it through the driver contract so the race can hold it alongside the others.

Parameters:

Name	Type	Description	Default
learning_rate	float	Step size for parameter updates.	0.01

Source code in src/dlhub/optimizers/comparison.py

class SGDOptimizer(BaseOptimizer):
    """
    Stochastic Gradient Descent optimizer.

    Basic gradient descent with fixed learning rate. Simple but often effective
    baseline for comparison with more sophisticated optimizers.

    The update rule itself lives in :mod:`dlhub.optimizers.mini_batch`, which is
    where the hub teaches plain gradient descent; this class exists to present it
    through the driver contract so the race can hold it alongside the others.

    Parameters
    ----------
    learning_rate : float, default=0.01
        Step size for parameter updates.
    """

    def __init__(self, learning_rate: float = 0.01):
        super().__init__(learning_rate)
        self.name = "SGD"
        self._optimizer = MiniBatchGradientDescent(learning_rate=learning_rate)

    def update_parameters(self, params: dict, grads: dict, t: int) -> dict:
        """
        Update parameters using vanilla gradient descent.

        Parameters
        ----------
        params : dict
            Current parameter values.
        grads : dict
            Gradients for each parameter.
        t : int
            Current iteration. Unused: the step is the same at every iteration,
            since plain gradient descent carries no state to correct for.

        Returns
        -------
        dict
            Updated parameters after SGD step.
        """
        return self._optimizer.update_parameters(params, grads)

update_parameters

update_parameters(params: dict, grads: dict, t: int) -> dict

Update parameters using vanilla gradient descent.

Parameters:

Name	Type	Description	Default
params	dict	Current parameter values.	required
grads	dict	Gradients for each parameter.	required
t	int	Current iteration. Unused: the step is the same at every iteration, since plain gradient descent carries no state to correct for.	required

Returns:

Type	Description
dict	Updated parameters after SGD step.

Source code in src/dlhub/optimizers/comparison.py

def update_parameters(self, params: dict, grads: dict, t: int) -> dict:
    """
    Update parameters using vanilla gradient descent.

    Parameters
    ----------
    params : dict
        Current parameter values.
    grads : dict
        Gradients for each parameter.
    t : int
        Current iteration. Unused: the step is the same at every iteration,
        since plain gradient descent carries no state to correct for.

    Returns
    -------
    dict
        Updated parameters after SGD step.
    """
    return self._optimizer.update_parameters(params, grads)

MomentumOptimizer

Bases: BaseOptimizer

Momentum optimizer using exponential moving averages.

Accelerates gradient descent by accumulating momentum in consistent directions and dampening oscillations. Particularly effective in ravines and saddle points.

Wraps :class:dlhub.optimizers.momentum.MomentumOptimizer, which is where the update rule is derived and written out.

Parameters:

Name	Type	Description	Default
learning_rate	float	Step size for parameter updates.	0.01
beta	float	Momentum coefficient for exponential moving average.	0.9
bias_correction	bool	Whether to divide out the bias of the zero-initialised velocity. See :data:RACE_BIAS_CORRECTION for why the race sets it rather than inheriting the canonical module's own default.	``RACE_BIAS_CORRECTION``

Source code in src/dlhub/optimizers/comparison.py

class MomentumOptimizer(BaseOptimizer):
    """
    Momentum optimizer using exponential moving averages.

    Accelerates gradient descent by accumulating momentum in consistent directions
    and dampening oscillations. Particularly effective in ravines and saddle points.

    Wraps :class:`dlhub.optimizers.momentum.MomentumOptimizer`, which is where the
    update rule is derived and written out.

    Parameters
    ----------
    learning_rate : float, default=0.01
        Step size for parameter updates.
    beta : float, default=0.9
        Momentum coefficient for exponential moving average.
    bias_correction : bool, default=``RACE_BIAS_CORRECTION``
        Whether to divide out the bias of the zero-initialised velocity. See
        :data:`RACE_BIAS_CORRECTION` for why the race sets it rather than
        inheriting the canonical module's own default.
    """

    def __init__(
        self,
        learning_rate: float = 0.01,
        beta: float = 0.9,
        bias_correction: bool = RACE_BIAS_CORRECTION,
    ):
        super().__init__(learning_rate)
        self.beta = beta
        self.bias_correction = bias_correction
        self.name = "Momentum"
        self._optimizer = CanonicalMomentum(
            learning_rate=learning_rate, beta=beta, bias_correction=bias_correction
        )

    def update_parameters(self, params: dict, grads: dict, t: int) -> dict:
        """
        Update parameters using momentum-based gradient descent.

        Parameters
        ----------
        params : dict
            Current parameter values.
        grads : dict
            Gradients for each parameter.
        t : int
            Current iteration. Unused: the canonical optimizer counts its own
            steps, and :meth:`reset` clears that count, so the two agree for any
            driver that counts from one and resets between runs.

        Returns
        -------
        dict
            Updated parameters after momentum step.
        """
        return self._optimizer.update_parameters(params, grads)

    def reset(self) -> None:
        """Reset momentum terms for new optimization run."""
        self._optimizer.reset_optimizer_state()

update_parameters

update_parameters(params: dict, grads: dict, t: int) -> dict

Update parameters using momentum-based gradient descent.

Parameters:

Name	Type	Description	Default
params	dict	Current parameter values.	required
grads	dict	Gradients for each parameter.	required
t	int	Current iteration. Unused: the canonical optimizer counts its own steps, and :meth:reset clears that count, so the two agree for any driver that counts from one and resets between runs.	required

Returns:

Type	Description
dict	Updated parameters after momentum step.

Source code in src/dlhub/optimizers/comparison.py

def update_parameters(self, params: dict, grads: dict, t: int) -> dict:
    """
    Update parameters using momentum-based gradient descent.

    Parameters
    ----------
    params : dict
        Current parameter values.
    grads : dict
        Gradients for each parameter.
    t : int
        Current iteration. Unused: the canonical optimizer counts its own
        steps, and :meth:`reset` clears that count, so the two agree for any
        driver that counts from one and resets between runs.

    Returns
    -------
    dict
        Updated parameters after momentum step.
    """
    return self._optimizer.update_parameters(params, grads)

reset

reset() -> None

Reset momentum terms for new optimization run.

Source code in src/dlhub/optimizers/comparison.py

def reset(self) -> None:
    """Reset momentum terms for new optimization run."""
    self._optimizer.reset_optimizer_state()

RMSpropOptimizer

Bases: BaseOptimizer

RMSprop (Root Mean Square Propagation) optimizer.

Adapts the learning rate per parameter using a running average of squared gradients, so that parameters with consistently large gradients take smaller steps.

Wraps :class:dlhub.optimizers.rmsprop.RMSpropOptimizer, which is where the update rule is derived and written out.

Parameters:

Name	Type	Description	Default
learning_rate	float	Step size for parameter updates.	0.001
beta	float	Decay rate for the running average of squared gradients.	0.9
epsilon	float	Small constant to prevent division by zero.	1e-8
bias_correction	bool	Whether to divide out the bias of the zero-initialised second moment. The canonical module defaults this off, which is how RMSprop is usually written; see :data:RACE_BIAS_CORRECTION for why the race overrides it.	``RACE_BIAS_CORRECTION``

Source code in src/dlhub/optimizers/comparison.py

class RMSpropOptimizer(BaseOptimizer):
    """
    RMSprop (Root Mean Square Propagation) optimizer.

    Adapts the learning rate per parameter using a running average of squared
    gradients, so that parameters with consistently large gradients take smaller
    steps.

    Wraps :class:`dlhub.optimizers.rmsprop.RMSpropOptimizer`, which is where the
    update rule is derived and written out.

    Parameters
    ----------
    learning_rate : float, default=0.001
        Step size for parameter updates.
    beta : float, default=0.9
        Decay rate for the running average of squared gradients.
    epsilon : float, default=1e-8
        Small constant to prevent division by zero.
    bias_correction : bool, default=``RACE_BIAS_CORRECTION``
        Whether to divide out the bias of the zero-initialised second moment.
        The canonical module defaults this off, which is how RMSprop is usually
        written; see :data:`RACE_BIAS_CORRECTION` for why the race overrides it.
    """

    def __init__(
        self,
        learning_rate: float = 0.001,
        beta: float = 0.9,
        epsilon: float = 1e-8,
        bias_correction: bool = RACE_BIAS_CORRECTION,
    ):
        super().__init__(learning_rate)
        self.beta = beta
        self.epsilon = epsilon
        self.bias_correction = bias_correction
        self.name = "RMSprop"
        self._optimizer = CanonicalRMSprop(
            learning_rate=learning_rate,
            beta=beta,
            epsilon=epsilon,
            bias_correction=bias_correction,
        )

    def update_parameters(self, params: dict, grads: dict, t: int) -> dict:
        """
        Update parameters using RMSprop optimization.

        Parameters
        ----------
        params : dict
            Current parameter values.
        grads : dict
            Gradients for each parameter.
        t : int
            Current iteration. Unused, for the reason given on
            :meth:`MomentumOptimizer.update_parameters`.

        Returns
        -------
        dict
            Updated parameters after RMSprop step.
        """
        return self._optimizer.update_parameters(params, grads)

    def reset(self) -> None:
        """Reset second moment estimates for new optimization run."""
        self._optimizer.reset_optimizer_state()

update_parameters

update_parameters(params: dict, grads: dict, t: int) -> dict

Update parameters using RMSprop optimization.

Parameters:

Name	Type	Description	Default
params	dict	Current parameter values.	required
grads	dict	Gradients for each parameter.	required
t	int	Current iteration. Unused, for the reason given on :meth:MomentumOptimizer.update_parameters.	required

Returns:

Type	Description
dict	Updated parameters after RMSprop step.

Source code in src/dlhub/optimizers/comparison.py

def update_parameters(self, params: dict, grads: dict, t: int) -> dict:
    """
    Update parameters using RMSprop optimization.

    Parameters
    ----------
    params : dict
        Current parameter values.
    grads : dict
        Gradients for each parameter.
    t : int
        Current iteration. Unused, for the reason given on
        :meth:`MomentumOptimizer.update_parameters`.

    Returns
    -------
    dict
        Updated parameters after RMSprop step.
    """
    return self._optimizer.update_parameters(params, grads)

reset

reset() -> None

Reset second moment estimates for new optimization run.

Source code in src/dlhub/optimizers/comparison.py

def reset(self) -> None:
    """Reset second moment estimates for new optimization run."""
    self._optimizer.reset_optimizer_state()

AdamOptimizer

Bases: BaseOptimizer

Adam (Adaptive Moment Estimation) optimizer.

Combines benefits of Momentum and RMSprop by maintaining both first and second moment estimates of gradients. Generally robust and effective across many problems.

Wraps :class:dlhub.optimizers.adam.AdamOptimizer, which is where the update rule is derived and written out. Adam bias-corrects both moments by construction, so it takes no flag: the correction is part of the method rather than an option on it.

Parameters:

Name	Type	Description	Default
learning_rate	float	Step size for parameter updates.	0.001
beta1	float	Exponential decay rate for first moment estimates.	0.9
beta2	float	Exponential decay rate for second moment estimates.	0.999
epsilon	float	Small constant to prevent division by zero.	1e-8

Source code in src/dlhub/optimizers/comparison.py

class AdamOptimizer(BaseOptimizer):
    """
    Adam (Adaptive Moment Estimation) optimizer.

    Combines benefits of Momentum and RMSprop by maintaining both first and second
    moment estimates of gradients. Generally robust and effective across many problems.

    Wraps :class:`dlhub.optimizers.adam.AdamOptimizer`, which is where the update
    rule is derived and written out. Adam bias-corrects both moments by
    construction, so it takes no flag: the correction is part of the method
    rather than an option on it.

    Parameters
    ----------
    learning_rate : float, default=0.001
        Step size for parameter updates.
    beta1 : float, default=0.9
        Exponential decay rate for first moment estimates.
    beta2 : float, default=0.999
        Exponential decay rate for second moment estimates.
    epsilon : float, default=1e-8
        Small constant to prevent division by zero.
    """

    def __init__(
        self,
        learning_rate: float = 0.001,
        beta1: float = 0.9,
        beta2: float = 0.999,
        epsilon: float = 1e-8,
    ):
        super().__init__(learning_rate)
        self.beta1 = beta1
        self.beta2 = beta2
        self.epsilon = epsilon
        self.name = "Adam"
        self._optimizer = CanonicalAdam(
            learning_rate=learning_rate, beta1=beta1, beta2=beta2, epsilon=epsilon
        )

    def update_parameters(self, params: dict, grads: dict, t: int) -> dict:
        """
        Update parameters using Adam optimization algorithm.

        Parameters
        ----------
        params : dict
            Current parameter values.
        grads : dict
            Gradients for each parameter.
        t : int
            Current iteration. Unused, for the reason given on
            :meth:`MomentumOptimizer.update_parameters`.

        Returns
        -------
        dict
            Updated parameters after Adam step.
        """
        return self._optimizer.update(params, grads)

    def reset(self) -> None:
        """Reset first and second moment estimates for new optimization run."""
        self._optimizer.reset_state()

update_parameters

update_parameters(params: dict, grads: dict, t: int) -> dict

Update parameters using Adam optimization algorithm.

Parameters:

Name	Type	Description	Default
params	dict	Current parameter values.	required
grads	dict	Gradients for each parameter.	required
t	int	Current iteration. Unused, for the reason given on :meth:MomentumOptimizer.update_parameters.	required

Returns:

Type	Description
dict	Updated parameters after Adam step.

Source code in src/dlhub/optimizers/comparison.py

def update_parameters(self, params: dict, grads: dict, t: int) -> dict:
    """
    Update parameters using Adam optimization algorithm.

    Parameters
    ----------
    params : dict
        Current parameter values.
    grads : dict
        Gradients for each parameter.
    t : int
        Current iteration. Unused, for the reason given on
        :meth:`MomentumOptimizer.update_parameters`.

    Returns
    -------
    dict
        Updated parameters after Adam step.
    """
    return self._optimizer.update(params, grads)

reset

reset() -> None

Reset first and second moment estimates for new optimization run.

Source code in src/dlhub/optimizers/comparison.py

def reset(self) -> None:
    """Reset first and second moment estimates for new optimization run."""
    self._optimizer.reset_state()

OptimizationProblem

Base class for defining optimization problems with loss functions and gradients.

Parameters:

Name	Type	Description	Default
name	str	Name of the optimization problem.	required

Source code in src/dlhub/optimizers/comparison.py

class OptimizationProblem:
    """
    Base class for defining optimization problems with loss functions and gradients.

    Parameters
    ----------
    name : str
        Name of the optimization problem.
    """

    def __init__(self, name: str):
        self.name = name

    def loss_function(self, params: dict) -> float:
        """
        Compute loss for given parameters.

        Parameters
        ----------
        params : dict
            Parameter values.

        Returns
        -------
        float
            Loss value.
        """
        raise NotImplementedError("Subclasses must implement loss_function")

    def gradients(self, params: dict) -> dict:
        """
        Compute gradients for given parameters.

        Parameters
        ----------
        params : dict
            Parameter values.

        Returns
        -------
        dict
            Gradients for each parameter.
        """
        raise NotImplementedError("Subclasses must implement gradients")

    def initial_parameters(self) -> dict:
        """
        Get initial parameter values for optimization.

        Returns
        -------
        dict
            Initial parameter values.
        """
        raise NotImplementedError("Subclasses must implement initial_parameters")

loss_function

loss_function(params: dict) -> float

Compute loss for given parameters.

Parameters:

Name	Type	Description	Default
params	dict	Parameter values.	required

Returns:

Type	Description
float	Loss value.

Source code in src/dlhub/optimizers/comparison.py

def loss_function(self, params: dict) -> float:
    """
    Compute loss for given parameters.

    Parameters
    ----------
    params : dict
        Parameter values.

    Returns
    -------
    float
        Loss value.
    """
    raise NotImplementedError("Subclasses must implement loss_function")

gradients

gradients(params: dict) -> dict

Compute gradients for given parameters.

Parameters:

Name	Type	Description	Default
params	dict	Parameter values.	required

Returns:

Type	Description
dict	Gradients for each parameter.

Source code in src/dlhub/optimizers/comparison.py

def gradients(self, params: dict) -> dict:
    """
    Compute gradients for given parameters.

    Parameters
    ----------
    params : dict
        Parameter values.

    Returns
    -------
    dict
        Gradients for each parameter.
    """
    raise NotImplementedError("Subclasses must implement gradients")

initial_parameters

initial_parameters() -> dict

Get initial parameter values for optimization.

Returns:

Type	Description
dict	Initial parameter values.

Source code in src/dlhub/optimizers/comparison.py

def initial_parameters(self) -> dict:
    """
    Get initial parameter values for optimization.

    Returns
    -------
    dict
        Initial parameter values.
    """
    raise NotImplementedError("Subclasses must implement initial_parameters")

QuadraticBowl

Bases: OptimizationProblem

Simple quadratic bowl optimization problem: f(x,y) = ax² + by².

Well-conditioned convex problem useful for demonstrating basic optimizer behavior.

Parameters:

Name	Type	Description	Default
a	float	Coefficient for x² term.	1.0
b	float	Coefficient for y² term.	1.0

Source code in src/dlhub/optimizers/comparison.py

class QuadraticBowl(OptimizationProblem):
    """
    Simple quadratic bowl optimization problem: f(x,y) = ax² + by².

    Well-conditioned convex problem useful for demonstrating basic optimizer behavior.

    Parameters
    ----------
    a : float, default=1.0
        Coefficient for x² term.
    b : float, default=1.0
        Coefficient for y² term.
    """

    def __init__(self, a: float = 1.0, b: float = 1.0):
        super().__init__(f"Quadratic Bowl (a={a}, b={b})")
        self.a = a
        self.b = b

    def loss_function(self, params: dict) -> float:
        """Compute quadratic loss: ax² + by²."""
        x, y = params["x"], params["y"]
        return self.a * x**2 + self.b * y**2

    def gradients(self, params: dict) -> dict:
        """Compute gradients: [2ax, 2by]."""
        x, y = params["x"], params["y"]
        return {"x": 2 * self.a * x, "y": 2 * self.b * y}

    def initial_parameters(self) -> dict:
        """Initialize at (5, 5) for clear visualization."""
        return {"x": 5.0, "y": 5.0}

loss_function

loss_function(params: dict) -> float

Compute quadratic loss: ax² + by².

Source code in src/dlhub/optimizers/comparison.py

def loss_function(self, params: dict) -> float:
    """Compute quadratic loss: ax² + by²."""
    x, y = params["x"], params["y"]
    return self.a * x**2 + self.b * y**2

gradients

gradients(params: dict) -> dict

Compute gradients: [2ax, 2by].

Source code in src/dlhub/optimizers/comparison.py

def gradients(self, params: dict) -> dict:
    """Compute gradients: [2ax, 2by]."""
    x, y = params["x"], params["y"]
    return {"x": 2 * self.a * x, "y": 2 * self.b * y}

initial_parameters

initial_parameters() -> dict

Initialize at (5, 5) for clear visualization.

Source code in src/dlhub/optimizers/comparison.py

def initial_parameters(self) -> dict:
    """Initialize at (5, 5) for clear visualization."""
    return {"x": 5.0, "y": 5.0}

RosenbrockFunction

Bases: OptimizationProblem

Rosenbrock function: f(x,y) = (a-x)² + b(y-x²)².

Classic non-convex optimization benchmark with narrow curved valley. Challenging for optimizers due to ill-conditioning and plateau regions.

Parameters:

Name	Type	Description	Default
a	float	Parameter controlling x-offset of minimum.	1.0
b	float	Parameter controlling valley curvature (higher = more challenging).	100.0

Source code in src/dlhub/optimizers/comparison.py

class RosenbrockFunction(OptimizationProblem):
    """
    Rosenbrock function: f(x,y) = (a-x)² + b(y-x²)².

    Classic non-convex optimization benchmark with narrow curved valley.
    Challenging for optimizers due to ill-conditioning and plateau regions.

    Parameters
    ----------
    a : float, default=1.0
        Parameter controlling x-offset of minimum.
    b : float, default=100.0
        Parameter controlling valley curvature (higher = more challenging).
    """

    def __init__(self, a: float = 1.0, b: float = 100.0):
        super().__init__(f"Rosenbrock Function (a={a}, b={b})")
        self.a = a
        self.b = b

    def loss_function(self, params: dict) -> float:
        """Compute Rosenbrock function value."""
        x, y = params["x"], params["y"]
        return (self.a - x) ** 2 + self.b * (y - x**2) ** 2

    def gradients(self, params: dict) -> dict:
        """Compute Rosenbrock gradients analytically."""
        x, y = params["x"], params["y"]
        dx = -2 * (self.a - x) - 4 * self.b * x * (y - x**2)
        dy = 2 * self.b * (y - x**2)
        return {"x": dx, "y": dy}

    def initial_parameters(self) -> dict:
        """Initialize away from minimum for interesting optimization path."""
        return {"x": -2.0, "y": 2.0}

loss_function

loss_function(params: dict) -> float

Compute Rosenbrock function value.

Source code in src/dlhub/optimizers/comparison.py

def loss_function(self, params: dict) -> float:
    """Compute Rosenbrock function value."""
    x, y = params["x"], params["y"]
    return (self.a - x) ** 2 + self.b * (y - x**2) ** 2

gradients

gradients(params: dict) -> dict

Compute Rosenbrock gradients analytically.

Source code in src/dlhub/optimizers/comparison.py

def gradients(self, params: dict) -> dict:
    """Compute Rosenbrock gradients analytically."""
    x, y = params["x"], params["y"]
    dx = -2 * (self.a - x) - 4 * self.b * x * (y - x**2)
    dy = 2 * self.b * (y - x**2)
    return {"x": dx, "y": dy}

initial_parameters

initial_parameters() -> dict

Initialize away from minimum for interesting optimization path.

Source code in src/dlhub/optimizers/comparison.py

def initial_parameters(self) -> dict:
    """Initialize away from minimum for interesting optimization path."""
    return {"x": -2.0, "y": 2.0}

BealeFunction

Bases: OptimizationProblem

Beale function: f(x,y) = (1.5 - x + xy)² + (2.25 - x + xy²)² + (2.625 - x + xy³)².

Multimodal function with global minimum and several local minima. Tests optimizer robustness to local minima and saddle points.

Source code in src/dlhub/optimizers/comparison.py

class BealeFunction(OptimizationProblem):
    """
    Beale function: f(x,y) = (1.5 - x + xy)² + (2.25 - x + x*y²)² + (2.625 - x + x*y³)².

    Multimodal function with global minimum and several local minima.
    Tests optimizer robustness to local minima and saddle points.
    """

    def __init__(self):
        super().__init__("Beale Function")

    def loss_function(self, params: dict) -> float:
        """Compute Beale function value."""
        x, y = params["x"], params["y"]
        term1 = (1.5 - x + x * y) ** 2
        term2 = (2.25 - x + x * y**2) ** 2
        term3 = (2.625 - x + x * y**3) ** 2
        return term1 + term2 + term3

    def gradients(self, params: dict) -> dict:
        """Compute Beale function gradients analytically."""
        x, y = params["x"], params["y"]

        # Partial derivatives computed analytically
        dx = (
            2 * (1.5 - x + x * y) * (-1 + y)
            + 2 * (2.25 - x + x * y**2) * (-1 + y**2)
            + 2 * (2.625 - x + x * y**3) * (-1 + y**3)
        )

        dy = (
            2 * (1.5 - x + x * y) * x
            + 2 * (2.25 - x + x * y**2) * (2 * x * y)
            + 2 * (2.625 - x + x * y**3) * (3 * x * y**2)
        )

        return {"x": dx, "y": dy}

    def initial_parameters(self) -> dict:
        """Initialize at challenging starting point."""
        return {"x": 4.0, "y": 4.0}

loss_function

loss_function(params: dict) -> float

Compute Beale function value.

Source code in src/dlhub/optimizers/comparison.py

def loss_function(self, params: dict) -> float:
    """Compute Beale function value."""
    x, y = params["x"], params["y"]
    term1 = (1.5 - x + x * y) ** 2
    term2 = (2.25 - x + x * y**2) ** 2
    term3 = (2.625 - x + x * y**3) ** 2
    return term1 + term2 + term3

gradients

gradients(params: dict) -> dict

Compute Beale function gradients analytically.

Source code in src/dlhub/optimizers/comparison.py

def gradients(self, params: dict) -> dict:
    """Compute Beale function gradients analytically."""
    x, y = params["x"], params["y"]

    # Partial derivatives computed analytically
    dx = (
        2 * (1.5 - x + x * y) * (-1 + y)
        + 2 * (2.25 - x + x * y**2) * (-1 + y**2)
        + 2 * (2.625 - x + x * y**3) * (-1 + y**3)
    )

    dy = (
        2 * (1.5 - x + x * y) * x
        + 2 * (2.25 - x + x * y**2) * (2 * x * y)
        + 2 * (2.625 - x + x * y**3) * (3 * x * y**2)
    )

    return {"x": dx, "y": dy}

initial_parameters

initial_parameters() -> dict

Initialize at challenging starting point.

Source code in src/dlhub/optimizers/comparison.py

def initial_parameters(self) -> dict:
    """Initialize at challenging starting point."""
    return {"x": 4.0, "y": 4.0}

OptimizationComparison

Comprehensive framework for comparing optimization algorithms.

Provides utilities to run multiple optimizers on various problems, collect performance metrics, and generate comparative visualizations.

Parameters:

Name	Type	Description	Default
max_iterations	int	Maximum number of optimization iterations.	1000
tolerance	float	Convergence tolerance for loss change.	1e-6
verbose	bool	Whether to print progress information.	True

Source code in src/dlhub/optimizers/comparison.py

class OptimizationComparison:
    """
    Comprehensive framework for comparing optimization algorithms.

    Provides utilities to run multiple optimizers on various problems,
    collect performance metrics, and generate comparative visualizations.

    Parameters
    ----------
    max_iterations : int, default=1000
        Maximum number of optimization iterations.
    tolerance : float, default=1e-6
        Convergence tolerance for loss change.
    verbose : bool, default=True
        Whether to print progress information.
    """

    def __init__(
        self, max_iterations: int = 1000, tolerance: float = 1e-6, verbose: bool = True
    ):
        self.max_iterations = max_iterations
        self.tolerance = tolerance
        self.verbose = verbose
        self.results = {}

    def create_optimizer(
        self, optimizer_type: OptimizerType, **kwargs
    ) -> BaseOptimizer:
        """
        Factory method to create optimizer instances.

        Parameters
        ----------
        optimizer_type : OptimizerType
            Type of optimizer to create.
        **kwargs
            Additional parameters for optimizer initialization.

        Returns
        -------
        BaseOptimizer
            Configured optimizer instance.
        """
        if optimizer_type == OptimizerType.SGD:
            return SGDOptimizer(**kwargs)
        elif optimizer_type == OptimizerType.MOMENTUM:
            return MomentumOptimizer(**kwargs)
        elif optimizer_type == OptimizerType.RMSPROP:
            return RMSpropOptimizer(**kwargs)
        elif optimizer_type == OptimizerType.ADAM:
            return AdamOptimizer(**kwargs)
        else:
            raise ValueError(f"Unknown optimizer type: {optimizer_type}")

    def run_optimization(
        self, problem: OptimizationProblem, optimizer: BaseOptimizer
    ) -> OptimizationRun:
        """
        Run single optimization experiment.

        Parameters
        ----------
        problem : OptimizationProblem
            Problem to optimize.
        optimizer : BaseOptimizer
            Optimizer to use.

        Returns
        -------
        OptimizationRun
            Results of optimization including metrics and trajectory.
        """
        optimizer.reset()

        # The benchmark problems below are two-dimensional surfaces, so they state
        # their starting points as plain floats, which reads naturally for a point
        # on a contour plot. Optimizers are written against arrays -- the contract
        # says so, and a neural network's parameters are never scalars -- so the
        # conversion happens once, here, where the problem's world meets the
        # optimizer's. Doing it at the boundary means every optimizer receives what
        # the contract promises, rather than each one having to tolerate floats.
        params = {
            name: np.asarray(value, dtype=float)
            for name, value in problem.initial_parameters().items()
        }
        losses = []
        parameter_history = []

        start_time = time.time()
        converged = False

        for iteration in range(1, self.max_iterations + 1):
            current_loss = problem.loss_function(params)
            grads = problem.gradients(params)

            losses.append(current_loss)
            parameter_history.append(params.copy())

            if len(losses) > 1:
                loss_change = abs(losses[-2] - losses[-1])
                if loss_change < self.tolerance:
                    converged = True
                    break

            params = optimizer.update_parameters(params, grads, iteration)

            # Prevent divergence
            if current_loss > 1e10 or np.any(
                [np.isnan(v) or np.isinf(v) for v in params.values()]
            ):
                if self.verbose:
                    print(f"{optimizer.name} diverged at iteration {iteration}")
                break

        end_time = time.time()

        result = OptimizationRun(
            optimizer_name=optimizer.name,
            losses=losses,
            parameters=parameter_history,
            convergence_time=end_time - start_time,
            final_loss=losses[-1] if losses else float("inf"),
            iterations_to_converge=len(losses) if converged else self.max_iterations,
        )

        if self.verbose:
            status = "converged" if converged else "max iterations reached"
            print(
                f"{optimizer.name}: {status} in {len(losses)} iterations, "
                f"final loss: {result.final_loss:.6f}, time: {result.convergence_time:.3f}s"
            )

        return result

    def compare_optimizers(
        self, problem: OptimizationProblem, optimizer_configs: dict[OptimizerType, dict]
    ) -> dict[str, OptimizationRun]:
        """
        Compare multiple optimizers on a single problem.

        Parameters
        ----------
        problem : OptimizationProblem
            Problem to optimize.
        optimizer_configs : dict
            Dictionary mapping optimizer types to their configuration parameters.

        Returns
        -------
        dict
            Results for each optimizer.
        """
        results = {}

        if self.verbose:
            print(f"\nOptimizing: {problem.name}")
            print("=" * 50)

        for opt_type, config in optimizer_configs.items():
            optimizer = self.create_optimizer(opt_type, **config)
            result = self.run_optimization(problem, optimizer)
            results[optimizer.name] = result

        return results

    def run_comprehensive_comparison(self) -> dict[str, dict[str, OptimizationRun]]:
        """
        Run comprehensive comparison across multiple problems and optimizers.

        Returns
        -------
        dict
            Nested dictionary: {problem_name: {optimizer_name: result}}
        """
        problems = [
            QuadraticBowl(a=1.0, b=1.0),  # Well-conditioned
            QuadraticBowl(a=1.0, b=100.0),  # Ill-conditioned
            RosenbrockFunction(),  # Non-convex valley
            BealeFunction(),  # Multimodal
        ]

        optimizer_configs = {
            OptimizerType.SGD: {"learning_rate": 0.01},
            OptimizerType.MOMENTUM: {"learning_rate": 0.01, "beta": 0.9},
            OptimizerType.RMSPROP: {"learning_rate": 0.01, "beta": 0.9},
            OptimizerType.ADAM: {"learning_rate": 0.05, "beta1": 0.9, "beta2": 0.999},
        }

        all_results = {}

        for problem in problems:
            results = self.compare_optimizers(problem, optimizer_configs)
            all_results[problem.name] = results

        self.results = all_results
        return all_results

    def plot_convergence_comparison(
        self,
        results: dict[str, OptimizationRun],
        problem_name: str,
        log_scale: bool = True,
    ):
        """
        Plot convergence curves for optimizer comparison.

        Parameters
        ----------
        results : dict
            Results from optimizer comparison.
        problem_name : str
            Name of the problem for plot title.
        log_scale : bool, default=True
            Whether to use logarithmic scale for loss.
        """
        plt.figure(figsize=(12, 8))

        colors = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728"]
        styles = ["-", "--", "-.", ":"]

        for i, (opt_name, result) in enumerate(results.items()):
            if result.losses:
                iterations = range(1, len(result.losses) + 1)
                plt.plot(
                    iterations,
                    result.losses,
                    color=colors[i % len(colors)],
                    linestyle=styles[i % len(styles)],
                    linewidth=2,
                    label=opt_name,
                    alpha=0.8,
                )

        plt.xlabel("Iterations", fontsize=12)
        plt.ylabel("Loss", fontsize=12)
        plt.title(
            f"Convergence Comparison: {problem_name}", fontsize=14, fontweight="bold"
        )
        plt.legend(fontsize=11)
        plt.grid(True, alpha=0.3)

        if log_scale:
            plt.yscale("log")
            plt.ylabel("Loss (log scale)", fontsize=12)

        plt.tight_layout()
        plt.show()

    def plot_optimization_paths(
        self,
        results: dict[str, OptimizationRun],
        problem: OptimizationProblem,
        contour_levels: int = 20,
    ):
        """
        Plot optimization trajectories on loss landscape contours.

        Parameters
        ----------
        results : dict
            Results from optimizer comparison.
        problem : OptimizationProblem
            Problem instance for computing loss landscape.
        contour_levels : int, default=20
            Number of contour levels to display.
        """
        plt.figure(figsize=(14, 10))

        # Create meshgrid for contour plot
        x_range = np.linspace(-6, 6, 100)
        y_range = np.linspace(-6, 6, 100)
        X, Y = np.meshgrid(x_range, y_range)
        Z = np.zeros_like(X)

        for i in range(X.shape[0]):
            for j in range(X.shape[1]):
                params = {"x": X[i, j], "y": Y[i, j]}
                Z[i, j] = problem.loss_function(params)

        contours = plt.contour(
            X, Y, Z, levels=contour_levels, alpha=0.6, cmap="viridis"
        )
        plt.clabel(contours, inline=True, fontsize=8, fmt="%.1f")

        colors = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728"]
        markers = ["o", "s", "^", "D"]

        for i, (opt_name, result) in enumerate(results.items()):
            if result.parameters:
                x_path = [p["x"] for p in result.parameters]
                y_path = [p["y"] for p in result.parameters]

                plt.plot(
                    x_path,
                    y_path,
                    color=colors[i % len(colors)],
                    marker=markers[i % len(markers)],
                    markersize=4,
                    linewidth=2,
                    label=f"{opt_name} ({len(x_path)} steps)",
                    alpha=0.8,
                )

                # Mark start and end points
                plt.plot(
                    x_path[0],
                    y_path[0],
                    marker="*",
                    color=colors[i % len(colors)],
                    markersize=12,
                    markeredgecolor="black",
                    markeredgewidth=1,
                )
                plt.plot(
                    x_path[-1],
                    y_path[-1],
                    marker="x",
                    color=colors[i % len(colors)],
                    markersize=10,
                    markeredgewidth=3,
                )

        plt.xlabel("x", fontsize=12)
        plt.ylabel("y", fontsize=12)
        plt.title(f"Optimization Paths: {problem.name}", fontsize=14, fontweight="bold")
        plt.legend(fontsize=11)
        plt.grid(True, alpha=0.3)
        plt.axis("equal")
        plt.tight_layout()
        plt.show()

    def generate_summary_table(
        self, all_results: dict[str, dict[str, OptimizationRun]]
    ) -> None:
        """
        Generate formatted summary table of optimization results.

        Parameters
        ----------
        all_results : dict
            Complete results from comprehensive comparison.
        """
        print("\n" + "=" * 80)
        print("OPTIMIZATION COMPARISON SUMMARY")
        print("=" * 80)

        for problem_name, results in all_results.items():
            print(f"\n{problem_name}")
            print("-" * len(problem_name))

            # Create formatted table
            headers = [
                "Optimizer",
                "Final Loss",
                "Iterations",
                "Conv. Time (s)",
                "Status",
            ]
            print(
                f"{headers[0]:<12} {headers[1]:<20} {headers[2]:<12} {headers[3]:<15} {headers[4]:<10}"
            )
            print("-" * 80)

            for opt_name, result in results.items():
                status = (
                    "✓"
                    if result.iterations_to_converge < self.max_iterations
                    else "Max iter"
                )
                print(
                    f"{opt_name:<12} {result.final_loss:<14.6f} "
                    f"{result.iterations_to_converge:<12} "
                    f"{result.convergence_time:<15.3f} {status:<10}"
                )

        print("\n" + "=" * 90)

create_optimizer

create_optimizer(optimizer_type: OptimizerType, **kwargs) -> BaseOptimizer

Factory method to create optimizer instances.

Parameters:

Name	Type	Description	Default
optimizer_type	OptimizerType	Type of optimizer to create.	required
**kwargs		Additional parameters for optimizer initialization.	{}

Returns:

Type	Description
BaseOptimizer	Configured optimizer instance.

Source code in src/dlhub/optimizers/comparison.py

def create_optimizer(
    self, optimizer_type: OptimizerType, **kwargs
) -> BaseOptimizer:
    """
    Factory method to create optimizer instances.

    Parameters
    ----------
    optimizer_type : OptimizerType
        Type of optimizer to create.
    **kwargs
        Additional parameters for optimizer initialization.

    Returns
    -------
    BaseOptimizer
        Configured optimizer instance.
    """
    if optimizer_type == OptimizerType.SGD:
        return SGDOptimizer(**kwargs)
    elif optimizer_type == OptimizerType.MOMENTUM:
        return MomentumOptimizer(**kwargs)
    elif optimizer_type == OptimizerType.RMSPROP:
        return RMSpropOptimizer(**kwargs)
    elif optimizer_type == OptimizerType.ADAM:
        return AdamOptimizer(**kwargs)
    else:
        raise ValueError(f"Unknown optimizer type: {optimizer_type}")

run_optimization

run_optimization(problem: OptimizationProblem, optimizer: BaseOptimizer) -> OptimizationRun

Run single optimization experiment.

Parameters:

Name	Type	Description	Default
problem	OptimizationProblem	Problem to optimize.	required
optimizer	BaseOptimizer	Optimizer to use.	required

Returns:

Type	Description
OptimizationRun	Results of optimization including metrics and trajectory.

Source code in src/dlhub/optimizers/comparison.py

def run_optimization(
    self, problem: OptimizationProblem, optimizer: BaseOptimizer
) -> OptimizationRun:
    """
    Run single optimization experiment.

    Parameters
    ----------
    problem : OptimizationProblem
        Problem to optimize.
    optimizer : BaseOptimizer
        Optimizer to use.

    Returns
    -------
    OptimizationRun
        Results of optimization including metrics and trajectory.
    """
    optimizer.reset()

    # The benchmark problems below are two-dimensional surfaces, so they state
    # their starting points as plain floats, which reads naturally for a point
    # on a contour plot. Optimizers are written against arrays -- the contract
    # says so, and a neural network's parameters are never scalars -- so the
    # conversion happens once, here, where the problem's world meets the
    # optimizer's. Doing it at the boundary means every optimizer receives what
    # the contract promises, rather than each one having to tolerate floats.
    params = {
        name: np.asarray(value, dtype=float)
        for name, value in problem.initial_parameters().items()
    }
    losses = []
    parameter_history = []

    start_time = time.time()
    converged = False

    for iteration in range(1, self.max_iterations + 1):
        current_loss = problem.loss_function(params)
        grads = problem.gradients(params)

        losses.append(current_loss)
        parameter_history.append(params.copy())

        if len(losses) > 1:
            loss_change = abs(losses[-2] - losses[-1])
            if loss_change < self.tolerance:
                converged = True
                break

        params = optimizer.update_parameters(params, grads, iteration)

        # Prevent divergence
        if current_loss > 1e10 or np.any(
            [np.isnan(v) or np.isinf(v) for v in params.values()]
        ):
            if self.verbose:
                print(f"{optimizer.name} diverged at iteration {iteration}")
            break

    end_time = time.time()

    result = OptimizationRun(
        optimizer_name=optimizer.name,
        losses=losses,
        parameters=parameter_history,
        convergence_time=end_time - start_time,
        final_loss=losses[-1] if losses else float("inf"),
        iterations_to_converge=len(losses) if converged else self.max_iterations,
    )

    if self.verbose:
        status = "converged" if converged else "max iterations reached"
        print(
            f"{optimizer.name}: {status} in {len(losses)} iterations, "
            f"final loss: {result.final_loss:.6f}, time: {result.convergence_time:.3f}s"
        )

    return result

compare_optimizers

compare_optimizers(problem: OptimizationProblem, optimizer_configs: dict[OptimizerType, dict]) -> dict[str, OptimizationRun]

Compare multiple optimizers on a single problem.

Parameters:

Name	Type	Description	Default
problem	OptimizationProblem	Problem to optimize.	required
optimizer_configs	dict	Dictionary mapping optimizer types to their configuration parameters.	required

Returns:

Type	Description
dict	Results for each optimizer.

Source code in src/dlhub/optimizers/comparison.py

def compare_optimizers(
    self, problem: OptimizationProblem, optimizer_configs: dict[OptimizerType, dict]
) -> dict[str, OptimizationRun]:
    """
    Compare multiple optimizers on a single problem.

    Parameters
    ----------
    problem : OptimizationProblem
        Problem to optimize.
    optimizer_configs : dict
        Dictionary mapping optimizer types to their configuration parameters.

    Returns
    -------
    dict
        Results for each optimizer.
    """
    results = {}

    if self.verbose:
        print(f"\nOptimizing: {problem.name}")
        print("=" * 50)

    for opt_type, config in optimizer_configs.items():
        optimizer = self.create_optimizer(opt_type, **config)
        result = self.run_optimization(problem, optimizer)
        results[optimizer.name] = result

    return results

run_comprehensive_comparison

run_comprehensive_comparison() -> dict[str, dict[str, OptimizationRun]]

Run comprehensive comparison across multiple problems and optimizers.

Returns:

Type	Description
dict	Nested dictionary: {problem_name: {optimizer_name: result}}

Source code in src/dlhub/optimizers/comparison.py

def run_comprehensive_comparison(self) -> dict[str, dict[str, OptimizationRun]]:
    """
    Run comprehensive comparison across multiple problems and optimizers.

    Returns
    -------
    dict
        Nested dictionary: {problem_name: {optimizer_name: result}}
    """
    problems = [
        QuadraticBowl(a=1.0, b=1.0),  # Well-conditioned
        QuadraticBowl(a=1.0, b=100.0),  # Ill-conditioned
        RosenbrockFunction(),  # Non-convex valley
        BealeFunction(),  # Multimodal
    ]

    optimizer_configs = {
        OptimizerType.SGD: {"learning_rate": 0.01},
        OptimizerType.MOMENTUM: {"learning_rate": 0.01, "beta": 0.9},
        OptimizerType.RMSPROP: {"learning_rate": 0.01, "beta": 0.9},
        OptimizerType.ADAM: {"learning_rate": 0.05, "beta1": 0.9, "beta2": 0.999},
    }

    all_results = {}

    for problem in problems:
        results = self.compare_optimizers(problem, optimizer_configs)
        all_results[problem.name] = results

    self.results = all_results
    return all_results

plot_convergence_comparison

plot_convergence_comparison(results: dict[str, OptimizationRun], problem_name: str, log_scale: bool = True)

Plot convergence curves for optimizer comparison.

Parameters:

Name	Type	Description	Default
results	dict	Results from optimizer comparison.	required
problem_name	str	Name of the problem for plot title.	required
log_scale	bool	Whether to use logarithmic scale for loss.	True

Source code in src/dlhub/optimizers/comparison.py

def plot_convergence_comparison(
    self,
    results: dict[str, OptimizationRun],
    problem_name: str,
    log_scale: bool = True,
):
    """
    Plot convergence curves for optimizer comparison.

    Parameters
    ----------
    results : dict
        Results from optimizer comparison.
    problem_name : str
        Name of the problem for plot title.
    log_scale : bool, default=True
        Whether to use logarithmic scale for loss.
    """
    plt.figure(figsize=(12, 8))

    colors = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728"]
    styles = ["-", "--", "-.", ":"]

    for i, (opt_name, result) in enumerate(results.items()):
        if result.losses:
            iterations = range(1, len(result.losses) + 1)
            plt.plot(
                iterations,
                result.losses,
                color=colors[i % len(colors)],
                linestyle=styles[i % len(styles)],
                linewidth=2,
                label=opt_name,
                alpha=0.8,
            )

    plt.xlabel("Iterations", fontsize=12)
    plt.ylabel("Loss", fontsize=12)
    plt.title(
        f"Convergence Comparison: {problem_name}", fontsize=14, fontweight="bold"
    )
    plt.legend(fontsize=11)
    plt.grid(True, alpha=0.3)

    if log_scale:
        plt.yscale("log")
        plt.ylabel("Loss (log scale)", fontsize=12)

    plt.tight_layout()
    plt.show()

plot_optimization_paths

plot_optimization_paths(results: dict[str, OptimizationRun], problem: OptimizationProblem, contour_levels: int = 20)

Plot optimization trajectories on loss landscape contours.

Parameters:

Name	Type	Description	Default
results	dict	Results from optimizer comparison.	required
problem	OptimizationProblem	Problem instance for computing loss landscape.	required
contour_levels	int	Number of contour levels to display.	20

Source code in src/dlhub/optimizers/comparison.py

def plot_optimization_paths(
    self,
    results: dict[str, OptimizationRun],
    problem: OptimizationProblem,
    contour_levels: int = 20,
):
    """
    Plot optimization trajectories on loss landscape contours.

    Parameters
    ----------
    results : dict
        Results from optimizer comparison.
    problem : OptimizationProblem
        Problem instance for computing loss landscape.
    contour_levels : int, default=20
        Number of contour levels to display.
    """
    plt.figure(figsize=(14, 10))

    # Create meshgrid for contour plot
    x_range = np.linspace(-6, 6, 100)
    y_range = np.linspace(-6, 6, 100)
    X, Y = np.meshgrid(x_range, y_range)
    Z = np.zeros_like(X)

    for i in range(X.shape[0]):
        for j in range(X.shape[1]):
            params = {"x": X[i, j], "y": Y[i, j]}
            Z[i, j] = problem.loss_function(params)

    contours = plt.contour(
        X, Y, Z, levels=contour_levels, alpha=0.6, cmap="viridis"
    )
    plt.clabel(contours, inline=True, fontsize=8, fmt="%.1f")

    colors = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728"]
    markers = ["o", "s", "^", "D"]

    for i, (opt_name, result) in enumerate(results.items()):
        if result.parameters:
            x_path = [p["x"] for p in result.parameters]
            y_path = [p["y"] for p in result.parameters]

            plt.plot(
                x_path,
                y_path,
                color=colors[i % len(colors)],
                marker=markers[i % len(markers)],
                markersize=4,
                linewidth=2,
                label=f"{opt_name} ({len(x_path)} steps)",
                alpha=0.8,
            )

            # Mark start and end points
            plt.plot(
                x_path[0],
                y_path[0],
                marker="*",
                color=colors[i % len(colors)],
                markersize=12,
                markeredgecolor="black",
                markeredgewidth=1,
            )
            plt.plot(
                x_path[-1],
                y_path[-1],
                marker="x",
                color=colors[i % len(colors)],
                markersize=10,
                markeredgewidth=3,
            )

    plt.xlabel("x", fontsize=12)
    plt.ylabel("y", fontsize=12)
    plt.title(f"Optimization Paths: {problem.name}", fontsize=14, fontweight="bold")
    plt.legend(fontsize=11)
    plt.grid(True, alpha=0.3)
    plt.axis("equal")
    plt.tight_layout()
    plt.show()

generate_summary_table

generate_summary_table(all_results: dict[str, dict[str, OptimizationRun]]) -> None

Generate formatted summary table of optimization results.

Parameters:

Name	Type	Description	Default
all_results	dict	Complete results from comprehensive comparison.	required

Source code in src/dlhub/optimizers/comparison.py

def generate_summary_table(
    self, all_results: dict[str, dict[str, OptimizationRun]]
) -> None:
    """
    Generate formatted summary table of optimization results.

    Parameters
    ----------
    all_results : dict
        Complete results from comprehensive comparison.
    """
    print("\n" + "=" * 80)
    print("OPTIMIZATION COMPARISON SUMMARY")
    print("=" * 80)

    for problem_name, results in all_results.items():
        print(f"\n{problem_name}")
        print("-" * len(problem_name))

        # Create formatted table
        headers = [
            "Optimizer",
            "Final Loss",
            "Iterations",
            "Conv. Time (s)",
            "Status",
        ]
        print(
            f"{headers[0]:<12} {headers[1]:<20} {headers[2]:<12} {headers[3]:<15} {headers[4]:<10}"
        )
        print("-" * 80)

        for opt_name, result in results.items():
            status = (
                "✓"
                if result.iterations_to_converge < self.max_iterations
                else "Max iter"
            )
            print(
                f"{opt_name:<12} {result.final_loss:<14.6f} "
                f"{result.iterations_to_converge:<12} "
                f"{result.convergence_time:<15.3f} {status:<10}"
            )

    print("\n" + "=" * 90)

OptimizationAnalytics

Advanced analytics utilities for optimization comparison results.

Provides statistical analysis, performance ranking, and detailed insights into optimizer behavior across different problem types.

Source code in src/dlhub/optimizers/comparison.py

class OptimizationAnalytics:
    """
    Advanced analytics utilities for optimization comparison results.

    Provides statistical analysis, performance ranking, and detailed
    insights into optimizer behavior across different problem types.
    """

    @staticmethod
    def compute_convergence_metrics(result: OptimizationRun) -> dict[str, float]:
        """
        Compute detailed convergence metrics for a single optimization run.

        Parameters
        ----------
        result : OptimizationRun
            Single optimization result to analyze.

        Returns
        -------
        dict
            Dictionary of computed metrics.
        """
        losses = np.array(result.losses)

        metrics = {
            "final_loss": result.final_loss,
            "initial_loss": losses[0] if len(losses) > 0 else float("inf"),
            "loss_reduction": losses[0] - result.final_loss if len(losses) > 0 else 0.0,
            "relative_improvement": ((losses[0] - result.final_loss) / losses[0]) * 100
            if len(losses) > 0 and losses[0] != 0
            else 0.0,
            "iterations_to_converge": result.iterations_to_converge,
            "convergence_time": result.convergence_time,
            "convergence_rate": 0.0,
            "stability_score": 0.0,
        }

        # Compute convergence rate (loss decrease per iteration)
        if len(losses) > 1:
            total_improvement = losses[0] - losses[-1]
            metrics["convergence_rate"] = total_improvement / len(losses)

            # Compute stability score (1 - coefficient of variation of loss changes)
            loss_changes = np.diff(losses)
            if len(loss_changes) > 0 and np.mean(loss_changes) != 0:
                cv = np.std(loss_changes) / abs(np.mean(loss_changes))
                metrics["stability_score"] = max(0, 1 - cv)

        return metrics

    @staticmethod
    def rank_optimizers(
        all_results: dict[str, dict[str, OptimizationRun]],
    ) -> dict[str, dict[str, int]]:
        """
        Rank optimizers across different problems and metrics.

        Parameters
        ----------
        all_results : dict
            Complete results from optimization comparison.

        Returns
        -------
        dict
            Rankings for each optimizer on each problem.
        """
        rankings = {}

        for problem_name, results in all_results.items():
            if not results:
                continue

            optimizer_metrics = {}
            for opt_name, result in results.items():
                optimizer_metrics[opt_name] = (
                    OptimizationAnalytics.compute_convergence_metrics(result)
                )

            rankings[problem_name] = {}
            sorted_by_loss = sorted(
                optimizer_metrics.items(), key=lambda x: x[1]["final_loss"]
            )
            for rank, (opt_name, _) in enumerate(sorted_by_loss, 1):
                rankings[problem_name][f"{opt_name}_loss_rank"] = rank

            sorted_by_speed = sorted(
                optimizer_metrics.items(), key=lambda x: x[1]["iterations_to_converge"]
            )
            for rank, (opt_name, _) in enumerate(sorted_by_speed, 1):
                rankings[problem_name][f"{opt_name}_speed_rank"] = rank

            sorted_by_stability = sorted(
                optimizer_metrics.items(),
                key=lambda x: x[1]["stability_score"],
                reverse=True,
            )
            for rank, (opt_name, _) in enumerate(sorted_by_stability, 1):
                rankings[problem_name][f"{opt_name}_stability_rank"] = rank

        return rankings

    @staticmethod
    def generate_performance_heatmap(
        all_results: dict[str, dict[str, OptimizationRun]],
    ):
        """
        Generate performance heatmap comparing optimizers across problems.

        Parameters
        ----------
        all_results : dict
            Complete results from optimization comparison.
        """
        import matplotlib.pyplot as plt
        import numpy as np

        optimizers = []
        problems = list(all_results.keys())

        for problem_results in all_results.values():
            for opt_name in problem_results.keys():
                if opt_name not in optimizers:
                    optimizers.append(opt_name)

        performance_matrix = np.zeros((len(optimizers), len(problems)))

        for j, problem_name in enumerate(problems):
            results = all_results[problem_name]
            losses = [results[opt].final_loss for opt in optimizers if opt in results]

            if losses:
                # Normalize losses (0 = best, 1 = worst)
                min_loss, max_loss = min(losses), max(losses)
                loss_range = max_loss - min_loss if max_loss != min_loss else 1

                for i, opt_name in enumerate(optimizers):
                    if opt_name in results:
                        normalized_loss = (
                            results[opt_name].final_loss - min_loss
                        ) / loss_range
                        performance_matrix[i, j] = normalized_loss
                    else:
                        performance_matrix[i, j] = (
                            1.0  # Worst performance if not available
                        )

        fig, ax = plt.subplots(figsize=(12, 8))
        im = ax.imshow(
            performance_matrix, cmap="RdYlGn_r", aspect="auto", vmin=0, vmax=1
        )

        ax.set_xticks(range(len(problems)))
        ax.set_yticks(range(len(optimizers)))
        ax.set_xticklabels(
            [p.split("(")[0].strip() for p in problems], rotation=45, ha="right"
        )
        ax.set_yticklabels(optimizers)

        cbar = plt.colorbar(im, ax=ax)
        cbar.set_label(
            "Normalized Performance (0=Best, 1=Worst)", rotation=270, labelpad=20
        )

        # Add text annotations
        for i in range(len(optimizers)):
            for j in range(len(problems)):
                text = ax.text(
                    j,
                    i,
                    f"{performance_matrix[i, j]:.2f}",
                    ha="center",
                    va="center",
                    color="black",
                    fontweight="bold",
                )

        ax.set_title(
            "Optimizer Performance Heatmap Across Problems",
            fontsize=14,
            fontweight="bold",
            pad=20,
        )
        plt.tight_layout()
        plt.show()

compute_convergence_metrics staticmethod

compute_convergence_metrics(result: OptimizationRun) -> dict[str, float]

Compute detailed convergence metrics for a single optimization run.

Parameters:

Name	Type	Description	Default
result	OptimizationRun	Single optimization result to analyze.	required

Returns:

Type	Description
dict	Dictionary of computed metrics.

Source code in src/dlhub/optimizers/comparison.py

@staticmethod
def compute_convergence_metrics(result: OptimizationRun) -> dict[str, float]:
    """
    Compute detailed convergence metrics for a single optimization run.

    Parameters
    ----------
    result : OptimizationRun
        Single optimization result to analyze.

    Returns
    -------
    dict
        Dictionary of computed metrics.
    """
    losses = np.array(result.losses)

    metrics = {
        "final_loss": result.final_loss,
        "initial_loss": losses[0] if len(losses) > 0 else float("inf"),
        "loss_reduction": losses[0] - result.final_loss if len(losses) > 0 else 0.0,
        "relative_improvement": ((losses[0] - result.final_loss) / losses[0]) * 100
        if len(losses) > 0 and losses[0] != 0
        else 0.0,
        "iterations_to_converge": result.iterations_to_converge,
        "convergence_time": result.convergence_time,
        "convergence_rate": 0.0,
        "stability_score": 0.0,
    }

    # Compute convergence rate (loss decrease per iteration)
    if len(losses) > 1:
        total_improvement = losses[0] - losses[-1]
        metrics["convergence_rate"] = total_improvement / len(losses)

        # Compute stability score (1 - coefficient of variation of loss changes)
        loss_changes = np.diff(losses)
        if len(loss_changes) > 0 and np.mean(loss_changes) != 0:
            cv = np.std(loss_changes) / abs(np.mean(loss_changes))
            metrics["stability_score"] = max(0, 1 - cv)

    return metrics

rank_optimizers staticmethod

rank_optimizers(all_results: dict[str, dict[str, OptimizationRun]]) -> dict[str, dict[str, int]]

Rank optimizers across different problems and metrics.

Parameters:

Name	Type	Description	Default
all_results	dict	Complete results from optimization comparison.	required

Returns:

Type	Description
dict	Rankings for each optimizer on each problem.

Source code in src/dlhub/optimizers/comparison.py

@staticmethod
def rank_optimizers(
    all_results: dict[str, dict[str, OptimizationRun]],
) -> dict[str, dict[str, int]]:
    """
    Rank optimizers across different problems and metrics.

    Parameters
    ----------
    all_results : dict
        Complete results from optimization comparison.

    Returns
    -------
    dict
        Rankings for each optimizer on each problem.
    """
    rankings = {}

    for problem_name, results in all_results.items():
        if not results:
            continue

        optimizer_metrics = {}
        for opt_name, result in results.items():
            optimizer_metrics[opt_name] = (
                OptimizationAnalytics.compute_convergence_metrics(result)
            )

        rankings[problem_name] = {}
        sorted_by_loss = sorted(
            optimizer_metrics.items(), key=lambda x: x[1]["final_loss"]
        )
        for rank, (opt_name, _) in enumerate(sorted_by_loss, 1):
            rankings[problem_name][f"{opt_name}_loss_rank"] = rank

        sorted_by_speed = sorted(
            optimizer_metrics.items(), key=lambda x: x[1]["iterations_to_converge"]
        )
        for rank, (opt_name, _) in enumerate(sorted_by_speed, 1):
            rankings[problem_name][f"{opt_name}_speed_rank"] = rank

        sorted_by_stability = sorted(
            optimizer_metrics.items(),
            key=lambda x: x[1]["stability_score"],
            reverse=True,
        )
        for rank, (opt_name, _) in enumerate(sorted_by_stability, 1):
            rankings[problem_name][f"{opt_name}_stability_rank"] = rank

    return rankings

generate_performance_heatmap staticmethod

generate_performance_heatmap(all_results: dict[str, dict[str, OptimizationRun]])

Generate performance heatmap comparing optimizers across problems.

Parameters:

Name	Type	Description	Default
all_results	dict	Complete results from optimization comparison.	required

Source code in src/dlhub/optimizers/comparison.py

@staticmethod
def generate_performance_heatmap(
    all_results: dict[str, dict[str, OptimizationRun]],
):
    """
    Generate performance heatmap comparing optimizers across problems.

    Parameters
    ----------
    all_results : dict
        Complete results from optimization comparison.
    """
    import matplotlib.pyplot as plt
    import numpy as np

    optimizers = []
    problems = list(all_results.keys())

    for problem_results in all_results.values():
        for opt_name in problem_results.keys():
            if opt_name not in optimizers:
                optimizers.append(opt_name)

    performance_matrix = np.zeros((len(optimizers), len(problems)))

    for j, problem_name in enumerate(problems):
        results = all_results[problem_name]
        losses = [results[opt].final_loss for opt in optimizers if opt in results]

        if losses:
            # Normalize losses (0 = best, 1 = worst)
            min_loss, max_loss = min(losses), max(losses)
            loss_range = max_loss - min_loss if max_loss != min_loss else 1

            for i, opt_name in enumerate(optimizers):
                if opt_name in results:
                    normalized_loss = (
                        results[opt_name].final_loss - min_loss
                    ) / loss_range
                    performance_matrix[i, j] = normalized_loss
                else:
                    performance_matrix[i, j] = (
                        1.0  # Worst performance if not available
                    )

    fig, ax = plt.subplots(figsize=(12, 8))
    im = ax.imshow(
        performance_matrix, cmap="RdYlGn_r", aspect="auto", vmin=0, vmax=1
    )

    ax.set_xticks(range(len(problems)))
    ax.set_yticks(range(len(optimizers)))
    ax.set_xticklabels(
        [p.split("(")[0].strip() for p in problems], rotation=45, ha="right"
    )
    ax.set_yticklabels(optimizers)

    cbar = plt.colorbar(im, ax=ax)
    cbar.set_label(
        "Normalized Performance (0=Best, 1=Worst)", rotation=270, labelpad=20
    )

    # Add text annotations
    for i in range(len(optimizers)):
        for j in range(len(problems)):
            text = ax.text(
                j,
                i,
                f"{performance_matrix[i, j]:.2f}",
                ha="center",
                va="center",
                color="black",
                fontweight="bold",
            )

    ax.set_title(
        "Optimizer Performance Heatmap Across Problems",
        fontsize=14,
        fontweight="bold",
        pad=20,
    )
    plt.tight_layout()
    plt.show()

main

main()

Main function demonstrating comprehensive optimization comparison.

Runs all optimizers on multiple test problems and generates comparative visualizations and summary statistics.

Source code in src/dlhub/optimizers/comparison.py

def main():
    """
    Main function demonstrating comprehensive optimization comparison.

    Runs all optimizers on multiple test problems and generates
    comparative visualizations and summary statistics.
    """
    print("🚀 Starting Comprehensive Optimization Algorithm Comparison")
    print("=" * 60)

    comparator = OptimizationComparison(
        max_iterations=500, tolerance=1e-8, verbose=True
    )
    all_results = comparator.run_comprehensive_comparison()
    comparator.generate_summary_table(all_results)

    problems = [
        QuadraticBowl(a=1.0, b=1.0),
        QuadraticBowl(a=1.0, b=100.0),
        RosenbrockFunction(),
        BealeFunction(),
    ]

    print("\n📊 Generating Visualization Plots...")
    for problem in problems:
        if problem.name in all_results:
            results = all_results[problem.name]

            print(f"Plotting convergence for: {problem.name}")
            comparator.plot_convergence_comparison(results, problem.name)

            print(f"Plotting optimization paths for: {problem.name}")
            comparator.plot_optimization_paths(results, problem)

    print("\n✅ Optimization comparison completed successfully!")
    print("Check the generated plots to analyze optimizer performance.")

    return all_results

run_custom_experiment

run_custom_experiment()

Example of running custom optimization experiment with specific configurations.

Demonstrates how to use the framework for targeted analysis with custom hyperparameters and problems.

Source code in src/dlhub/optimizers/comparison.py

def run_custom_experiment():
    """
    Example of running custom optimization experiment with specific configurations.

    Demonstrates how to use the framework for targeted analysis with
    custom hyperparameters and problems.
    """
    print("\n🔬 Running Custom Optimization Experiment")
    print("=" * 50)

    problem = RosenbrockFunction(a=1.0, b=10.0)  # Less challenging than default

    custom_configs = {
        OptimizerType.SGD: {"learning_rate": 0.001},
        OptimizerType.MOMENTUM: {"learning_rate": 0.001, "beta": 0.95},
        OptimizerType.RMSPROP: {"learning_rate": 0.001, "beta": 0.99},
        OptimizerType.ADAM: {"learning_rate": 0.05, "beta1": 0.9, "beta2": 0.999},
    }

    comparator = OptimizationComparison(
        max_iterations=2000, tolerance=1e-10, verbose=True
    )
    results = comparator.compare_optimizers(problem, custom_configs)

    print("\n📈 Custom Experiment Results:")
    for opt_name, result in results.items():
        metrics = OptimizationAnalytics.compute_convergence_metrics(result)
        print(f"{opt_name}:")
        print(f"  Final Loss: {metrics['final_loss']:.8f}")
        print(f"  Improvement: {metrics['relative_improvement']:.2f}%")
        print(f"  Convergence Rate: {metrics['convergence_rate']:.6f} loss/iter")
        print(f"  Stability Score: {metrics['stability_score']:.3f}")
        print()

    comparator.plot_convergence_comparison(results, "Custom Rosenbrock Experiment")
    comparator.plot_optimization_paths(results, problem)

    return results

demonstrate_hyperparameter_sensitivity

demonstrate_hyperparameter_sensitivity()

Demonstrate sensitivity of optimizers to hyperparameter choices.

Shows how different learning rates affect optimizer performance on the same problem, highlighting the importance of tuning.

Source code in src/dlhub/optimizers/comparison.py

def demonstrate_hyperparameter_sensitivity():
    """
    Demonstrate sensitivity of optimizers to hyperparameter choices.

    Shows how different learning rates affect optimizer performance
    on the same problem, highlighting the importance of tuning.
    """
    print("\n⚙️  Hyperparameter Sensitivity Analysis")
    print("=" * 50)

    problem = QuadraticBowl(a=1.0, b=10.0)  # Moderately ill-conditioned
    learning_rates = [0.001, 0.01, 0.1, 0.5]

    plt.figure(figsize=(15, 10))

    for i, optimizer_type in enumerate([OptimizerType.SGD, OptimizerType.ADAM]):
        plt.subplot(2, 2, i * 2 + 1)

        for lr in learning_rates:
            comparator = OptimizationComparison(max_iterations=200, verbose=False)

            if optimizer_type == OptimizerType.SGD:
                optimizer = SGDOptimizer(learning_rate=lr)
            else:
                optimizer = AdamOptimizer(learning_rate=lr)

            result = comparator.run_optimization(problem, optimizer)

            if result.losses:
                plt.plot(result.losses, label=f"LR={lr}", linewidth=2, alpha=0.8)

        plt.xlabel("Iterations")
        plt.ylabel("Loss")
        plt.title(f"{optimizer_type.value.upper()} - Learning Rate Sensitivity")
        plt.yscale("log")
        plt.legend()
        plt.grid(True, alpha=0.3)

        # Show final loss comparison
        plt.subplot(2, 2, i * 2 + 2)
        final_losses = []

        for lr in learning_rates:
            comparator = OptimizationComparison(max_iterations=200, verbose=False)

            if optimizer_type == OptimizerType.SGD:
                optimizer = SGDOptimizer(learning_rate=lr)
            else:
                optimizer = AdamOptimizer(learning_rate=lr)

            result = comparator.run_optimization(problem, optimizer)
            final_losses.append(result.final_loss)

        plt.bar(range(len(learning_rates)), final_losses, alpha=0.7)
        plt.xlabel("Learning Rate")
        plt.ylabel("Final Loss")
        plt.title(f"{optimizer_type.value.upper()} - Final Loss vs Learning Rate")
        plt.xticks(range(len(learning_rates)), [str(lr) for lr in learning_rates])
        plt.yscale("log")
        plt.grid(True, alpha=0.3)

    plt.tight_layout()
    plt.show()

exponential_weighted_averages

Exponential Weighted Averages (EWA) Implementation

A comprehensive implementation of exponential weighted averages with bias correction, multiple averaging strategies, and numerical stability features.

Exponential weighted averages are fundamental building blocks for modern optimization algorithms like Adam, RMSprop, and momentum-based optimizers.

References

• Used in Adam, RMSprop, Momentum optimizers
• Bias correction technique from Adam paper (Kingma & Ba, 2014)

Author

Deep Learning Reference Hub

License

MIT

Notes

Bias correction divides the accumulator by the total weight it has applied to its samples. Textbooks write that weight as 1 - beta**t, which is its closed form when beta is the same at every step.

This implementation tracks the weight directly, through weight = beta_t * weight + (1 - beta_t). The two agree exactly for a constant beta. They diverge once beta varies with t, which is what warmup_steps and the EXPONENTIAL_DECAY strategy both do: there the closed form understates the applied weight and the correction inflates the result.

AveragingStrategy

Bases: Enum

Enumeration of different averaging strategies.

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

class AveragingStrategy(Enum):
    """Enumeration of different averaging strategies."""

    SIMPLE = "simple"
    BIAS_CORRECTED = "bias_corrected"
    VARIANCE_CORRECTED = "variance_corrected"
    EXPONENTIAL_DECAY = "exponential_decay"

ExponentialWeightedAverage

Exponential Weighted Average with bias correction and multiple strategies.

This class implements exponential weighted averages (also known as exponentially weighted moving averages) with various correction techniques commonly used in deep learning optimization algorithms.

The basic formula is: v_t = beta * v_{t-1} + (1 - beta) * theta_t

Where: - v_t is the average at time t - beta is the decay parameter - theta_t is the current value - v_0 = 0 (initial value)

Parameters:

Name	Type	Description	Default
beta	float	Decay parameter (0 < beta < 1). Higher values give more weight to past values. Common values: 0.9 (momentum), 0.999 (second moments in Adam)	0.9
bias_correction	bool	Whether to apply bias correction to account for initialization bias	True
strategy	AveragingStrategy	Averaging strategy to use	AveragingStrategy.BIAS_CORRECTED
epsilon	float	Small constant for numerical stability	1e-8
warmup_steps	int	Number of warmup steps before applying full averaging	0

Attributes:

Name	Type	Description
v	float or ndarray	Current average value
t	int	Time step (number of updates)
history	list	History of average values

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

class ExponentialWeightedAverage:
    """
    Exponential Weighted Average with bias correction and multiple strategies.

    This class implements exponential weighted averages (also known as exponentially
    weighted moving averages) with various correction techniques commonly used in
    deep learning optimization algorithms.

    The basic formula is:
        v_t = beta * v_{t-1} + (1 - beta) * theta_t

    Where:
        - v_t is the average at time t
        - beta is the decay parameter
        - theta_t is the current value
        - v_0 = 0 (initial value)

    Parameters
    ----------
    beta : float, default=0.9
        Decay parameter (0 < beta < 1). Higher values give more weight to past values.
        Common values: 0.9 (momentum), 0.999 (second moments in Adam)
    bias_correction : bool, default=True
        Whether to apply bias correction to account for initialization bias
    strategy : AveragingStrategy, default=AveragingStrategy.BIAS_CORRECTED
        Averaging strategy to use
    epsilon : float, default=1e-8
        Small constant for numerical stability
    warmup_steps : int, default=0
        Number of warmup steps before applying full averaging

    Attributes
    ----------
    v : float or np.ndarray
        Current average value
    t : int
        Time step (number of updates)
    history : list
        History of average values
    """

    def __init__(
        self,
        beta: float = 0.9,
        bias_correction: bool = True,
        strategy: AveragingStrategy = AveragingStrategy.BIAS_CORRECTED,
        epsilon: float = 1e-8,
        warmup_steps: int = 0,
    ):
        if not 0.0 < beta < 1.0:
            raise ValueError(f"Beta must be in (0, 1), got {beta}")
        if epsilon <= 0:
            raise ValueError(f"Epsilon must be positive, got {epsilon}")
        if warmup_steps < 0:
            raise ValueError(f"Warmup steps must be non-negative, got {warmup_steps}")

        self.beta = beta
        self.bias_correction = bias_correction
        self.strategy = strategy
        self.epsilon = epsilon
        self.warmup_steps = warmup_steps

        self.v = None
        self.t = 0
        self.history = []

        # Cumulative weight the accumulator has actually applied to its samples.
        # The textbook correction factor 1 - beta**t is this quantity's closed form
        # for a *constant* beta. Warm-up varies beta with t, which invalidates that
        # closed form, so the weight is tracked directly instead. See Notes.
        self.weight = 0.0

        self.squared_avg = None
        self.variance_history = []

    def update(self, value: float | np.ndarray) -> float | np.ndarray:
        """
        Update the exponential weighted average with a new value.

        Parameters
        ----------
        value : float or np.ndarray
            New value to incorporate into the average

        Returns
        -------
        float or np.ndarray
            Updated average value
        """
        self.t += 1

        if isinstance(value, np.ndarray):
            if np.any(np.isnan(value)) or np.any(np.isinf(value)):
                warnings.warn("NaN or Inf detected in input value")
                value = np.nan_to_num(value, nan=0.0, posinf=1e6, neginf=-1e6)
        else:
            if np.isnan(value) or np.isinf(value):
                warnings.warn("NaN or Inf detected in input value")
                value = 0.0 if np.isnan(value) else (1e6 if value > 0 else -1e6)

        if self.v is None:
            if isinstance(value, np.ndarray):
                self.v = np.zeros_like(value, dtype=np.float64)
                if self.strategy == AveragingStrategy.VARIANCE_CORRECTED:
                    self.squared_avg = np.zeros_like(value, dtype=np.float64)
            else:
                self.v = 0.0
                if self.strategy == AveragingStrategy.VARIANCE_CORRECTED:
                    self.squared_avg = 0.0

        if self.t <= self.warmup_steps:
            effective_beta = min(self.beta, (self.t - 1) / self.t)
        else:
            effective_beta = self.beta

        if self.strategy == AveragingStrategy.EXPONENTIAL_DECAY:
            effective_beta = effective_beta ** (self.t / 1000)  # Decay over time

        # Track the weight with whichever coefficient this step actually applied,
        # so the two stay consistent for every strategy.
        self.weight = effective_beta * self.weight + (1 - effective_beta)

        if self.strategy == AveragingStrategy.SIMPLE:
            self.v = effective_beta * self.v + (1 - effective_beta) * value
            result = self.v

        elif self.strategy == AveragingStrategy.BIAS_CORRECTED:
            self.v = effective_beta * self.v + (1 - effective_beta) * value
            if self.bias_correction:
                result = self.v / (self.weight + self.epsilon)
            else:
                result = self.v

        elif self.strategy == AveragingStrategy.VARIANCE_CORRECTED:
            self.v = effective_beta * self.v + (1 - effective_beta) * value
            self.squared_avg = effective_beta * self.squared_avg + (
                1 - effective_beta
            ) * (value**2)

            if self.bias_correction:
                mean_corrected = self.v / (self.weight + self.epsilon)
                variance_corrected = self.squared_avg / (self.weight + self.epsilon)

                variance = variance_corrected - mean_corrected**2
                self.variance_history.append(variance)
                result = mean_corrected
            else:
                result = self.v

        elif self.strategy == AveragingStrategy.EXPONENTIAL_DECAY:
            self.v = effective_beta * self.v + (1 - effective_beta) * value
            result = self.v

        else:
            raise ValueError(f"Unknown averaging strategy: {self.strategy}")

        self.history.append(result.copy() if isinstance(result, np.ndarray) else result)
        return result

    def get_current_average(self) -> float | np.ndarray | None:
        """
        Get the current average value.

        Returns
        -------
        float, np.ndarray, or None
            Current average value, None if no updates have been made
        """
        if self.v is None:
            return None

        if self.strategy == AveragingStrategy.BIAS_CORRECTED and self.bias_correction:
            return self.v / (self.weight + self.epsilon)
        else:
            return self.v

    def get_variance(self) -> float | np.ndarray | None:
        """
        Get the current variance estimate (only available with VARIANCE_CORRECTED strategy).

        Returns
        -------
        float, np.ndarray, or None
            Current variance estimate, None if not available
        """
        if (
            self.strategy != AveragingStrategy.VARIANCE_CORRECTED
            or self.squared_avg is None
        ):
            return None

        if self.bias_correction:
            mean_corrected = self.v / (self.weight + self.epsilon)
            variance_corrected = self.squared_avg / (self.weight + self.epsilon)
            return variance_corrected - mean_corrected**2
        else:
            return self.squared_avg - self.v**2

    def reset(self) -> None:
        """Reset the exponential weighted average to initial state."""
        self.v = None
        self.squared_avg = None
        self.t = 0
        self.weight = 0.0
        self.history.clear()
        self.variance_history.clear()

    def get_effective_window_size(self) -> float:
        """
        Get the effective window size of the exponential weighted average.

        The effective window size is approximately 1/(1-beta).

        Returns
        -------
        float
            Effective window size
        """
        return 1.0 / (1.0 - self.beta)

    def get_config(self) -> dict[str, Any]:
        """
        Get configuration dictionary.

        Returns
        -------
        dict
            Configuration dictionary
        """
        return {
            "beta": self.beta,
            "bias_correction": self.bias_correction,
            "strategy": self.strategy.value,
            "epsilon": self.epsilon,
            "warmup_steps": self.warmup_steps,
            "time_step": self.t,
        }

    def get_state(self) -> dict[str, Any]:
        """
        Get complete state dictionary.

        Returns
        -------
        dict
            Complete state dictionary
        """
        return {
            "config": self.get_config(),
            "v": self.v,
            "weight": self.weight,
            "squared_avg": self.squared_avg,
            "history": self.history.copy(),
            "variance_history": self.variance_history.copy(),
        }

    def load_state(self, state: dict[str, Any]) -> None:
        """
        Load state from dictionary.

        Parameters
        ----------
        state : dict
            State dictionary from get_state()
        """
        config = state["config"]
        self.beta = config["beta"]
        self.bias_correction = config["bias_correction"]
        self.strategy = AveragingStrategy(config["strategy"])
        self.epsilon = config["epsilon"]
        self.warmup_steps = config["warmup_steps"]
        self.t = config["time_step"]

        self.v = state["v"]
        # Older states predate the tracked weight. Fall back to the constant-beta
        # closed form, which is exact whenever warmup_steps is 0.
        self.weight = state.get("weight", 1 - self.beta**self.t if self.t else 0.0)
        self.squared_avg = state["squared_avg"]
        self.history = state["history"].copy()
        self.variance_history = state["variance_history"].copy()

update

update(value: float | ndarray) -> float | np.ndarray

Update the exponential weighted average with a new value.

Parameters:

Name	Type	Description	Default
value	float or ndarray	New value to incorporate into the average	required

Returns:

Type	Description
float or ndarray	Updated average value

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def update(self, value: float | np.ndarray) -> float | np.ndarray:
    """
    Update the exponential weighted average with a new value.

    Parameters
    ----------
    value : float or np.ndarray
        New value to incorporate into the average

    Returns
    -------
    float or np.ndarray
        Updated average value
    """
    self.t += 1

    if isinstance(value, np.ndarray):
        if np.any(np.isnan(value)) or np.any(np.isinf(value)):
            warnings.warn("NaN or Inf detected in input value")
            value = np.nan_to_num(value, nan=0.0, posinf=1e6, neginf=-1e6)
    else:
        if np.isnan(value) or np.isinf(value):
            warnings.warn("NaN or Inf detected in input value")
            value = 0.0 if np.isnan(value) else (1e6 if value > 0 else -1e6)

    if self.v is None:
        if isinstance(value, np.ndarray):
            self.v = np.zeros_like(value, dtype=np.float64)
            if self.strategy == AveragingStrategy.VARIANCE_CORRECTED:
                self.squared_avg = np.zeros_like(value, dtype=np.float64)
        else:
            self.v = 0.0
            if self.strategy == AveragingStrategy.VARIANCE_CORRECTED:
                self.squared_avg = 0.0

    if self.t <= self.warmup_steps:
        effective_beta = min(self.beta, (self.t - 1) / self.t)
    else:
        effective_beta = self.beta

    if self.strategy == AveragingStrategy.EXPONENTIAL_DECAY:
        effective_beta = effective_beta ** (self.t / 1000)  # Decay over time

    # Track the weight with whichever coefficient this step actually applied,
    # so the two stay consistent for every strategy.
    self.weight = effective_beta * self.weight + (1 - effective_beta)

    if self.strategy == AveragingStrategy.SIMPLE:
        self.v = effective_beta * self.v + (1 - effective_beta) * value
        result = self.v

    elif self.strategy == AveragingStrategy.BIAS_CORRECTED:
        self.v = effective_beta * self.v + (1 - effective_beta) * value
        if self.bias_correction:
            result = self.v / (self.weight + self.epsilon)
        else:
            result = self.v

    elif self.strategy == AveragingStrategy.VARIANCE_CORRECTED:
        self.v = effective_beta * self.v + (1 - effective_beta) * value
        self.squared_avg = effective_beta * self.squared_avg + (
            1 - effective_beta
        ) * (value**2)

        if self.bias_correction:
            mean_corrected = self.v / (self.weight + self.epsilon)
            variance_corrected = self.squared_avg / (self.weight + self.epsilon)

            variance = variance_corrected - mean_corrected**2
            self.variance_history.append(variance)
            result = mean_corrected
        else:
            result = self.v

    elif self.strategy == AveragingStrategy.EXPONENTIAL_DECAY:
        self.v = effective_beta * self.v + (1 - effective_beta) * value
        result = self.v

    else:
        raise ValueError(f"Unknown averaging strategy: {self.strategy}")

    self.history.append(result.copy() if isinstance(result, np.ndarray) else result)
    return result

get_current_average

get_current_average() -> float | np.ndarray | None

Get the current average value.

Returns:

Type	Description
float, np.ndarray, or None	Current average value, None if no updates have been made

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def get_current_average(self) -> float | np.ndarray | None:
    """
    Get the current average value.

    Returns
    -------
    float, np.ndarray, or None
        Current average value, None if no updates have been made
    """
    if self.v is None:
        return None

    if self.strategy == AveragingStrategy.BIAS_CORRECTED and self.bias_correction:
        return self.v / (self.weight + self.epsilon)
    else:
        return self.v

get_variance

get_variance() -> float | np.ndarray | None

Get the current variance estimate (only available with VARIANCE_CORRECTED strategy).

Returns:

Type	Description
float, np.ndarray, or None	Current variance estimate, None if not available

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def get_variance(self) -> float | np.ndarray | None:
    """
    Get the current variance estimate (only available with VARIANCE_CORRECTED strategy).

    Returns
    -------
    float, np.ndarray, or None
        Current variance estimate, None if not available
    """
    if (
        self.strategy != AveragingStrategy.VARIANCE_CORRECTED
        or self.squared_avg is None
    ):
        return None

    if self.bias_correction:
        mean_corrected = self.v / (self.weight + self.epsilon)
        variance_corrected = self.squared_avg / (self.weight + self.epsilon)
        return variance_corrected - mean_corrected**2
    else:
        return self.squared_avg - self.v**2

reset

reset() -> None

Reset the exponential weighted average to initial state.

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def reset(self) -> None:
    """Reset the exponential weighted average to initial state."""
    self.v = None
    self.squared_avg = None
    self.t = 0
    self.weight = 0.0
    self.history.clear()
    self.variance_history.clear()

get_effective_window_size

get_effective_window_size() -> float

Get the effective window size of the exponential weighted average.

The effective window size is approximately 1/(1-beta).

Returns:

Type	Description
float	Effective window size

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def get_effective_window_size(self) -> float:
    """
    Get the effective window size of the exponential weighted average.

    The effective window size is approximately 1/(1-beta).

    Returns
    -------
    float
        Effective window size
    """
    return 1.0 / (1.0 - self.beta)

get_config

get_config() -> dict[str, Any]

Get configuration dictionary.

Returns:

Type	Description
dict	Configuration dictionary

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def get_config(self) -> dict[str, Any]:
    """
    Get configuration dictionary.

    Returns
    -------
    dict
        Configuration dictionary
    """
    return {
        "beta": self.beta,
        "bias_correction": self.bias_correction,
        "strategy": self.strategy.value,
        "epsilon": self.epsilon,
        "warmup_steps": self.warmup_steps,
        "time_step": self.t,
    }

get_state

get_state() -> dict[str, Any]

Get complete state dictionary.

Returns:

Type	Description
dict	Complete state dictionary

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def get_state(self) -> dict[str, Any]:
    """
    Get complete state dictionary.

    Returns
    -------
    dict
        Complete state dictionary
    """
    return {
        "config": self.get_config(),
        "v": self.v,
        "weight": self.weight,
        "squared_avg": self.squared_avg,
        "history": self.history.copy(),
        "variance_history": self.variance_history.copy(),
    }

load_state

load_state(state: dict[str, Any]) -> None

Load state from dictionary.

Parameters:

Name	Type	Description	Default
state	dict	State dictionary from get_state()	required

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def load_state(self, state: dict[str, Any]) -> None:
    """
    Load state from dictionary.

    Parameters
    ----------
    state : dict
        State dictionary from get_state()
    """
    config = state["config"]
    self.beta = config["beta"]
    self.bias_correction = config["bias_correction"]
    self.strategy = AveragingStrategy(config["strategy"])
    self.epsilon = config["epsilon"]
    self.warmup_steps = config["warmup_steps"]
    self.t = config["time_step"]

    self.v = state["v"]
    # Older states predate the tracked weight. Fall back to the constant-beta
    # closed form, which is exact whenever warmup_steps is 0.
    self.weight = state.get("weight", 1 - self.beta**self.t if self.t else 0.0)
    self.squared_avg = state["squared_avg"]
    self.history = state["history"].copy()
    self.variance_history = state["variance_history"].copy()

MultiVariateEWA

Multi-variate Exponential Weighted Average for handling multiple variables simultaneously.

This class manages multiple exponential weighted averages, commonly used in optimization algorithms where different parameters need separate averages.

Parameters:

Name	Type	Description	Default
beta	float	Common decay parameter for all variables	0.9
bias_correction	bool	Whether to apply bias correction	True
strategy	AveragingStrategy	Averaging strategy	AveragingStrategy.BIAS_CORRECTED
**kwargs		Additional parameters passed to individual EWA instances	{}

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

class MultiVariateEWA:
    """
    Multi-variate Exponential Weighted Average for handling multiple variables simultaneously.

    This class manages multiple exponential weighted averages, commonly used in
    optimization algorithms where different parameters need separate averages.

    Parameters
    ----------
    beta : float, default=0.9
        Common decay parameter for all variables
    bias_correction : bool, default=True
        Whether to apply bias correction
    strategy : AveragingStrategy, default=AveragingStrategy.BIAS_CORRECTED
        Averaging strategy
    **kwargs
        Additional parameters passed to individual EWA instances
    """

    def __init__(
        self,
        beta: float = 0.9,
        bias_correction: bool = True,
        strategy: AveragingStrategy = AveragingStrategy.BIAS_CORRECTED,
        **kwargs,
    ):
        self.beta = beta
        self.bias_correction = bias_correction
        self.strategy = strategy
        self.kwargs = kwargs

        self.averages: dict[str, ExponentialWeightedAverage] = {}

    def update(
        self, values: dict[str, float | np.ndarray]
    ) -> dict[str, float | np.ndarray]:
        """
        Update all averages with new values.

        Parameters
        ----------
        values : dict
            Dictionary of new values for each variable

        Returns
        -------
        dict
            Dictionary of updated averages
        """
        results = {}

        for name, value in values.items():
            if name not in self.averages:
                self.averages[name] = ExponentialWeightedAverage(
                    beta=self.beta,
                    bias_correction=self.bias_correction,
                    strategy=self.strategy,
                    **self.kwargs,
                )

            results[name] = self.averages[name].update(value)

        return results

    def get_averages(self) -> dict[str, float | np.ndarray]:
        """Get current averages for all variables."""
        return {name: ewa.get_current_average() for name, ewa in self.averages.items()}

    def reset(self) -> None:
        """Reset all averages."""
        for ewa in self.averages.values():
            ewa.reset()

    def get_state(self) -> dict[str, Any]:
        """Get complete state for all averages."""
        return {
            "config": {
                "beta": self.beta,
                "bias_correction": self.bias_correction,
                "strategy": self.strategy.value,
                "kwargs": self.kwargs,
            },
            "averages": {name: ewa.get_state() for name, ewa in self.averages.items()},
        }

update

update(values: dict[str, float | ndarray]) -> dict[str, float | np.ndarray]

Update all averages with new values.

Parameters:

Name	Type	Description	Default
values	dict	Dictionary of new values for each variable	required

Returns:

Type	Description
dict	Dictionary of updated averages

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def update(
    self, values: dict[str, float | np.ndarray]
) -> dict[str, float | np.ndarray]:
    """
    Update all averages with new values.

    Parameters
    ----------
    values : dict
        Dictionary of new values for each variable

    Returns
    -------
    dict
        Dictionary of updated averages
    """
    results = {}

    for name, value in values.items():
        if name not in self.averages:
            self.averages[name] = ExponentialWeightedAverage(
                beta=self.beta,
                bias_correction=self.bias_correction,
                strategy=self.strategy,
                **self.kwargs,
            )

        results[name] = self.averages[name].update(value)

    return results

get_averages

get_averages() -> dict[str, float | np.ndarray]

Get current averages for all variables.

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def get_averages(self) -> dict[str, float | np.ndarray]:
    """Get current averages for all variables."""
    return {name: ewa.get_current_average() for name, ewa in self.averages.items()}

reset

reset() -> None

Reset all averages.

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def reset(self) -> None:
    """Reset all averages."""
    for ewa in self.averages.values():
        ewa.reset()

get_state

get_state() -> dict[str, Any]

Get complete state for all averages.

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def get_state(self) -> dict[str, Any]:
    """Get complete state for all averages."""
    return {
        "config": {
            "beta": self.beta,
            "bias_correction": self.bias_correction,
            "strategy": self.strategy.value,
            "kwargs": self.kwargs,
        },
        "averages": {name: ewa.get_state() for name, ewa in self.averages.items()},
    }

create_momentum_ewa

create_momentum_ewa(beta: float = 0.9) -> ExponentialWeightedAverage

Create EWA for momentum optimization.

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def create_momentum_ewa(beta: float = 0.9) -> ExponentialWeightedAverage:
    """Create EWA for momentum optimization."""
    return ExponentialWeightedAverage(
        beta=beta, bias_correction=True, strategy=AveragingStrategy.BIAS_CORRECTED
    )

create_rmsprop_ewa

create_rmsprop_ewa(beta: float = 0.999) -> ExponentialWeightedAverage

Create EWA for RMSprop (second moments).

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def create_rmsprop_ewa(beta: float = 0.999) -> ExponentialWeightedAverage:
    """Create EWA for RMSprop (second moments)."""
    return ExponentialWeightedAverage(
        beta=beta, bias_correction=True, strategy=AveragingStrategy.BIAS_CORRECTED
    )

create_adam_ewa_pair

create_adam_ewa_pair(beta1: float = 0.9, beta2: float = 0.999) -> tuple[ExponentialWeightedAverage, ExponentialWeightedAverage]

Create EWA pair for Adam optimizer (first and second moments).

Source code in src/dlhub/optimizers/exponential_weighted_averages.py

def create_adam_ewa_pair(
    beta1: float = 0.9, beta2: float = 0.999
) -> tuple[ExponentialWeightedAverage, ExponentialWeightedAverage]:
    """Create EWA pair for Adam optimizer (first and second moments)."""
    first_moment = ExponentialWeightedAverage(
        beta=beta1, bias_correction=True, strategy=AveragingStrategy.BIAS_CORRECTED
    )

    second_moment = ExponentialWeightedAverage(
        beta=beta2, bias_correction=True, strategy=AveragingStrategy.BIAS_CORRECTED
    )

    return first_moment, second_moment

mini_batch

Mini-batch Gradient Descent Implementation

This module implements efficient mini-batch gradient descent with proper shuffling and batch creation for neural network training.

Author

Deep Learning Reference Hub

License

MIT

MiniBatchGradientDescent

Mini-batch Gradient Descent optimizer with configurable batch size and shuffling.

This implementation provides efficient mini-batch processing with proper data shuffling and batch creation for neural network training.

Parameters:

Name	Type	Description	Default
learning_rate	float	Learning rate for gradient descent updates	0.001
batch_size	int	Size of mini-batches for training	64
shuffle	bool	Whether to shuffle data at each epoch	True
random_seed	int	Random seed for reproducibility	None

Attributes:

Name	Type	Description
learning_rate	float	Current learning rate
batch_size	int	Mini-batch size
shuffle	bool	Shuffling flag
history	dict	Training history including losses and metrics

Source code in src/dlhub/optimizers/mini_batch.py

class MiniBatchGradientDescent:
    """
    Mini-batch Gradient Descent optimizer with configurable batch size and shuffling.

    This implementation provides efficient mini-batch processing with proper data
    shuffling and batch creation for neural network training.

    Parameters
    ----------
    learning_rate : float, default=0.001
        Learning rate for gradient descent updates
    batch_size : int, default=64
        Size of mini-batches for training
    shuffle : bool, default=True
        Whether to shuffle data at each epoch
    random_seed : int, optional
        Random seed for reproducibility

    Attributes
    ----------
    learning_rate : float
        Current learning rate
    batch_size : int
        Mini-batch size
    shuffle : bool
        Shuffling flag
    history : dict
        Training history including losses and metrics
    """

    def __init__(
        self,
        learning_rate: float = 0.001,
        batch_size: int = 64,
        shuffle: bool = True,
        random_seed: int | None = None,
    ):
        self.learning_rate = learning_rate
        self.batch_size = batch_size
        self.shuffle = shuffle
        self.history = {"loss": [], "accuracy": []}

        if random_seed is not None:
            np.random.seed(random_seed)
            random.seed(random_seed)

    def create_mini_batches(
        self, X: np.ndarray, Y: np.ndarray
    ) -> list[tuple[np.ndarray, np.ndarray]]:
        """
        Create mini-batches from training data with optional shuffling.

        Parameters
        ----------
        X : np.ndarray
            Input features of shape (n_features, m_examples)
        Y : np.ndarray
            Target labels of shape (n_classes, m_examples)

        Returns
        -------
        List[Tuple[np.ndarray, np.ndarray]]
            List of (X_batch, Y_batch) tuples

        Notes
        -----
        If shuffle is True, data is randomly permuted before creating batches.
        The last batch may be smaller if the dataset size is not divisible by batch_size.
        """
        m = X.shape[1]
        mini_batchs = []

        if self.shuffle:
            permutation = np.random.permutation(m)
            X_shuffled = X[:, permutation]
            Y_shuffled = Y[:, permutation]
        else:
            X_shuffled = X
            Y_shuffled = Y

        full_batches = m // self.batch_size
        for k in range(full_batches):
            start = k * self.batch_size
            end = start + self.batch_size

            X_batch = X_shuffled[:, start:end]
            Y_batch = Y_shuffled[:, start:end]

            mini_batchs.append((X_batch, Y_batch))

        if m % self.batch_size:
            start = full_batches * self.batch_size
            X_batch = X_shuffled[:, start:]
            Y_batch = Y_shuffled[:, start:]
            mini_batchs.append((X_batch, Y_batch))

        return mini_batchs

    def update_parameters(
        self, parameters: dict[str, np.ndarray], gradients: dict[str, np.ndarray]
    ) -> dict[str, np.ndarray]:
        """
        Update model parameters using gradient descent.

        Parameters
        ----------
        parameters : Dict[str, np.ndarray]
            Current model parameters
        gradients : Dict[str, np.ndarray]
            Computed gradients for each parameter

        Returns
        -------
        Dict[str, np.ndarray]
            Updated parameters

        Notes
        -----
        Updates parameters using the standard gradient descent rule:
        θ = θ - α * ∇J(θ)
        """
        updated_parameters = {}

        for key in parameters:
            updated_parameters[key] = (
                parameters[key] - self.learning_rate * gradients[key]
            )

        return updated_parameters

    def train_epoch(
        self,
        X: np.ndarray,
        Y: np.ndarray,
        parameters: dict[str, np.ndarray],
        forward_propagation_fn: callable,
        backward_propagation_fn: callable,
        compute_cost_fn: callable,
    ) -> tuple[dict[str, np.ndarray], float]:
        """
        Train for one epoch using mini-batch gradient descent.

        Parameters
        ----------
        X : np.ndarray
            Input features of shape (n_features, m_examples)
        Y : np.ndarray
            Target labels of shape (n_classes, m_examples)
        parameters : Dict[str, np.ndarray]
            Current model parameters
        forward_propagation_fn : callable
            Function to compute forward propagation
        backward_propagation_fn : callable
            Function to compute backward propagation
        compute_cost_fn : callable
            Function to compute cost/loss

        Returns
        -------
        Tuple[Dict[str, np.ndarray], float]
            Updated parameters and epoch loss

        Notes
        -----
        Performs one complete epoch of mini-batch gradient descent training.
        """
        epoch_cost = 0.0
        mini_batches = self.create_mini_batches(X, Y)

        for X_batch, Y_batch in mini_batches:
            AL, caches = forward_propagation_fn(X_batch, parameters)
            epoch_cost += compute_cost_fn(AL, Y_batch)
            gradients = backward_propagation_fn(AL, Y_batch, caches)
            parameters = self.update_parameters(parameters, gradients)

        epoch_cost /= len(mini_batches)
        return parameters, epoch_cost

    def fit(
        self,
        X: np.ndarray,
        Y: np.ndarray,
        parameters: dict[str, np.ndarray],
        forward_propagation_fn: callable,
        backward_propagation_fn: callable,
        compute_cost_fn: callable,
        epochs: int = 1000,
        print_cost: bool = True,
        print_every: int = 100,
    ) -> dict[str, np.ndarray]:
        """
        Train the model using mini-batch gradient descent.

        Parameters
        ----------
        X : np.ndarray
            Input features of shape (n_features, m_examples)
        Y : np.ndarray
            Target labels of shape (n_classes, m_examples)
        parameters : Dict[str, np.ndarray]
            Initial model parameters
        forward_propagation_fn : callable
            Function to compute forward propagation
        backward_propagation_fn : callable
            Function to compute backward propagation
        compute_cost_fn : callable
            Function to compute cost/loss
        epochs : int, default=1000
            Number of training epochs
        print_cost : bool, default=True
            Whether to print cost during training
        print_every : int, default=100
            Print cost every N epochs

        Returns
        -------
        Dict[str, np.ndarray]
            Trained parameters

        Notes
        -----
        Trains the model for the specified number of epochs using mini-batch
        gradient descent. Training history is stored in self.history.
        """
        for epoch in range(epochs):
            parameters, epoch_cost = self.train_epoch(
                X,
                Y,
                parameters,
                forward_propagation_fn,
                backward_propagation_fn,
                compute_cost_fn,
            )

            self.history["loss"].append(epoch_cost)

            if print_cost and epoch % print_every == 0:
                print(f"Epoch {epoch}: Cost = {epoch_cost:.6f}")

        return parameters

    def get_config(self) -> dict[str, Any]:
        """
        Get optimizer configuration.

        Returns
        -------
        Dict[str, Any]
            Configuration dictionary
        """
        return {
            "learning_rate": self.learning_rate,
            "batch_size": self.batch_size,
            "shuffle": self.shuffle,
            "optimizer": "MiniBatchGradientDescent",
        }

create_mini_batches

create_mini_batches(X: ndarray, Y: ndarray) -> list[tuple[np.ndarray, np.ndarray]]

Create mini-batches from training data with optional shuffling.

Parameters:

Name	Type	Description	Default
X	ndarray	Input features of shape (n_features, m_examples)	required
Y	ndarray	Target labels of shape (n_classes, m_examples)	required

Returns:

Type	Description
List[Tuple[ndarray, ndarray]]	List of (X_batch, Y_batch) tuples

Notes

If shuffle is True, data is randomly permuted before creating batches. The last batch may be smaller if the dataset size is not divisible by batch_size.

Source code in src/dlhub/optimizers/mini_batch.py

def create_mini_batches(
    self, X: np.ndarray, Y: np.ndarray
) -> list[tuple[np.ndarray, np.ndarray]]:
    """
    Create mini-batches from training data with optional shuffling.

    Parameters
    ----------
    X : np.ndarray
        Input features of shape (n_features, m_examples)
    Y : np.ndarray
        Target labels of shape (n_classes, m_examples)

    Returns
    -------
    List[Tuple[np.ndarray, np.ndarray]]
        List of (X_batch, Y_batch) tuples

    Notes
    -----
    If shuffle is True, data is randomly permuted before creating batches.
    The last batch may be smaller if the dataset size is not divisible by batch_size.
    """
    m = X.shape[1]
    mini_batchs = []

    if self.shuffle:
        permutation = np.random.permutation(m)
        X_shuffled = X[:, permutation]
        Y_shuffled = Y[:, permutation]
    else:
        X_shuffled = X
        Y_shuffled = Y

    full_batches = m // self.batch_size
    for k in range(full_batches):
        start = k * self.batch_size
        end = start + self.batch_size

        X_batch = X_shuffled[:, start:end]
        Y_batch = Y_shuffled[:, start:end]

        mini_batchs.append((X_batch, Y_batch))

    if m % self.batch_size:
        start = full_batches * self.batch_size
        X_batch = X_shuffled[:, start:]
        Y_batch = Y_shuffled[:, start:]
        mini_batchs.append((X_batch, Y_batch))

    return mini_batchs

update_parameters

update_parameters(parameters: dict[str, ndarray], gradients: dict[str, ndarray]) -> dict[str, np.ndarray]

Update model parameters using gradient descent.

Parameters:

Name	Type	Description	Default
parameters	Dict[str, ndarray]	Current model parameters	required
gradients	Dict[str, ndarray]	Computed gradients for each parameter	required

Returns:

Type	Description
Dict[str, ndarray]	Updated parameters

Notes

Updates parameters using the standard gradient descent rule: θ = θ - α * ∇J(θ)

Source code in src/dlhub/optimizers/mini_batch.py

def update_parameters(
    self, parameters: dict[str, np.ndarray], gradients: dict[str, np.ndarray]
) -> dict[str, np.ndarray]:
    """
    Update model parameters using gradient descent.

    Parameters
    ----------
    parameters : Dict[str, np.ndarray]
        Current model parameters
    gradients : Dict[str, np.ndarray]
        Computed gradients for each parameter

    Returns
    -------
    Dict[str, np.ndarray]
        Updated parameters

    Notes
    -----
    Updates parameters using the standard gradient descent rule:
    θ = θ - α * ∇J(θ)
    """
    updated_parameters = {}

    for key in parameters:
        updated_parameters[key] = (
            parameters[key] - self.learning_rate * gradients[key]
        )

    return updated_parameters

train_epoch

train_epoch(X: ndarray, Y: ndarray, parameters: dict[str, ndarray], forward_propagation_fn: callable, backward_propagation_fn: callable, compute_cost_fn: callable) -> tuple[dict[str, np.ndarray], float]

Train for one epoch using mini-batch gradient descent.

Parameters:

Name	Type	Description	Default
X	ndarray	Input features of shape (n_features, m_examples)	required
Y	ndarray	Target labels of shape (n_classes, m_examples)	required
parameters	Dict[str, ndarray]	Current model parameters	required
forward_propagation_fn	callable	Function to compute forward propagation	required
backward_propagation_fn	callable	Function to compute backward propagation	required
compute_cost_fn	callable	Function to compute cost/loss	required

Returns:

Type	Description
Tuple[Dict[str, ndarray], float]	Updated parameters and epoch loss

Notes

Performs one complete epoch of mini-batch gradient descent training.

Source code in src/dlhub/optimizers/mini_batch.py

def train_epoch(
    self,
    X: np.ndarray,
    Y: np.ndarray,
    parameters: dict[str, np.ndarray],
    forward_propagation_fn: callable,
    backward_propagation_fn: callable,
    compute_cost_fn: callable,
) -> tuple[dict[str, np.ndarray], float]:
    """
    Train for one epoch using mini-batch gradient descent.

    Parameters
    ----------
    X : np.ndarray
        Input features of shape (n_features, m_examples)
    Y : np.ndarray
        Target labels of shape (n_classes, m_examples)
    parameters : Dict[str, np.ndarray]
        Current model parameters
    forward_propagation_fn : callable
        Function to compute forward propagation
    backward_propagation_fn : callable
        Function to compute backward propagation
    compute_cost_fn : callable
        Function to compute cost/loss

    Returns
    -------
    Tuple[Dict[str, np.ndarray], float]
        Updated parameters and epoch loss

    Notes
    -----
    Performs one complete epoch of mini-batch gradient descent training.
    """
    epoch_cost = 0.0
    mini_batches = self.create_mini_batches(X, Y)

    for X_batch, Y_batch in mini_batches:
        AL, caches = forward_propagation_fn(X_batch, parameters)
        epoch_cost += compute_cost_fn(AL, Y_batch)
        gradients = backward_propagation_fn(AL, Y_batch, caches)
        parameters = self.update_parameters(parameters, gradients)

    epoch_cost /= len(mini_batches)
    return parameters, epoch_cost

fit

fit(X: ndarray, Y: ndarray, parameters: dict[str, ndarray], forward_propagation_fn: callable, backward_propagation_fn: callable, compute_cost_fn: callable, epochs: int = 1000, print_cost: bool = True, print_every: int = 100) -> dict[str, np.ndarray]

Train the model using mini-batch gradient descent.

Parameters:

Name	Type	Description	Default
X	ndarray	Input features of shape (n_features, m_examples)	required
Y	ndarray	Target labels of shape (n_classes, m_examples)	required
parameters	Dict[str, ndarray]	Initial model parameters	required
forward_propagation_fn	callable	Function to compute forward propagation	required
backward_propagation_fn	callable	Function to compute backward propagation	required
compute_cost_fn	callable	Function to compute cost/loss	required
epochs	int	Number of training epochs	1000
print_cost	bool	Whether to print cost during training	True
print_every	int	Print cost every N epochs	100

Returns:

Type	Description
Dict[str, ndarray]	Trained parameters

Notes

Trains the model for the specified number of epochs using mini-batch gradient descent. Training history is stored in self.history.

Source code in src/dlhub/optimizers/mini_batch.py

def fit(
    self,
    X: np.ndarray,
    Y: np.ndarray,
    parameters: dict[str, np.ndarray],
    forward_propagation_fn: callable,
    backward_propagation_fn: callable,
    compute_cost_fn: callable,
    epochs: int = 1000,
    print_cost: bool = True,
    print_every: int = 100,
) -> dict[str, np.ndarray]:
    """
    Train the model using mini-batch gradient descent.

    Parameters
    ----------
    X : np.ndarray
        Input features of shape (n_features, m_examples)
    Y : np.ndarray
        Target labels of shape (n_classes, m_examples)
    parameters : Dict[str, np.ndarray]
        Initial model parameters
    forward_propagation_fn : callable
        Function to compute forward propagation
    backward_propagation_fn : callable
        Function to compute backward propagation
    compute_cost_fn : callable
        Function to compute cost/loss
    epochs : int, default=1000
        Number of training epochs
    print_cost : bool, default=True
        Whether to print cost during training
    print_every : int, default=100
        Print cost every N epochs

    Returns
    -------
    Dict[str, np.ndarray]
        Trained parameters

    Notes
    -----
    Trains the model for the specified number of epochs using mini-batch
    gradient descent. Training history is stored in self.history.
    """
    for epoch in range(epochs):
        parameters, epoch_cost = self.train_epoch(
            X,
            Y,
            parameters,
            forward_propagation_fn,
            backward_propagation_fn,
            compute_cost_fn,
        )

        self.history["loss"].append(epoch_cost)

        if print_cost and epoch % print_every == 0:
            print(f"Epoch {epoch}: Cost = {epoch_cost:.6f}")

    return parameters

get_config

get_config() -> dict[str, Any]

Get optimizer configuration.

Returns:

Type	Description
Dict[str, Any]	Configuration dictionary

Source code in src/dlhub/optimizers/mini_batch.py

def get_config(self) -> dict[str, Any]:
    """
    Get optimizer configuration.

    Returns
    -------
    Dict[str, Any]
        Configuration dictionary
    """
    return {
        "learning_rate": self.learning_rate,
        "batch_size": self.batch_size,
        "shuffle": self.shuffle,
        "optimizer": "MiniBatchGradientDescent",
    }

initialize_parameters

initialize_parameters(layer_dims: list[int]) -> dict[str, np.ndarray]

Initialize parameters for a neural network with given layer dimensions.

Parameters:

Name	Type	Description	Default
layer_dims	List[int]	List containing the dimensions of each layer	required

Returns:

Type	Description
Dict[str, ndarray]	Dictionary containing initialized parameters

Source code in src/dlhub/optimizers/mini_batch.py

def initialize_parameters(layer_dims: list[int]) -> dict[str, np.ndarray]:
    """
    Initialize parameters for a neural network with given layer dimensions.

    Parameters
    ----------
    layer_dims : List[int]
        List containing the dimensions of each layer

    Returns
    -------
    Dict[str, np.ndarray]
        Dictionary containing initialized parameters
    """
    parameters = {}
    L = len(layer_dims)

    for l in range(1, L):
        parameters[f"W{l}"] = np.random.randn(
            layer_dims[l], layer_dims[l - 1]
        ) * np.sqrt(2.0 / layer_dims[l - 1])
        parameters[f"b{l}"] = np.zeros((layer_dims[l], 1))

    return parameters

example_usage

example_usage()

Example demonstrating how to use MiniBatchGradientDescent.

Source code in src/dlhub/optimizers/mini_batch.py

def example_usage():
    """Example demonstrating how to use MiniBatchGradientDescent."""
    np.random.seed(42)
    X = np.random.randn(10, 1000)  # 10 features, 1000 examples
    Y = (X[0:1, :] > 0).astype(int)  # Binary classification

    layer_dims = [10, 5, 1]
    parameters = initialize_parameters(layer_dims)

    # Dummy functions (replace with actual implementations)
    def forward_propagation(X, parameters):
        return np.random.randn(1, X.shape[1]), {}

    def backward_propagation(AL, Y, caches):
        return {key: np.random.randn(*val.shape) for key, val in parameters.items()}

    def compute_cost(AL, Y):
        return np.random.rand()

    optimizer = MiniBatchGradientDescent(
        learning_rate=0.01, batch_size=32, shuffle=True
    )

    trained_parameters = optimizer.fit(
        X,
        Y,
        parameters,
        forward_propagation,
        backward_propagation,
        compute_cost,
        epochs=100,
        print_every=20,
    )

    print(f"\nFinal cost: {optimizer.history['loss'][-1]:.6f}")
    print(f"\nOptimizer config: {optimizer.get_config()}")

momentum

Momentum Optimizer Implementation

This module implements gradient descent with momentum, including exponential weighted averages and bias correction for efficient neural network training.

Author

Deep Learning Reference Hub

License

MIT

MomentumOptimizer

Gradient Descent with Momentum optimizer.

This implementation uses exponential weighted averages to accumulate gradients and includes bias correction for better convergence, especially in early training.

Parameters:

Name	Type	Description	Default
learning_rate	float	Learning rate for parameter updates	0.001
beta	float	Momentum parameter (exponential decay rate)	0.9
bias_correction	bool	Whether to apply bias correction to momentum estimates	True
epsilon	float	Small constant for numerical stability	1e-8

Attributes:

Name	Type	Description
learning_rate	float	Current learning rate
beta	float	Momentum parameter
bias_correction	bool	Bias correction flag
epsilon	float	Numerical stability constant
v	Dict[str, ndarray]	Momentum (velocity) estimates for each parameter
t	int	Time step counter for bias correction
history	Dict[str, List[float]]	Training history

Source code in src/dlhub/optimizers/momentum.py

class MomentumOptimizer:
    """
    Gradient Descent with Momentum optimizer.

    This implementation uses exponential weighted averages to accumulate gradients
    and includes bias correction for better convergence, especially in early training.

    Parameters
    ----------
    learning_rate : float, default=0.001
        Learning rate for parameter updates
    beta : float, default=0.9
        Momentum parameter (exponential decay rate)
    bias_correction : bool, default=True
        Whether to apply bias correction to momentum estimates
    epsilon : float, default=1e-8
        Small constant for numerical stability

    Attributes
    ----------
    learning_rate : float
        Current learning rate
    beta : float
        Momentum parameter
    bias_correction : bool
        Bias correction flag
    epsilon : float
        Numerical stability constant
    v : Dict[str, np.ndarray]
        Momentum (velocity) estimates for each parameter
    t : int
        Time step counter for bias correction
    history : Dict[str, List[float]]
        Training history
    """

    def __init__(
        self,
        learning_rate: float = 0.001,
        beta: float = 0.9,
        bias_correction: bool = True,
        epsilon: float = 1e-8,
    ):
        self.learning_rate = learning_rate
        self.beta = beta
        self.bias_correction = bias_correction
        self.epsilon = epsilon

        self.v = {}  # Momentum estimates
        self.t = 0  # Time step
        self.history = {"loss": [], "gradient_norm": []}

    def initialize_velocity(self, parameters: dict[str, np.ndarray]) -> None:
        """
        Initialize velocity (momentum) estimates for all parameters.

        Parameters
        ----------
        parameters : Dict[str, np.ndarray]
            Model parameters to initialize velocity for

        Notes
        -----
        Velocities are initialized to zero arrays with the same shape as parameters.
        """
        for key in parameters:
            self.v[key] = np.zeros_like(parameters[key])

    def update_parameters(
        self, parameters: dict[str, np.ndarray], gradients: dict[str, np.ndarray]
    ) -> dict[str, np.ndarray]:
        """
        Update parameters using momentum-based gradient descent.

        Parameters
        ----------
        parameters : Dict[str, np.ndarray]
            Current model parameters
        gradients : Dict[str, np.ndarray]
            Computed gradients for each parameter

        Returns
        -------
        Dict[str, np.ndarray]
            Updated parameters

        Notes
        -----
        Updates parameters using momentum:
        v_t = β * v_{t-1} + (1-β) * g_t
        θ_t = θ_{t-1} - α * v_t_corrected

        Where v_t_corrected includes bias correction if enabled.
        """
        if not self.v:
            self.initialize_velocity(parameters)

        self.t += 1
        updated_parameters = {}

        for key in parameters:
            self.v[key] = self.beta * self.v[key] + (1 - self.beta) * gradients[key]

            if self.bias_correction:
                v_corrected = self.v[key] / (1 - self.beta**self.t)
            else:
                v_corrected = self.v[key]

            updated_parameters[key] = parameters[key] - self.learning_rate * v_corrected

        return updated_parameters

    def compute_gradient_norm(self, gradients: dict[str, np.ndarray]) -> float:
        """
        Compute the L2 norm of gradients for monitoring convergence.

        Parameters
        ----------
        gradients : Dict[str, np.ndarray]
            Gradients for each parameter

        Returns
        -------
        float
            L2 norm of all gradients
        """
        total_norm = 0.0
        for grad in gradients.values():
            total_norm += np.sum(grad**2)
        return np.sqrt(total_norm)

    def train_step(
        self,
        X: np.ndarray,
        Y: np.ndarray,
        parameters: dict[str, np.ndarray],
        forward_propagation_fn: callable,
        backward_propagation_fn: callable,
        compute_cost_fn: callable,
    ) -> tuple[dict[str, np.ndarray]]:
        """
        Perform one training step with momentum optimizer.

        Parameters
        ----------
        X : np.ndarray
            Input features
        Y : np.ndarray
            Target labels
        parameters : Dict[str, np.ndarray]
            Current model parameters
        forward_propagation_fn : callable
            Function to compute forward propagation
        backward_propagation_fn : callable
            Function to compute backward propagation
        compute_cost_fn : callable
            Function to compute cost/loss

        Returns
        -------
        Tuple[Dict[str, np.ndarray], float]
            Updated parameters and current loss
        """
        AL, caches = forward_propagation_fn(X, parameters)
        cost = compute_cost_fn(AL, Y)
        gradients = backward_propagation_fn(AL, Y, caches)
        parameters = self.update_parameters(parameters, gradients)

        grad_norm = self.compute_gradient_norm(gradients)
        self.history["gradient_norm"].append(grad_norm)

        return parameters, cost

    def fit(
        self,
        X: np.ndarray,
        Y: np.ndarray,
        parameters: dict[str, np.ndarray],
        forward_propagation_fn: callable,
        backward_propagation_fn: callable,
        compute_cost_fn: callable,
        epochs: int = 1000,
        print_cost: bool = True,
        print_every: int = 100,
    ) -> dict[str, np.ndarray]:
        """
        Train the model using momentum optimizer.

        Parameters
        ----------
        X : np.ndarray
            Input features
        Y : np.ndarray
            Target labels
        parameters : Dict[str, np.ndarray]
            Initial model parameters
        forward_propagation_fn : callable
            Function to compute forward propagation
        backward_propagation_fn : callable
            Function to compute backward propagation
        compute_cost_fn : callable
            Function to compute cost/loss
        epochs : int, default=1000
            Number of training epochs
        print_cost : bool, default=True
            Whether to print cost during training
        print_every : int, default=100
            Print cost every N epochs

        Returns
        -------
        Dict[str, np.ndarray]
            Trained parameters
        """
        for epoch in range(epochs):
            parameters, cost = self.train_step(
                X,
                Y,
                parameters,
                forward_propagation_fn,
                backward_propagation_fn,
                compute_cost_fn,
            )

            self.history["loss"].append(cost)

            if print_cost and epoch % print_every == 0:
                grad_norm = self.history["gradient_norm"][-1]
                print(
                    f"Epoch {epoch}: Cost = {cost:.6f}, Gradient Norm = {grad_norm:.6f}"
                )

        return parameters

    def get_momentum_statistics(self) -> dict[str, dict[str, float]]:
        """
        Get statistics about momentum estimates.

        Returns
        -------
        Dict[str, Dict[str, float]]
            Statistics for each parameter's momentum
        """
        stats = {}
        for key, v in self.v.items():
            stats[key] = {
                "mean": np.mean(v),
                "std": np.std(v),
                "max": np.max(v),
                "min": np.min(v),
                "norm": np.linalg.norm(v),
            }
        return stats

    def reset_optimizer_state(self) -> None:
        """Reset optimizer state including velocity estimates and time step."""
        self.v = {}
        self.t = 0
        self.history = {"loss": [], "gradient_norm": []}

    def get_config(self) -> dict[str, Any]:
        """
        Get optimizer configuration.

        Returns
        -------
        Dict[str, Any]
            Configuration dictionary
        """
        return {
            "learning_rate": self.learning_rate,
            "beta": self.beta,
            "bias_correction": self.bias_correction,
            "epsilon": self.epsilon,
            "optimizer": "MomentumOptimizer",
        }

initialize_velocity

initialize_velocity(parameters: dict[str, ndarray]) -> None

Initialize velocity (momentum) estimates for all parameters.

Parameters:

Name	Type	Description	Default
parameters	Dict[str, ndarray]	Model parameters to initialize velocity for	required

Notes

Velocities are initialized to zero arrays with the same shape as parameters.

Source code in src/dlhub/optimizers/momentum.py

def initialize_velocity(self, parameters: dict[str, np.ndarray]) -> None:
    """
    Initialize velocity (momentum) estimates for all parameters.

    Parameters
    ----------
    parameters : Dict[str, np.ndarray]
        Model parameters to initialize velocity for

    Notes
    -----
    Velocities are initialized to zero arrays with the same shape as parameters.
    """
    for key in parameters:
        self.v[key] = np.zeros_like(parameters[key])

update_parameters

update_parameters(parameters: dict[str, ndarray], gradients: dict[str, ndarray]) -> dict[str, np.ndarray]

Update parameters using momentum-based gradient descent.

Parameters:

Name	Type	Description	Default
parameters	Dict[str, ndarray]	Current model parameters	required
gradients	Dict[str, ndarray]	Computed gradients for each parameter	required

Returns:

Type	Description
Dict[str, ndarray]	Updated parameters

Notes

Updates parameters using momentum: v_t = β * v_{t-1} + (1-β) * g_t θ_t = θ_{t-1} - α * v_t_corrected

Where v_t_corrected includes bias correction if enabled.

Source code in src/dlhub/optimizers/momentum.py

def update_parameters(
    self, parameters: dict[str, np.ndarray], gradients: dict[str, np.ndarray]
) -> dict[str, np.ndarray]:
    """
    Update parameters using momentum-based gradient descent.

    Parameters
    ----------
    parameters : Dict[str, np.ndarray]
        Current model parameters
    gradients : Dict[str, np.ndarray]
        Computed gradients for each parameter

    Returns
    -------
    Dict[str, np.ndarray]
        Updated parameters

    Notes
    -----
    Updates parameters using momentum:
    v_t = β * v_{t-1} + (1-β) * g_t
    θ_t = θ_{t-1} - α * v_t_corrected

    Where v_t_corrected includes bias correction if enabled.
    """
    if not self.v:
        self.initialize_velocity(parameters)

    self.t += 1
    updated_parameters = {}

    for key in parameters:
        self.v[key] = self.beta * self.v[key] + (1 - self.beta) * gradients[key]

        if self.bias_correction:
            v_corrected = self.v[key] / (1 - self.beta**self.t)
        else:
            v_corrected = self.v[key]

        updated_parameters[key] = parameters[key] - self.learning_rate * v_corrected

    return updated_parameters

compute_gradient_norm

compute_gradient_norm(gradients: dict[str, ndarray]) -> float

Compute the L2 norm of gradients for monitoring convergence.

Parameters:

Name	Type	Description	Default
gradients	Dict[str, ndarray]	Gradients for each parameter	required

Returns:

Type	Description
float	L2 norm of all gradients

Source code in src/dlhub/optimizers/momentum.py

def compute_gradient_norm(self, gradients: dict[str, np.ndarray]) -> float:
    """
    Compute the L2 norm of gradients for monitoring convergence.

    Parameters
    ----------
    gradients : Dict[str, np.ndarray]
        Gradients for each parameter

    Returns
    -------
    float
        L2 norm of all gradients
    """
    total_norm = 0.0
    for grad in gradients.values():
        total_norm += np.sum(grad**2)
    return np.sqrt(total_norm)

train_step

train_step(X: ndarray, Y: ndarray, parameters: dict[str, ndarray], forward_propagation_fn: callable, backward_propagation_fn: callable, compute_cost_fn: callable) -> tuple[dict[str, np.ndarray]]

Perform one training step with momentum optimizer.

Parameters:

Name	Type	Description	Default
X	ndarray	Input features	required
Y	ndarray	Target labels	required
parameters	Dict[str, ndarray]	Current model parameters	required
forward_propagation_fn	callable	Function to compute forward propagation	required
backward_propagation_fn	callable	Function to compute backward propagation	required
compute_cost_fn	callable	Function to compute cost/loss	required

Returns:

Type	Description
Tuple[Dict[str, ndarray], float]	Updated parameters and current loss

Source code in src/dlhub/optimizers/momentum.py

def train_step(
    self,
    X: np.ndarray,
    Y: np.ndarray,
    parameters: dict[str, np.ndarray],
    forward_propagation_fn: callable,
    backward_propagation_fn: callable,
    compute_cost_fn: callable,
) -> tuple[dict[str, np.ndarray]]:
    """
    Perform one training step with momentum optimizer.

    Parameters
    ----------
    X : np.ndarray
        Input features
    Y : np.ndarray
        Target labels
    parameters : Dict[str, np.ndarray]
        Current model parameters
    forward_propagation_fn : callable
        Function to compute forward propagation
    backward_propagation_fn : callable
        Function to compute backward propagation
    compute_cost_fn : callable
        Function to compute cost/loss

    Returns
    -------
    Tuple[Dict[str, np.ndarray], float]
        Updated parameters and current loss
    """
    AL, caches = forward_propagation_fn(X, parameters)
    cost = compute_cost_fn(AL, Y)
    gradients = backward_propagation_fn(AL, Y, caches)
    parameters = self.update_parameters(parameters, gradients)

    grad_norm = self.compute_gradient_norm(gradients)
    self.history["gradient_norm"].append(grad_norm)

    return parameters, cost

fit

fit(X: ndarray, Y: ndarray, parameters: dict[str, ndarray], forward_propagation_fn: callable, backward_propagation_fn: callable, compute_cost_fn: callable, epochs: int = 1000, print_cost: bool = True, print_every: int = 100) -> dict[str, np.ndarray]

Train the model using momentum optimizer.

Parameters:

Name	Type	Description	Default
X	ndarray	Input features	required
Y	ndarray	Target labels	required
parameters	Dict[str, ndarray]	Initial model parameters	required
forward_propagation_fn	callable	Function to compute forward propagation	required
backward_propagation_fn	callable	Function to compute backward propagation	required
compute_cost_fn	callable	Function to compute cost/loss	required
epochs	int	Number of training epochs	1000
print_cost	bool	Whether to print cost during training	True
print_every	int	Print cost every N epochs	100

Returns:

Type	Description
Dict[str, ndarray]	Trained parameters

Source code in src/dlhub/optimizers/momentum.py

def fit(
    self,
    X: np.ndarray,
    Y: np.ndarray,
    parameters: dict[str, np.ndarray],
    forward_propagation_fn: callable,
    backward_propagation_fn: callable,
    compute_cost_fn: callable,
    epochs: int = 1000,
    print_cost: bool = True,
    print_every: int = 100,
) -> dict[str, np.ndarray]:
    """
    Train the model using momentum optimizer.

    Parameters
    ----------
    X : np.ndarray
        Input features
    Y : np.ndarray
        Target labels
    parameters : Dict[str, np.ndarray]
        Initial model parameters
    forward_propagation_fn : callable
        Function to compute forward propagation
    backward_propagation_fn : callable
        Function to compute backward propagation
    compute_cost_fn : callable
        Function to compute cost/loss
    epochs : int, default=1000
        Number of training epochs
    print_cost : bool, default=True
        Whether to print cost during training
    print_every : int, default=100
        Print cost every N epochs

    Returns
    -------
    Dict[str, np.ndarray]
        Trained parameters
    """
    for epoch in range(epochs):
        parameters, cost = self.train_step(
            X,
            Y,
            parameters,
            forward_propagation_fn,
            backward_propagation_fn,
            compute_cost_fn,
        )

        self.history["loss"].append(cost)

        if print_cost and epoch % print_every == 0:
            grad_norm = self.history["gradient_norm"][-1]
            print(
                f"Epoch {epoch}: Cost = {cost:.6f}, Gradient Norm = {grad_norm:.6f}"
            )

    return parameters

get_momentum_statistics

get_momentum_statistics() -> dict[str, dict[str, float]]

Get statistics about momentum estimates.

Returns:

Type	Description
Dict[str, Dict[str, float]]	Statistics for each parameter's momentum

Source code in src/dlhub/optimizers/momentum.py

def get_momentum_statistics(self) -> dict[str, dict[str, float]]:
    """
    Get statistics about momentum estimates.

    Returns
    -------
    Dict[str, Dict[str, float]]
        Statistics for each parameter's momentum
    """
    stats = {}
    for key, v in self.v.items():
        stats[key] = {
            "mean": np.mean(v),
            "std": np.std(v),
            "max": np.max(v),
            "min": np.min(v),
            "norm": np.linalg.norm(v),
        }
    return stats

reset_optimizer_state

reset_optimizer_state() -> None

Reset optimizer state including velocity estimates and time step.

Source code in src/dlhub/optimizers/momentum.py

def reset_optimizer_state(self) -> None:
    """Reset optimizer state including velocity estimates and time step."""
    self.v = {}
    self.t = 0
    self.history = {"loss": [], "gradient_norm": []}

get_config

get_config() -> dict[str, Any]

Get optimizer configuration.

Returns:

Type	Description
Dict[str, Any]	Configuration dictionary

Source code in src/dlhub/optimizers/momentum.py

def get_config(self) -> dict[str, Any]:
    """
    Get optimizer configuration.

    Returns
    -------
    Dict[str, Any]
        Configuration dictionary
    """
    return {
        "learning_rate": self.learning_rate,
        "beta": self.beta,
        "bias_correction": self.bias_correction,
        "epsilon": self.epsilon,
        "optimizer": "MomentumOptimizer",
    }

example_usage

example_usage()

Example demonstrating how to use MomentumOptimizer.

Source code in src/dlhub/optimizers/momentum.py

def example_usage():
    """Example demonstrating how to use MomentumOptimizer."""
    np.random.seed(42)
    X = np.random.randn(10, 1000)  # 10 features, 1000 examples
    Y = (X[0:1, :] > 0).astype(int)  # Binary classification

    def initialize_parameters(layer_dims):
        parameters = {}
        L = len(layer_dims)
        for l in range(1, L):
            parameters[f"W{l}"] = (
                np.random.randn(layer_dims[l], layer_dims[l - 1]) * 0.01
            )
            parameters[f"b{l}"] = np.zeros((layer_dims[l], 1))
        return parameters

    layer_dims = [10, 5, 1]
    parameters = initialize_parameters(layer_dims)

    # Dummy functions (replace with actual implementations)
    def forward_propagation(X, parameters):
        return np.random.randn(1, X.shape[1]), {}

    def backward_propagation(AL, Y, caches):
        return {
            key: np.random.randn(*val.shape) * 0.01 for key, val in parameters.items()
        }

    def compute_cost(AL, Y):
        return np.random.rand()

    optimizer = MomentumOptimizer(learning_rate=0.01, beta=0.9, bias_correction=True)

    trained_parameters = optimizer.fit(
        X,
        Y,
        parameters,
        forward_propagation,
        backward_propagation,
        compute_cost,
        epochs=100,
        print_every=20,
    )

    print(f"\nFinal cost: {optimizer.history['loss'][-1]:.6f}")
    print(f"Momentum statistics: \n{optimizer.get_momentum_statistics()}")
    print(f"\nOptimizer config: {optimizer.get_config()}")

rmsprop

RMSprop Optimizer Implementation

This module implements the RMSprop (Root Mean Square Propagation) optimizer with adaptive learning rate optimization using squared gradient accumulation and parameter-wise learning rate adjustment.

Author

Deep Learning Reference Hub

License

MIT

RMSpropOptimizer

RMSprop (Root Mean Square Propagation) optimizer.

RMSprop adapts the learning rate for each parameter by dividing by a running average of the magnitudes of recent gradients. This helps with convergence on non-convex functions and handles different scaling of parameters. It also applies a learning rate decay technique based on current step.

Parameters:

Name	Type	Description	Default
learning_rate	float	Learning rate for parameter updates	0.001
beta	float	Exponential decay rate for the second moment estimates	0.9
epsilon	float	Small constant for numerical stability	1e-8
bias_correction	bool	Whether to apply bias correction (not standard in RMSprop)	False
decay	float	Learning rate decay factor	0.0

Attributes:

Name	Type	Description
learning_rate	float	Current learning rate
beta	float	Decay rate for second moment estimates
epsilon	float	Numerical stability constant
s	Dict[str, ndarray]	Second moment estimates (squared gradients) for each parameter
t	int	Time step counter
history	Dict[str, List[float]]	Training history including losses and learning rates

Source code in src/dlhub/optimizers/rmsprop.py

class RMSpropOptimizer:
    """
    RMSprop (Root Mean Square Propagation) optimizer.

    RMSprop adapts the learning rate for each parameter by dividing by a running
    average of the magnitudes of recent gradients. This helps with convergence
    on non-convex functions and handles different scaling of parameters. It also
    applies a learning rate decay technique based on current step.

    Parameters
    ----------
    learning_rate : float, default=0.001
        Learning rate for parameter updates
    beta : float, default=0.9
        Exponential decay rate for the second moment estimates
    epsilon : float, default=1e-8
        Small constant for numerical stability
    bias_correction : bool, default=False
        Whether to apply bias correction (not standard in RMSprop)
    decay : float, default=0.0
        Learning rate decay factor

    Attributes
    ----------
    learning_rate : float
        Current learning rate
    beta : float
        Decay rate for second moment estimates
    epsilon : float
        Numerical stability constant
    s : Dict[str, np.ndarray]
        Second moment estimates (squared gradients) for each parameter
    t : int
        Time step counter
    history : Dict[str, List[float]]
        Training history including losses and learning rates
    """

    def __init__(
        self,
        learning_rate: float = 0.001,
        beta: float = 0.9,
        epsilon: float = 1e-8,
        bias_correction: bool = False,
        decay: float = 0.0,
    ):
        self.learning_rate = learning_rate
        self.initial_learning_rate = learning_rate
        self.beta = beta
        self.epsilon = epsilon
        self.bias_correction = bias_correction
        self.decay = decay

        self.s = {}  # Second moment estimates
        self.t = 0  # Time step
        self.history = {
            "loss": [],
            "gradient_norm": [],
            "learning_rate": [],
            "rms_grad": [],
        }

    def initialize_second_moments(self, parameters: dict[str, np.ndarray]) -> None:
        """
        Initialize second moment estimates for all parameters.

        Parameters
        ----------
        parameters : Dict[str, np.ndarray]
            Model parameters to initialize second moments for

        Notes
        -----
        Second moments are initialized to zero arrays with the same shape as parameters.
        """
        for key in parameters:
            self.s[key] = np.zeros_like(parameters[key])

    def update_learning_rate(self) -> None:
        """
        Update learning rate with decay if specified.

        Notes
        -----
        Applies learning rate decay: lr = lr_initial / (1 + decay * t)
        """
        if self.decay > 0:
            self.learning_rate = self.initial_learning_rate / (1 + self.decay * self.t)

    def update_parameters(
        self, parameters: dict[str, np.ndarray], gradients: dict[str, np.ndarray]
    ) -> dict[str, np.ndarray]:
        """
        Update parameters using RMSprop optimization.

        Parameters
        ----------
        parameters : Dict[str, np.ndarray]
            Current model parameters
        gradients : Dict[str, np.ndarray]
            Computed gradients for each parameter

        Returns
        -------
        Dict[str, np.ndarray]
            Updated parameters

        Notes
        -----
        Updates parameters using RMSprop:
        s_t = β * s_{t-1} + (1-β) * g_t²
        θ_t = θ_{t-1} - α * g_t / (√s_t + ε)

        Where s_t is the exponential weighted average of squared gradients.
        Stores average RMS gradient for monitoring
        """
        if not self.s:
            self.initialize_second_moments(parameters)

        self.t += 1
        self.update_learning_rate()

        updated_parameters = {}
        rms_gradients = {}

        for key in parameters:
            self.s[key] = self.beta * self.s[key] + (1 - self.beta) * (
                gradients[key] ** 2
            )

            if self.bias_correction:
                s_corrected = self.s[key] / (1 - self.beta**self.t)
            else:
                s_corrected = self.s[key]

            rms_grad = np.sqrt(np.mean(s_corrected))
            rms_gradients[key] = rms_grad

            updated_parameters[key] = parameters[key] - self.learning_rate * gradients[
                key
            ] / (np.sqrt(s_corrected) + self.epsilon)

        avg_rms_grad = np.mean(list(rms_gradients.values()))
        self.history["rms_grad"].append(avg_rms_grad)
        self.history["learning_rate"].append(self.learning_rate)

        return updated_parameters

    def compute_gradient_norm(self, gradients: dict[str, np.ndarray]) -> float:
        """
        Compute the L2 norm of gradients for monitoring convergence.

        Parameters
        ----------
        gradients : Dict[str, np.ndarray]
            Gradients for each parameter

        Returns
        -------
        float
            L2 norm of all gradients
        """
        total_norm = 0.0
        for grad in gradients.values():
            total_norm += np.sum(grad**2)
        return np.sqrt(total_norm)

    def get_effective_learning_rates(
        self, parameters: dict[str, np.ndarray]
    ) -> dict[str, np.ndarray]:
        """
        Compute effective learning rates for each parameter.

        Parameters
        ----------
        parameters : Dict[str, np.ndarray]
            Current model parameters

        Returns
        -------
        Dict[str, np.ndarray]
            Effective learning rates for each parameter
        """
        effective_lrs = {}

        if not self.s:
            for key in parameters:  # If not initialized, return base learning rate
                effective_lrs[key] = np.full_like(parameters[key], self.learning_rate)
        else:
            for key in parameters:
                if self.bias_correction:
                    s_corrected = self.s[key] / (1 - self.beta**self.t)
                else:
                    s_corrected = self.s[key]

                effective_lrs[key] = self.learning_rate / (
                    np.sqrt(s_corrected) + self.epsilon
                )

        return effective_lrs

    def train_step(
        self,
        X: np.ndarray,
        Y: np.ndarray,
        parameters: dict[str, np.ndarray],
        forward_propagation_fn: callable,
        backward_propagation_fn: callable,
        compute_cost_fn: callable,
    ) -> tuple[dict[str, np.ndarray], float]:
        """
        Perform one training step with RMSprop optimizer.

        Parameters
        ----------
        X : np.ndarray
            Input features
        Y : np.ndarray
            Target labels
        parameters : Dict[str, np.ndarray]
            Current model parameters
        forward_propagation_fn : callable
            Function to compute forward propagation
        backward_propagation_fn : callable
            Function to compute backward propagation
        compute_cost_fn : callable
            Function to compute cost/loss

        Returns
        -------
        Tuple[Dict[str, np.ndarray], float]
            Updated parameters and current loss
        """
        AL, caches = forward_propagation_fn(X, parameters)
        cost = compute_cost_fn(AL, Y)
        gradients = backward_propagation_fn(AL, Y, caches)
        parameters = self.update_parameters(parameters, gradients)

        grad_norm = self.compute_gradient_norm(gradients)
        self.history["gradient_norm"].append(grad_norm)

        return parameters, cost

    def fit(
        self,
        X: np.ndarray,
        Y: np.ndarray,
        parameters: dict[str, np.ndarray],
        forward_propagation_fn: callable,
        backward_propagation_fn: callable,
        compute_cost_fn: callable,
        epochs: int = 1000,
        print_cost: bool = True,
        print_every: int = 100,
    ) -> dict[str, np.ndarray]:
        """
        Train the model using RMSprop optimizer.

        Parameters
        ----------
        X : np.ndarray
            Input features
        Y : np.ndarray
            Target labels
        parameters : Dict[str, np.ndarray]
            Initial model parameters
        forward_propagation_fn : callable
            Function to compute forward propagation
        backward_propagation_fn : callable
            Function to compute backward propagation
        compute_cost_fn : callable
            Function to compute cost/loss
        epochs : int, default=1000
            Number of training epochs
        print_cost : bool, default=True
            Whether to print cost during training
        print_every : int, default=100
            Print cost every N epochs

        Returns
        -------
        Dict[str, np.ndarray]
            Trained parameters
        """
        for epoch in range(epochs):
            parameters, cost = self.train_step(
                X,
                Y,
                parameters,
                forward_propagation_fn,
                backward_propagation_fn,
                compute_cost_fn,
            )

            self.history["loss"].append(cost)

            if print_cost and epoch % print_every == 0:
                grad_norm = self.history["gradient_norm"][-1]
                rms_grad = self.history["rms_grad"][-1]
                lr = self.history["learning_rate"][-1]
                print(
                    f"Epoch {epoch}: Cost = {cost:.6f}, "
                    f"Gradient Norm = {grad_norm:.6f}, "
                    f"RMS Grad = {rms_grad:.6f}, "
                    f"LR = {lr:.6f}"
                )

        return parameters

    def get_second_moment_statistics(self) -> dict[str, dict[str, float]]:
        """
        Get statistics about second moment estimates.

        Returns
        -------
        Dict[str, Dict[str, float]]
            Statistics for each parameter's second moments
        """
        stats = {}
        for key, s in self.s.items():
            stats[key] = {
                "mean": np.mean(s),
                "std": np.std(s),
                "max": np.max(s),
                "min": np.min(s),
                "norm": np.linalg.norm(s),
            }
        return stats

    def reset_optimizer_state(self) -> None:
        """Reset optimizer state including second moment estimates and time step."""
        self.s = {}
        self.t = 0
        self.learning_rate = self.initial_learning_rate
        self.history = {
            "loss": [],
            "gradient_norm": [],
            "learning_rate": [],
            "rms_grad": [],
        }

    def get_config(self) -> dict[str, Any]:
        """
        Get optimizer configuration.

        Returns
        -------
        Dict[str, Any]
            Configuration dictionary
        """
        return {
            "learning_rate": self.learning_rate,
            "initial_learning_rate": self.initial_learning_rate,
            "beta": self.beta,
            "epsilon": self.epsilon,
            "bias_correction": self.bias_correction,
            "decay": self.decay,
            "optimizer": "RMSpropOptimizer",
        }

initialize_second_moments

initialize_second_moments(parameters: dict[str, ndarray]) -> None

Initialize second moment estimates for all parameters.

Parameters:

Name	Type	Description	Default
parameters	Dict[str, ndarray]	Model parameters to initialize second moments for	required

Notes

Second moments are initialized to zero arrays with the same shape as parameters.

Source code in src/dlhub/optimizers/rmsprop.py

def initialize_second_moments(self, parameters: dict[str, np.ndarray]) -> None:
    """
    Initialize second moment estimates for all parameters.

    Parameters
    ----------
    parameters : Dict[str, np.ndarray]
        Model parameters to initialize second moments for

    Notes
    -----
    Second moments are initialized to zero arrays with the same shape as parameters.
    """
    for key in parameters:
        self.s[key] = np.zeros_like(parameters[key])

update_learning_rate

update_learning_rate() -> None

Update learning rate with decay if specified.

Notes

Applies learning rate decay: lr = lr_initial / (1 + decay * t)

Source code in src/dlhub/optimizers/rmsprop.py

def update_learning_rate(self) -> None:
    """
    Update learning rate with decay if specified.

    Notes
    -----
    Applies learning rate decay: lr = lr_initial / (1 + decay * t)
    """
    if self.decay > 0:
        self.learning_rate = self.initial_learning_rate / (1 + self.decay * self.t)

update_parameters

update_parameters(parameters: dict[str, ndarray], gradients: dict[str, ndarray]) -> dict[str, np.ndarray]

Update parameters using RMSprop optimization.

Parameters:

Name	Type	Description	Default
parameters	Dict[str, ndarray]	Current model parameters	required
gradients	Dict[str, ndarray]	Computed gradients for each parameter	required

Returns:

Type	Description
Dict[str, ndarray]	Updated parameters

Notes

Updates parameters using RMSprop: s_t = β * s_{t-1} + (1-β) * g_t² θ_t = θ_{t-1} - α * g_t / (√s_t + ε)

Where s_t is the exponential weighted average of squared gradients. Stores average RMS gradient for monitoring

Source code in src/dlhub/optimizers/rmsprop.py

def update_parameters(
    self, parameters: dict[str, np.ndarray], gradients: dict[str, np.ndarray]
) -> dict[str, np.ndarray]:
    """
    Update parameters using RMSprop optimization.

    Parameters
    ----------
    parameters : Dict[str, np.ndarray]
        Current model parameters
    gradients : Dict[str, np.ndarray]
        Computed gradients for each parameter

    Returns
    -------
    Dict[str, np.ndarray]
        Updated parameters

    Notes
    -----
    Updates parameters using RMSprop:
    s_t = β * s_{t-1} + (1-β) * g_t²
    θ_t = θ_{t-1} - α * g_t / (√s_t + ε)

    Where s_t is the exponential weighted average of squared gradients.
    Stores average RMS gradient for monitoring
    """
    if not self.s:
        self.initialize_second_moments(parameters)

    self.t += 1
    self.update_learning_rate()

    updated_parameters = {}
    rms_gradients = {}

    for key in parameters:
        self.s[key] = self.beta * self.s[key] + (1 - self.beta) * (
            gradients[key] ** 2
        )

        if self.bias_correction:
            s_corrected = self.s[key] / (1 - self.beta**self.t)
        else:
            s_corrected = self.s[key]

        rms_grad = np.sqrt(np.mean(s_corrected))
        rms_gradients[key] = rms_grad

        updated_parameters[key] = parameters[key] - self.learning_rate * gradients[
            key
        ] / (np.sqrt(s_corrected) + self.epsilon)

    avg_rms_grad = np.mean(list(rms_gradients.values()))
    self.history["rms_grad"].append(avg_rms_grad)
    self.history["learning_rate"].append(self.learning_rate)

    return updated_parameters

compute_gradient_norm

compute_gradient_norm(gradients: dict[str, ndarray]) -> float

Compute the L2 norm of gradients for monitoring convergence.

Parameters:

Name	Type	Description	Default
gradients	Dict[str, ndarray]	Gradients for each parameter	required

Returns:

Type	Description
float	L2 norm of all gradients

Source code in src/dlhub/optimizers/rmsprop.py

def compute_gradient_norm(self, gradients: dict[str, np.ndarray]) -> float:
    """
    Compute the L2 norm of gradients for monitoring convergence.

    Parameters
    ----------
    gradients : Dict[str, np.ndarray]
        Gradients for each parameter

    Returns
    -------
    float
        L2 norm of all gradients
    """
    total_norm = 0.0
    for grad in gradients.values():
        total_norm += np.sum(grad**2)
    return np.sqrt(total_norm)

get_effective_learning_rates

get_effective_learning_rates(parameters: dict[str, ndarray]) -> dict[str, np.ndarray]

Compute effective learning rates for each parameter.

Parameters:

Name	Type	Description	Default
parameters	Dict[str, ndarray]	Current model parameters	required

Returns:

Type	Description
Dict[str, ndarray]	Effective learning rates for each parameter

Source code in src/dlhub/optimizers/rmsprop.py

def get_effective_learning_rates(
    self, parameters: dict[str, np.ndarray]
) -> dict[str, np.ndarray]:
    """
    Compute effective learning rates for each parameter.

    Parameters
    ----------
    parameters : Dict[str, np.ndarray]
        Current model parameters

    Returns
    -------
    Dict[str, np.ndarray]
        Effective learning rates for each parameter
    """
    effective_lrs = {}

    if not self.s:
        for key in parameters:  # If not initialized, return base learning rate
            effective_lrs[key] = np.full_like(parameters[key], self.learning_rate)
    else:
        for key in parameters:
            if self.bias_correction:
                s_corrected = self.s[key] / (1 - self.beta**self.t)
            else:
                s_corrected = self.s[key]

            effective_lrs[key] = self.learning_rate / (
                np.sqrt(s_corrected) + self.epsilon
            )

    return effective_lrs

train_step

train_step(X: ndarray, Y: ndarray, parameters: dict[str, ndarray], forward_propagation_fn: callable, backward_propagation_fn: callable, compute_cost_fn: callable) -> tuple[dict[str, np.ndarray], float]

Perform one training step with RMSprop optimizer.

Parameters:

Name	Type	Description	Default
X	ndarray	Input features	required
Y	ndarray	Target labels	required
parameters	Dict[str, ndarray]	Current model parameters	required
forward_propagation_fn	callable	Function to compute forward propagation	required
backward_propagation_fn	callable	Function to compute backward propagation	required
compute_cost_fn	callable	Function to compute cost/loss	required

Returns:

Type	Description
Tuple[Dict[str, ndarray], float]	Updated parameters and current loss

Source code in src/dlhub/optimizers/rmsprop.py

def train_step(
    self,
    X: np.ndarray,
    Y: np.ndarray,
    parameters: dict[str, np.ndarray],
    forward_propagation_fn: callable,
    backward_propagation_fn: callable,
    compute_cost_fn: callable,
) -> tuple[dict[str, np.ndarray], float]:
    """
    Perform one training step with RMSprop optimizer.

    Parameters
    ----------
    X : np.ndarray
        Input features
    Y : np.ndarray
        Target labels
    parameters : Dict[str, np.ndarray]
        Current model parameters
    forward_propagation_fn : callable
        Function to compute forward propagation
    backward_propagation_fn : callable
        Function to compute backward propagation
    compute_cost_fn : callable
        Function to compute cost/loss

    Returns
    -------
    Tuple[Dict[str, np.ndarray], float]
        Updated parameters and current loss
    """
    AL, caches = forward_propagation_fn(X, parameters)
    cost = compute_cost_fn(AL, Y)
    gradients = backward_propagation_fn(AL, Y, caches)
    parameters = self.update_parameters(parameters, gradients)

    grad_norm = self.compute_gradient_norm(gradients)
    self.history["gradient_norm"].append(grad_norm)

    return parameters, cost

fit

fit(X: ndarray, Y: ndarray, parameters: dict[str, ndarray], forward_propagation_fn: callable, backward_propagation_fn: callable, compute_cost_fn: callable, epochs: int = 1000, print_cost: bool = True, print_every: int = 100) -> dict[str, np.ndarray]

Train the model using RMSprop optimizer.

Parameters:

Name	Type	Description	Default
X	ndarray	Input features	required
Y	ndarray	Target labels	required
parameters	Dict[str, ndarray]	Initial model parameters	required
forward_propagation_fn	callable	Function to compute forward propagation	required
backward_propagation_fn	callable	Function to compute backward propagation	required
compute_cost_fn	callable	Function to compute cost/loss	required
epochs	int	Number of training epochs	1000
print_cost	bool	Whether to print cost during training	True
print_every	int	Print cost every N epochs	100

Returns:

Type	Description
Dict[str, ndarray]	Trained parameters

Source code in src/dlhub/optimizers/rmsprop.py

def fit(
    self,
    X: np.ndarray,
    Y: np.ndarray,
    parameters: dict[str, np.ndarray],
    forward_propagation_fn: callable,
    backward_propagation_fn: callable,
    compute_cost_fn: callable,
    epochs: int = 1000,
    print_cost: bool = True,
    print_every: int = 100,
) -> dict[str, np.ndarray]:
    """
    Train the model using RMSprop optimizer.

    Parameters
    ----------
    X : np.ndarray
        Input features
    Y : np.ndarray
        Target labels
    parameters : Dict[str, np.ndarray]
        Initial model parameters
    forward_propagation_fn : callable
        Function to compute forward propagation
    backward_propagation_fn : callable
        Function to compute backward propagation
    compute_cost_fn : callable
        Function to compute cost/loss
    epochs : int, default=1000
        Number of training epochs
    print_cost : bool, default=True
        Whether to print cost during training
    print_every : int, default=100
        Print cost every N epochs

    Returns
    -------
    Dict[str, np.ndarray]
        Trained parameters
    """
    for epoch in range(epochs):
        parameters, cost = self.train_step(
            X,
            Y,
            parameters,
            forward_propagation_fn,
            backward_propagation_fn,
            compute_cost_fn,
        )

        self.history["loss"].append(cost)

        if print_cost and epoch % print_every == 0:
            grad_norm = self.history["gradient_norm"][-1]
            rms_grad = self.history["rms_grad"][-1]
            lr = self.history["learning_rate"][-1]
            print(
                f"Epoch {epoch}: Cost = {cost:.6f}, "
                f"Gradient Norm = {grad_norm:.6f}, "
                f"RMS Grad = {rms_grad:.6f}, "
                f"LR = {lr:.6f}"
            )

    return parameters

get_second_moment_statistics

get_second_moment_statistics() -> dict[str, dict[str, float]]

Get statistics about second moment estimates.

Returns:

Type	Description
Dict[str, Dict[str, float]]	Statistics for each parameter's second moments

Source code in src/dlhub/optimizers/rmsprop.py

def get_second_moment_statistics(self) -> dict[str, dict[str, float]]:
    """
    Get statistics about second moment estimates.

    Returns
    -------
    Dict[str, Dict[str, float]]
        Statistics for each parameter's second moments
    """
    stats = {}
    for key, s in self.s.items():
        stats[key] = {
            "mean": np.mean(s),
            "std": np.std(s),
            "max": np.max(s),
            "min": np.min(s),
            "norm": np.linalg.norm(s),
        }
    return stats

reset_optimizer_state

reset_optimizer_state() -> None

Reset optimizer state including second moment estimates and time step.

Source code in src/dlhub/optimizers/rmsprop.py

def reset_optimizer_state(self) -> None:
    """Reset optimizer state including second moment estimates and time step."""
    self.s = {}
    self.t = 0
    self.learning_rate = self.initial_learning_rate
    self.history = {
        "loss": [],
        "gradient_norm": [],
        "learning_rate": [],
        "rms_grad": [],
    }

get_config

get_config() -> dict[str, Any]

Get optimizer configuration.

Returns:

Type	Description
Dict[str, Any]	Configuration dictionary

Source code in src/dlhub/optimizers/rmsprop.py

def get_config(self) -> dict[str, Any]:
    """
    Get optimizer configuration.

    Returns
    -------
    Dict[str, Any]
        Configuration dictionary
    """
    return {
        "learning_rate": self.learning_rate,
        "initial_learning_rate": self.initial_learning_rate,
        "beta": self.beta,
        "epsilon": self.epsilon,
        "bias_correction": self.bias_correction,
        "decay": self.decay,
        "optimizer": "RMSpropOptimizer",
    }

adaptive_learning_rate_analysis

adaptive_learning_rate_analysis(optimizer: RMSpropOptimizer, parameters: dict[str, ndarray]) -> None

Analyze and visualize adaptive learning rates in RMSprop.

Parameters:

Name	Type	Description	Default
optimizer	RMSpropOptimizer	Trained RMSprop optimizer	required
parameters	Dict[str, ndarray]	Model parameters	required

Source code in src/dlhub/optimizers/rmsprop.py

def adaptive_learning_rate_analysis(
    optimizer: RMSpropOptimizer, parameters: dict[str, np.ndarray]
) -> None:
    """
    Analyze and visualize adaptive learning rates in RMSprop.

    Parameters
    ----------
    optimizer : RMSpropOptimizer
        Trained RMSprop optimizer
    parameters : Dict[str, np.ndarray]
        Model parameters
    """
    print("=== Adaptive Learning Rate Analysis ===")
    effective_lrs = optimizer.get_effective_learning_rates(parameters)

    for key, lr_array in effective_lrs.items():
        print(f"\nParameter {key}:")
        print(f"  Base LR: {optimizer.learning_rate:.6f}")
        print(f"  Effective LR - Mean: {np.mean(lr_array):.6f}")
        print(f"  Effective LR - Std: {np.std(lr_array):.6f}")
        print(f"  Effective LR - Min: {np.min(lr_array):.6f}")
        print(f"  Effective LR - Max: {np.max(lr_array):.6f}")
        print(f"  Adaptation Ratio: {np.mean(lr_array) / optimizer.learning_rate:.4f}")

example_usage

example_usage()

Example demonstrating how to use RMSpropOptimizer.

Source code in src/dlhub/optimizers/rmsprop.py

def example_usage():
    """Example demonstrating how to use RMSpropOptimizer."""
    np.random.seed(42)
    X = np.random.randn(10, 1000)  # 10 features, 1000 examples
    Y = (X[0:1, :] > 0).astype(int)  # Binary classification

    X[5:7, :] *= 10  # Larger scale features!
    X[8:, :] *= 0.1  # Smaller scale features!

    def initialize_parameters(layer_dims):
        parameters = {}
        L = len(layer_dims)
        for l in range(1, L):
            parameters[f"W{l}"] = (
                np.random.randn(layer_dims[l], layer_dims[l - 1]) * 0.01
            )
            parameters[f"b{l}"] = np.zeros((layer_dims[l], 1))
        return parameters

    layer_dims = [10, 5, 1]
    parameters = initialize_parameters(layer_dims)

    # Dummy functions (replace with actual implementations)
    def forward_propagation(X, parameters):
        return np.random.randn(1, X.shape[1]), {}

    def backward_propagation(AL, Y, caches):
        gradients = {}  # Simulating gradients with different scales
        for key, val in parameters.items():
            if "W" in key:
                gradients[key] = np.random.randn(*val.shape) * 0.1
            else:  # bias terms
                gradients[key] = np.random.randn(*val.shape) * 0.01
        return gradients

    def compute_cost(AL, Y):
        return np.random.rand()

    optimizer = RMSpropOptimizer(
        learning_rate=0.001, beta=0.9, epsilon=1e-8, bias_correction=False, decay=1e-3
    )

    trained_parameters = optimizer.fit(
        X,
        Y,
        parameters,
        forward_propagation,
        backward_propagation,
        compute_cost,
        epochs=1000,
        print_every=100,
    )

    print(f"\nFinal cost: {optimizer.history['loss'][-1]:.6f}")
    print(f"Final learning rate: {optimizer.learning_rate:.6f}")
    print(f"\nSecond moment statistics: {optimizer.get_second_moment_statistics()}")
    print(f"\nOptimizer config:\n{optimizer.get_config()}\n")

    adaptive_learning_rate_analysis(optimizer, trained_parameters)

schedules

Learning Rate Scheduler Implementation

A comprehensive implementation of various learning rate scheduling strategies commonly used in deep learning optimization.

Learning rate scheduling is crucial for training stability and convergence. This module provides multiple scheduling strategies with configurable parameters.

References

• Loshchilov, I., & Hutter, F. (2016). SGDR: Stochastic Gradient Descent with Warm Restarts
• Smith, L. N. (2017). Cyclical Learning Rates for Training Neural Networks
• Goyal, P. et al. (2017). Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour

Author

Deep Learning Reference Hub

License

MIT

SchedulerType

Bases: Enum

Enumeration of different scheduling strategies.

Source code in src/dlhub/optimizers/schedules.py

class SchedulerType(Enum):
    """Enumeration of different scheduling strategies."""

    CONSTANT = "constant"
    STEP_DECAY = "step_decay"
    EXPONENTIAL_DECAY = "exponential_decay"
    POLYNOMIAL_DECAY = "polynomial_decay"
    COSINE_ANNEALING = "cosine_annealing"
    COSINE_ANNEALING_WARM_RESTARTS = "cosine_annealing_warm_restarts"
    CYCLICAL = "cyclical"
    ONE_CYCLE = "one_cycle"
    REDUCE_ON_PLATEAU = "reduce_on_plateau"
    WARMUP_COSINE = "warmup_cosine"
    LINEAR_WARMUP = "linear_warmup"
    CUSTOM = "custom"

LearningRateScheduler

Comprehensive Learning Rate Scheduler with multiple scheduling strategies.

This class provides various learning rate scheduling strategies commonly used in deep learning training, including step decay, cosine annealing, cyclical learning rates, and warm restarts.

Parameters:

Name	Type	Description	Default
initial_lr	float	Initial learning rate	required
scheduler_type	SchedulerType	Type of scheduling strategy to use	CONSTANT
total_steps	int	Total number of training steps (required for some schedulers)	None
**kwargs		Additional parameters specific to each scheduler type	{}

Attributes:

Name	Type	Description
current_lr	float	Current learning rate
step_count	int	Number of steps taken
history	list	History of learning rates

Source code in src/dlhub/optimizers/schedules.py

class LearningRateScheduler:
    """
    Comprehensive Learning Rate Scheduler with multiple scheduling strategies.

    This class provides various learning rate scheduling strategies commonly used
    in deep learning training, including step decay, cosine annealing, cyclical
    learning rates, and warm restarts.

    Parameters
    ----------
    initial_lr : float
        Initial learning rate
    scheduler_type : SchedulerType
        Type of scheduling strategy to use
    total_steps : int, optional
        Total number of training steps (required for some schedulers)
    **kwargs
        Additional parameters specific to each scheduler type

    Attributes
    ----------
    current_lr : float
        Current learning rate
    step_count : int
        Number of steps taken
    history : list
        History of learning rates
    """

    def __init__(
        self,
        initial_lr: float,
        scheduler_type: SchedulerType = SchedulerType.CONSTANT,
        total_steps: int | None = None,
        **kwargs,
    ):
        if initial_lr <= 0:
            raise ValueError(
                f"Initial learning rate must be positive, got {initial_lr}"
            )

        self.initial_lr = initial_lr
        self.scheduler_type = scheduler_type
        self.total_steps = total_steps
        self.kwargs = kwargs

        self.current_lr = initial_lr
        self.step_count = 0
        self.history = [initial_lr]

        self._plateau_count = 0
        self._best_metric = None
        self._cycle_count = 0
        self._restart_count = 0
        self._cooldown_counter = 0

        self._validate_parameters()

    def _validate_parameters(self) -> None:
        """Validate scheduler-specific parameters."""
        if self.scheduler_type == SchedulerType.STEP_DECAY:
            if "step_size" not in self.kwargs:
                raise ValueError("step_size required for STEP_DECAY scheduler")
            if "gamma" not in self.kwargs:
                self.kwargs["gamma"] = 0.1

        elif self.scheduler_type == SchedulerType.EXPONENTIAL_DECAY:
            if "gamma" not in self.kwargs:
                raise ValueError("gamma required for EXPONENTIAL_DECAY scheduler")

        elif self.scheduler_type == SchedulerType.POLYNOMIAL_DECAY:
            if "power" not in self.kwargs:
                self.kwargs["power"] = 1.0
            if self.total_steps is None:
                raise ValueError("total_steps required for POLYNOMIAL_DECAY scheduler")

        elif self.scheduler_type == SchedulerType.COSINE_ANNEALING:
            if "T_max" not in self.kwargs and self.total_steps is None:
                raise ValueError(
                    "Either T_max or total_steps required for COSINE_ANNEALING"
                )
            if "eta_min" not in self.kwargs:
                self.kwargs["eta_min"] = 0.0

        elif self.scheduler_type == SchedulerType.COSINE_ANNEALING_WARM_RESTARTS:
            if "T_0" not in self.kwargs:
                self.kwargs["T_0"] = 10
            if "T_mult" not in self.kwargs:
                self.kwargs["T_mult"] = 2
            if "eta_min" not in self.kwargs:
                self.kwargs["eta_min"] = 0.0

        elif self.scheduler_type == SchedulerType.CYCLICAL:
            if "base_lr" not in self.kwargs:
                self.kwargs["base_lr"] = self.initial_lr * 0.1
            if "max_lr" not in self.kwargs:
                self.kwargs["max_lr"] = self.initial_lr
            if "step_size_up" not in self.kwargs:
                self.kwargs["step_size_up"] = 2000
            if "mode" not in self.kwargs:
                self.kwargs["mode"] = "triangular"

        elif self.scheduler_type == SchedulerType.ONE_CYCLE:
            if "max_lr" not in self.kwargs:
                self.kwargs["max_lr"] = self.initial_lr * 10
            if self.total_steps is None:
                raise ValueError("total_steps required for ONE_CYCLE scheduler")
            if "pct_start" not in self.kwargs:
                self.kwargs["pct_start"] = 0.3
            if "anneal_strategy" not in self.kwargs:
                self.kwargs["anneal_strategy"] = "cos"

        elif self.scheduler_type == SchedulerType.REDUCE_ON_PLATEAU:
            if "factor" not in self.kwargs:
                self.kwargs["factor"] = 0.1
            if "patience" not in self.kwargs:
                self.kwargs["patience"] = 10
            if "threshold" not in self.kwargs:
                self.kwargs["threshold"] = 1e-4
            if "cooldown" not in self.kwargs:
                self.kwargs["cooldown"] = 0
            if "min_lr" not in self.kwargs:
                self.kwargs["min_lr"] = 0.0

        elif self.scheduler_type == SchedulerType.WARMUP_COSINE:
            if "warmup_steps" not in self.kwargs:
                self.kwargs["warmup_steps"] = 1000
            if self.total_steps is None:
                raise ValueError("total_steps required for WARMUP_COSINE scheduler")

        elif self.scheduler_type == SchedulerType.LINEAR_WARMUP:
            if "warmup_steps" not in self.kwargs:
                raise ValueError("warmup_steps required for LINEAR_WARMUP scheduler")

        elif self.scheduler_type == SchedulerType.CUSTOM:
            if "custom_func" not in self.kwargs:
                raise ValueError("custom_func required for CUSTOM scheduler")

    def step(self, metric: float | None = None) -> float:
        """
        Update the learning rate for one step.

        Parameters
        ----------
        metric : float, optional
            Current metric value (required for REDUCE_ON_PLATEAU)

        Returns
        -------
        float
            Updated learning rate
        """
        self.step_count += 1

        if self.scheduler_type == SchedulerType.CONSTANT:
            self.current_lr = self.initial_lr

        elif self.scheduler_type == SchedulerType.STEP_DECAY:
            self.current_lr = self._step_decay()

        elif self.scheduler_type == SchedulerType.EXPONENTIAL_DECAY:
            self.current_lr = self._exponential_decay()

        elif self.scheduler_type == SchedulerType.POLYNOMIAL_DECAY:
            self.current_lr = self._polynomial_decay()

        elif self.scheduler_type == SchedulerType.COSINE_ANNEALING:
            self.current_lr = self._cosine_annealing()

        elif self.scheduler_type == SchedulerType.COSINE_ANNEALING_WARM_RESTARTS:
            self.current_lr = self._cosine_annealing_warm_restarts()

        elif self.scheduler_type == SchedulerType.CYCLICAL:
            self.current_lr = self._cyclical()

        elif self.scheduler_type == SchedulerType.ONE_CYCLE:
            self.current_lr = self._one_cycle()

        elif self.scheduler_type == SchedulerType.REDUCE_ON_PLATEAU:
            self.current_lr = self._reduce_on_plateau(metric)

        elif self.scheduler_type == SchedulerType.WARMUP_COSINE:
            self.current_lr = self._warmup_cosine()

        elif self.scheduler_type == SchedulerType.LINEAR_WARMUP:
            self.current_lr = self._linear_warmup()

        elif self.scheduler_type == SchedulerType.CUSTOM:
            self.current_lr = self._custom()

        else:
            raise ValueError(f"Unknown scheduler type: {self.scheduler_type}")

        self.current_lr = max(self.current_lr, 0.0)
        self.history.append(self.current_lr)
        return self.current_lr

    def _step_decay(self) -> float:
        """Step decay scheduler."""
        step_size = self.kwargs["step_size"]
        gamma = self.kwargs["gamma"]
        return self.initial_lr * (gamma ** (self.step_count // step_size))

    def _exponential_decay(self) -> float:
        """Exponential decay scheduler."""
        gamma = self.kwargs["gamma"]
        return self.initial_lr * (gamma**self.step_count)

    def _polynomial_decay(self) -> float:
        """Polynomial decay scheduler."""
        power = self.kwargs["power"]
        if self.step_count >= self.total_steps:
            return 0.0
        return self.initial_lr * ((1 - self.step_count / self.total_steps) ** power)

    def _cosine_annealing(self) -> float:
        """Cosine annealing scheduler."""
        T_max = self.kwargs.get("T_max", self.total_steps)
        eta_min = self.kwargs["eta_min"]

        if T_max is None:
            T_max = self.total_steps

        return (
            eta_min
            + (self.initial_lr - eta_min)
            * (1 + math.cos(math.pi * self.step_count / T_max))
            / 2
        )

    def _cosine_annealing_warm_restarts(self) -> float:
        """Cosine annealing with warm restarts (SGDR)."""
        T_0 = self.kwargs["T_0"]
        T_mult = self.kwargs["T_mult"]
        eta_min = self.kwargs["eta_min"]

        # The cycle is recomputed from step_count on every call, so the restart
        # count has to be assigned, not accumulated: incrementing inside this
        # loop adds the whole cycle history again at every step.
        T_cur = self.step_count
        T_i = T_0
        restarts = 0

        while T_cur >= T_i:
            T_cur -= T_i
            T_i *= T_mult
            restarts += 1

        self._restart_count = restarts

        return (
            eta_min
            + (self.initial_lr - eta_min) * (1 + math.cos(math.pi * T_cur / T_i)) / 2
        )

    def _cyclical(self) -> float:
        """Cyclical learning rate scheduler."""
        base_lr = self.kwargs["base_lr"]
        max_lr = self.kwargs["max_lr"]
        step_size_up = self.kwargs["step_size_up"]
        mode = self.kwargs["mode"]

        cycle = math.floor(1 + self.step_count / (2 * step_size_up))
        x = abs(self.step_count / step_size_up - 2 * cycle + 1)
        self._cycle_count = cycle

        if mode == "triangular":
            scale_fn = lambda x: 1.0
            scale_mode = "cycle"
        elif mode == "triangular2":
            scale_fn = lambda x: 1 / (2.0 ** (cycle - 1))
            scale_mode = "cycle"
        elif mode == "exp_range":
            gamma = self.kwargs.get("gamma", 1.0)
            scale_fn = lambda x: gamma**self.step_count
            scale_mode = "iterations"
        else:
            raise ValueError(f"Unknown cyclical mode: {mode}")

        if scale_mode == "cycle":
            scale_factor = scale_fn(cycle)
        else:
            scale_factor = scale_fn(self.step_count)

        return base_lr + (max_lr - base_lr) * max(0, (1 - x)) * scale_factor

    def _one_cycle(self) -> float:
        """One cycle learning rate scheduler."""
        max_lr = self.kwargs["max_lr"]
        pct_start = self.kwargs["pct_start"]
        anneal_strategy = self.kwargs["anneal_strategy"]

        step_ratio = self.step_count / self.total_steps

        if step_ratio <= pct_start:
            # Warmup phase
            if anneal_strategy == "linear":
                return (
                    self.initial_lr
                    + (max_lr - self.initial_lr) * step_ratio / pct_start
                )
            else:  # cosine
                return (
                    self.initial_lr
                    + (max_lr - self.initial_lr)
                    * (1 - math.cos(math.pi * step_ratio / pct_start))
                    / 2
                )
        else:
            # Annealing phase
            remaining_ratio = (step_ratio - pct_start) / (1 - pct_start)
            if anneal_strategy == "linear":
                return max_lr - (max_lr - self.initial_lr) * remaining_ratio
            else:  # cosine
                return (
                    self.initial_lr
                    + (max_lr - self.initial_lr)
                    * (1 + math.cos(math.pi * remaining_ratio))
                    / 2
                )

    def _reduce_on_plateau(self, metric: float | None) -> float:
        """Reduce on plateau scheduler."""
        if metric is None:
            warnings.warn("Metric required for REDUCE_ON_PLATEAU scheduler")
            return self.current_lr

        factor = self.kwargs["factor"]
        patience = self.kwargs["patience"]
        threshold = self.kwargs["threshold"]
        cooldown = self.kwargs["cooldown"]
        min_lr = self.kwargs["min_lr"]
        mode = self.kwargs.get("mode", "min")

        if self._best_metric is None:
            self._best_metric = metric
            return self.current_lr

        # Check if metric improved
        if mode == "min":
            improved = metric < self._best_metric - threshold
        else:  # mode == 'max'
            improved = metric > self._best_metric + threshold

        if improved:
            self._best_metric = metric
            self._plateau_count = 0
            return self.current_lr

        # A reduction takes time to show up in the metric, so `cooldown` steps
        # after one are not counted against patience. Without this the next
        # reduction can fire before the previous one has had any effect.
        if self._cooldown_counter > 0:
            self._cooldown_counter -= 1
            self._plateau_count = 0
            return self.current_lr

        self._plateau_count += 1

        # Reduce learning rate if patience exceeded
        if self._plateau_count > patience:
            new_lr = max(self.current_lr * factor, min_lr)
            if new_lr < self.current_lr:
                self._plateau_count = 0
                self._cooldown_counter = cooldown
            return new_lr

        return self.current_lr

    def _warmup_cosine(self) -> float:
        """Warmup followed by cosine annealing."""
        warmup_steps = self.kwargs["warmup_steps"]

        if self.step_count <= warmup_steps:
            # Linear warmup
            return self.initial_lr * self.step_count / warmup_steps
        else:
            # Cosine annealing
            progress = (self.step_count - warmup_steps) / (
                self.total_steps - warmup_steps
            )
            return self.initial_lr * (1 + math.cos(math.pi * progress)) / 2

    def _linear_warmup(self) -> float:
        """Linear warmup scheduler."""
        warmup_steps = self.kwargs["warmup_steps"]

        if self.step_count <= warmup_steps:
            return self.initial_lr * self.step_count / warmup_steps
        else:
            return self.initial_lr

    def _custom(self) -> float:
        """Create custom scheduler using user-provided function."""
        custom_func = self.kwargs["custom_func"]
        return custom_func(self.step_count, self.initial_lr, **self.kwargs)

    def get_lr(self) -> float:
        """Get current learning rate."""
        return self.current_lr

    def reset(self) -> None:
        """Reset scheduler to initial state."""
        self.current_lr = self.initial_lr
        self.step_count = 0
        self.history = [self.initial_lr]
        self._plateau_count = 0
        self._best_metric = None
        self._cycle_count = 0
        self._restart_count = 0
        self._cooldown_counter = 0

    def get_config(self) -> dict[str, Any]:
        """Get scheduler configuration."""
        return {
            "initial_lr": self.initial_lr,
            "scheduler_type": self.scheduler_type.value,
            "total_steps": self.total_steps,
            "step_count": self.step_count,
            "kwargs": self.kwargs.copy(),
        }

    def get_state(self) -> dict[str, Any]:
        """Get complete scheduler state."""
        return {
            "config": self.get_config(),
            "current_lr": self.current_lr,
            "history": self.history.copy(),
            "plateau_count": self._plateau_count,
            "best_metric": self._best_metric,
            "cycle_count": self._cycle_count,
            "restart_count": self._restart_count,
            "cooldown_counter": self._cooldown_counter,
        }

    def load_state(self, state: dict[str, Any]) -> None:
        """Load scheduler state."""
        config = state["config"]
        self.initial_lr = config["initial_lr"]
        self.scheduler_type = SchedulerType(config["scheduler_type"])
        self.total_steps = config["total_steps"]
        self.step_count = config["step_count"]
        self.kwargs = config["kwargs"]

        self.current_lr = state["current_lr"]
        self.history = state["history"]
        self._plateau_count = state["plateau_count"]
        self._best_metric = state["best_metric"]
        self._cycle_count = state["cycle_count"]
        self._restart_count = state["restart_count"]
        self._cooldown_counter = state.get("cooldown_counter", 0)

step

step(metric: float | None = None) -> float

Update the learning rate for one step.

Parameters:

Name	Type	Description	Default
metric	float	Current metric value (required for REDUCE_ON_PLATEAU)	None

Returns:

Type	Description
float	Updated learning rate

Source code in src/dlhub/optimizers/schedules.py

def step(self, metric: float | None = None) -> float:
    """
    Update the learning rate for one step.

    Parameters
    ----------
    metric : float, optional
        Current metric value (required for REDUCE_ON_PLATEAU)

    Returns
    -------
    float
        Updated learning rate
    """
    self.step_count += 1

    if self.scheduler_type == SchedulerType.CONSTANT:
        self.current_lr = self.initial_lr

    elif self.scheduler_type == SchedulerType.STEP_DECAY:
        self.current_lr = self._step_decay()

    elif self.scheduler_type == SchedulerType.EXPONENTIAL_DECAY:
        self.current_lr = self._exponential_decay()

    elif self.scheduler_type == SchedulerType.POLYNOMIAL_DECAY:
        self.current_lr = self._polynomial_decay()

    elif self.scheduler_type == SchedulerType.COSINE_ANNEALING:
        self.current_lr = self._cosine_annealing()

    elif self.scheduler_type == SchedulerType.COSINE_ANNEALING_WARM_RESTARTS:
        self.current_lr = self._cosine_annealing_warm_restarts()

    elif self.scheduler_type == SchedulerType.CYCLICAL:
        self.current_lr = self._cyclical()

    elif self.scheduler_type == SchedulerType.ONE_CYCLE:
        self.current_lr = self._one_cycle()

    elif self.scheduler_type == SchedulerType.REDUCE_ON_PLATEAU:
        self.current_lr = self._reduce_on_plateau(metric)

    elif self.scheduler_type == SchedulerType.WARMUP_COSINE:
        self.current_lr = self._warmup_cosine()

    elif self.scheduler_type == SchedulerType.LINEAR_WARMUP:
        self.current_lr = self._linear_warmup()

    elif self.scheduler_type == SchedulerType.CUSTOM:
        self.current_lr = self._custom()

    else:
        raise ValueError(f"Unknown scheduler type: {self.scheduler_type}")

    self.current_lr = max(self.current_lr, 0.0)
    self.history.append(self.current_lr)
    return self.current_lr

get_lr

get_lr() -> float

Get current learning rate.

Source code in src/dlhub/optimizers/schedules.py

def get_lr(self) -> float:
    """Get current learning rate."""
    return self.current_lr

reset

reset() -> None

Reset scheduler to initial state.

Source code in src/dlhub/optimizers/schedules.py

def reset(self) -> None:
    """Reset scheduler to initial state."""
    self.current_lr = self.initial_lr
    self.step_count = 0
    self.history = [self.initial_lr]
    self._plateau_count = 0
    self._best_metric = None
    self._cycle_count = 0
    self._restart_count = 0
    self._cooldown_counter = 0

get_config

get_config() -> dict[str, Any]

Get scheduler configuration.

Source code in src/dlhub/optimizers/schedules.py

def get_config(self) -> dict[str, Any]:
    """Get scheduler configuration."""
    return {
        "initial_lr": self.initial_lr,
        "scheduler_type": self.scheduler_type.value,
        "total_steps": self.total_steps,
        "step_count": self.step_count,
        "kwargs": self.kwargs.copy(),
    }

get_state

get_state() -> dict[str, Any]

Get complete scheduler state.

Source code in src/dlhub/optimizers/schedules.py

def get_state(self) -> dict[str, Any]:
    """Get complete scheduler state."""
    return {
        "config": self.get_config(),
        "current_lr": self.current_lr,
        "history": self.history.copy(),
        "plateau_count": self._plateau_count,
        "best_metric": self._best_metric,
        "cycle_count": self._cycle_count,
        "restart_count": self._restart_count,
        "cooldown_counter": self._cooldown_counter,
    }

load_state

load_state(state: dict[str, Any]) -> None

Load scheduler state.

Source code in src/dlhub/optimizers/schedules.py

def load_state(self, state: dict[str, Any]) -> None:
    """Load scheduler state."""
    config = state["config"]
    self.initial_lr = config["initial_lr"]
    self.scheduler_type = SchedulerType(config["scheduler_type"])
    self.total_steps = config["total_steps"]
    self.step_count = config["step_count"]
    self.kwargs = config["kwargs"]

    self.current_lr = state["current_lr"]
    self.history = state["history"]
    self._plateau_count = state["plateau_count"]
    self._best_metric = state["best_metric"]
    self._cycle_count = state["cycle_count"]
    self._restart_count = state["restart_count"]
    self._cooldown_counter = state.get("cooldown_counter", 0)

create_step_scheduler

create_step_scheduler(initial_lr: float, step_size: int, gamma: float = 0.1) -> LearningRateScheduler

Create step decay scheduler.

Source code in src/dlhub/optimizers/schedules.py

def create_step_scheduler(
    initial_lr: float, step_size: int, gamma: float = 0.1
) -> LearningRateScheduler:
    """Create step decay scheduler."""
    return LearningRateScheduler(
        initial_lr=initial_lr,
        scheduler_type=SchedulerType.STEP_DECAY,
        step_size=step_size,
        gamma=gamma,
    )

create_cosine_scheduler

create_cosine_scheduler(initial_lr: float, total_steps: int, eta_min: float = 0.0) -> LearningRateScheduler

Create cosine annealing scheduler.

Source code in src/dlhub/optimizers/schedules.py

def create_cosine_scheduler(
    initial_lr: float, total_steps: int, eta_min: float = 0.0
) -> LearningRateScheduler:
    """Create cosine annealing scheduler."""
    return LearningRateScheduler(
        initial_lr=initial_lr,
        scheduler_type=SchedulerType.COSINE_ANNEALING,
        total_steps=total_steps,
        eta_min=eta_min,
    )

create_one_cycle_scheduler

create_one_cycle_scheduler(initial_lr: float, max_lr: float, total_steps: int, pct_start: float = 0.3) -> LearningRateScheduler

Create one cycle scheduler.

Source code in src/dlhub/optimizers/schedules.py

def create_one_cycle_scheduler(
    initial_lr: float, max_lr: float, total_steps: int, pct_start: float = 0.3
) -> LearningRateScheduler:
    """Create one cycle scheduler."""
    return LearningRateScheduler(
        initial_lr=initial_lr,
        scheduler_type=SchedulerType.ONE_CYCLE,
        total_steps=total_steps,
        max_lr=max_lr,
        pct_start=pct_start,
    )

create_warmup_cosine_scheduler

create_warmup_cosine_scheduler(initial_lr: float, total_steps: int, warmup_steps: int) -> LearningRateScheduler

Create warmup + cosine scheduler.

Source code in src/dlhub/optimizers/schedules.py

def create_warmup_cosine_scheduler(
    initial_lr: float, total_steps: int, warmup_steps: int
) -> LearningRateScheduler:
    """Create warmup + cosine scheduler."""
    return LearningRateScheduler(
        initial_lr=initial_lr,
        scheduler_type=SchedulerType.WARMUP_COSINE,
        total_steps=total_steps,
        warmup_steps=warmup_steps,
    )