Neural networks

Network construction, forward and backward propagation, and the training loop that drives them.

dlhub.nn

Neural Networks

Network construction, forward and backward propagation, and the training loop that drives them.

Author

Deep Learning Reference Hub

License

MIT

DeepNeuralNetwork

A comprehensive Deep Neural Network implementation with modern techniques.

This implementation includes: - Multiple initialization methods (He, Xavier, Random) - Regularization techniques (L1, L2, Dropout) - Batch Normalization - Gradient Clipping - Learning Rate Scheduling - Early Stopping - Comprehensive metrics tracking

Parameters:

Name	Type	Description	Default
layer_dims	List	Layer dimensions [n_x, n_h1, n_h2, ..., n_y]	required
initialization	str	Initialization method ('he', 'xavier', 'random')	'he'
regularization	str(optional)	Regularization type (None, 'l1', 'l2', 'dropout')	None
lambda_reg	float	Regularization strength parameter	0.01
keep_prob	float	Dropout keep probability (0 < keep_prob <= 1)	0.8
use_batch_norm	bool	Whether to use batch normalization	True
gradient_clipping	bool	Whether to apply gradient clipping	True
clip_value	float	Maximum gradient norm for clipping	True

Attributes:

Name	Type	Description
layer_dims	List[int]	Dimensions of each layer
L	int	Number of layers (excluding input)
parameters	Dict	Network weights and biases
bn_params	Dict	Batch normalization parameters
costs_history	List	Training cost history
val_costs_history	List	Validation cost history
accuracies_history	List	Training accuracy history
val_accuracies_history	List	Validation accuracy history

Notes

Raises ValueError: If invalid parameters are provided

Source code in src/dlhub/nn/fully_connected.py

class DeepNeuralNetwork:
    """
    A comprehensive Deep Neural Network implementation with modern techniques.

    This implementation includes:
    - Multiple initialization methods (He, Xavier, Random)
    - Regularization techniques (L1, L2, Dropout)
    - Batch Normalization
    - Gradient Clipping
    - Learning Rate Scheduling
    - Early Stopping
    - Comprehensive metrics tracking

    Parameters
    ----------
    layer_dims : List
        Layer dimensions [n_x, n_h1, n_h2, ..., n_y]
    initialization : str
        Initialization method ('he', 'xavier', 'random')
    regularization : str(optional)
        Regularization type (None, 'l1', 'l2', 'dropout')
    lambda_reg : float, default=0.01
        Regularization strength parameter
    keep_prob : float, default=0.8
        Dropout keep probability (0 < keep_prob <= 1)
    use_batch_norm : bool, default=True
        Whether to use batch normalization
    gradient_clipping : bool, default=True
        Whether to apply gradient clipping
    clip_value : float, default=True
        Maximum gradient norm for clipping

    Attributes
    ----------
    layer_dims : List[int]
        Dimensions of each layer
    L : int
        Number of layers (excluding input)
    parameters : Dict
        Network weights and biases
    bn_params : Dict
        Batch normalization parameters
    costs_history : List
        Training cost history
    val_costs_history: List
        Validation cost history
    accuracies_history : List
        Training accuracy history
    val_accuracies_history : List
        Validation accuracy history

    Notes
    -----
    Raises ValueError: If invalid parameters are provided
    """

    def __init__(
        self,
        layer_dims: list[int],
        initialization: str = "he",
        regularization: str | None = None,
        lambda_reg: float = 0.01,
        keep_prob: float = 0.8,
        use_batch_norm: bool = True,
        gradient_clipping: bool = True,
        clip_value: float = 5.0,
    ):
        self._validate_inputs(
            layer_dims, initialization, regularization, lambda_reg, keep_prob
        )

        self.layer_dims = layer_dims
        self.L = len(layer_dims) - 1  # Number of layers (excluding input)
        self.regularization = regularization
        self.lambda_reg = lambda_reg
        self.keep_prob = keep_prob
        self.use_batch_norm = use_batch_norm
        self.gradient_clipping = gradient_clipping
        self.clip_value = clip_value

        self.parameters = self._initialize_parameters(initialization)

        if use_batch_norm:
            self.running_mean = {}
            self.running_var = {}
            self.momentum = 0.9
            self.bn_params = self._initialize_batch_norm()

        self.costs_history = []
        self.val_costs_history = []
        self.accuracies_history = []
        self.val_accuracies_history = []

        self.best_val_cost = float("inf")
        self.patience_counter = 0

    def _validate_inputs(
        self,
        layer_dims: list[int],
        initialization: str,
        regularization: str | None,
        lambda_reg: float,
        keep_prob: float,
    ) -> None:
        """Validate input parameters."""
        if len(layer_dims) < 2:
            raise ValueError("Network must have at least 2 layers (input and output)")

        if any(dim <= 0 for dim in layer_dims):
            raise ValueError("All layer dimensions must be positive")

        if initialization not in ["he", "xavier", "random"]:
            raise ValueError("Initialization must be 'he', 'xavier', or 'random'")

        if regularization not in [None, "l1", "l2", "dropout"]:
            raise ValueError("Regularization must be None, 'l1', 'l2', or 'dropout'")

        if lambda_reg < 0:
            raise ValueError("Regularization parameter must be non-negative")

        if not 0 < keep_prob <= 1:
            raise ValueError("Keep probability must be in (0, 1]")

    def _initialize_parameters(self, method: str) -> dict[str, np.ndarray]:
        """
        Initialize network parameters using specified method.

        Parameters
        ----------
        method : str
            Initialization method ('he', 'xavier', 'random')

        Returns
        -------
        Dict[str, np.ndarray]
            Contains initialized weights and biases
        """
        np.random.seed(42)
        parameters = {}

        for l in range(1, self.L + 1):
            fan_in = self.layer_dims[l - 1]
            fan_out = self.layer_dims[l]

            if method == "he":
                std = np.sqrt(2.0 / fan_in)
            elif method == "xavier":
                std = np.sqrt(1.0 / fan_in)
            elif method == "random":
                std = 0.01

            parameters[f"W{l}"] = np.random.randn(fan_out, fan_in) * std
            parameters[f"b{l}"] = np.zeros((fan_out, 1))

        return parameters

    def _initialize_batch_norm(self) -> dict[str, np.ndarray]:
        """
        Initialize batch normalization parameters.

        Returns
        -------
        Dict[str, np.ndarray]
            Contains gamma and beta parameters
        """
        bn_params = {}

        for l in range(1, self.L):  # Not applied to output layer
            bn_params[f"gamma{l}"] = np.ones((self.layer_dims[l], 1))
            bn_params[f"beta{l}"] = np.zeros((self.layer_dims[l], 1))

            self.running_mean[f"mean{l}"] = np.zeros((self.layer_dims[l], 1))
            self.running_var[f"var{l}"] = np.ones((self.layer_dims[l], 1))

        return bn_params

    def _relu(self, Z: np.ndarray) -> np.ndarray:
        """Relu activation function."""
        return np.maximum(0, Z)

    def _relu_derivative(self, Z: np.ndarray) -> np.ndarray:
        """Relu derivative."""
        return (Z > 0).astype(float)

    def _sigmoid(self, Z: np.ndarray) -> np.ndarray:
        """Sigmoid activation function with numerical stability."""
        Z_clipped = np.clip(Z, -500, 500)
        return 1 / (1 + np.exp(-Z_clipped))

    def _sigmoid_derivative(self, A: np.ndarray) -> np.ndarray:
        """Sigmoid derivative."""
        return A * (1 - A)

    def forward_propagation(
        self, X: np.ndarray, training: bool = True
    ) -> tuple[np.ndarray, list]:
        """
        Perform forward propagation through the network.

        Parameters
        ----------
        X : np.ndarray
            Input data of shape (n_features, m_samples)
        training : bool, default=True
            Whether in training mode (affects dropout and batch norm)

        Returns
        -------
        Tuple[np.ndarray, List]
            Contains final output and caches for backprob
        """
        if X.shape[0] != self.layer_dims[0]:
            raise ValueError(
                f"Input shape {X.shape[0]} doesn't match expected {self.layer_dims[0]}"
            )

        caches = []
        A = X

        for l in range(1, self.L):
            A_prev = A
            Z = np.dot(self.parameters[f"W{l}"], A_prev) + self.parameters[f"b{l}"]

            if self.use_batch_norm:
                Z, bn_cache = self._batch_norm_forward(Z, l, training)
            else:
                bn_cache = None

            A = self._relu(Z)

            if self.regularization == "dropout" and training:
                A, dropout_cache = self._dropout_forward(A, self.keep_prob)
            else:
                dropout_cache = None

            cache = (A_prev, Z, A, bn_cache, dropout_cache)
            caches.append(cache)

        A_prev = A
        ZL = (
            np.dot(self.parameters[f"W{self.L}"], A_prev)
            + self.parameters[f"b{self.L}"]
        )
        AL = self._sigmoid(ZL)

        cache = (A_prev, ZL, AL, None, None)
        caches.append(cache)

        return AL, caches

    def _batch_norm_forward(
        self, Z: np.ndarray, l: int, training: bool, eps: float = 1e-8
    ) -> tuple[np.ndarray, tuple | None]:
        """
        Batch normalization forward pass.

        Parameters
        ----------
        Z : np.ndarray
            Pre-activation values
        l : int
            Layer index
        training : bool, default=True
            Whether in training mode
        eps : float, default=1e-8
            Small constant for numerical stability

        Returns
        -------
        Tuple[np.ndarray, tuple]
            Contains normalized output and cache for backprop
        """
        if training:
            mu = np.mean(Z, axis=1, keepdims=True)
            var = np.var(Z, axis=1, keepdims=True)

            self.running_mean[f"mean{l}"] = (
                self.momentum * self.running_mean[f"mean{l}"] + (1 - self.momentum) * mu
            )
            self.running_var[f"var{l}"] = (
                self.momentum * self.running_var[f"var{l}"] + (1 - self.momentum) * var
            )

            Z_norm = (Z - mu) / np.sqrt(var + eps)
            Z_out = self.bn_params[f"gamma{l}"] * Z_norm + self.bn_params[f"beta{l}"]

            cache = (Z, Z_norm, mu, var, eps)
            return Z_out, cache
        else:
            # Use running statistics for inference
            Z_norm = (Z - self.running_mean[f"mean{l}"]) / np.sqrt(
                self.running_var[f"var{l}"] + eps
            )
            Z_out = self.bn_params[f"gamma{l}"] * Z_norm + self.bn_params[f"beta{l}"]
            return Z_out, None

    def _dropout_forward(
        self, A: np.ndarray, keep_prob: float
    ) -> tuple[np.ndarray, np.ndarray]:
        """
        Dropout forward pass.

        Parameters
        ----------
        A : np.ndarray
            Activations
        keep_prob : float
            Probability of keeping each neuron

        Returns
        -------
        Tuple[np.ndarray, np.ndarray]
            Contains dropped activations and dropout mask
        """
        mask = np.random.binomial(1, keep_prob, A.shape) / keep_prob
        A_drop = A * mask
        return A_drop, mask

    def compute_cost(self, AL: np.ndarray, Y: np.ndarray) -> float:
        """
        Compute the cost function with regularization.

        Parameters
        ----------
        AL : np.ndarray
            Network output of shape (1, m_samples)
        Y : np.ndarray
            True labels of shape (1, m_samples)

        Returns
        -------
        float
            Total cost including regularization
        """
        m = Y.shape[1]
        AL_clipped = np.clip(AL, 1e-8, 1 - 1e-8)  # Clip predictions to prevent log(0)

        cost = -(1 / m) * (
            np.dot(Y, np.log(AL_clipped).T) + np.dot(1 - Y, np.log(1 - AL_clipped).T)
        )

        reg_cost = 0
        if self.regularization == "l2":
            weights = np.concatenate(
                [self.parameters[f"W{l}"].flatten() for l in range(1, self.L + 1)]
            )
            reg_cost = self.lambda_reg / (2 * m) * np.sum(weights**2)
            cost += reg_cost

        elif self.regularization == "l1":
            weights = np.concatenate(
                [self.parameters[f"W{l}"].flatten() for l in range(1, self.L + 1)]
            )
            reg_cost = self.lambda_reg / m * np.sum(np.abs(weights))
            cost += reg_cost

        return np.squeeze(cost)

    def backward_propagation(
        self, AL: np.ndarray, Y: np.ndarray, caches: list
    ) -> dict[str, np.ndarray]:
        """
        Perform backward propagation to compute gradients.

        Parameters
        ----------
        AL : np.ndarray
            Network output
        Y : np.ndarray
            True labels
        caches : List
            Forward propagation caches

        Returns
        -------
        Dict[str, np.ndarray]
            Contains gradients for all parameters
        """
        grads = {}
        m = AL.shape[1]

        # Carried from one iteration to the next: layer l reads the dA that
        # layer l+1 produced. Bound here so the handoff is visible rather than
        # implied by the loop.
        dA_next = None

        for l in reversed(range(1, self.L + 1)):
            A_prev, Z, A, bn_cache, dropout_cache = caches[l - 1]

            if l == self.L:
                dZ = AL - Y  # Cross-Entropy Derivative
            else:
                dA = dA_next

                if dropout_cache is not None:
                    dA = dA * dropout_cache

                dZ = dA * self._relu_derivative(Z)

                if bn_cache is not None:
                    dZ = self._batch_norm_backward(dZ, bn_cache, l)

            dW = (1 / m) * np.dot(dZ, A_prev.T)
            db = (1 / m) * np.sum(dZ, axis=1, keepdims=True)
            dA_prev = np.dot(self.parameters[f"W{l}"].T, dZ)

            if self.regularization == "l2":
                dW += (self.lambda_reg / m) * self.parameters[f"W{l}"]
            elif self.regularization == "l1":
                dW += (self.lambda_reg / m) * np.sign(self.parameters[f"W{l}"])

            grads[f"dW{l}"] = dW
            grads[f"db{l}"] = db

            dA_next = dA_prev

        return grads

    def _batch_norm_backward(
        self, dZ_out: np.ndarray, cache: tuple, l: int
    ) -> np.ndarray:
        """
        Batch normalization backward pass.

        Parameters
        ----------
        dZ_out : np.ndarray
            Gradient from next layer
        cache : Tuple
            Forward pass cache
        l : int
            Layer index

        Returns
        -------
        np.ndarray
            Gradient with respect to input
        """
        Z, Z_norm, mu, var, eps = cache
        m = Z.shape[1]

        dgamma = np.sum(dZ_out * Z_norm, axis=1, keepdims=True)
        dbeta = np.sum(dZ_out, axis=1, keepdims=True)

        if not hasattr(self, "bn_grads"):
            self.bn_grads = {}
        self.bn_grads[f"dgamma{l}"] = dgamma
        self.bn_grads[f"dbeta{l}"] = dbeta

        dZ_norm = dZ_out * self.bn_params[f"gamma{l}"]

        dvar = np.sum(
            dZ_norm * (Z - mu) * -0.5 * (var + eps) ** (-3 / 2), axis=1, keepdims=True
        )
        dmu = (
            np.sum(dZ_norm * -1 / np.sqrt(var + eps), axis=1, keepdims=True)
            + dvar * np.sum(-2 * (Z - mu), axis=1, keepdims=True) / m
        )

        dZ = dZ_norm / np.sqrt(var + eps) + dvar * 2 * (Z - mu) / m + dmu / m

        return dZ

    def _clip_gradients(self, grads: dict[str, np.ndarray]) -> dict[str, np.ndarray]:
        """
        Apply gradient clipping to prevent exploding gradients.

        Parameters
        ----------
        grads : Dict[str, np.ndarray]
            Dictionary of gradients

        Returns
        -------
        Dict[str, np.ndarray]
            Dictionary of clipped gradients
        """
        total_norm = 0
        for grad in grads.values():
            total_norm += np.sum(grad**2)
        total_norm = np.sqrt(total_norm)

        if total_norm > self.clip_value:
            clip_coeff = self.clip_value / total_norm
            for key in grads:
                grads[key] = grads[key] * clip_coeff

        return grads

    def update_parameters(
        self, grads: dict[str, np.ndarray], learning_rate: float
    ) -> None:
        """
        Update network parameters using gradients.

        Parameters
        ----------
        grads : Dict[str, np.ndarray]
            Dictionary of gradients
        learning_rate : float
            Learning rate for parameter updates
        """
        if self.gradient_clipping:
            grads = self._clip_gradients(grads)

        for l in range(1, self.L + 1):
            self.parameters[f"W{l}"] -= learning_rate * grads[f"dW{l}"]
            self.parameters[f"b{l}"] -= learning_rate * grads[f"db{l}"]

        if self.use_batch_norm and hasattr(self, "bn_grads"):
            for l in range(1, self.L):
                self.bn_params[f"gamma{l}"] -= (
                    learning_rate * self.bn_grads[f"dgamma{l}"]
                )
                self.bn_params[f"beta{l}"] -= learning_rate * self.bn_grads[f"dbeta{l}"]

    def compute_accuracy(self, AL: np.ndarray, Y: np.ndarray) -> float:
        """
        Compute classification accuracy.

        Parameters
        ----------
        AL : np.ndarray
            Network predictions
        Y : np.ndarray
            True labels

        Returns
        -------
        float
            Accuracy percentage
        """
        predictions = (AL > 0.5).astype(int)
        accuracy = np.mean(predictions == Y)
        return accuracy

    def train(
        self,
        X: np.ndarray,
        Y: np.ndarray,
        X_val: np.ndarray,
        Y_val: np.ndarray,
        learning_rate: float = 0.01,
        num_epochs: int = 1000,
        print_cost: bool = True,
        learning_rate_decay: float = 0.95,
        decay_step: int = 100,
        early_stopping: bool = True,
        patience: int = 50,
    ) -> dict[str, list]:
        """
        Train the neural network with advanced techniques.

        Parameters
        ----------
        X : np.ndarray
            Training data of shape (n_features, m_samples)
        Y : np.ndarray
            Training labels of shape (1, m_samples)
        X_val : np.ndarray
            Validation data
        Y_val : np.ndarray
            Validation labels
        learning_rate : float, default=0.01
            Initial learning rate
        num_epochs : int, default=1000
            Number of training epochs
        print_cost : bool, default=True
            Whether to print cost during training
        learning_rate_decay : float, default=0.95
            Learning rate decay factor
        decay_step : int, default=100
            Steps between learning rate decay
        early_stopping : bool, default=True
            Whether to use early stopping
        patience : int, default=50
            Early stopping patience

        Returns
        -------
        Dict[str, List]
            Contains training history
        """
        if X.shape[0] != self.layer_dims[0]:
            raise ValueError(
                f"Input features {X.shape[0]} don't match network input {self.layer_dims[0]}"
            )

        self.costs_history = []
        self.val_costs_history = []
        self.accuracies_history = []
        self.val_accuracies_history = []

        current_lr = learning_rate

        for epoch in range(num_epochs):
            AL, caches = self.forward_propagation(X, training=True)

            cost = self.compute_cost(AL, Y)
            accuracy = self.compute_accuracy(AL, Y)

            self.costs_history.append(cost)
            self.accuracies_history.append(accuracy)

            grads = self.backward_propagation(AL, Y, caches)

            self.update_parameters(grads, current_lr)

            AL_val, _ = self.forward_propagation(X_val, training=False)
            val_cost = self.compute_cost(AL_val, Y_val)
            val_accuracy = self.compute_accuracy(AL_val, Y_val)

            self.val_costs_history.append(val_cost)
            self.val_accuracies_history.append(val_accuracy)

            if epoch % decay_step == 0 and epoch > 0:
                current_lr *= learning_rate_decay

            if early_stopping:
                if val_cost < self.best_val_cost:
                    self.best_val_cost = val_cost
                    self.patience_counter = 0
                else:
                    self.patience_counter += 1

                if self.patience_counter >= patience:
                    if print_cost:
                        print(f"Early stopping at epoch {epoch}")
                    break

            if print_cost and epoch % 100 == 0:
                print(
                    f"Epoch {epoch:4d}: Cost = {cost:.6f}, Acc = {accuracy:.4f}, "
                    f"Val Cost = {val_cost:.6f}, Val Acc = {val_accuracy:.4f}, LR = {current_lr:.6f}"
                )

        return {
            "costs": self.costs_history,
            "val_costs": self.val_costs_history,
            "accuracies": self.accuracies_history,
            "val_accuracies": self.val_accuracies_history,
        }

    def predict(self, X: np.ndarray) -> np.ndarray:
        """
        Make predictions on new data.

        Parameters
        ----------
        X : np.ndarray
            Input data of shape (n_features, m_samples)

        Returns
        -------
        np.ndarray
            Binary predictions of shape (1, m_samples)
        """
        AL, _ = self.forward_propagation(X, training=False)
        predictions = (AL > 0.5).astype(int)
        return predictions

    def predict_proba(self, X: np.ndarray) -> np.ndarray:
        """
        Get prediction probabilities.

        Parameters
        ----------
        X : np.ndarray
            Input data of shape (n_features, m_samples)

        Returns
        -------
        np.ndarray
            Probabilities of shape (1, m_samples)
        """
        AL, _ = self.forward_propagation(X, training=False)
        return AL

    def plot_training_history(self, figsize: tuple[int, int] = (15, 5)) -> None:
        """
        Plot training history including cost and accuracy curves.

        Parameters
        ----------
        figsize : Tuple[int, int]
            Figure size tuple
        """
        if not self.costs_history:
            print("No training history available. Train the model first.")
            return

        fig, axes = plt.subplots(1, 2, figsize=figsize)

        # Cost Curves
        axes[0].plot(self.costs_history, label="Training Cost", linewidth=2)
        axes[0].plot(self.val_costs_history, label="Validation Cost", linewidth=2)
        axes[0].set_title("Training and Validation Cost")
        axes[0].set_xlabel("Epoch")
        axes[0].set_ylabel("Cost")
        axes[0].legend()
        axes[0].grid(True, alpha=0.3)

        # Accuracy Curves
        axes[1].plot(self.accuracies_history, label="Training Accuracy", linewidth=2)
        axes[1].plot(
            self.val_accuracies_history, label="Validation Accuracy", linewidth=2
        )
        axes[1].set_title("Training and Validation Accuracy")
        axes[1].set_xlabel("Epoch")
        axes[1].set_ylabel("Accuracy")
        axes[1].legend()
        axes[1].grid(True, alpha=0.3)

        plt.tight_layout()
        plt.show()

    def get_model_info(self) -> dict[str, Any]:
        """
        Get comprehensive model information.

        Returns
        -------
        Dict[str, Any]
            Contains model configuration and statistics
        """
        total_params = sum(param.size for param in self.parameters.values())
        if self.use_batch_norm:
            total_params += sum(param.size for param in self.bn_params.values())

        return {
            "architecture": self.layer_dims,
            "total_parameters": total_params,
            "regularization": self.regularization,
            "lambda_reg": self.lambda_reg,
            "batch_normalization": self.use_batch_norm,
            "dropout_keep_prob": (
                self.keep_prob if self.regularization == "dropout" else None
            ),
            "gradient_clipping": self.gradient_clipping,
            "clip_value": self.clip_value if self.gradient_clipping else None,
        }

forward_propagation

forward_propagation(X: ndarray, training: bool = True) -> tuple[np.ndarray, list]

Perform forward propagation through the network.

Parameters:

Name	Type	Description	Default
X	ndarray	Input data of shape (n_features, m_samples)	required
training	bool	Whether in training mode (affects dropout and batch norm)	True

Returns:

Type	Description
Tuple[ndarray, List]	Contains final output and caches for backprob

Source code in src/dlhub/nn/fully_connected.py

def forward_propagation(
    self, X: np.ndarray, training: bool = True
) -> tuple[np.ndarray, list]:
    """
    Perform forward propagation through the network.

    Parameters
    ----------
    X : np.ndarray
        Input data of shape (n_features, m_samples)
    training : bool, default=True
        Whether in training mode (affects dropout and batch norm)

    Returns
    -------
    Tuple[np.ndarray, List]
        Contains final output and caches for backprob
    """
    if X.shape[0] != self.layer_dims[0]:
        raise ValueError(
            f"Input shape {X.shape[0]} doesn't match expected {self.layer_dims[0]}"
        )

    caches = []
    A = X

    for l in range(1, self.L):
        A_prev = A
        Z = np.dot(self.parameters[f"W{l}"], A_prev) + self.parameters[f"b{l}"]

        if self.use_batch_norm:
            Z, bn_cache = self._batch_norm_forward(Z, l, training)
        else:
            bn_cache = None

        A = self._relu(Z)

        if self.regularization == "dropout" and training:
            A, dropout_cache = self._dropout_forward(A, self.keep_prob)
        else:
            dropout_cache = None

        cache = (A_prev, Z, A, bn_cache, dropout_cache)
        caches.append(cache)

    A_prev = A
    ZL = (
        np.dot(self.parameters[f"W{self.L}"], A_prev)
        + self.parameters[f"b{self.L}"]
    )
    AL = self._sigmoid(ZL)

    cache = (A_prev, ZL, AL, None, None)
    caches.append(cache)

    return AL, caches

compute_cost

compute_cost(AL: ndarray, Y: ndarray) -> float

Compute the cost function with regularization.

Parameters:

Name	Type	Description	Default
AL	ndarray	Network output of shape (1, m_samples)	required
Y	ndarray	True labels of shape (1, m_samples)	required

Returns:

Type	Description
float	Total cost including regularization

Source code in src/dlhub/nn/fully_connected.py

def compute_cost(self, AL: np.ndarray, Y: np.ndarray) -> float:
    """
    Compute the cost function with regularization.

    Parameters
    ----------
    AL : np.ndarray
        Network output of shape (1, m_samples)
    Y : np.ndarray
        True labels of shape (1, m_samples)

    Returns
    -------
    float
        Total cost including regularization
    """
    m = Y.shape[1]
    AL_clipped = np.clip(AL, 1e-8, 1 - 1e-8)  # Clip predictions to prevent log(0)

    cost = -(1 / m) * (
        np.dot(Y, np.log(AL_clipped).T) + np.dot(1 - Y, np.log(1 - AL_clipped).T)
    )

    reg_cost = 0
    if self.regularization == "l2":
        weights = np.concatenate(
            [self.parameters[f"W{l}"].flatten() for l in range(1, self.L + 1)]
        )
        reg_cost = self.lambda_reg / (2 * m) * np.sum(weights**2)
        cost += reg_cost

    elif self.regularization == "l1":
        weights = np.concatenate(
            [self.parameters[f"W{l}"].flatten() for l in range(1, self.L + 1)]
        )
        reg_cost = self.lambda_reg / m * np.sum(np.abs(weights))
        cost += reg_cost

    return np.squeeze(cost)

backward_propagation

backward_propagation(AL: ndarray, Y: ndarray, caches: list) -> dict[str, np.ndarray]

Perform backward propagation to compute gradients.

Parameters:

Name	Type	Description	Default
AL	ndarray	Network output	required
Y	ndarray	True labels	required
caches	List	Forward propagation caches	required

Returns:

Type	Description
Dict[str, ndarray]	Contains gradients for all parameters

Source code in src/dlhub/nn/fully_connected.py

def backward_propagation(
    self, AL: np.ndarray, Y: np.ndarray, caches: list
) -> dict[str, np.ndarray]:
    """
    Perform backward propagation to compute gradients.

    Parameters
    ----------
    AL : np.ndarray
        Network output
    Y : np.ndarray
        True labels
    caches : List
        Forward propagation caches

    Returns
    -------
    Dict[str, np.ndarray]
        Contains gradients for all parameters
    """
    grads = {}
    m = AL.shape[1]

    # Carried from one iteration to the next: layer l reads the dA that
    # layer l+1 produced. Bound here so the handoff is visible rather than
    # implied by the loop.
    dA_next = None

    for l in reversed(range(1, self.L + 1)):
        A_prev, Z, A, bn_cache, dropout_cache = caches[l - 1]

        if l == self.L:
            dZ = AL - Y  # Cross-Entropy Derivative
        else:
            dA = dA_next

            if dropout_cache is not None:
                dA = dA * dropout_cache

            dZ = dA * self._relu_derivative(Z)

            if bn_cache is not None:
                dZ = self._batch_norm_backward(dZ, bn_cache, l)

        dW = (1 / m) * np.dot(dZ, A_prev.T)
        db = (1 / m) * np.sum(dZ, axis=1, keepdims=True)
        dA_prev = np.dot(self.parameters[f"W{l}"].T, dZ)

        if self.regularization == "l2":
            dW += (self.lambda_reg / m) * self.parameters[f"W{l}"]
        elif self.regularization == "l1":
            dW += (self.lambda_reg / m) * np.sign(self.parameters[f"W{l}"])

        grads[f"dW{l}"] = dW
        grads[f"db{l}"] = db

        dA_next = dA_prev

    return grads

update_parameters

update_parameters(grads: dict[str, ndarray], learning_rate: float) -> None

Update network parameters using gradients.

Parameters:

Name	Type	Description	Default
grads	Dict[str, ndarray]	Dictionary of gradients	required
learning_rate	float	Learning rate for parameter updates	required

Source code in src/dlhub/nn/fully_connected.py

def update_parameters(
    self, grads: dict[str, np.ndarray], learning_rate: float
) -> None:
    """
    Update network parameters using gradients.

    Parameters
    ----------
    grads : Dict[str, np.ndarray]
        Dictionary of gradients
    learning_rate : float
        Learning rate for parameter updates
    """
    if self.gradient_clipping:
        grads = self._clip_gradients(grads)

    for l in range(1, self.L + 1):
        self.parameters[f"W{l}"] -= learning_rate * grads[f"dW{l}"]
        self.parameters[f"b{l}"] -= learning_rate * grads[f"db{l}"]

    if self.use_batch_norm and hasattr(self, "bn_grads"):
        for l in range(1, self.L):
            self.bn_params[f"gamma{l}"] -= (
                learning_rate * self.bn_grads[f"dgamma{l}"]
            )
            self.bn_params[f"beta{l}"] -= learning_rate * self.bn_grads[f"dbeta{l}"]

compute_accuracy

compute_accuracy(AL: ndarray, Y: ndarray) -> float

Compute classification accuracy.

Parameters:

Name	Type	Description	Default
AL	ndarray	Network predictions	required
Y	ndarray	True labels	required

Returns:

Type	Description
float	Accuracy percentage

Source code in src/dlhub/nn/fully_connected.py

def compute_accuracy(self, AL: np.ndarray, Y: np.ndarray) -> float:
    """
    Compute classification accuracy.

    Parameters
    ----------
    AL : np.ndarray
        Network predictions
    Y : np.ndarray
        True labels

    Returns
    -------
    float
        Accuracy percentage
    """
    predictions = (AL > 0.5).astype(int)
    accuracy = np.mean(predictions == Y)
    return accuracy

train

train(X: ndarray, Y: ndarray, X_val: ndarray, Y_val: ndarray, learning_rate: float = 0.01, num_epochs: int = 1000, print_cost: bool = True, learning_rate_decay: float = 0.95, decay_step: int = 100, early_stopping: bool = True, patience: int = 50) -> dict[str, list]

Train the neural network with advanced techniques.

Parameters:

Name	Type	Description	Default
X	ndarray	Training data of shape (n_features, m_samples)	required
Y	ndarray	Training labels of shape (1, m_samples)	required
X_val	ndarray	Validation data	required
Y_val	ndarray	Validation labels	required
learning_rate	float	Initial learning rate	0.01
num_epochs	int	Number of training epochs	1000
print_cost	bool	Whether to print cost during training	True
learning_rate_decay	float	Learning rate decay factor	0.95
decay_step	int	Steps between learning rate decay	100
early_stopping	bool	Whether to use early stopping	True
patience	int	Early stopping patience	50

Returns:

Type	Description
Dict[str, List]	Contains training history

Source code in src/dlhub/nn/fully_connected.py

def train(
    self,
    X: np.ndarray,
    Y: np.ndarray,
    X_val: np.ndarray,
    Y_val: np.ndarray,
    learning_rate: float = 0.01,
    num_epochs: int = 1000,
    print_cost: bool = True,
    learning_rate_decay: float = 0.95,
    decay_step: int = 100,
    early_stopping: bool = True,
    patience: int = 50,
) -> dict[str, list]:
    """
    Train the neural network with advanced techniques.

    Parameters
    ----------
    X : np.ndarray
        Training data of shape (n_features, m_samples)
    Y : np.ndarray
        Training labels of shape (1, m_samples)
    X_val : np.ndarray
        Validation data
    Y_val : np.ndarray
        Validation labels
    learning_rate : float, default=0.01
        Initial learning rate
    num_epochs : int, default=1000
        Number of training epochs
    print_cost : bool, default=True
        Whether to print cost during training
    learning_rate_decay : float, default=0.95
        Learning rate decay factor
    decay_step : int, default=100
        Steps between learning rate decay
    early_stopping : bool, default=True
        Whether to use early stopping
    patience : int, default=50
        Early stopping patience

    Returns
    -------
    Dict[str, List]
        Contains training history
    """
    if X.shape[0] != self.layer_dims[0]:
        raise ValueError(
            f"Input features {X.shape[0]} don't match network input {self.layer_dims[0]}"
        )

    self.costs_history = []
    self.val_costs_history = []
    self.accuracies_history = []
    self.val_accuracies_history = []

    current_lr = learning_rate

    for epoch in range(num_epochs):
        AL, caches = self.forward_propagation(X, training=True)

        cost = self.compute_cost(AL, Y)
        accuracy = self.compute_accuracy(AL, Y)

        self.costs_history.append(cost)
        self.accuracies_history.append(accuracy)

        grads = self.backward_propagation(AL, Y, caches)

        self.update_parameters(grads, current_lr)

        AL_val, _ = self.forward_propagation(X_val, training=False)
        val_cost = self.compute_cost(AL_val, Y_val)
        val_accuracy = self.compute_accuracy(AL_val, Y_val)

        self.val_costs_history.append(val_cost)
        self.val_accuracies_history.append(val_accuracy)

        if epoch % decay_step == 0 and epoch > 0:
            current_lr *= learning_rate_decay

        if early_stopping:
            if val_cost < self.best_val_cost:
                self.best_val_cost = val_cost
                self.patience_counter = 0
            else:
                self.patience_counter += 1

            if self.patience_counter >= patience:
                if print_cost:
                    print(f"Early stopping at epoch {epoch}")
                break

        if print_cost and epoch % 100 == 0:
            print(
                f"Epoch {epoch:4d}: Cost = {cost:.6f}, Acc = {accuracy:.4f}, "
                f"Val Cost = {val_cost:.6f}, Val Acc = {val_accuracy:.4f}, LR = {current_lr:.6f}"
            )

    return {
        "costs": self.costs_history,
        "val_costs": self.val_costs_history,
        "accuracies": self.accuracies_history,
        "val_accuracies": self.val_accuracies_history,
    }

predict

predict(X: ndarray) -> np.ndarray

Make predictions on new data.

Parameters:

Name	Type	Description	Default
X	ndarray	Input data of shape (n_features, m_samples)	required

Returns:

Type	Description
ndarray	Binary predictions of shape (1, m_samples)

Source code in src/dlhub/nn/fully_connected.py

def predict(self, X: np.ndarray) -> np.ndarray:
    """
    Make predictions on new data.

    Parameters
    ----------
    X : np.ndarray
        Input data of shape (n_features, m_samples)

    Returns
    -------
    np.ndarray
        Binary predictions of shape (1, m_samples)
    """
    AL, _ = self.forward_propagation(X, training=False)
    predictions = (AL > 0.5).astype(int)
    return predictions

predict_proba

predict_proba(X: ndarray) -> np.ndarray

Get prediction probabilities.

Parameters:

Name	Type	Description	Default
X	ndarray	Input data of shape (n_features, m_samples)	required

Returns:

Type	Description
ndarray	Probabilities of shape (1, m_samples)

Source code in src/dlhub/nn/fully_connected.py

def predict_proba(self, X: np.ndarray) -> np.ndarray:
    """
    Get prediction probabilities.

    Parameters
    ----------
    X : np.ndarray
        Input data of shape (n_features, m_samples)

    Returns
    -------
    np.ndarray
        Probabilities of shape (1, m_samples)
    """
    AL, _ = self.forward_propagation(X, training=False)
    return AL

plot_training_history

plot_training_history(figsize: tuple[int, int] = (15, 5)) -> None

Plot training history including cost and accuracy curves.

Parameters:

Name	Type	Description	Default
figsize	Tuple[int, int]	Figure size tuple	(15, 5)

Source code in src/dlhub/nn/fully_connected.py

def plot_training_history(self, figsize: tuple[int, int] = (15, 5)) -> None:
    """
    Plot training history including cost and accuracy curves.

    Parameters
    ----------
    figsize : Tuple[int, int]
        Figure size tuple
    """
    if not self.costs_history:
        print("No training history available. Train the model first.")
        return

    fig, axes = plt.subplots(1, 2, figsize=figsize)

    # Cost Curves
    axes[0].plot(self.costs_history, label="Training Cost", linewidth=2)
    axes[0].plot(self.val_costs_history, label="Validation Cost", linewidth=2)
    axes[0].set_title("Training and Validation Cost")
    axes[0].set_xlabel("Epoch")
    axes[0].set_ylabel("Cost")
    axes[0].legend()
    axes[0].grid(True, alpha=0.3)

    # Accuracy Curves
    axes[1].plot(self.accuracies_history, label="Training Accuracy", linewidth=2)
    axes[1].plot(
        self.val_accuracies_history, label="Validation Accuracy", linewidth=2
    )
    axes[1].set_title("Training and Validation Accuracy")
    axes[1].set_xlabel("Epoch")
    axes[1].set_ylabel("Accuracy")
    axes[1].legend()
    axes[1].grid(True, alpha=0.3)

    plt.tight_layout()
    plt.show()

get_model_info

get_model_info() -> dict[str, Any]

Get comprehensive model information.

Returns:

Type	Description
Dict[str, Any]	Contains model configuration and statistics

Source code in src/dlhub/nn/fully_connected.py

def get_model_info(self) -> dict[str, Any]:
    """
    Get comprehensive model information.

    Returns
    -------
    Dict[str, Any]
        Contains model configuration and statistics
    """
    total_params = sum(param.size for param in self.parameters.values())
    if self.use_batch_norm:
        total_params += sum(param.size for param in self.bn_params.values())

    return {
        "architecture": self.layer_dims,
        "total_parameters": total_params,
        "regularization": self.regularization,
        "lambda_reg": self.lambda_reg,
        "batch_normalization": self.use_batch_norm,
        "dropout_keep_prob": (
            self.keep_prob if self.regularization == "dropout" else None
        ),
        "gradient_clipping": self.gradient_clipping,
        "clip_value": self.clip_value if self.gradient_clipping else None,
    }

create_sample_data

create_sample_data(n_features: int = 128, n_train: int = 1000, n_val: int = 200) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]

Create sample dataset for testing.

Parameters:

Name	Type	Description	Default
n_features	int	Number of input features	128
n_train	int	Number of training samples	1000
n_val	int	Number of validation samples	200

Returns:

Type	Description
Tuple[ndarray, ndarray, ndarray, ndarray]	(X_train, Y_train, X_val, Y_val)

Source code in src/dlhub/nn/fully_connected.py

def create_sample_data(
    n_features: int = 128, n_train: int = 1000, n_val: int = 200
) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
    """
    Create sample dataset for testing.

    Parameters
    ----------
    n_features : int, default=128
        Number of input features
    n_train : int, default=1000
        Number of training samples
    n_val : int, default=200
        Number of validation samples

    Returns
    -------
    Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]
        (X_train, Y_train, X_val, Y_val)
    """
    np.random.seed(42)

    X_train = np.random.randn(n_features, n_train)
    Y_train = (np.sum(X_train[:10], axis=0, keepdims=True) > 0).astype(int)

    X_val = np.random.randn(n_features, n_val)
    Y_val = (np.sum(X_val[:10], axis=0, keepdims=True) > 0).astype(int)

    return X_train, Y_train, X_val, Y_val

fully_connected

Deep Neural Network Implementation

Defines a configurable fully-connected deep neural network from scratch using NumPy, including forward/backward propagation, parameter initialization, optimization, and prediction routines.

References

• Goodfellow, I., Bengio, Y., & Courville, A. (2016). Deep Learning. MIT Press.
• Ng, A. (2017). Deep Learning Specialization: Course on Neural Networks. (gradient checking, initialization, forward/backward propagation)

Author

Deep Learning Reference Hub

License

MIT License

Notes

• Supports L-layer architectures with ReLU and Sigmoid activations
• Includes optional gradient checking and loss tracking

DeepNeuralNetwork

A comprehensive Deep Neural Network implementation with modern techniques.

This implementation includes: - Multiple initialization methods (He, Xavier, Random) - Regularization techniques (L1, L2, Dropout) - Batch Normalization - Gradient Clipping - Learning Rate Scheduling - Early Stopping - Comprehensive metrics tracking

Parameters:

Name	Type	Description	Default
layer_dims	List	Layer dimensions [n_x, n_h1, n_h2, ..., n_y]	required
initialization	str	Initialization method ('he', 'xavier', 'random')	'he'
regularization	str(optional)	Regularization type (None, 'l1', 'l2', 'dropout')	None
lambda_reg	float	Regularization strength parameter	0.01
keep_prob	float	Dropout keep probability (0 < keep_prob <= 1)	0.8
use_batch_norm	bool	Whether to use batch normalization	True
gradient_clipping	bool	Whether to apply gradient clipping	True
clip_value	float	Maximum gradient norm for clipping	True

Attributes:

Name	Type	Description
layer_dims	List[int]	Dimensions of each layer
L	int	Number of layers (excluding input)
parameters	Dict	Network weights and biases
bn_params	Dict	Batch normalization parameters
costs_history	List	Training cost history
val_costs_history	List	Validation cost history
accuracies_history	List	Training accuracy history
val_accuracies_history	List	Validation accuracy history

Notes

Raises ValueError: If invalid parameters are provided

Source code in src/dlhub/nn/fully_connected.py

class DeepNeuralNetwork:
    """
    A comprehensive Deep Neural Network implementation with modern techniques.

    This implementation includes:
    - Multiple initialization methods (He, Xavier, Random)
    - Regularization techniques (L1, L2, Dropout)
    - Batch Normalization
    - Gradient Clipping
    - Learning Rate Scheduling
    - Early Stopping
    - Comprehensive metrics tracking

    Parameters
    ----------
    layer_dims : List
        Layer dimensions [n_x, n_h1, n_h2, ..., n_y]
    initialization : str
        Initialization method ('he', 'xavier', 'random')
    regularization : str(optional)
        Regularization type (None, 'l1', 'l2', 'dropout')
    lambda_reg : float, default=0.01
        Regularization strength parameter
    keep_prob : float, default=0.8
        Dropout keep probability (0 < keep_prob <= 1)
    use_batch_norm : bool, default=True
        Whether to use batch normalization
    gradient_clipping : bool, default=True
        Whether to apply gradient clipping
    clip_value : float, default=True
        Maximum gradient norm for clipping

    Attributes
    ----------
    layer_dims : List[int]
        Dimensions of each layer
    L : int
        Number of layers (excluding input)
    parameters : Dict
        Network weights and biases
    bn_params : Dict
        Batch normalization parameters
    costs_history : List
        Training cost history
    val_costs_history: List
        Validation cost history
    accuracies_history : List
        Training accuracy history
    val_accuracies_history : List
        Validation accuracy history

    Notes
    -----
    Raises ValueError: If invalid parameters are provided
    """

    def __init__(
        self,
        layer_dims: list[int],
        initialization: str = "he",
        regularization: str | None = None,
        lambda_reg: float = 0.01,
        keep_prob: float = 0.8,
        use_batch_norm: bool = True,
        gradient_clipping: bool = True,
        clip_value: float = 5.0,
    ):
        self._validate_inputs(
            layer_dims, initialization, regularization, lambda_reg, keep_prob
        )

        self.layer_dims = layer_dims
        self.L = len(layer_dims) - 1  # Number of layers (excluding input)
        self.regularization = regularization
        self.lambda_reg = lambda_reg
        self.keep_prob = keep_prob
        self.use_batch_norm = use_batch_norm
        self.gradient_clipping = gradient_clipping
        self.clip_value = clip_value

        self.parameters = self._initialize_parameters(initialization)

        if use_batch_norm:
            self.running_mean = {}
            self.running_var = {}
            self.momentum = 0.9
            self.bn_params = self._initialize_batch_norm()

        self.costs_history = []
        self.val_costs_history = []
        self.accuracies_history = []
        self.val_accuracies_history = []

        self.best_val_cost = float("inf")
        self.patience_counter = 0

    def _validate_inputs(
        self,
        layer_dims: list[int],
        initialization: str,
        regularization: str | None,
        lambda_reg: float,
        keep_prob: float,
    ) -> None:
        """Validate input parameters."""
        if len(layer_dims) < 2:
            raise ValueError("Network must have at least 2 layers (input and output)")

        if any(dim <= 0 for dim in layer_dims):
            raise ValueError("All layer dimensions must be positive")

        if initialization not in ["he", "xavier", "random"]:
            raise ValueError("Initialization must be 'he', 'xavier', or 'random'")

        if regularization not in [None, "l1", "l2", "dropout"]:
            raise ValueError("Regularization must be None, 'l1', 'l2', or 'dropout'")

        if lambda_reg < 0:
            raise ValueError("Regularization parameter must be non-negative")

        if not 0 < keep_prob <= 1:
            raise ValueError("Keep probability must be in (0, 1]")

    def _initialize_parameters(self, method: str) -> dict[str, np.ndarray]:
        """
        Initialize network parameters using specified method.

        Parameters
        ----------
        method : str
            Initialization method ('he', 'xavier', 'random')

        Returns
        -------
        Dict[str, np.ndarray]
            Contains initialized weights and biases
        """
        np.random.seed(42)
        parameters = {}

        for l in range(1, self.L + 1):
            fan_in = self.layer_dims[l - 1]
            fan_out = self.layer_dims[l]

            if method == "he":
                std = np.sqrt(2.0 / fan_in)
            elif method == "xavier":
                std = np.sqrt(1.0 / fan_in)
            elif method == "random":
                std = 0.01

            parameters[f"W{l}"] = np.random.randn(fan_out, fan_in) * std
            parameters[f"b{l}"] = np.zeros((fan_out, 1))

        return parameters

    def _initialize_batch_norm(self) -> dict[str, np.ndarray]:
        """
        Initialize batch normalization parameters.

        Returns
        -------
        Dict[str, np.ndarray]
            Contains gamma and beta parameters
        """
        bn_params = {}

        for l in range(1, self.L):  # Not applied to output layer
            bn_params[f"gamma{l}"] = np.ones((self.layer_dims[l], 1))
            bn_params[f"beta{l}"] = np.zeros((self.layer_dims[l], 1))

            self.running_mean[f"mean{l}"] = np.zeros((self.layer_dims[l], 1))
            self.running_var[f"var{l}"] = np.ones((self.layer_dims[l], 1))

        return bn_params

    def _relu(self, Z: np.ndarray) -> np.ndarray:
        """Relu activation function."""
        return np.maximum(0, Z)

    def _relu_derivative(self, Z: np.ndarray) -> np.ndarray:
        """Relu derivative."""
        return (Z > 0).astype(float)

    def _sigmoid(self, Z: np.ndarray) -> np.ndarray:
        """Sigmoid activation function with numerical stability."""
        Z_clipped = np.clip(Z, -500, 500)
        return 1 / (1 + np.exp(-Z_clipped))

    def _sigmoid_derivative(self, A: np.ndarray) -> np.ndarray:
        """Sigmoid derivative."""
        return A * (1 - A)

    def forward_propagation(
        self, X: np.ndarray, training: bool = True
    ) -> tuple[np.ndarray, list]:
        """
        Perform forward propagation through the network.

        Parameters
        ----------
        X : np.ndarray
            Input data of shape (n_features, m_samples)
        training : bool, default=True
            Whether in training mode (affects dropout and batch norm)

        Returns
        -------
        Tuple[np.ndarray, List]
            Contains final output and caches for backprob
        """
        if X.shape[0] != self.layer_dims[0]:
            raise ValueError(
                f"Input shape {X.shape[0]} doesn't match expected {self.layer_dims[0]}"
            )

        caches = []
        A = X

        for l in range(1, self.L):
            A_prev = A
            Z = np.dot(self.parameters[f"W{l}"], A_prev) + self.parameters[f"b{l}"]

            if self.use_batch_norm:
                Z, bn_cache = self._batch_norm_forward(Z, l, training)
            else:
                bn_cache = None

            A = self._relu(Z)

            if self.regularization == "dropout" and training:
                A, dropout_cache = self._dropout_forward(A, self.keep_prob)
            else:
                dropout_cache = None

            cache = (A_prev, Z, A, bn_cache, dropout_cache)
            caches.append(cache)

        A_prev = A
        ZL = (
            np.dot(self.parameters[f"W{self.L}"], A_prev)
            + self.parameters[f"b{self.L}"]
        )
        AL = self._sigmoid(ZL)

        cache = (A_prev, ZL, AL, None, None)
        caches.append(cache)

        return AL, caches

    def _batch_norm_forward(
        self, Z: np.ndarray, l: int, training: bool, eps: float = 1e-8
    ) -> tuple[np.ndarray, tuple | None]:
        """
        Batch normalization forward pass.

        Parameters
        ----------
        Z : np.ndarray
            Pre-activation values
        l : int
            Layer index
        training : bool, default=True
            Whether in training mode
        eps : float, default=1e-8
            Small constant for numerical stability

        Returns
        -------
        Tuple[np.ndarray, tuple]
            Contains normalized output and cache for backprop
        """
        if training:
            mu = np.mean(Z, axis=1, keepdims=True)
            var = np.var(Z, axis=1, keepdims=True)

            self.running_mean[f"mean{l}"] = (
                self.momentum * self.running_mean[f"mean{l}"] + (1 - self.momentum) * mu
            )
            self.running_var[f"var{l}"] = (
                self.momentum * self.running_var[f"var{l}"] + (1 - self.momentum) * var
            )

            Z_norm = (Z - mu) / np.sqrt(var + eps)
            Z_out = self.bn_params[f"gamma{l}"] * Z_norm + self.bn_params[f"beta{l}"]

            cache = (Z, Z_norm, mu, var, eps)
            return Z_out, cache
        else:
            # Use running statistics for inference
            Z_norm = (Z - self.running_mean[f"mean{l}"]) / np.sqrt(
                self.running_var[f"var{l}"] + eps
            )
            Z_out = self.bn_params[f"gamma{l}"] * Z_norm + self.bn_params[f"beta{l}"]
            return Z_out, None

    def _dropout_forward(
        self, A: np.ndarray, keep_prob: float
    ) -> tuple[np.ndarray, np.ndarray]:
        """
        Dropout forward pass.

        Parameters
        ----------
        A : np.ndarray
            Activations
        keep_prob : float
            Probability of keeping each neuron

        Returns
        -------
        Tuple[np.ndarray, np.ndarray]
            Contains dropped activations and dropout mask
        """
        mask = np.random.binomial(1, keep_prob, A.shape) / keep_prob
        A_drop = A * mask
        return A_drop, mask

    def compute_cost(self, AL: np.ndarray, Y: np.ndarray) -> float:
        """
        Compute the cost function with regularization.

        Parameters
        ----------
        AL : np.ndarray
            Network output of shape (1, m_samples)
        Y : np.ndarray
            True labels of shape (1, m_samples)

        Returns
        -------
        float
            Total cost including regularization
        """
        m = Y.shape[1]
        AL_clipped = np.clip(AL, 1e-8, 1 - 1e-8)  # Clip predictions to prevent log(0)

        cost = -(1 / m) * (
            np.dot(Y, np.log(AL_clipped).T) + np.dot(1 - Y, np.log(1 - AL_clipped).T)
        )

        reg_cost = 0
        if self.regularization == "l2":
            weights = np.concatenate(
                [self.parameters[f"W{l}"].flatten() for l in range(1, self.L + 1)]
            )
            reg_cost = self.lambda_reg / (2 * m) * np.sum(weights**2)
            cost += reg_cost

        elif self.regularization == "l1":
            weights = np.concatenate(
                [self.parameters[f"W{l}"].flatten() for l in range(1, self.L + 1)]
            )
            reg_cost = self.lambda_reg / m * np.sum(np.abs(weights))
            cost += reg_cost

        return np.squeeze(cost)

    def backward_propagation(
        self, AL: np.ndarray, Y: np.ndarray, caches: list
    ) -> dict[str, np.ndarray]:
        """
        Perform backward propagation to compute gradients.

        Parameters
        ----------
        AL : np.ndarray
            Network output
        Y : np.ndarray
            True labels
        caches : List
            Forward propagation caches

        Returns
        -------
        Dict[str, np.ndarray]
            Contains gradients for all parameters
        """
        grads = {}
        m = AL.shape[1]

        # Carried from one iteration to the next: layer l reads the dA that
        # layer l+1 produced. Bound here so the handoff is visible rather than
        # implied by the loop.
        dA_next = None

        for l in reversed(range(1, self.L + 1)):
            A_prev, Z, A, bn_cache, dropout_cache = caches[l - 1]

            if l == self.L:
                dZ = AL - Y  # Cross-Entropy Derivative
            else:
                dA = dA_next

                if dropout_cache is not None:
                    dA = dA * dropout_cache

                dZ = dA * self._relu_derivative(Z)

                if bn_cache is not None:
                    dZ = self._batch_norm_backward(dZ, bn_cache, l)

            dW = (1 / m) * np.dot(dZ, A_prev.T)
            db = (1 / m) * np.sum(dZ, axis=1, keepdims=True)
            dA_prev = np.dot(self.parameters[f"W{l}"].T, dZ)

            if self.regularization == "l2":
                dW += (self.lambda_reg / m) * self.parameters[f"W{l}"]
            elif self.regularization == "l1":
                dW += (self.lambda_reg / m) * np.sign(self.parameters[f"W{l}"])

            grads[f"dW{l}"] = dW
            grads[f"db{l}"] = db

            dA_next = dA_prev

        return grads

    def _batch_norm_backward(
        self, dZ_out: np.ndarray, cache: tuple, l: int
    ) -> np.ndarray:
        """
        Batch normalization backward pass.

        Parameters
        ----------
        dZ_out : np.ndarray
            Gradient from next layer
        cache : Tuple
            Forward pass cache
        l : int
            Layer index

        Returns
        -------
        np.ndarray
            Gradient with respect to input
        """
        Z, Z_norm, mu, var, eps = cache
        m = Z.shape[1]

        dgamma = np.sum(dZ_out * Z_norm, axis=1, keepdims=True)
        dbeta = np.sum(dZ_out, axis=1, keepdims=True)

        if not hasattr(self, "bn_grads"):
            self.bn_grads = {}
        self.bn_grads[f"dgamma{l}"] = dgamma
        self.bn_grads[f"dbeta{l}"] = dbeta

        dZ_norm = dZ_out * self.bn_params[f"gamma{l}"]

        dvar = np.sum(
            dZ_norm * (Z - mu) * -0.5 * (var + eps) ** (-3 / 2), axis=1, keepdims=True
        )
        dmu = (
            np.sum(dZ_norm * -1 / np.sqrt(var + eps), axis=1, keepdims=True)
            + dvar * np.sum(-2 * (Z - mu), axis=1, keepdims=True) / m
        )

        dZ = dZ_norm / np.sqrt(var + eps) + dvar * 2 * (Z - mu) / m + dmu / m

        return dZ

    def _clip_gradients(self, grads: dict[str, np.ndarray]) -> dict[str, np.ndarray]:
        """
        Apply gradient clipping to prevent exploding gradients.

        Parameters
        ----------
        grads : Dict[str, np.ndarray]
            Dictionary of gradients

        Returns
        -------
        Dict[str, np.ndarray]
            Dictionary of clipped gradients
        """
        total_norm = 0
        for grad in grads.values():
            total_norm += np.sum(grad**2)
        total_norm = np.sqrt(total_norm)

        if total_norm > self.clip_value:
            clip_coeff = self.clip_value / total_norm
            for key in grads:
                grads[key] = grads[key] * clip_coeff

        return grads

    def update_parameters(
        self, grads: dict[str, np.ndarray], learning_rate: float
    ) -> None:
        """
        Update network parameters using gradients.

        Parameters
        ----------
        grads : Dict[str, np.ndarray]
            Dictionary of gradients
        learning_rate : float
            Learning rate for parameter updates
        """
        if self.gradient_clipping:
            grads = self._clip_gradients(grads)

        for l in range(1, self.L + 1):
            self.parameters[f"W{l}"] -= learning_rate * grads[f"dW{l}"]
            self.parameters[f"b{l}"] -= learning_rate * grads[f"db{l}"]

        if self.use_batch_norm and hasattr(self, "bn_grads"):
            for l in range(1, self.L):
                self.bn_params[f"gamma{l}"] -= (
                    learning_rate * self.bn_grads[f"dgamma{l}"]
                )
                self.bn_params[f"beta{l}"] -= learning_rate * self.bn_grads[f"dbeta{l}"]

    def compute_accuracy(self, AL: np.ndarray, Y: np.ndarray) -> float:
        """
        Compute classification accuracy.

        Parameters
        ----------
        AL : np.ndarray
            Network predictions
        Y : np.ndarray
            True labels

        Returns
        -------
        float
            Accuracy percentage
        """
        predictions = (AL > 0.5).astype(int)
        accuracy = np.mean(predictions == Y)
        return accuracy

    def train(
        self,
        X: np.ndarray,
        Y: np.ndarray,
        X_val: np.ndarray,
        Y_val: np.ndarray,
        learning_rate: float = 0.01,
        num_epochs: int = 1000,
        print_cost: bool = True,
        learning_rate_decay: float = 0.95,
        decay_step: int = 100,
        early_stopping: bool = True,
        patience: int = 50,
    ) -> dict[str, list]:
        """
        Train the neural network with advanced techniques.

        Parameters
        ----------
        X : np.ndarray
            Training data of shape (n_features, m_samples)
        Y : np.ndarray
            Training labels of shape (1, m_samples)
        X_val : np.ndarray
            Validation data
        Y_val : np.ndarray
            Validation labels
        learning_rate : float, default=0.01
            Initial learning rate
        num_epochs : int, default=1000
            Number of training epochs
        print_cost : bool, default=True
            Whether to print cost during training
        learning_rate_decay : float, default=0.95
            Learning rate decay factor
        decay_step : int, default=100
            Steps between learning rate decay
        early_stopping : bool, default=True
            Whether to use early stopping
        patience : int, default=50
            Early stopping patience

        Returns
        -------
        Dict[str, List]
            Contains training history
        """
        if X.shape[0] != self.layer_dims[0]:
            raise ValueError(
                f"Input features {X.shape[0]} don't match network input {self.layer_dims[0]}"
            )

        self.costs_history = []
        self.val_costs_history = []
        self.accuracies_history = []
        self.val_accuracies_history = []

        current_lr = learning_rate

        for epoch in range(num_epochs):
            AL, caches = self.forward_propagation(X, training=True)

            cost = self.compute_cost(AL, Y)
            accuracy = self.compute_accuracy(AL, Y)

            self.costs_history.append(cost)
            self.accuracies_history.append(accuracy)

            grads = self.backward_propagation(AL, Y, caches)

            self.update_parameters(grads, current_lr)

            AL_val, _ = self.forward_propagation(X_val, training=False)
            val_cost = self.compute_cost(AL_val, Y_val)
            val_accuracy = self.compute_accuracy(AL_val, Y_val)

            self.val_costs_history.append(val_cost)
            self.val_accuracies_history.append(val_accuracy)

            if epoch % decay_step == 0 and epoch > 0:
                current_lr *= learning_rate_decay

            if early_stopping:
                if val_cost < self.best_val_cost:
                    self.best_val_cost = val_cost
                    self.patience_counter = 0
                else:
                    self.patience_counter += 1

                if self.patience_counter >= patience:
                    if print_cost:
                        print(f"Early stopping at epoch {epoch}")
                    break

            if print_cost and epoch % 100 == 0:
                print(
                    f"Epoch {epoch:4d}: Cost = {cost:.6f}, Acc = {accuracy:.4f}, "
                    f"Val Cost = {val_cost:.6f}, Val Acc = {val_accuracy:.4f}, LR = {current_lr:.6f}"
                )

        return {
            "costs": self.costs_history,
            "val_costs": self.val_costs_history,
            "accuracies": self.accuracies_history,
            "val_accuracies": self.val_accuracies_history,
        }

    def predict(self, X: np.ndarray) -> np.ndarray:
        """
        Make predictions on new data.

        Parameters
        ----------
        X : np.ndarray
            Input data of shape (n_features, m_samples)

        Returns
        -------
        np.ndarray
            Binary predictions of shape (1, m_samples)
        """
        AL, _ = self.forward_propagation(X, training=False)
        predictions = (AL > 0.5).astype(int)
        return predictions

    def predict_proba(self, X: np.ndarray) -> np.ndarray:
        """
        Get prediction probabilities.

        Parameters
        ----------
        X : np.ndarray
            Input data of shape (n_features, m_samples)

        Returns
        -------
        np.ndarray
            Probabilities of shape (1, m_samples)
        """
        AL, _ = self.forward_propagation(X, training=False)
        return AL

    def plot_training_history(self, figsize: tuple[int, int] = (15, 5)) -> None:
        """
        Plot training history including cost and accuracy curves.

        Parameters
        ----------
        figsize : Tuple[int, int]
            Figure size tuple
        """
        if not self.costs_history:
            print("No training history available. Train the model first.")
            return

        fig, axes = plt.subplots(1, 2, figsize=figsize)

        # Cost Curves
        axes[0].plot(self.costs_history, label="Training Cost", linewidth=2)
        axes[0].plot(self.val_costs_history, label="Validation Cost", linewidth=2)
        axes[0].set_title("Training and Validation Cost")
        axes[0].set_xlabel("Epoch")
        axes[0].set_ylabel("Cost")
        axes[0].legend()
        axes[0].grid(True, alpha=0.3)

        # Accuracy Curves
        axes[1].plot(self.accuracies_history, label="Training Accuracy", linewidth=2)
        axes[1].plot(
            self.val_accuracies_history, label="Validation Accuracy", linewidth=2
        )
        axes[1].set_title("Training and Validation Accuracy")
        axes[1].set_xlabel("Epoch")
        axes[1].set_ylabel("Accuracy")
        axes[1].legend()
        axes[1].grid(True, alpha=0.3)

        plt.tight_layout()
        plt.show()

    def get_model_info(self) -> dict[str, Any]:
        """
        Get comprehensive model information.

        Returns
        -------
        Dict[str, Any]
            Contains model configuration and statistics
        """
        total_params = sum(param.size for param in self.parameters.values())
        if self.use_batch_norm:
            total_params += sum(param.size for param in self.bn_params.values())

        return {
            "architecture": self.layer_dims,
            "total_parameters": total_params,
            "regularization": self.regularization,
            "lambda_reg": self.lambda_reg,
            "batch_normalization": self.use_batch_norm,
            "dropout_keep_prob": (
                self.keep_prob if self.regularization == "dropout" else None
            ),
            "gradient_clipping": self.gradient_clipping,
            "clip_value": self.clip_value if self.gradient_clipping else None,
        }

forward_propagation

forward_propagation(X: ndarray, training: bool = True) -> tuple[np.ndarray, list]

Perform forward propagation through the network.

Parameters:

Name	Type	Description	Default
X	ndarray	Input data of shape (n_features, m_samples)	required
training	bool	Whether in training mode (affects dropout and batch norm)	True

Returns:

Type	Description
Tuple[ndarray, List]	Contains final output and caches for backprob

Source code in src/dlhub/nn/fully_connected.py

def forward_propagation(
    self, X: np.ndarray, training: bool = True
) -> tuple[np.ndarray, list]:
    """
    Perform forward propagation through the network.

    Parameters
    ----------
    X : np.ndarray
        Input data of shape (n_features, m_samples)
    training : bool, default=True
        Whether in training mode (affects dropout and batch norm)

    Returns
    -------
    Tuple[np.ndarray, List]
        Contains final output and caches for backprob
    """
    if X.shape[0] != self.layer_dims[0]:
        raise ValueError(
            f"Input shape {X.shape[0]} doesn't match expected {self.layer_dims[0]}"
        )

    caches = []
    A = X

    for l in range(1, self.L):
        A_prev = A
        Z = np.dot(self.parameters[f"W{l}"], A_prev) + self.parameters[f"b{l}"]

        if self.use_batch_norm:
            Z, bn_cache = self._batch_norm_forward(Z, l, training)
        else:
            bn_cache = None

        A = self._relu(Z)

        if self.regularization == "dropout" and training:
            A, dropout_cache = self._dropout_forward(A, self.keep_prob)
        else:
            dropout_cache = None

        cache = (A_prev, Z, A, bn_cache, dropout_cache)
        caches.append(cache)

    A_prev = A
    ZL = (
        np.dot(self.parameters[f"W{self.L}"], A_prev)
        + self.parameters[f"b{self.L}"]
    )
    AL = self._sigmoid(ZL)

    cache = (A_prev, ZL, AL, None, None)
    caches.append(cache)

    return AL, caches

compute_cost

compute_cost(AL: ndarray, Y: ndarray) -> float

Compute the cost function with regularization.

Parameters:

Name	Type	Description	Default
AL	ndarray	Network output of shape (1, m_samples)	required
Y	ndarray	True labels of shape (1, m_samples)	required

Returns:

Type	Description
float	Total cost including regularization

Source code in src/dlhub/nn/fully_connected.py

def compute_cost(self, AL: np.ndarray, Y: np.ndarray) -> float:
    """
    Compute the cost function with regularization.

    Parameters
    ----------
    AL : np.ndarray
        Network output of shape (1, m_samples)
    Y : np.ndarray
        True labels of shape (1, m_samples)

    Returns
    -------
    float
        Total cost including regularization
    """
    m = Y.shape[1]
    AL_clipped = np.clip(AL, 1e-8, 1 - 1e-8)  # Clip predictions to prevent log(0)

    cost = -(1 / m) * (
        np.dot(Y, np.log(AL_clipped).T) + np.dot(1 - Y, np.log(1 - AL_clipped).T)
    )

    reg_cost = 0
    if self.regularization == "l2":
        weights = np.concatenate(
            [self.parameters[f"W{l}"].flatten() for l in range(1, self.L + 1)]
        )
        reg_cost = self.lambda_reg / (2 * m) * np.sum(weights**2)
        cost += reg_cost

    elif self.regularization == "l1":
        weights = np.concatenate(
            [self.parameters[f"W{l}"].flatten() for l in range(1, self.L + 1)]
        )
        reg_cost = self.lambda_reg / m * np.sum(np.abs(weights))
        cost += reg_cost

    return np.squeeze(cost)

backward_propagation

backward_propagation(AL: ndarray, Y: ndarray, caches: list) -> dict[str, np.ndarray]

Perform backward propagation to compute gradients.

Parameters:

Name	Type	Description	Default
AL	ndarray	Network output	required
Y	ndarray	True labels	required
caches	List	Forward propagation caches	required

Returns:

Type	Description
Dict[str, ndarray]	Contains gradients for all parameters

Source code in src/dlhub/nn/fully_connected.py

def backward_propagation(
    self, AL: np.ndarray, Y: np.ndarray, caches: list
) -> dict[str, np.ndarray]:
    """
    Perform backward propagation to compute gradients.

    Parameters
    ----------
    AL : np.ndarray
        Network output
    Y : np.ndarray
        True labels
    caches : List
        Forward propagation caches

    Returns
    -------
    Dict[str, np.ndarray]
        Contains gradients for all parameters
    """
    grads = {}
    m = AL.shape[1]

    # Carried from one iteration to the next: layer l reads the dA that
    # layer l+1 produced. Bound here so the handoff is visible rather than
    # implied by the loop.
    dA_next = None

    for l in reversed(range(1, self.L + 1)):
        A_prev, Z, A, bn_cache, dropout_cache = caches[l - 1]

        if l == self.L:
            dZ = AL - Y  # Cross-Entropy Derivative
        else:
            dA = dA_next

            if dropout_cache is not None:
                dA = dA * dropout_cache

            dZ = dA * self._relu_derivative(Z)

            if bn_cache is not None:
                dZ = self._batch_norm_backward(dZ, bn_cache, l)

        dW = (1 / m) * np.dot(dZ, A_prev.T)
        db = (1 / m) * np.sum(dZ, axis=1, keepdims=True)
        dA_prev = np.dot(self.parameters[f"W{l}"].T, dZ)

        if self.regularization == "l2":
            dW += (self.lambda_reg / m) * self.parameters[f"W{l}"]
        elif self.regularization == "l1":
            dW += (self.lambda_reg / m) * np.sign(self.parameters[f"W{l}"])

        grads[f"dW{l}"] = dW
        grads[f"db{l}"] = db

        dA_next = dA_prev

    return grads

update_parameters

update_parameters(grads: dict[str, ndarray], learning_rate: float) -> None

Update network parameters using gradients.

Parameters:

Name	Type	Description	Default
grads	Dict[str, ndarray]	Dictionary of gradients	required
learning_rate	float	Learning rate for parameter updates	required

Source code in src/dlhub/nn/fully_connected.py

def update_parameters(
    self, grads: dict[str, np.ndarray], learning_rate: float
) -> None:
    """
    Update network parameters using gradients.

    Parameters
    ----------
    grads : Dict[str, np.ndarray]
        Dictionary of gradients
    learning_rate : float
        Learning rate for parameter updates
    """
    if self.gradient_clipping:
        grads = self._clip_gradients(grads)

    for l in range(1, self.L + 1):
        self.parameters[f"W{l}"] -= learning_rate * grads[f"dW{l}"]
        self.parameters[f"b{l}"] -= learning_rate * grads[f"db{l}"]

    if self.use_batch_norm and hasattr(self, "bn_grads"):
        for l in range(1, self.L):
            self.bn_params[f"gamma{l}"] -= (
                learning_rate * self.bn_grads[f"dgamma{l}"]
            )
            self.bn_params[f"beta{l}"] -= learning_rate * self.bn_grads[f"dbeta{l}"]

compute_accuracy

compute_accuracy(AL: ndarray, Y: ndarray) -> float

Compute classification accuracy.

Parameters:

Name	Type	Description	Default
AL	ndarray	Network predictions	required
Y	ndarray	True labels	required

Returns:

Type	Description
float	Accuracy percentage

Source code in src/dlhub/nn/fully_connected.py

def compute_accuracy(self, AL: np.ndarray, Y: np.ndarray) -> float:
    """
    Compute classification accuracy.

    Parameters
    ----------
    AL : np.ndarray
        Network predictions
    Y : np.ndarray
        True labels

    Returns
    -------
    float
        Accuracy percentage
    """
    predictions = (AL > 0.5).astype(int)
    accuracy = np.mean(predictions == Y)
    return accuracy

train

train(X: ndarray, Y: ndarray, X_val: ndarray, Y_val: ndarray, learning_rate: float = 0.01, num_epochs: int = 1000, print_cost: bool = True, learning_rate_decay: float = 0.95, decay_step: int = 100, early_stopping: bool = True, patience: int = 50) -> dict[str, list]

Train the neural network with advanced techniques.

Parameters:

Name	Type	Description	Default
X	ndarray	Training data of shape (n_features, m_samples)	required
Y	ndarray	Training labels of shape (1, m_samples)	required
X_val	ndarray	Validation data	required
Y_val	ndarray	Validation labels	required
learning_rate	float	Initial learning rate	0.01
num_epochs	int	Number of training epochs	1000
print_cost	bool	Whether to print cost during training	True
learning_rate_decay	float	Learning rate decay factor	0.95
decay_step	int	Steps between learning rate decay	100
early_stopping	bool	Whether to use early stopping	True
patience	int	Early stopping patience	50

Returns:

Type	Description
Dict[str, List]	Contains training history

Source code in src/dlhub/nn/fully_connected.py

def train(
    self,
    X: np.ndarray,
    Y: np.ndarray,
    X_val: np.ndarray,
    Y_val: np.ndarray,
    learning_rate: float = 0.01,
    num_epochs: int = 1000,
    print_cost: bool = True,
    learning_rate_decay: float = 0.95,
    decay_step: int = 100,
    early_stopping: bool = True,
    patience: int = 50,
) -> dict[str, list]:
    """
    Train the neural network with advanced techniques.

    Parameters
    ----------
    X : np.ndarray
        Training data of shape (n_features, m_samples)
    Y : np.ndarray
        Training labels of shape (1, m_samples)
    X_val : np.ndarray
        Validation data
    Y_val : np.ndarray
        Validation labels
    learning_rate : float, default=0.01
        Initial learning rate
    num_epochs : int, default=1000
        Number of training epochs
    print_cost : bool, default=True
        Whether to print cost during training
    learning_rate_decay : float, default=0.95
        Learning rate decay factor
    decay_step : int, default=100
        Steps between learning rate decay
    early_stopping : bool, default=True
        Whether to use early stopping
    patience : int, default=50
        Early stopping patience

    Returns
    -------
    Dict[str, List]
        Contains training history
    """
    if X.shape[0] != self.layer_dims[0]:
        raise ValueError(
            f"Input features {X.shape[0]} don't match network input {self.layer_dims[0]}"
        )

    self.costs_history = []
    self.val_costs_history = []
    self.accuracies_history = []
    self.val_accuracies_history = []

    current_lr = learning_rate

    for epoch in range(num_epochs):
        AL, caches = self.forward_propagation(X, training=True)

        cost = self.compute_cost(AL, Y)
        accuracy = self.compute_accuracy(AL, Y)

        self.costs_history.append(cost)
        self.accuracies_history.append(accuracy)

        grads = self.backward_propagation(AL, Y, caches)

        self.update_parameters(grads, current_lr)

        AL_val, _ = self.forward_propagation(X_val, training=False)
        val_cost = self.compute_cost(AL_val, Y_val)
        val_accuracy = self.compute_accuracy(AL_val, Y_val)

        self.val_costs_history.append(val_cost)
        self.val_accuracies_history.append(val_accuracy)

        if epoch % decay_step == 0 and epoch > 0:
            current_lr *= learning_rate_decay

        if early_stopping:
            if val_cost < self.best_val_cost:
                self.best_val_cost = val_cost
                self.patience_counter = 0
            else:
                self.patience_counter += 1

            if self.patience_counter >= patience:
                if print_cost:
                    print(f"Early stopping at epoch {epoch}")
                break

        if print_cost and epoch % 100 == 0:
            print(
                f"Epoch {epoch:4d}: Cost = {cost:.6f}, Acc = {accuracy:.4f}, "
                f"Val Cost = {val_cost:.6f}, Val Acc = {val_accuracy:.4f}, LR = {current_lr:.6f}"
            )

    return {
        "costs": self.costs_history,
        "val_costs": self.val_costs_history,
        "accuracies": self.accuracies_history,
        "val_accuracies": self.val_accuracies_history,
    }

predict

predict(X: ndarray) -> np.ndarray

Make predictions on new data.

Parameters:

Name	Type	Description	Default
X	ndarray	Input data of shape (n_features, m_samples)	required

Returns:

Type	Description
ndarray	Binary predictions of shape (1, m_samples)

Source code in src/dlhub/nn/fully_connected.py

def predict(self, X: np.ndarray) -> np.ndarray:
    """
    Make predictions on new data.

    Parameters
    ----------
    X : np.ndarray
        Input data of shape (n_features, m_samples)

    Returns
    -------
    np.ndarray
        Binary predictions of shape (1, m_samples)
    """
    AL, _ = self.forward_propagation(X, training=False)
    predictions = (AL > 0.5).astype(int)
    return predictions

predict_proba

predict_proba(X: ndarray) -> np.ndarray

Get prediction probabilities.

Parameters:

Name	Type	Description	Default
X	ndarray	Input data of shape (n_features, m_samples)	required

Returns:

Type	Description
ndarray	Probabilities of shape (1, m_samples)

Source code in src/dlhub/nn/fully_connected.py

def predict_proba(self, X: np.ndarray) -> np.ndarray:
    """
    Get prediction probabilities.

    Parameters
    ----------
    X : np.ndarray
        Input data of shape (n_features, m_samples)

    Returns
    -------
    np.ndarray
        Probabilities of shape (1, m_samples)
    """
    AL, _ = self.forward_propagation(X, training=False)
    return AL

plot_training_history

plot_training_history(figsize: tuple[int, int] = (15, 5)) -> None

Plot training history including cost and accuracy curves.

Parameters:

Name	Type	Description	Default
figsize	Tuple[int, int]	Figure size tuple	(15, 5)

Source code in src/dlhub/nn/fully_connected.py

def plot_training_history(self, figsize: tuple[int, int] = (15, 5)) -> None:
    """
    Plot training history including cost and accuracy curves.

    Parameters
    ----------
    figsize : Tuple[int, int]
        Figure size tuple
    """
    if not self.costs_history:
        print("No training history available. Train the model first.")
        return

    fig, axes = plt.subplots(1, 2, figsize=figsize)

    # Cost Curves
    axes[0].plot(self.costs_history, label="Training Cost", linewidth=2)
    axes[0].plot(self.val_costs_history, label="Validation Cost", linewidth=2)
    axes[0].set_title("Training and Validation Cost")
    axes[0].set_xlabel("Epoch")
    axes[0].set_ylabel("Cost")
    axes[0].legend()
    axes[0].grid(True, alpha=0.3)

    # Accuracy Curves
    axes[1].plot(self.accuracies_history, label="Training Accuracy", linewidth=2)
    axes[1].plot(
        self.val_accuracies_history, label="Validation Accuracy", linewidth=2
    )
    axes[1].set_title("Training and Validation Accuracy")
    axes[1].set_xlabel("Epoch")
    axes[1].set_ylabel("Accuracy")
    axes[1].legend()
    axes[1].grid(True, alpha=0.3)

    plt.tight_layout()
    plt.show()

get_model_info

get_model_info() -> dict[str, Any]

Get comprehensive model information.

Returns:

Type	Description
Dict[str, Any]	Contains model configuration and statistics

Source code in src/dlhub/nn/fully_connected.py

def get_model_info(self) -> dict[str, Any]:
    """
    Get comprehensive model information.

    Returns
    -------
    Dict[str, Any]
        Contains model configuration and statistics
    """
    total_params = sum(param.size for param in self.parameters.values())
    if self.use_batch_norm:
        total_params += sum(param.size for param in self.bn_params.values())

    return {
        "architecture": self.layer_dims,
        "total_parameters": total_params,
        "regularization": self.regularization,
        "lambda_reg": self.lambda_reg,
        "batch_normalization": self.use_batch_norm,
        "dropout_keep_prob": (
            self.keep_prob if self.regularization == "dropout" else None
        ),
        "gradient_clipping": self.gradient_clipping,
        "clip_value": self.clip_value if self.gradient_clipping else None,
    }

create_sample_data

create_sample_data(n_features: int = 128, n_train: int = 1000, n_val: int = 200) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]

Create sample dataset for testing.

Parameters:

Name	Type	Description	Default
n_features	int	Number of input features	128
n_train	int	Number of training samples	1000
n_val	int	Number of validation samples	200

Returns:

Type	Description
Tuple[ndarray, ndarray, ndarray, ndarray]	(X_train, Y_train, X_val, Y_val)

Source code in src/dlhub/nn/fully_connected.py

def create_sample_data(
    n_features: int = 128, n_train: int = 1000, n_val: int = 200
) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
    """
    Create sample dataset for testing.

    Parameters
    ----------
    n_features : int, default=128
        Number of input features
    n_train : int, default=1000
        Number of training samples
    n_val : int, default=200
        Number of validation samples

    Returns
    -------
    Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]
        (X_train, Y_train, X_val, Y_val)
    """
    np.random.seed(42)

    X_train = np.random.randn(n_features, n_train)
    Y_train = (np.sum(X_train[:10], axis=0, keepdims=True) > 0).astype(int)

    X_val = np.random.randn(n_features, n_val)
    Y_val = (np.sum(X_val[:10], axis=0, keepdims=True) > 0).astype(int)

    return X_train, Y_train, X_val, Y_val

main

main()

Example usage of the Deep Neural Network.

Source code in src/dlhub/nn/fully_connected.py

def main():
    """Example usage of the Deep Neural Network."""
    print("Creating sample dataset...")
    X_train, Y_train, X_val, Y_val = create_sample_data()

    print(f"Training data shape: {X_train.shape}")
    print(f"Training labels shape: {Y_train.shape}")
    print(f"Validation data shape: {X_val.shape}")
    print(f"Validation labels shape: {Y_val.shape}")

    print("\nCreating Deep Neural Network...")
    model = DeepNeuralNetwork(
        layer_dims=[128, 64, 1],
        initialization="he",
        regularization="l2",
        lambda_reg=0.4,
        keep_prob=0.9,
        use_batch_norm=True,
        gradient_clipping=False,
        clip_value=5.0,
    )

    print("\nModel Information:")
    info = model.get_model_info()
    for key, value in info.items():
        print(f"  {key}: {value}")

    print("\nTraining model...")
    history = model.train(
        X_train,
        Y_train,
        X_val,
        Y_val,
        learning_rate=0.02,
        num_epochs=5000,
        print_cost=True,
        learning_rate_decay=0.99,
        decay_step=100,
        early_stopping=False,
        patience=50,
    )

    print("\nMaking predictions...")
    train_predictions = model.predict(X_train)
    val_predictions = model.predict(X_val)

    train_accuracy = model.compute_accuracy(model.predict_proba(X_train), Y_train)
    val_accuracy = model.compute_accuracy(model.predict_proba(X_val), Y_val)

    print(f"\nFinal Results:")
    print(f"Training Accuracy: {train_accuracy:.4f}")
    print(f"Validation Accuracy: {val_accuracy:.4f}")

    model.plot_training_history()

sequence

Sequence Models

Recurrent architectures for sequential data, written to be read alongside the documents that derive them. The vanilla RNN here is the from-scratch reference that the framework parity port is checked against; its notation follows Andrew Ng's Course 5 so the gated cells (LSTM, GRU) extend the same symbols.

Each module carries its own make_fixture. The name re-exported here is the vanilla RNN's, because a package-level name can only mean one thing; import the gated ones from their modules, from dlhub.nn.sequence.lstm import make_fixture.

Author

Deep Learning Reference Hub

License

MIT

compute_loss

compute_loss(y_pred: ndarray, y: ndarray) -> float

Total softmax cross-entropy loss, summed over timesteps.

Implements L = sum_t L^{<t>} with each per-step loss the batch-mean cross-entropy L^{<t>} = -(1/m) sum_{i,c} y^{<t>}_{c,i} log y_hat^{<t>}_{c,i}. No epsilon guards the log: the stable softmax output is strictly positive (see module Notes).

Parameters:

Name	Type	Description	Default
y_pred	ndarray	Predicted distributions, shape (n_y, m, T_x).	required
y	ndarray	One-hot targets, same shape (n_y, m, T_x).	required

Returns:

Type	Description
float	The scalar loss summed over all timesteps.

Source code in src/dlhub/nn/sequence/_common.py

def compute_loss(y_pred: np.ndarray, y: np.ndarray) -> float:
    """
    Total softmax cross-entropy loss, summed over timesteps.

    Implements ``L = sum_t L^{<t>}`` with each per-step loss the batch-mean
    cross-entropy ``L^{<t>} = -(1/m) sum_{i,c} y^{<t>}_{c,i} log y_hat^{<t>}_{c,i}``.
    No epsilon guards the ``log``: the stable softmax output is strictly
    positive (see module ``Notes``).

    Parameters
    ----------
    y_pred : np.ndarray
        Predicted distributions, shape ``(n_y, m, T_x)``.
    y : np.ndarray
        One-hot targets, same shape ``(n_y, m, T_x)``.

    Returns
    -------
    float
        The scalar loss summed over all timesteps.
    """
    m = y_pred.shape[1]
    return float(-np.sum(y * np.log(y_pred)) / m)

sigmoid

sigmoid(z: ndarray) -> np.ndarray

Element-wise logistic sigmoid, in the overflow-free form.

Computes sigma(z) = 1 / (1 + exp(-z)) without ever exponentiating a positive number. See the module Notes for why the two branches are written this way.

Parameters:

Name	Type	Description	Default
z	ndarray	Pre-activations of any shape.	required

Returns:

Type	Description
ndarray	Values in (0, 1), the same shape as z.

Source code in src/dlhub/nn/sequence/_common.py

def sigmoid(z: np.ndarray) -> np.ndarray:
    """
    Element-wise logistic sigmoid, in the overflow-free form.

    Computes ``sigma(z) = 1 / (1 + exp(-z))`` without ever exponentiating a
    positive number. See the module ``Notes`` for why the two branches are
    written this way.

    Parameters
    ----------
    z : np.ndarray
        Pre-activations of any shape.

    Returns
    -------
    np.ndarray
        Values in ``(0, 1)``, the same shape as ``z``.
    """
    exp_neg_abs = np.exp(-np.abs(z))  # in (0, 1] for every finite z
    return np.where(
        z >= 0,
        1.0 / (1.0 + exp_neg_abs),  # z >= 0: divide through by e^{z}
        exp_neg_abs / (1.0 + exp_neg_abs),  # z <  0: the unshifted form
    )

softmax

softmax(z: ndarray) -> np.ndarray

Column-wise softmax with the max-subtraction stability shift.

Normalizes over axis=0 (the class/feature axis), so each column is a probability distribution that sums to one. See the module Notes for why the maximum is subtracted first.

Parameters:

Name	Type	Description	Default
z	ndarray	Logits of shape (n_y, m).	required

Returns:

Type	Description
ndarray	Probabilities of shape (n_y, m), strictly positive, columns summing to one.

Source code in src/dlhub/nn/sequence/_common.py

def softmax(z: np.ndarray) -> np.ndarray:
    """
    Column-wise softmax with the max-subtraction stability shift.

    Normalizes over ``axis=0`` (the class/feature axis), so each column is a
    probability distribution that sums to one. See the module ``Notes`` for why
    the maximum is subtracted first.

    Parameters
    ----------
    z : np.ndarray
        Logits of shape ``(n_y, m)``.

    Returns
    -------
    np.ndarray
        Probabilities of shape ``(n_y, m)``, strictly positive, columns summing
        to one.
    """
    z_shifted = z - np.max(z, axis=0, keepdims=True)
    exp_z = np.exp(z_shifted)
    return exp_z / np.sum(exp_z, axis=0, keepdims=True)

split_gate_matrix

split_gate_matrix(matrix: ndarray, n_a: int) -> tuple[np.ndarray, np.ndarray]

Split a concatenated gate matrix into its recurrent and input halves.

The gated cells use one weight per gate acting on the stacked vector [a^{<t-1>}; x^{<t>}], so W_g = [W_ga | W_gx]. The two halves are never needed separately by the forward or backward pass -- the point of the concatenated form is that they are not -- but they are worth naming when reporting a gradient, because dW_ga is the recurrent path and dW_gx the input path, which is the split the vanilla RNN keeps explicit.

Parameters:

Name	Type	Description	Default
matrix	ndarray	A gate weight or gate weight gradient, shape (n_a, n_a + n_x).	required
n_a	int	The hidden width, which is the column the split falls on.	required

Returns:

Name	Type	Description
recurrent	ndarray	W_ga, the first n_a columns, shape (n_a, n_a).
input_	ndarray	W_gx, the remaining columns, shape (n_a, n_x).

Raises:

Type	Description
ValueError	If matrix has no more than n_a columns, which would leave the input half empty.

Source code in src/dlhub/nn/sequence/_common.py

def split_gate_matrix(matrix: np.ndarray, n_a: int) -> tuple[np.ndarray, np.ndarray]:
    """
    Split a concatenated gate matrix into its recurrent and input halves.

    The gated cells use one weight per gate acting on the stacked vector
    ``[a^{<t-1>}; x^{<t>}]``, so ``W_g = [W_ga | W_gx]``. The two halves are
    never needed separately by the forward or backward pass -- the point of the
    concatenated form is that they are not -- but they are worth naming when
    reporting a gradient, because ``dW_ga`` is the recurrent path and ``dW_gx``
    the input path, which is the split the vanilla RNN keeps explicit.

    Parameters
    ----------
    matrix : np.ndarray
        A gate weight or gate weight gradient, shape ``(n_a, n_a + n_x)``.
    n_a : int
        The hidden width, which is the column the split falls on.

    Returns
    -------
    recurrent : np.ndarray
        ``W_ga``, the first ``n_a`` columns, shape ``(n_a, n_a)``.
    input_ : np.ndarray
        ``W_gx``, the remaining columns, shape ``(n_a, n_x)``.

    Raises
    ------
    ValueError
        If ``matrix`` has no more than ``n_a`` columns, which would leave the
        input half empty.
    """
    if matrix.ndim != 2 or matrix.shape[1] <= n_a:
        raise ValueError(
            f"expected a 2-D matrix with more than {n_a} columns; got {matrix.shape}"
        )
    return matrix[:, :n_a], matrix[:, n_a:]

gru_backward

gru_backward(y: ndarray, caches: list[dict[str, ndarray]], parameters: dict[str, ndarray]) -> dict[str, np.ndarray]

Backpropagation through time (BPTT) for the GRU.

Walks the unrolled graph from t = T_x back to t = 1. Every gate weight is shared across time, so each receives an accumulated gradient summed over all timesteps, and the hidden-state gradient is threaded backward so each step adds its recurrence contribution (da_prev) to the next step's incoming gradient.

Parameters:

Name	Type	Description	Default
y	ndarray	One-hot targets, shape (n_y, m, T_x).	required
caches	list[dict[str, ndarray]]	Per-timestep caches from :func:gru_forward.	required
parameters	dict[str, ndarray]	The shared weights and biases (see :func:gru_cell_forward).	required

Returns:

Type	Description
dict[str, ndarray]	Accumulated gradients dWr, dWu, dWc, dbr, dbu, dbc, dWy, dby; the gradient da0 w.r.t. the initial hidden state; and dx, the gradient w.r.t. the whole input sequence, shape (n_x, m, T_x).

Source code in src/dlhub/nn/sequence/gru.py

def gru_backward(
    y: np.ndarray,
    caches: list[dict[str, np.ndarray]],
    parameters: dict[str, np.ndarray],
) -> dict[str, np.ndarray]:
    """
    Backpropagation through time (BPTT) for the GRU.

    Walks the unrolled graph from ``t = T_x`` back to ``t = 1``. Every gate
    weight is shared across time, so each receives an **accumulated** gradient
    summed over all timesteps, and the hidden-state gradient is threaded
    backward so each step adds its recurrence contribution (``da_prev``) to the
    next step's incoming gradient.

    Parameters
    ----------
    y : np.ndarray
        One-hot targets, shape ``(n_y, m, T_x)``.
    caches : list[dict[str, np.ndarray]]
        Per-timestep caches from :func:`gru_forward`.
    parameters : dict[str, np.ndarray]
        The shared weights and biases (see :func:`gru_cell_forward`).

    Returns
    -------
    dict[str, np.ndarray]
        Accumulated gradients ``dWr``, ``dWu``, ``dWc``, ``dbr``, ``dbu``,
        ``dbc``, ``dWy``, ``dby``; the gradient ``da0`` w.r.t. the initial hidden
        state; and ``dx``, the gradient w.r.t. the whole input sequence, shape
        ``(n_x, m, T_x)``.
    """
    T_x = len(caches)
    _, m, _ = y.shape
    n_x = caches[0]["x_t"].shape[0]
    n_a = parameters["Wu"].shape[0]

    accumulated = {f"d{key}": np.zeros_like(parameters[key]) for key in PARAMETER_KEYS}
    dx = np.zeros((n_x, m, T_x))

    # The recurrence contribution into the current step, carried back one step
    # at a time. Zero at the final timestep, which has no successor.
    da_next = np.zeros((n_a, m))

    for t in reversed(range(T_x)):
        grads_t = gru_cell_backward(y[:, :, t], da_next, caches[t], parameters)

        # The shared weights accumulate: every timestep used the same matrix,
        # so every timestep adds a term to its gradient.
        for key in PARAMETER_KEYS:
            accumulated[f"d{key}"] += grads_t[f"d{key}"]

        dx[:, :, t] = grads_t["dxt"]
        da_next = grads_t["da_prev"]  # hand the recurrence back to t-1

    # After the loop, the recurrence gradient has reached a^{<0>}.
    return {**accumulated, "da0": da_next, "dx": dx}

gru_cell_backward

gru_cell_backward(y_t: ndarray, da_next: ndarray, cache: dict[str, ndarray], parameters: dict[str, ndarray]) -> dict[str, np.ndarray]

Backward pass for a single GRU timestep.

Walks the forward pass backwards, one equation at a time. Three separate paths deposit gradient into a^{<t-1>}, and the whole point of the cell is that the first of them is a plain elementwise factor:

1. the blend, through (1 - Gamma_u) -- no weight matrix, no saturating nonlinearity, so gradient crosses the step essentially undamped when the update gate is near zero;
2. the candidate, through W_c and then the reset gate;
3. the gates themselves, through W_r and W_u.

Parameters:

Name	Type	Description	Default
y_t	ndarray	One-hot target at this timestep, shape (n_y, m).	required
da_next	ndarray	Gradient of the loss w.r.t. this step's hidden state arriving from the recurrence at t+1, shape (n_a, m). Zero for the final step.	required
cache	dict[str, ndarray]	This step's cache from :func:gru_cell_forward.	required
parameters	dict[str, ndarray]	The shared weights and biases (see :func:gru_cell_forward).	required

Returns:

Type	Description
dict[str, ndarray]	This step's gradient contributions: dWr, dWu, dWc, dbr, dbu, dbc, dWy, dby, together with dxt (into the input) and da_prev (the recurrence handed back to t-1).

Source code in src/dlhub/nn/sequence/gru.py

def gru_cell_backward(
    y_t: np.ndarray,
    da_next: np.ndarray,
    cache: dict[str, np.ndarray],
    parameters: dict[str, np.ndarray],
) -> dict[str, np.ndarray]:
    """
    Backward pass for a single GRU timestep.

    Walks the forward pass backwards, one equation at a time. Three separate
    paths deposit gradient into ``a^{<t-1>}``, and the whole point of the cell is
    that the first of them is a plain elementwise factor:

    1. the **blend**, through ``(1 - Gamma_u)`` -- no weight matrix, no
       saturating nonlinearity, so gradient crosses the step essentially undamped
       when the update gate is near zero;
    2. the **candidate**, through ``W_c`` and then the reset gate;
    3. the **gates** themselves, through ``W_r`` and ``W_u``.

    Parameters
    ----------
    y_t : np.ndarray
        One-hot target at this timestep, shape ``(n_y, m)``.
    da_next : np.ndarray
        Gradient of the loss w.r.t. this step's hidden state arriving from the
        recurrence at ``t+1``, shape ``(n_a, m)``. Zero for the final step.
    cache : dict[str, np.ndarray]
        This step's cache from :func:`gru_cell_forward`.
    parameters : dict[str, np.ndarray]
        The shared weights and biases (see :func:`gru_cell_forward`).

    Returns
    -------
    dict[str, np.ndarray]
        This step's gradient contributions: ``dWr``, ``dWu``, ``dWc``, ``dbr``,
        ``dbu``, ``dbc``, ``dWy``, ``dby``, together with ``dxt`` (into the
        input) and ``da_prev`` (the recurrence handed back to ``t-1``).
    """
    a_next, a_prev = cache["a_next"], cache["a_prev"]
    gamma_r, gamma_u = cache["gamma_r"], cache["gamma_u"]
    a_tilde = cache["a_tilde"]
    concat, concat_reset = cache["concat"], cache["concat_reset"]
    y_pred_t = cache["y_pred_t"]

    Wr, Wu, Wc, Wy = (
        parameters["Wr"],
        parameters["Wu"],
        parameters["Wc"],
        parameters["Wy"],
    )
    n_a, m = a_next.shape

    # --- Output path: softmax cross-entropy -------------------------------
    # For softmax + cross-entropy the logit gradient collapses to (y_hat - y),
    # scaled by the 1/m batch mean used in compute_loss.
    dz_y = (y_pred_t - y_t) / m  # (n_y, m)
    dWy = dz_y @ a_next.T  # (n_y, n_a)
    dby = np.sum(dz_y, axis=1, keepdims=True)  # (n_y, 1)
    da_from_output = Wy.T @ dz_y  # (n_a, m)

    # --- Merge the two gradient sources at a^{<t>} ------------------------
    da_t = da_from_output + da_next  # (n_a, m)

    # --- Blend: a^{<t>} = Gamma_u * a_tilde + (1 - Gamma_u) * a^{<t-1>} ---
    # Differentiate the convex combination in its three arguments.
    dgamma_u = da_t * (a_tilde - a_prev)  # (n_a, m)
    da_tilde = da_t * gamma_u  # (n_a, m)
    da_prev = da_t * (1 - gamma_u)  # path 1: the undamped carry

    # --- Candidate: a_tilde = tanh(W_c [Gamma_r * a^{<t-1>}; x^{<t>}] + b_c)
    dz_c = da_tilde * (1 - a_tilde**2)  # tanh', read off the activation
    dWc = dz_c @ concat_reset.T  # (n_a, n_a + n_x)
    dbc = np.sum(dz_c, axis=1, keepdims=True)  # (n_a, 1)
    dconcat_reset = Wc.T @ dz_c  # (n_a + n_x, m)

    # The top n_a rows are the gradient w.r.t. the *reset* state, which the
    # product rule then splits between the gate and the state itself.
    d_reset_state = dconcat_reset[:n_a, :]  # d/d(Gamma_r * a^{<t-1>})
    dgamma_r = d_reset_state * a_prev
    da_prev = da_prev + d_reset_state * gamma_r  # path 2: through the candidate
    dxt = dconcat_reset[n_a:, :]  # (n_x, m)

    # --- Gates: sigma'(z) = Gamma (1 - Gamma), read off the cached gate ----
    dz_r = dgamma_r * gamma_r * (1 - gamma_r)
    dWr = dz_r @ concat.T
    dbr = np.sum(dz_r, axis=1, keepdims=True)
    dconcat_r = Wr.T @ dz_r

    dz_u = dgamma_u * gamma_u * (1 - gamma_u)
    dWu = dz_u @ concat.T
    dbu = np.sum(dz_u, axis=1, keepdims=True)
    dconcat_u = Wu.T @ dz_u

    # Both gates read the same stacked vector, so each contributes to both the
    # recurrent gradient and the input gradient. Path 3.
    da_prev = da_prev + dconcat_r[:n_a, :] + dconcat_u[:n_a, :]
    dxt = dxt + dconcat_r[n_a:, :] + dconcat_u[n_a:, :]

    return {
        "dWr": dWr,
        "dWu": dWu,
        "dWc": dWc,
        "dbr": dbr,
        "dbu": dbu,
        "dbc": dbc,
        "dWy": dWy,
        "dby": dby,
        "dxt": dxt,
        "da_prev": da_prev,
    }

gru_cell_forward

gru_cell_forward(x_t: ndarray, a_prev: ndarray, parameters: dict[str, ndarray]) -> tuple[np.ndarray, np.ndarray, dict[str, np.ndarray]]

One timestep of the GRU forward recurrence.

Computes the two gates, the candidate activation, the blended state, and the softmax read-out for a single step, in that order.

Parameters:

Name	Type	Description	Default
x_t	ndarray	Input at this timestep, shape (n_x, m).	required
a_prev	ndarray	Hidden state carried in from the previous step, a^{<t-1>}, shape (n_a, m).	required
parameters	dict[str, ndarray]	The shared weights and biases: Wr, Wu, Wc each (n_a, n_a + n_x); br, bu, bc each (n_a, 1); Wy (n_y, n_a); by (n_y, 1).	required

Returns:

Name	Type	Description
a_next	ndarray	Updated hidden state a^{<t>}, shape (n_a, m).
y_pred_t	ndarray	Output distribution y_hat^{<t>}, shape (n_y, m).
cache	dict[str, ndarray]	Everything this step's backward pass needs, each array named.

Source code in src/dlhub/nn/sequence/gru.py

def gru_cell_forward(
    x_t: np.ndarray, a_prev: np.ndarray, parameters: dict[str, np.ndarray]
) -> tuple[np.ndarray, np.ndarray, dict[str, np.ndarray]]:
    """
    One timestep of the GRU forward recurrence.

    Computes the two gates, the candidate activation, the blended state, and the
    softmax read-out for a single step, in that order.

    Parameters
    ----------
    x_t : np.ndarray
        Input at this timestep, shape ``(n_x, m)``.
    a_prev : np.ndarray
        Hidden state carried in from the previous step, ``a^{<t-1>}``, shape
        ``(n_a, m)``.
    parameters : dict[str, np.ndarray]
        The shared weights and biases: ``Wr``, ``Wu``, ``Wc`` each
        ``(n_a, n_a + n_x)``; ``br``, ``bu``, ``bc`` each ``(n_a, 1)``; ``Wy``
        ``(n_y, n_a)``; ``by`` ``(n_y, 1)``.

    Returns
    -------
    a_next : np.ndarray
        Updated hidden state ``a^{<t>}``, shape ``(n_a, m)``.
    y_pred_t : np.ndarray
        Output distribution ``y_hat^{<t>}``, shape ``(n_y, m)``.
    cache : dict[str, np.ndarray]
        Everything this step's backward pass needs, each array named.
    """
    Wr, Wu, Wc = parameters["Wr"], parameters["Wu"], parameters["Wc"]
    br, bu, bc = parameters["br"], parameters["bu"], parameters["bc"]
    Wy, by = parameters["Wy"], parameters["by"]

    # The concatenated form: one stacked vector [a^{<t-1>}; x^{<t>}], shape
    # (n_a + n_x, m), that every gate weight multiplies.
    concat = np.concatenate([a_prev, x_t], axis=0)

    # --- Gates ------------------------------------------------------------
    gamma_r = sigmoid(Wr @ concat + br)  # reset: how much state the candidate sees
    gamma_u = sigmoid(Wu @ concat + bu)  # update: keep the old state, or take the new

    # --- Candidate --------------------------------------------------------
    # The reset gate is applied *before* the hidden transform: it scales
    # a^{<t-1>} inside the concatenation W_c acts on. See the module Notes.
    concat_reset = np.concatenate([gamma_r * a_prev, x_t], axis=0)
    a_tilde = np.tanh(Wc @ concat_reset + bc)

    # --- Blend ------------------------------------------------------------
    # A convex combination, elementwise: gate near 1 takes the candidate, gate
    # near 0 copies the previous state forward unchanged.
    a_next = gamma_u * a_tilde + (1 - gamma_u) * a_prev

    # --- Read-out ---------------------------------------------------------
    y_pred_t = softmax(Wy @ a_next + by)

    cache = {
        "a_next": a_next,
        "a_prev": a_prev,
        "gamma_r": gamma_r,
        "gamma_u": gamma_u,
        "a_tilde": a_tilde,
        "concat": concat,
        "concat_reset": concat_reset,
        "x_t": x_t,
        "y_pred_t": y_pred_t,
    }
    return a_next, y_pred_t, cache

gru_forward

gru_forward(x: ndarray, parameters: dict[str, ndarray], a0: ndarray | None = None) -> tuple[np.ndarray, np.ndarray, list[dict[str, np.ndarray]]]

Forward propagation through time across a whole sequence.

Unrolls :func:gru_cell_forward over T_x timesteps, threading the hidden state from one step to the next.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence of shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	The shared weights and biases (see :func:gru_cell_forward).	required
a0	ndarray or None	Initial hidden state a^{<0>} of shape (n_a, m). Defaults to the zero vector, matching the standard convention a^{<0>} = 0.	None

Returns:

Name	Type	Description
a	ndarray	All hidden states stacked over time, shape (n_a, m, T_x).
y_pred	ndarray	All output distributions, shape (n_y, m, T_x).
caches	list[dict[str, ndarray]]	Per-timestep caches, in order, for :func:gru_backward.

Raises:

Type	Description
ValueError	If x is not three-dimensional, or if a supplied a0 does not have shape (n_a, m).

Source code in src/dlhub/nn/sequence/gru.py

def gru_forward(
    x: np.ndarray,
    parameters: dict[str, np.ndarray],
    a0: np.ndarray | None = None,
) -> tuple[np.ndarray, np.ndarray, list[dict[str, np.ndarray]]]:
    """
    Forward propagation through time across a whole sequence.

    Unrolls :func:`gru_cell_forward` over ``T_x`` timesteps, threading the hidden
    state from one step to the next.

    Parameters
    ----------
    x : np.ndarray
        Input sequence of shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        The shared weights and biases (see :func:`gru_cell_forward`).
    a0 : np.ndarray or None, optional
        Initial hidden state ``a^{<0>}`` of shape ``(n_a, m)``. Defaults to the
        zero vector, matching the standard convention ``a^{<0>} = 0``.

    Returns
    -------
    a : np.ndarray
        All hidden states stacked over time, shape ``(n_a, m, T_x)``.
    y_pred : np.ndarray
        All output distributions, shape ``(n_y, m, T_x)``.
    caches : list[dict[str, np.ndarray]]
        Per-timestep caches, in order, for :func:`gru_backward`.

    Raises
    ------
    ValueError
        If ``x`` is not three-dimensional, or if a supplied ``a0`` does not have
        shape ``(n_a, m)``.
    """
    if x.ndim != 3:
        raise ValueError(f"x must have shape (n_x, m, T_x); got ndim {x.ndim}")

    n_x, m, T_x = x.shape
    n_a = parameters["Wu"].shape[0]
    n_y = parameters["Wy"].shape[0]

    if a0 is None:
        a0 = np.zeros((n_a, m))
    elif a0.shape != (n_a, m):
        raise ValueError(f"a0 must have shape {(n_a, m)}; got {a0.shape}")

    a = np.zeros((n_a, m, T_x))
    y_pred = np.zeros((n_y, m, T_x))
    caches: list[dict[str, np.ndarray]] = []

    a_prev = a0
    for t in range(T_x):
        a_next, y_pred_t, cache = gru_cell_forward(x[:, :, t], a_prev, parameters)
        a[:, :, t] = a_next
        y_pred[:, :, t] = y_pred_t
        caches.append(cache)
        a_prev = a_next  # carry the state forward to t+1

    return a, y_pred, caches

lstm_backward

lstm_backward(y: ndarray, caches: list[dict[str, ndarray]], parameters: dict[str, ndarray]) -> dict[str, np.ndarray]

Backpropagation through time (BPTT) for the LSTM.

Walks the unrolled graph from t = T_x back to t = 1, threading two recurrent gradients backward -- one along the hidden state and one along the cell state. Every gate weight is shared across time, so each receives an accumulated gradient summed over all timesteps.

Parameters:

Name	Type	Description	Default
y	ndarray	One-hot targets, shape (n_y, m, T_x).	required
caches	list[dict[str, ndarray]]	Per-timestep caches from :func:lstm_forward.	required
parameters	dict[str, ndarray]	The shared weights and biases (see :func:lstm_cell_forward).	required

Returns:

Type	Description
dict[str, ndarray]	Accumulated gradients dWf, dWu, dWc, dWo, dbf, dbu, dbc, dbo, dWy, dby; the gradients da0 and dc0 w.r.t. the two initial states; and dx, the gradient w.r.t. the whole input sequence, shape (n_x, m, T_x).

Source code in src/dlhub/nn/sequence/lstm.py

def lstm_backward(
    y: np.ndarray,
    caches: list[dict[str, np.ndarray]],
    parameters: dict[str, np.ndarray],
) -> dict[str, np.ndarray]:
    """
    Backpropagation through time (BPTT) for the LSTM.

    Walks the unrolled graph from ``t = T_x`` back to ``t = 1``, threading
    **two** recurrent gradients backward -- one along the hidden state and one
    along the cell state. Every gate weight is shared across time, so each
    receives an accumulated gradient summed over all timesteps.

    Parameters
    ----------
    y : np.ndarray
        One-hot targets, shape ``(n_y, m, T_x)``.
    caches : list[dict[str, np.ndarray]]
        Per-timestep caches from :func:`lstm_forward`.
    parameters : dict[str, np.ndarray]
        The shared weights and biases (see :func:`lstm_cell_forward`).

    Returns
    -------
    dict[str, np.ndarray]
        Accumulated gradients ``dWf``, ``dWu``, ``dWc``, ``dWo``, ``dbf``,
        ``dbu``, ``dbc``, ``dbo``, ``dWy``, ``dby``; the gradients ``da0`` and
        ``dc0`` w.r.t. the two initial states; and ``dx``, the gradient w.r.t.
        the whole input sequence, shape ``(n_x, m, T_x)``.
    """
    T_x = len(caches)
    _, m, _ = y.shape
    n_x = caches[0]["x_t"].shape[0]
    n_a = parameters["Wf"].shape[0]

    accumulated = {f"d{key}": np.zeros_like(parameters[key]) for key in PARAMETER_KEYS}
    dx = np.zeros((n_x, m, T_x))

    # The two recurrence contributions into the current step, carried back one
    # step at a time. Both zero at the final timestep, which has no successor.
    da_next = np.zeros((n_a, m))
    dc_next = np.zeros((n_a, m))

    for t in reversed(range(T_x)):
        grads_t = lstm_cell_backward(
            y[:, :, t], da_next, dc_next, caches[t], parameters
        )

        # The shared weights accumulate: every timestep used the same matrix,
        # so every timestep adds a term to its gradient.
        for key in PARAMETER_KEYS:
            accumulated[f"d{key}"] += grads_t[f"d{key}"]

        dx[:, :, t] = grads_t["dxt"]
        da_next = grads_t["da_prev"]  # hand both recurrences back to t-1
        dc_next = grads_t["dc_prev"]

    # After the loop, the two recurrence gradients have reached the initial
    # states a^{<0>} and c^{<0>}.
    return {**accumulated, "da0": da_next, "dc0": dc_next, "dx": dx}

lstm_cell_backward

lstm_cell_backward(y_t: ndarray, da_next: ndarray, dc_next: ndarray, cache: dict[str, ndarray], parameters: dict[str, ndarray]) -> dict[str, np.ndarray]

Backward pass for a single LSTM timestep.

Walks the forward pass backwards, one equation at a time. Two gradients arrive from the future, not one: da_next along the hidden state and dc_next along the cell state. They meet at c^{<t>}, because the hidden state is a function of the cell state at the same step.

The line worth reading twice is dc_prev = dc_t * Gamma_f. That is the whole thesis of the architecture as one statement: the gradient crossing from t to t-1 along the cell path is scaled elementwise by the forget gate, with no weight matrix and no saturating nonlinearity in the way.

Parameters:

Name	Type	Description	Default
y_t	ndarray	One-hot target at this timestep, shape (n_y, m).	required
da_next	ndarray	Gradient w.r.t. this step's hidden state arriving from the recurrence at t+1, shape (n_a, m). Zero for the final step.	required
dc_next	ndarray	Gradient w.r.t. this step's cell state arriving from the recurrence at t+1, shape (n_a, m). Zero for the final step.	required
cache	dict[str, ndarray]	This step's cache from :func:lstm_cell_forward.	required
parameters	dict[str, ndarray]	The shared weights and biases (see :func:lstm_cell_forward).	required

Returns:

Type	Description
dict[str, ndarray]	This step's gradient contributions: dWf, dWu, dWc, dWo, dbf, dbu, dbc, dbo, dWy, dby, together with dxt (into the input) and da_prev, dc_prev (the two recurrences handed back to t-1).

Source code in src/dlhub/nn/sequence/lstm.py

def lstm_cell_backward(
    y_t: np.ndarray,
    da_next: np.ndarray,
    dc_next: np.ndarray,
    cache: dict[str, np.ndarray],
    parameters: dict[str, np.ndarray],
) -> dict[str, np.ndarray]:
    """
    Backward pass for a single LSTM timestep.

    Walks the forward pass backwards, one equation at a time. Two gradients
    arrive from the future, not one: ``da_next`` along the hidden state and
    ``dc_next`` along the cell state. They meet at ``c^{<t>}``, because the
    hidden state is a function of the cell state at the same step.

    The line worth reading twice is ``dc_prev = dc_t * Gamma_f``. That is the
    whole thesis of the architecture as one statement: the gradient crossing
    from ``t`` to ``t-1`` along the cell path is scaled elementwise by the forget
    gate, with no weight matrix and no saturating nonlinearity in the way.

    Parameters
    ----------
    y_t : np.ndarray
        One-hot target at this timestep, shape ``(n_y, m)``.
    da_next : np.ndarray
        Gradient w.r.t. this step's hidden state arriving from the recurrence at
        ``t+1``, shape ``(n_a, m)``. Zero for the final step.
    dc_next : np.ndarray
        Gradient w.r.t. this step's cell state arriving from the recurrence at
        ``t+1``, shape ``(n_a, m)``. Zero for the final step.
    cache : dict[str, np.ndarray]
        This step's cache from :func:`lstm_cell_forward`.
    parameters : dict[str, np.ndarray]
        The shared weights and biases (see :func:`lstm_cell_forward`).

    Returns
    -------
    dict[str, np.ndarray]
        This step's gradient contributions: ``dWf``, ``dWu``, ``dWc``, ``dWo``,
        ``dbf``, ``dbu``, ``dbc``, ``dbo``, ``dWy``, ``dby``, together with
        ``dxt`` (into the input) and ``da_prev``, ``dc_prev`` (the two
        recurrences handed back to ``t-1``).
    """
    a_next, c_prev = cache["a_next"], cache["c_prev"]
    gamma_f, gamma_u = cache["gamma_f"], cache["gamma_u"]
    c_tilde, gamma_o = cache["c_tilde"], cache["gamma_o"]
    tanh_c_next, concat = cache["tanh_c_next"], cache["concat"]
    y_pred_t = cache["y_pred_t"]

    Wf, Wu, Wc, Wo, Wy = (
        parameters["Wf"],
        parameters["Wu"],
        parameters["Wc"],
        parameters["Wo"],
        parameters["Wy"],
    )
    n_a, m = a_next.shape

    # --- Output path: softmax cross-entropy -------------------------------
    # For softmax + cross-entropy the logit gradient collapses to (y_hat - y),
    # scaled by the 1/m batch mean used in compute_loss.
    dz_y = (y_pred_t - y_t) / m  # (n_y, m)
    dWy = dz_y @ a_next.T  # (n_y, n_a)
    dby = np.sum(dz_y, axis=1, keepdims=True)  # (n_y, 1)
    da_from_output = Wy.T @ dz_y  # (n_a, m)

    # --- Merge the two gradient sources at a^{<t>} ------------------------
    da_t = da_from_output + da_next  # (n_a, m)

    # --- Hidden state: a^{<t>} = Gamma_o * tanh(c^{<t>}) ------------------
    dgamma_o = da_t * tanh_c_next
    # The hidden state feeds the cell state's gradient: differentiating the
    # tanh gives 1 - tanh(c)^2, read off the cached tanh. This joins the
    # gradient that arrived along the cell path from t+1.
    dc_t = dc_next + da_t * gamma_o * (1 - tanh_c_next**2)  # (n_a, m)

    # --- Cell state: c^{<t>} = Gamma_u * c_tilde + Gamma_f * c^{<t-1>} ----
    # An addition of two products, so each factor's gradient is the other.
    dgamma_u = dc_t * c_tilde
    dc_tilde = dc_t * gamma_u
    dgamma_f = dc_t * c_prev
    dc_prev = dc_t * gamma_f  # the undamped carry: d c^{<t>} / d c^{<t-1>}

    # --- Pre-activations, in the topic's gate order -----------------------
    # sigma'(z) = Gamma (1 - Gamma) and tanh'(z) = 1 - c_tilde^2, both read off
    # the cached activation rather than recomputed.
    dz_f = dgamma_f * gamma_f * (1 - gamma_f)
    dz_u = dgamma_u * gamma_u * (1 - gamma_u)
    dz_c = dc_tilde * (1 - c_tilde**2)
    dz_o = dgamma_o * gamma_o * (1 - gamma_o)

    # --- Gate weights and biases ------------------------------------------
    # Each gate read the same stacked vector [a^{<t-1>}; x^{<t>}], so each gate
    # weight's gradient is its pre-activation gradient times that vector.
    dWf, dbf = dz_f @ concat.T, np.sum(dz_f, axis=1, keepdims=True)
    dWu, dbu = dz_u @ concat.T, np.sum(dz_u, axis=1, keepdims=True)
    dWc, dbc = dz_c @ concat.T, np.sum(dz_c, axis=1, keepdims=True)
    dWo, dbo = dz_o @ concat.T, np.sum(dz_o, axis=1, keepdims=True)

    # --- Back through the concatenation -----------------------------------
    # All four gates read the same vector, so their contributions add; the top
    # n_a rows are the recurrence into a^{<t-1>} and the rest is the input.
    dconcat = Wf.T @ dz_f + Wu.T @ dz_u + Wc.T @ dz_c + Wo.T @ dz_o
    da_prev = dconcat[:n_a, :]  # (n_a, m)
    dxt = dconcat[n_a:, :]  # (n_x, m)

    return {
        "dWf": dWf,
        "dWu": dWu,
        "dWc": dWc,
        "dWo": dWo,
        "dbf": dbf,
        "dbu": dbu,
        "dbc": dbc,
        "dbo": dbo,
        "dWy": dWy,
        "dby": dby,
        "dxt": dxt,
        "da_prev": da_prev,
        "dc_prev": dc_prev,
    }

lstm_cell_forward

lstm_cell_forward(x_t: ndarray, a_prev: ndarray, c_prev: ndarray, parameters: dict[str, ndarray]) -> tuple[np.ndarray, np.ndarray, np.ndarray, dict[str, np.ndarray]]

One timestep of the LSTM forward recurrence.

Computes the four gate quantities in the topic's order (forget, update, candidate, output), then the cell state, the hidden state, and the softmax read-out.

Parameters:

Name	Type	Description	Default
x_t	ndarray	Input at this timestep, shape (n_x, m).	required
a_prev	ndarray	Hidden state carried in from the previous step, a^{<t-1>}, shape (n_a, m).	required
c_prev	ndarray	Cell state carried in from the previous step, c^{<t-1>}, shape (n_a, m).	required
parameters	dict[str, ndarray]	The shared weights and biases: Wf, Wu, Wc, Wo each (n_a, n_a + n_x); bf, bu, bc, bo each (n_a, 1); Wy (n_y, n_a); by (n_y, 1).	required

Returns:

Name	Type	Description
a_next	ndarray	Updated hidden state a^{<t>}, shape (n_a, m).
c_next	ndarray	Updated cell state c^{<t>}, shape (n_a, m).
y_pred_t	ndarray	Output distribution y_hat^{<t>}, shape (n_y, m).
cache	dict[str, ndarray]	Everything this step's backward pass needs, each array named.

Source code in src/dlhub/nn/sequence/lstm.py

def lstm_cell_forward(
    x_t: np.ndarray,
    a_prev: np.ndarray,
    c_prev: np.ndarray,
    parameters: dict[str, np.ndarray],
) -> tuple[np.ndarray, np.ndarray, np.ndarray, dict[str, np.ndarray]]:
    """
    One timestep of the LSTM forward recurrence.

    Computes the four gate quantities in the topic's order (forget, update,
    candidate, output), then the cell state, the hidden state, and the softmax
    read-out.

    Parameters
    ----------
    x_t : np.ndarray
        Input at this timestep, shape ``(n_x, m)``.
    a_prev : np.ndarray
        Hidden state carried in from the previous step, ``a^{<t-1>}``, shape
        ``(n_a, m)``.
    c_prev : np.ndarray
        Cell state carried in from the previous step, ``c^{<t-1>}``, shape
        ``(n_a, m)``.
    parameters : dict[str, np.ndarray]
        The shared weights and biases: ``Wf``, ``Wu``, ``Wc``, ``Wo`` each
        ``(n_a, n_a + n_x)``; ``bf``, ``bu``, ``bc``, ``bo`` each ``(n_a, 1)``;
        ``Wy`` ``(n_y, n_a)``; ``by`` ``(n_y, 1)``.

    Returns
    -------
    a_next : np.ndarray
        Updated hidden state ``a^{<t>}``, shape ``(n_a, m)``.
    c_next : np.ndarray
        Updated cell state ``c^{<t>}``, shape ``(n_a, m)``.
    y_pred_t : np.ndarray
        Output distribution ``y_hat^{<t>}``, shape ``(n_y, m)``.
    cache : dict[str, np.ndarray]
        Everything this step's backward pass needs, each array named.
    """
    Wf, Wu, Wc, Wo = (
        parameters["Wf"],
        parameters["Wu"],
        parameters["Wc"],
        parameters["Wo"],
    )
    bf, bu, bc, bo = (
        parameters["bf"],
        parameters["bu"],
        parameters["bc"],
        parameters["bo"],
    )
    Wy, by = parameters["Wy"], parameters["by"]

    # The concatenated form: one stacked vector [a^{<t-1>}; x^{<t>}], shape
    # (n_a + n_x, m), that all four gate weights multiply. Unlike the GRU, no
    # gate is applied inside this concatenation -- the LSTM has no reset gate.
    concat = np.concatenate([a_prev, x_t], axis=0)

    # --- The four gate quantities, in the topic's order -------------------
    gamma_f = sigmoid(Wf @ concat + bf)  # forget: how much of c^{<t-1>} survives
    gamma_u = sigmoid(Wu @ concat + bu)  # update ("input"): how much candidate enters
    c_tilde = np.tanh(Wc @ concat + bc)  # candidate: what could be written
    gamma_o = sigmoid(Wo @ concat + bo)  # output: how much of the cell is exposed

    # --- Cell state: the additive path ------------------------------------
    # Two gated terms are *added*. Nothing multiplies c^{<t-1>} but an
    # elementwise gate, which is the whole architectural point (module Notes).
    c_next = gamma_u * c_tilde + gamma_f * c_prev

    # --- Hidden state: a squashed, gated view of the cell -----------------
    tanh_c_next = np.tanh(c_next)
    a_next = gamma_o * tanh_c_next

    # --- Read-out ---------------------------------------------------------
    y_pred_t = softmax(Wy @ a_next + by)

    cache = {
        "a_next": a_next,
        "c_next": c_next,
        "a_prev": a_prev,
        "c_prev": c_prev,
        "gamma_f": gamma_f,
        "gamma_u": gamma_u,
        "c_tilde": c_tilde,
        "gamma_o": gamma_o,
        "tanh_c_next": tanh_c_next,
        "concat": concat,
        "x_t": x_t,
        "y_pred_t": y_pred_t,
    }
    return a_next, c_next, y_pred_t, cache

lstm_forward

lstm_forward(x: ndarray, parameters: dict[str, ndarray], a0: ndarray | None = None, c0: ndarray | None = None) -> tuple[np.ndarray, np.ndarray, np.ndarray, list[dict[str, np.ndarray]]]

Forward propagation through time across a whole sequence.

Unrolls :func:lstm_cell_forward over T_x timesteps, threading two states from one step to the next: the hidden state a and the cell state c.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence of shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	The shared weights and biases (see :func:lstm_cell_forward).	required
a0	ndarray or None	Initial hidden state a^{<0>} of shape (n_a, m). Defaults to the zero vector.	None
c0	ndarray or None	Initial cell state c^{<0>} of shape (n_a, m). Defaults to the zero vector.	None

Returns:

Name	Type	Description
a	ndarray	All hidden states stacked over time, shape (n_a, m, T_x).
c	ndarray	All cell states stacked over time, shape (n_a, m, T_x).
y_pred	ndarray	All output distributions, shape (n_y, m, T_x).
caches	list[dict[str, ndarray]]	Per-timestep caches, in order, for :func:lstm_backward.

Raises:

Type	Description
ValueError	If x is not three-dimensional, or if a supplied a0 or c0 does not have shape (n_a, m).

Source code in src/dlhub/nn/sequence/lstm.py

def lstm_forward(
    x: np.ndarray,
    parameters: dict[str, np.ndarray],
    a0: np.ndarray | None = None,
    c0: np.ndarray | None = None,
) -> tuple[np.ndarray, np.ndarray, np.ndarray, list[dict[str, np.ndarray]]]:
    """
    Forward propagation through time across a whole sequence.

    Unrolls :func:`lstm_cell_forward` over ``T_x`` timesteps, threading **two**
    states from one step to the next: the hidden state ``a`` and the cell state
    ``c``.

    Parameters
    ----------
    x : np.ndarray
        Input sequence of shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        The shared weights and biases (see :func:`lstm_cell_forward`).
    a0 : np.ndarray or None, optional
        Initial hidden state ``a^{<0>}`` of shape ``(n_a, m)``. Defaults to the
        zero vector.
    c0 : np.ndarray or None, optional
        Initial cell state ``c^{<0>}`` of shape ``(n_a, m)``. Defaults to the
        zero vector.

    Returns
    -------
    a : np.ndarray
        All hidden states stacked over time, shape ``(n_a, m, T_x)``.
    c : np.ndarray
        All cell states stacked over time, shape ``(n_a, m, T_x)``.
    y_pred : np.ndarray
        All output distributions, shape ``(n_y, m, T_x)``.
    caches : list[dict[str, np.ndarray]]
        Per-timestep caches, in order, for :func:`lstm_backward`.

    Raises
    ------
    ValueError
        If ``x`` is not three-dimensional, or if a supplied ``a0`` or ``c0`` does
        not have shape ``(n_a, m)``.
    """
    if x.ndim != 3:
        raise ValueError(f"x must have shape (n_x, m, T_x); got ndim {x.ndim}")

    n_x, m, T_x = x.shape
    n_a = parameters["Wf"].shape[0]
    n_y = parameters["Wy"].shape[0]

    if a0 is None:
        a0 = np.zeros((n_a, m))
    elif a0.shape != (n_a, m):
        raise ValueError(f"a0 must have shape {(n_a, m)}; got {a0.shape}")

    if c0 is None:
        c0 = np.zeros((n_a, m))
    elif c0.shape != (n_a, m):
        raise ValueError(f"c0 must have shape {(n_a, m)}; got {c0.shape}")

    a = np.zeros((n_a, m, T_x))
    c = np.zeros((n_a, m, T_x))
    y_pred = np.zeros((n_y, m, T_x))
    caches: list[dict[str, np.ndarray]] = []

    a_prev, c_prev = a0, c0
    for t in range(T_x):
        a_next, c_next, y_pred_t, cache = lstm_cell_forward(
            x[:, :, t], a_prev, c_prev, parameters
        )
        a[:, :, t] = a_next
        c[:, :, t] = c_next
        y_pred[:, :, t] = y_pred_t
        caches.append(cache)
        a_prev, c_prev = a_next, c_next  # carry both states forward to t+1

    return a, c, y_pred, caches

g

g(k: ndarray) -> np.ndarray

The strictly positive candidate activation of Sec. B.2.1.

::

g(k) = k + 0.5      for k >= 0
     = sigma(k)     for k <  0

Continuous at zero (both branches give 0.5), positive everywhere, and still slope-1 linear on the positive half -- which is what lets the log-space scan take log(beta) at all.

Parameters:

Name	Type	Description	Default
k	ndarray	The candidate pre-activation W_c x.	required

Returns:

Type	Description
ndarray	Strictly positive values, same shape as k.

Source code in src/dlhub/nn/sequence/min_gated.py

def g(k: np.ndarray) -> np.ndarray:
    """
    The strictly positive candidate activation of Sec. B.2.1.

    ::

        g(k) = k + 0.5      for k >= 0
             = sigma(k)     for k <  0

    Continuous at zero (both branches give ``0.5``), positive everywhere, and
    still slope-1 linear on the positive half -- which is what lets the
    log-space scan take ``log(beta)`` at all.

    Parameters
    ----------
    k : np.ndarray
        The candidate pre-activation ``W_c x``.

    Returns
    -------
    np.ndarray
        Strictly positive values, same shape as ``k``.
    """
    return np.where(k >= 0.0, k + 0.5, sigmoid(k))

identity_candidate

identity_candidate(k: ndarray) -> np.ndarray

The minimal cells' candidate activation: the identity (Sec. 3.1.2).

The classical GRU and LSTM squash their candidate with tanh. The minimal cells do not: the candidate is the plain linear map a_tilde^{<t>} = W_c x^{<t>}. This function exists so the deletion has a name and a docstring, rather than being an absence a later reader restores by accident.

Parameters:

Name	Type	Description	Default
k	ndarray	The candidate pre-activation W_c x.	required

Returns:

Type	Description
ndarray	k itself, unchanged.

Source code in src/dlhub/nn/sequence/min_gated.py

def identity_candidate(k: np.ndarray) -> np.ndarray:
    """
    The minimal cells' candidate activation: the identity (Sec. 3.1.2).

    The classical GRU and LSTM squash their candidate with ``tanh``. The
    minimal cells **do not**: the candidate is the plain linear map
    ``a_tilde^{<t>} = W_c x^{<t>}``. This function exists so the deletion has a
    name and a docstring, rather than being an absence a later reader restores
    by accident.

    Parameters
    ----------
    k : np.ndarray
        The candidate pre-activation ``W_c x``.

    Returns
    -------
    np.ndarray
        ``k`` itself, unchanged.
    """
    return k

log_g

log_g(k: ndarray) -> np.ndarray

log(g(k)), computed without forming g(k) first (Sec. B.2.1).

::

log g(k) = log(k + 0.5)     for k >= 0
         = -softplus(-k)    for k <  0

The negative branch reuses the identity log sigma(k) = -softplus(-k), so no logarithm of an underflowed sigmoid is ever taken.

Parameters:

Name	Type	Description	Default
k	ndarray	The candidate pre-activation W_c x.	required

Returns:

Type	Description
ndarray	log(g(k)), same shape as k.

Source code in src/dlhub/nn/sequence/min_gated.py

def log_g(k: np.ndarray) -> np.ndarray:
    """
    ``log(g(k))``, computed without forming ``g(k)`` first (Sec. B.2.1).

    ::

        log g(k) = log(k + 0.5)     for k >= 0
                 = -softplus(-k)    for k <  0

    The negative branch reuses the identity ``log sigma(k) = -softplus(-k)``,
    so no logarithm of an underflowed sigmoid is ever taken.

    Parameters
    ----------
    k : np.ndarray
        The candidate pre-activation ``W_c x``.

    Returns
    -------
    np.ndarray
        ``log(g(k))``, same shape as ``k``.
    """
    # The inner `where` keeps `log` away from the negative branch's inputs, so
    # the unused half of the expression cannot emit an invalid-value warning.
    positive_branch = np.where(k >= 0.0, k + 0.5, 1.0)
    return np.where(k >= 0.0, np.log(positive_branch), -softplus(-k))

log_space_scan

log_space_scan(log_alpha: ndarray, log_beta: ndarray, a0: ndarray | None = None) -> np.ndarray

Resolve the recurrence in log space -- the stable scan of Sec. B.1.

Writing A_t = sum_{i <= t} log alpha_i for the cumulative log-decay, the closed form of the recurrence is::

a^{<t>} = exp( A_t + log( sum_{j <= t} exp(log beta_j - A_j) ) )

with the initial state entering as the j = 0 term (coefficient one, hence log alpha_0 = 0). Every product has become a sum and every sum a logaddexp, so nothing underflows on the way. This is Heinsen's (2023) formulation, which the paper adopts.

Parameters:

Name	Type	Description	Default
log_alpha	ndarray	Logs of the per-step coefficients, each (n_a, m, T_x). Both require positive coefficients, which is why the candidate becomes :func:g.	required
log_beta	ndarray	Logs of the per-step coefficients, each (n_a, m, T_x). Both require positive coefficients, which is why the candidate becomes :func:g.	required
a0	ndarray or None	Initial state a^{<0>}, shape (n_a, m), and non-negative: the log of a negative state does not exist. Defaults to zero.	None

Returns:

Type	Description
ndarray	The hidden states, shape (n_a, m, T_x), all strictly positive.

Raises:

Type	Description
ValueError	If the two coefficient arrays disagree in shape, if a0 is misshapen, or if any entry of a0 is negative.

Source code in src/dlhub/nn/sequence/min_gated.py

def log_space_scan(
    log_alpha: np.ndarray, log_beta: np.ndarray, a0: np.ndarray | None = None
) -> np.ndarray:
    """
    Resolve the recurrence in log space -- the stable scan of Sec. B.1.

    Writing ``A_t = sum_{i <= t} log alpha_i`` for the cumulative log-decay, the
    closed form of the recurrence is::

        a^{<t>} = exp( A_t + log( sum_{j <= t} exp(log beta_j - A_j) ) )

    with the initial state entering as the ``j = 0`` term (coefficient one,
    hence ``log alpha_0 = 0``). Every product has become a sum and every sum a
    ``logaddexp``, so nothing underflows on the way. This is Heinsen's (2023)
    formulation, which the paper adopts.

    Parameters
    ----------
    log_alpha, log_beta : np.ndarray
        Logs of the per-step coefficients, each ``(n_a, m, T_x)``. Both require
        positive coefficients, which is why the candidate becomes :func:`g`.
    a0 : np.ndarray or None, optional
        Initial state ``a^{<0>}``, shape ``(n_a, m)``, and **non-negative**:
        the log of a negative state does not exist. Defaults to zero.

    Returns
    -------
    np.ndarray
        The hidden states, shape ``(n_a, m, T_x)``, all strictly positive.

    Raises
    ------
    ValueError
        If the two coefficient arrays disagree in shape, if ``a0`` is misshapen,
        or if any entry of ``a0`` is negative.
    """
    if log_alpha.shape != log_beta.shape:
        raise ValueError(
            f"log_alpha {log_alpha.shape} and log_beta {log_beta.shape} must match"
        )

    n_a, m, T_x = log_alpha.shape
    a_start = _resolve_initial_state(a0, n_a, m)
    if np.any(a_start < 0):
        raise ValueError("log_space_scan needs a non-negative a0; see module Notes")

    # Prepend the initial state as step 0: it is injected with coefficient one,
    # so its log-coefficient is zero and its log-injection is log(a0).
    zero_log_alpha = np.zeros((n_a, m, 1))
    with np.errstate(divide="ignore"):  # a0 == 0 is the default, and log 0 = -inf
        log_a_start = np.log(a_start)[:, :, np.newaxis]

    padded_log_alpha = np.concatenate([zero_log_alpha, log_alpha], axis=2)
    padded_log_beta = np.concatenate([log_a_start, log_beta], axis=2)

    cumulative_log_alpha = np.cumsum(padded_log_alpha, axis=2)  # A_t
    log_a = cumulative_log_alpha + logcumsumexp(padded_log_beta - cumulative_log_alpha)

    return np.exp(log_a[:, :, 1:])  # drop the prepended step 0

logcumsumexp

logcumsumexp(z: ndarray) -> np.ndarray

Cumulative log(sum(exp(.))) along the last (time) axis.

Built with the same stride-doubling as :func:parallel_scan, with np.logaddexp -- itself associative -- in place of affine composition. NumPy has no built-in for this, and the naive log(cumsum(exp(z))) overflows exactly where a log-space scan is worth having.

Parameters:

Name	Type	Description	Default
z	ndarray	Log-domain values, shape (..., T).	required

Returns:

Type	Description
ndarray	out[..., t] = log(sum_{i <= t} exp(z[..., i])), same shape as z.

Source code in src/dlhub/nn/sequence/min_gated.py

def logcumsumexp(z: np.ndarray) -> np.ndarray:
    """
    Cumulative ``log(sum(exp(.)))`` along the last (time) axis.

    Built with the same stride-doubling as :func:`parallel_scan`, with
    ``np.logaddexp`` -- itself associative -- in place of affine composition.
    NumPy has no built-in for this, and the naive ``log(cumsum(exp(z)))``
    overflows exactly where a log-space scan is worth having.

    Parameters
    ----------
    z : np.ndarray
        Log-domain values, shape ``(..., T)``.

    Returns
    -------
    np.ndarray
        ``out[..., t] = log(sum_{i <= t} exp(z[..., i]))``, same shape as ``z``.
    """
    T = z.shape[-1]
    out = z.copy()

    stride = 1
    while stride < T:
        combined = np.logaddexp(out[..., stride:], out[..., :-stride])
        out = np.concatenate([out[..., :stride], combined], axis=-1)
        stride *= 2
    return out

min_gru_coefficients

min_gru_coefficients(x: ndarray, parameters: dict[str, ndarray], candidate: Callable[[ndarray], ndarray] = identity_candidate) -> tuple[np.ndarray, np.ndarray]

The minGRU recurrence coefficients (alpha, beta) (Sec. 3.1.3).

::

Gamma_u = sigma(W_u x)
a_tilde = candidate(W_c x)
alpha   = 1 - Gamma_u
beta    = Gamma_u * a_tilde

so that a^{<t>} = alpha^{<t>} * a^{<t-1>} + beta^{<t>} reproduces the cell's blend. There is no reset gate: the candidate never reads a^{<t-1>}.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	Wu and Wc, each (n_a, n_x).	required
candidate	Callable[[ndarray], ndarray]	Candidate activation applied to W_c x. Defaults to :func:identity_candidate, the paper's minimal form. Pass :func:g to match :func:log_space_scan's positivity requirement, or np.tanh to recover the classical GRU's candidate.	identity_candidate

Returns:

Name	Type	Description
alpha	ndarray	Per-step decay 1 - Gamma_u, shape (n_a, m, T_x).
beta	ndarray	Per-step injection Gamma_u * a_tilde, shape (n_a, m, T_x).

Source code in src/dlhub/nn/sequence/min_gated.py

def min_gru_coefficients(
    x: np.ndarray,
    parameters: dict[str, np.ndarray],
    candidate: Callable[[np.ndarray], np.ndarray] = identity_candidate,
) -> tuple[np.ndarray, np.ndarray]:
    """
    The minGRU recurrence coefficients ``(alpha, beta)`` (Sec. 3.1.3).

    ::

        Gamma_u = sigma(W_u x)
        a_tilde = candidate(W_c x)
        alpha   = 1 - Gamma_u
        beta    = Gamma_u * a_tilde

    so that ``a^{<t>} = alpha^{<t>} * a^{<t-1>} + beta^{<t>}`` reproduces the
    cell's blend. There is no reset gate: the candidate never reads
    ``a^{<t-1>}``.

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        ``Wu`` and ``Wc``, each ``(n_a, n_x)``.
    candidate : Callable[[np.ndarray], np.ndarray], optional
        Candidate activation applied to ``W_c x``. Defaults to
        :func:`identity_candidate`, the paper's minimal form. Pass :func:`g` to
        match :func:`log_space_scan`'s positivity requirement, or ``np.tanh``
        to recover the classical GRU's candidate.

    Returns
    -------
    alpha : np.ndarray
        Per-step decay ``1 - Gamma_u``, shape ``(n_a, m, T_x)``.
    beta : np.ndarray
        Per-step injection ``Gamma_u * a_tilde``, shape ``(n_a, m, T_x)``.
    """
    k_u = project_over_time(parameters["Wu"], x)  # (n_a, m, T_x)
    k_c = project_over_time(parameters["Wc"], x)  # (n_a, m, T_x)

    gamma_u = sigmoid(k_u)  # update gate
    a_tilde = candidate(k_c)  # candidate state -- linear by default

    alpha = 1.0 - gamma_u  # weight on a^{<t-1>}
    beta = gamma_u * a_tilde  # weight on the candidate
    return alpha, beta

min_gru_forward_log

min_gru_forward_log(x: ndarray, parameters: dict[str, ndarray], a0: ndarray | None = None) -> np.ndarray

Forward pass for minGRU through the log-space scan (Sec. B.1).

This is the positive-candidate cell: the candidate is :func:g, not the identity, because a log-space scan needs beta > 0. It therefore matches :func:min_gru_forward_sequential called with candidate=g, and not the default linear one.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	Wu and Wc, each (n_a, n_x).	required
a0	ndarray or None	Non-negative initial state, shape (n_a, m). Defaults to zero.	None

Returns:

Type	Description
ndarray	Hidden states a, shape (n_a, m, T_x), strictly positive.

Source code in src/dlhub/nn/sequence/min_gated.py

def min_gru_forward_log(
    x: np.ndarray, parameters: dict[str, np.ndarray], a0: np.ndarray | None = None
) -> np.ndarray:
    """
    Forward pass for minGRU through the log-space scan (Sec. B.1).

    This is the *positive-candidate* cell: the candidate is :func:`g`, not the
    identity, because a log-space scan needs ``beta > 0``. It therefore matches
    :func:`min_gru_forward_sequential` called with ``candidate=g``, and not the
    default linear one.

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        ``Wu`` and ``Wc``, each ``(n_a, n_x)``.
    a0 : np.ndarray or None, optional
        Non-negative initial state, shape ``(n_a, m)``. Defaults to zero.

    Returns
    -------
    np.ndarray
        Hidden states ``a``, shape ``(n_a, m, T_x)``, strictly positive.
    """
    log_alpha, log_beta = min_gru_log_coefficients(x, parameters)
    return log_space_scan(log_alpha, log_beta, a0)

min_gru_forward_parallel

min_gru_forward_parallel(x: ndarray, parameters: dict[str, ndarray], a0: ndarray | None = None, candidate: Callable[[ndarray], ndarray] = identity_candidate) -> np.ndarray

Forward pass for minGRU, resolved by the plain-space associative scan.

Same function as :func:min_gru_forward_sequential, evaluated in a different association order.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	Wu and Wc, each (n_a, n_x).	required
a0	ndarray or None	Initial state, shape (n_a, m). Defaults to zero.	None
candidate	Callable[[ndarray], ndarray]	Candidate activation, default :func:identity_candidate.	identity_candidate

Returns:

Type	Description
ndarray	Hidden states a, shape (n_a, m, T_x).

Source code in src/dlhub/nn/sequence/min_gated.py

def min_gru_forward_parallel(
    x: np.ndarray,
    parameters: dict[str, np.ndarray],
    a0: np.ndarray | None = None,
    candidate: Callable[[np.ndarray], np.ndarray] = identity_candidate,
) -> np.ndarray:
    """
    Forward pass for minGRU, resolved by the plain-space associative scan.

    Same function as :func:`min_gru_forward_sequential`, evaluated in a
    different association order.

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        ``Wu`` and ``Wc``, each ``(n_a, n_x)``.
    a0 : np.ndarray or None, optional
        Initial state, shape ``(n_a, m)``. Defaults to zero.
    candidate : Callable[[np.ndarray], np.ndarray], optional
        Candidate activation, default :func:`identity_candidate`.

    Returns
    -------
    np.ndarray
        Hidden states ``a``, shape ``(n_a, m, T_x)``.
    """
    alpha, beta = min_gru_coefficients(x, parameters, candidate)
    return parallel_scan(alpha, beta, a0)

min_gru_forward_sequential

min_gru_forward_sequential(x: ndarray, parameters: dict[str, ndarray], a0: ndarray | None = None, candidate: Callable[[ndarray], ndarray] = identity_candidate) -> np.ndarray

Forward pass for minGRU, resolved by the serial recurrence.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	Wu and Wc, each (n_a, n_x).	required
a0	ndarray or None	Initial state, shape (n_a, m). Defaults to zero.	None
candidate	Callable[[ndarray], ndarray]	Candidate activation, default :func:identity_candidate.	identity_candidate

Returns:

Type	Description
ndarray	Hidden states a, shape (n_a, m, T_x).

Source code in src/dlhub/nn/sequence/min_gated.py

def min_gru_forward_sequential(
    x: np.ndarray,
    parameters: dict[str, np.ndarray],
    a0: np.ndarray | None = None,
    candidate: Callable[[np.ndarray], np.ndarray] = identity_candidate,
) -> np.ndarray:
    """
    Forward pass for minGRU, resolved by the serial recurrence.

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        ``Wu`` and ``Wc``, each ``(n_a, n_x)``.
    a0 : np.ndarray or None, optional
        Initial state, shape ``(n_a, m)``. Defaults to zero.
    candidate : Callable[[np.ndarray], np.ndarray], optional
        Candidate activation, default :func:`identity_candidate`.

    Returns
    -------
    np.ndarray
        Hidden states ``a``, shape ``(n_a, m, T_x)``.
    """
    alpha, beta = min_gru_coefficients(x, parameters, candidate)
    return sequential_scan(alpha, beta, a0)

min_gru_log_coefficients

min_gru_log_coefficients(x: ndarray, parameters: dict[str, ndarray]) -> tuple[np.ndarray, np.ndarray]

The minGRU coefficients in log space (Sec. B.1), using :func:g.

Takes the gate logarithms straight from the pre-activation k_u = W_u x, never forming the gate itself::

log alpha = log(1 - Gamma_u) = -softplus(k_u)
log beta  = log Gamma_u + log g(k_c) = -softplus(-k_u) + log_g(k_c)

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	Wu and Wc, each (n_a, n_x).	required

Returns:

Name	Type	Description
log_alpha	ndarray	log(alpha), shape (n_a, m, T_x).
log_beta	ndarray	log(beta), shape (n_a, m, T_x).

Source code in src/dlhub/nn/sequence/min_gated.py

def min_gru_log_coefficients(
    x: np.ndarray, parameters: dict[str, np.ndarray]
) -> tuple[np.ndarray, np.ndarray]:
    """
    The minGRU coefficients in log space (Sec. B.1), using :func:`g`.

    Takes the gate logarithms straight from the pre-activation
    ``k_u = W_u x``, never forming the gate itself::

        log alpha = log(1 - Gamma_u) = -softplus(k_u)
        log beta  = log Gamma_u + log g(k_c) = -softplus(-k_u) + log_g(k_c)

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        ``Wu`` and ``Wc``, each ``(n_a, n_x)``.

    Returns
    -------
    log_alpha : np.ndarray
        ``log(alpha)``, shape ``(n_a, m, T_x)``.
    log_beta : np.ndarray
        ``log(beta)``, shape ``(n_a, m, T_x)``.
    """
    k_u = project_over_time(parameters["Wu"], x)
    k_c = project_over_time(parameters["Wc"], x)

    log_alpha = -softplus(k_u)  # log(1 - Gamma_u)
    log_beta = -softplus(-k_u) + log_g(k_c)  # log Gamma_u + log g(k_c)
    return log_alpha, log_beta

min_lstm_coefficients

min_lstm_coefficients(x: ndarray, parameters: dict[str, ndarray], candidate: Callable[[ndarray], ndarray] = identity_candidate) -> tuple[np.ndarray, np.ndarray]

The minLSTM recurrence coefficients (alpha, beta) (Sec. 3.2.4).

::

Gamma_f  = sigma(W_f x)
Gamma_u  = sigma(W_u x)
a_tilde  = candidate(W_c x)
Gamma_f' = Gamma_f / (Gamma_f + Gamma_u)
Gamma_u' = Gamma_u / (Gamma_f + Gamma_u)
alpha    = Gamma_f'
beta     = Gamma_u' * a_tilde

The normalisation is what replaces the missing output gate: because Gamma_f' + Gamma_u' = 1 exactly, the state is a convex blend at every step and so keeps a time-independent scale.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	Wf, Wu, and Wc, each (n_a, n_x).	required
candidate	Callable[[ndarray], ndarray]	Candidate activation applied to W_c x. Defaults to :func:identity_candidate.	identity_candidate

Returns:

Name	Type	Description
alpha	ndarray	The normalised forget gate Gamma_f', shape (n_a, m, T_x).
beta	ndarray	Gamma_u' * a_tilde, shape (n_a, m, T_x).

Source code in src/dlhub/nn/sequence/min_gated.py

def min_lstm_coefficients(
    x: np.ndarray,
    parameters: dict[str, np.ndarray],
    candidate: Callable[[np.ndarray], np.ndarray] = identity_candidate,
) -> tuple[np.ndarray, np.ndarray]:
    """
    The minLSTM recurrence coefficients ``(alpha, beta)`` (Sec. 3.2.4).

    ::

        Gamma_f  = sigma(W_f x)
        Gamma_u  = sigma(W_u x)
        a_tilde  = candidate(W_c x)
        Gamma_f' = Gamma_f / (Gamma_f + Gamma_u)
        Gamma_u' = Gamma_u / (Gamma_f + Gamma_u)
        alpha    = Gamma_f'
        beta     = Gamma_u' * a_tilde

    The normalisation is what replaces the missing output gate: because
    ``Gamma_f' + Gamma_u' = 1`` exactly, the state is a convex blend at every
    step and so keeps a time-independent scale.

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        ``Wf``, ``Wu``, and ``Wc``, each ``(n_a, n_x)``.
    candidate : Callable[[np.ndarray], np.ndarray], optional
        Candidate activation applied to ``W_c x``. Defaults to
        :func:`identity_candidate`.

    Returns
    -------
    alpha : np.ndarray
        The normalised forget gate ``Gamma_f'``, shape ``(n_a, m, T_x)``.
    beta : np.ndarray
        ``Gamma_u' * a_tilde``, shape ``(n_a, m, T_x)``.
    """
    k_f = project_over_time(parameters["Wf"], x)  # (n_a, m, T_x)
    k_u = project_over_time(parameters["Wu"], x)  # (n_a, m, T_x)
    k_c = project_over_time(parameters["Wc"], x)  # (n_a, m, T_x)

    gamma_f = sigmoid(k_f)  # forget gate
    gamma_u = sigmoid(k_u)  # update (input) gate
    a_tilde = candidate(k_c)  # candidate state -- linear by default

    # Normalise the two gates so they sum to one. No epsilon: the denominator
    # is a sum of two strictly positive sigmoids (see the module Notes).
    normaliser = gamma_f + gamma_u
    gamma_f_prime = gamma_f / normaliser
    gamma_u_prime = gamma_u / normaliser

    alpha = gamma_f_prime
    beta = gamma_u_prime * a_tilde
    return alpha, beta

min_lstm_forward_log

min_lstm_forward_log(x: ndarray, parameters: dict[str, ndarray], a0: ndarray | None = None) -> np.ndarray

Forward pass for minLSTM through the log-space scan (Sec. B.1).

As with :func:min_gru_forward_log, the candidate is :func:g, so this matches :func:min_lstm_forward_sequential called with candidate=g.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	Wf, Wu, and Wc, each (n_a, n_x).	required
a0	ndarray or None	Non-negative initial state, shape (n_a, m). Defaults to zero.	None

Returns:

Type	Description
ndarray	Hidden states a, shape (n_a, m, T_x), strictly positive.

Source code in src/dlhub/nn/sequence/min_gated.py

def min_lstm_forward_log(
    x: np.ndarray, parameters: dict[str, np.ndarray], a0: np.ndarray | None = None
) -> np.ndarray:
    """
    Forward pass for minLSTM through the log-space scan (Sec. B.1).

    As with :func:`min_gru_forward_log`, the candidate is :func:`g`, so this
    matches :func:`min_lstm_forward_sequential` called with ``candidate=g``.

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        ``Wf``, ``Wu``, and ``Wc``, each ``(n_a, n_x)``.
    a0 : np.ndarray or None, optional
        Non-negative initial state, shape ``(n_a, m)``. Defaults to zero.

    Returns
    -------
    np.ndarray
        Hidden states ``a``, shape ``(n_a, m, T_x)``, strictly positive.
    """
    log_alpha, log_beta = min_lstm_log_coefficients(x, parameters)
    return log_space_scan(log_alpha, log_beta, a0)

min_lstm_forward_parallel

min_lstm_forward_parallel(x: ndarray, parameters: dict[str, ndarray], a0: ndarray | None = None, candidate: Callable[[ndarray], ndarray] = identity_candidate) -> np.ndarray

Forward pass for minLSTM, resolved by the plain-space associative scan.

Same function as :func:min_lstm_forward_sequential, evaluated in a different association order.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	Wf, Wu, and Wc, each (n_a, n_x).	required
a0	ndarray or None	Initial state, shape (n_a, m). Defaults to zero.	None
candidate	Callable[[ndarray], ndarray]	Candidate activation, default :func:identity_candidate.	identity_candidate

Returns:

Type	Description
ndarray	Hidden states a, shape (n_a, m, T_x).

Source code in src/dlhub/nn/sequence/min_gated.py

def min_lstm_forward_parallel(
    x: np.ndarray,
    parameters: dict[str, np.ndarray],
    a0: np.ndarray | None = None,
    candidate: Callable[[np.ndarray], np.ndarray] = identity_candidate,
) -> np.ndarray:
    """
    Forward pass for minLSTM, resolved by the plain-space associative scan.

    Same function as :func:`min_lstm_forward_sequential`, evaluated in a
    different association order.

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        ``Wf``, ``Wu``, and ``Wc``, each ``(n_a, n_x)``.
    a0 : np.ndarray or None, optional
        Initial state, shape ``(n_a, m)``. Defaults to zero.
    candidate : Callable[[np.ndarray], np.ndarray], optional
        Candidate activation, default :func:`identity_candidate`.

    Returns
    -------
    np.ndarray
        Hidden states ``a``, shape ``(n_a, m, T_x)``.
    """
    alpha, beta = min_lstm_coefficients(x, parameters, candidate)
    return parallel_scan(alpha, beta, a0)

min_lstm_forward_sequential

min_lstm_forward_sequential(x: ndarray, parameters: dict[str, ndarray], a0: ndarray | None = None, candidate: Callable[[ndarray], ndarray] = identity_candidate) -> np.ndarray

Forward pass for minLSTM, resolved by the serial recurrence.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	Wf, Wu, and Wc, each (n_a, n_x).	required
a0	ndarray or None	Initial state, shape (n_a, m). Defaults to zero.	None
candidate	Callable[[ndarray], ndarray]	Candidate activation, default :func:identity_candidate.	identity_candidate

Returns:

Type	Description
ndarray	Hidden states a, shape (n_a, m, T_x).

Source code in src/dlhub/nn/sequence/min_gated.py

def min_lstm_forward_sequential(
    x: np.ndarray,
    parameters: dict[str, np.ndarray],
    a0: np.ndarray | None = None,
    candidate: Callable[[np.ndarray], np.ndarray] = identity_candidate,
) -> np.ndarray:
    """
    Forward pass for minLSTM, resolved by the serial recurrence.

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        ``Wf``, ``Wu``, and ``Wc``, each ``(n_a, n_x)``.
    a0 : np.ndarray or None, optional
        Initial state, shape ``(n_a, m)``. Defaults to zero.
    candidate : Callable[[np.ndarray], np.ndarray], optional
        Candidate activation, default :func:`identity_candidate`.

    Returns
    -------
    np.ndarray
        Hidden states ``a``, shape ``(n_a, m, T_x)``.
    """
    alpha, beta = min_lstm_coefficients(x, parameters, candidate)
    return sequential_scan(alpha, beta, a0)

min_lstm_log_coefficients

min_lstm_log_coefficients(x: ndarray, parameters: dict[str, ndarray]) -> tuple[np.ndarray, np.ndarray]

The minLSTM coefficients in log space (Sec. B.1), using :func:g.

The gate normalisation becomes a subtraction::

log Gamma_f = -softplus(-k_f)
log Gamma_u = -softplus(-k_u)
log(Gamma_f + Gamma_u) = logaddexp(log Gamma_f, log Gamma_u)
log alpha = log Gamma_f - log(Gamma_f + Gamma_u)
log beta  = log Gamma_u - log(Gamma_f + Gamma_u) + log_g(k_c)

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	Wf, Wu, and Wc, each (n_a, n_x).	required

Returns:

Name	Type	Description
log_alpha	ndarray	log(alpha), shape (n_a, m, T_x).
log_beta	ndarray	log(beta), shape (n_a, m, T_x).

Source code in src/dlhub/nn/sequence/min_gated.py

def min_lstm_log_coefficients(
    x: np.ndarray, parameters: dict[str, np.ndarray]
) -> tuple[np.ndarray, np.ndarray]:
    """
    The minLSTM coefficients in log space (Sec. B.1), using :func:`g`.

    The gate normalisation becomes a subtraction::

        log Gamma_f = -softplus(-k_f)
        log Gamma_u = -softplus(-k_u)
        log(Gamma_f + Gamma_u) = logaddexp(log Gamma_f, log Gamma_u)
        log alpha = log Gamma_f - log(Gamma_f + Gamma_u)
        log beta  = log Gamma_u - log(Gamma_f + Gamma_u) + log_g(k_c)

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        ``Wf``, ``Wu``, and ``Wc``, each ``(n_a, n_x)``.

    Returns
    -------
    log_alpha : np.ndarray
        ``log(alpha)``, shape ``(n_a, m, T_x)``.
    log_beta : np.ndarray
        ``log(beta)``, shape ``(n_a, m, T_x)``.
    """
    k_f = project_over_time(parameters["Wf"], x)
    k_u = project_over_time(parameters["Wu"], x)
    k_c = project_over_time(parameters["Wc"], x)

    log_gamma_f = -softplus(-k_f)
    log_gamma_u = -softplus(-k_u)
    log_normaliser = np.logaddexp(log_gamma_f, log_gamma_u)

    log_alpha = log_gamma_f - log_normaliser
    log_beta = log_gamma_u - log_normaliser + log_g(k_c)
    return log_alpha, log_beta

parallel_scan

parallel_scan(alpha: ndarray, beta: ndarray, a0: ndarray | None = None) -> np.ndarray

Resolve the same recurrence by an associative scan, in plain space.

Each timestep is an affine map a -> alpha * a + beta, and composing two of them -- first (alpha_1, beta_1), then (alpha_2, beta_2) -- gives another affine map::

(alpha_1, beta_1) . (alpha_2, beta_2)
    = (alpha_2 * alpha_1,  alpha_2 * beta_1 + beta_2)

Composition is associative, so the prefix compositions can be built by doubling the stride (the Hillis-Steele inclusive scan) in ceil(log2 T_x) rounds instead of T_x serial steps. After the doubling, entry t holds the single affine map from a^{<0>} to a^{<t>}, which is then applied to the initial state.

Parameters:

Name	Type	Description	Default
alpha	ndarray	Per-step coefficients, each (n_a, m, T_x).	required
beta	ndarray	Per-step coefficients, each (n_a, m, T_x).	required
a0	ndarray or None	Initial state a^{<0>}, shape (n_a, m). Defaults to zero.	None

Returns:

Type	Description
ndarray	The hidden states, shape (n_a, m, T_x), equal to :func:sequential_scan's output up to floating-point association.

Raises:

Type	Description
ValueError	If alpha and beta disagree in shape, or a0 is misshapen.

Source code in src/dlhub/nn/sequence/min_gated.py

def parallel_scan(
    alpha: np.ndarray, beta: np.ndarray, a0: np.ndarray | None = None
) -> np.ndarray:
    """
    Resolve the same recurrence by an associative scan, in plain space.

    Each timestep is an affine map ``a -> alpha * a + beta``, and composing two
    of them -- first ``(alpha_1, beta_1)``, then ``(alpha_2, beta_2)`` -- gives
    another affine map::

        (alpha_1, beta_1) . (alpha_2, beta_2)
            = (alpha_2 * alpha_1,  alpha_2 * beta_1 + beta_2)

    Composition is associative, so the prefix compositions can be built by
    doubling the stride (the Hillis-Steele inclusive scan) in
    ``ceil(log2 T_x)`` rounds instead of ``T_x`` serial steps. After the
    doubling, entry ``t`` holds the single affine map from ``a^{<0>}`` to
    ``a^{<t>}``, which is then applied to the initial state.

    Parameters
    ----------
    alpha, beta : np.ndarray
        Per-step coefficients, each ``(n_a, m, T_x)``.
    a0 : np.ndarray or None, optional
        Initial state ``a^{<0>}``, shape ``(n_a, m)``. Defaults to zero.

    Returns
    -------
    np.ndarray
        The hidden states, shape ``(n_a, m, T_x)``, equal to
        :func:`sequential_scan`'s output up to floating-point association.

    Raises
    ------
    ValueError
        If ``alpha`` and ``beta`` disagree in shape, or ``a0`` is misshapen.
    """
    if alpha.shape != beta.shape:
        raise ValueError(f"alpha {alpha.shape} and beta {beta.shape} must match")

    n_a, m, T_x = alpha.shape
    a_start = _resolve_initial_state(a0, n_a, m)

    # prefix[t] is the composition of steps [t - span + 1 .. t]; it starts as
    # the single step t and its span doubles each round.
    prefix_alpha, prefix_beta = alpha.copy(), beta.copy()

    stride = 1
    while stride < T_x:
        # The earlier half of each pair: what steps [t - 2*stride + 1 .. t - stride]
        # already compose to. The later half is the entry being updated.
        left_alpha, left_beta = (
            prefix_alpha[:, :, :-stride],
            prefix_beta[:, :, :-stride],
        )
        right_alpha = prefix_alpha[:, :, stride:]
        right_beta = prefix_beta[:, :, stride:]

        composed_alpha = right_alpha * left_alpha
        composed_beta = right_alpha * left_beta + right_beta

        # Entries before `stride` already span all the history there is, so they
        # are carried through untouched.
        prefix_alpha = np.concatenate(
            [prefix_alpha[:, :, :stride], composed_alpha], axis=2
        )
        prefix_beta = np.concatenate(
            [prefix_beta[:, :, :stride], composed_beta], axis=2
        )
        stride *= 2

    return prefix_alpha * a_start[:, :, np.newaxis] + prefix_beta

project_over_time

project_over_time(W: ndarray, x: ndarray) -> np.ndarray

Apply one weight matrix to every timestep in a single contraction.

Computes W x^{<t>} for all t, which is exactly the freedom the reduction buys: with no dependence on a^{<t-1>}, nothing in a gate has to wait for the step before it.

Parameters:

Name	Type	Description	Default
W	ndarray	Weight matrix of shape (n_a, n_x).	required
x	ndarray	Input sequence of shape (n_x, m, T_x).	required

Returns:

Type	Description
ndarray	Pre-activations of shape (n_a, m, T_x), whose slice at t equals W @ x[:, :, t].

Raises:

Type	Description
ValueError	If x is not three-dimensional.

Source code in src/dlhub/nn/sequence/min_gated.py

def project_over_time(W: np.ndarray, x: np.ndarray) -> np.ndarray:
    """
    Apply one weight matrix to every timestep in a single contraction.

    Computes ``W x^{<t>}`` for all ``t``, which is exactly the freedom the
    reduction buys: with no dependence on ``a^{<t-1>}``, nothing in a gate has
    to wait for the step before it.

    Parameters
    ----------
    W : np.ndarray
        Weight matrix of shape ``(n_a, n_x)``.
    x : np.ndarray
        Input sequence of shape ``(n_x, m, T_x)``.

    Returns
    -------
    np.ndarray
        Pre-activations of shape ``(n_a, m, T_x)``, whose slice at ``t`` equals
        ``W @ x[:, :, t]``.

    Raises
    ------
    ValueError
        If ``x`` is not three-dimensional.
    """
    if x.ndim != 3:
        raise ValueError(f"x must have shape (n_x, m, T_x); got ndim {x.ndim}")
    return np.tensordot(W, x, axes=([1], [0]))

sequential_scan

sequential_scan(alpha: ndarray, beta: ndarray, a0: ndarray | None = None) -> np.ndarray

Resolve a^{<t>} = alpha^{<t>} a^{<t-1>} + beta^{<t>} one step at a time.

This is the definition, written as the loop the recurrence describes. It is the reference the two scans below are checked against.

Parameters:

Name	Type	Description	Default
alpha	ndarray	Per-step coefficients, each (n_a, m, T_x).	required
beta	ndarray	Per-step coefficients, each (n_a, m, T_x).	required
a0	ndarray or None	Initial state a^{<0>}, shape (n_a, m). Defaults to zero.	None

Returns:

Type	Description
ndarray	The hidden states, shape (n_a, m, T_x).

Raises:

Type	Description
ValueError	If alpha and beta disagree in shape, or a0 is misshapen.

Source code in src/dlhub/nn/sequence/min_gated.py

def sequential_scan(
    alpha: np.ndarray, beta: np.ndarray, a0: np.ndarray | None = None
) -> np.ndarray:
    """
    Resolve ``a^{<t>} = alpha^{<t>} a^{<t-1>} + beta^{<t>}`` one step at a time.

    This is the definition, written as the loop the recurrence describes. It is
    the reference the two scans below are checked against.

    Parameters
    ----------
    alpha, beta : np.ndarray
        Per-step coefficients, each ``(n_a, m, T_x)``.
    a0 : np.ndarray or None, optional
        Initial state ``a^{<0>}``, shape ``(n_a, m)``. Defaults to zero.

    Returns
    -------
    np.ndarray
        The hidden states, shape ``(n_a, m, T_x)``.

    Raises
    ------
    ValueError
        If ``alpha`` and ``beta`` disagree in shape, or ``a0`` is misshapen.
    """
    if alpha.shape != beta.shape:
        raise ValueError(f"alpha {alpha.shape} and beta {beta.shape} must match")

    n_a, m, T_x = alpha.shape
    a_prev = _resolve_initial_state(a0, n_a, m)

    a = np.zeros((n_a, m, T_x))
    for t in range(T_x):
        a_t = alpha[:, :, t] * a_prev + beta[:, :, t]
        a[:, :, t] = a_t
        a_prev = a_t  # carry the state forward to t+1
    return a

softplus

softplus(z: ndarray) -> np.ndarray

Stable log(1 + exp(z)).

Rewritten as log1p(exp(-|z|)) + max(z, 0), which is algebraically the same function but only ever exponentiates a non-positive number. See the module Notes.

Parameters:

Name	Type	Description	Default
z	ndarray	Pre-activation of any shape.	required

Returns:

Type	Description
ndarray	log(1 + exp(z)), elementwise, same shape as z.

Source code in src/dlhub/nn/sequence/min_gated.py

def softplus(z: np.ndarray) -> np.ndarray:
    """
    Stable ``log(1 + exp(z))``.

    Rewritten as ``log1p(exp(-|z|)) + max(z, 0)``, which is algebraically the
    same function but only ever exponentiates a non-positive number. See the
    module ``Notes``.

    Parameters
    ----------
    z : np.ndarray
        Pre-activation of any shape.

    Returns
    -------
    np.ndarray
        ``log(1 + exp(z))``, elementwise, same shape as ``z``.
    """
    return np.log1p(np.exp(-np.abs(z))) + np.maximum(z, 0.0)

bidirectional_rnn_forward

bidirectional_rnn_forward(x: ndarray, parameters_forward: dict[str, ndarray], parameters_backward: dict[str, ndarray], a0_forward: ndarray | None = None, a0_backward: ndarray | None = None) -> tuple[np.ndarray, dict[str, np.ndarray]]

Bidirectional RNN forward pass, as a composition of the vanilla cell.

Runs two independent recurrences over the sequence -- one left-to-right (a->) and one right-to-left (a<-) -- and concatenates their hidden states at each timestep as [a->^{<t>}; a<-^{<t>}]. There is no new cell mathematics: this is :func:rnn_forward applied twice, once to the time-reversed input. A bidirectional model attaches its own output layer, mapping the 2 n_a-dimensional concatenation to n_y; the per-direction read-outs computed by the reused forward pass are not used here.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters_forward	dict[str, ndarray]	Independent weight sets for the two directions.	required
parameters_backward	dict[str, ndarray]	Independent weight sets for the two directions.	required
a0_forward	ndarray or None	Initial hidden states for each direction, each shape (n_a, m). Default to zero.	None
a0_backward	ndarray or None	Initial hidden states for each direction, each shape (n_a, m). Default to zero.	None

Returns:

Name	Type	Description
a_bidirectional	ndarray	Concatenated hidden states, shape (2 * n_a, m, T_x).
parts	dict[str, ndarray]	The two direction sequences, {"a_forward": ..., "a_backward": ...}, each (n_a, m, T_x) and both in forward time order.

Source code in src/dlhub/nn/sequence/rnn.py

def bidirectional_rnn_forward(
    x: np.ndarray,
    parameters_forward: dict[str, np.ndarray],
    parameters_backward: dict[str, np.ndarray],
    a0_forward: np.ndarray | None = None,
    a0_backward: np.ndarray | None = None,
) -> tuple[np.ndarray, dict[str, np.ndarray]]:
    """
    Bidirectional RNN forward pass, as a composition of the vanilla cell.

    Runs two independent recurrences over the sequence -- one left-to-right
    (``a->``) and one right-to-left (``a<-``) -- and concatenates their hidden
    states at each timestep as ``[a->^{<t>}; a<-^{<t>}]``. There is no new cell
    mathematics: this is :func:`rnn_forward` applied twice, once to the
    time-reversed input. A bidirectional model attaches its own output layer,
    mapping the ``2 n_a``-dimensional concatenation to ``n_y``; the per-direction
    read-outs computed by the reused forward pass are not used here.

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters_forward, parameters_backward : dict[str, np.ndarray]
        Independent weight sets for the two directions.
    a0_forward, a0_backward : np.ndarray or None, optional
        Initial hidden states for each direction, each shape ``(n_a, m)``.
        Default to zero.

    Returns
    -------
    a_bidirectional : np.ndarray
        Concatenated hidden states, shape ``(2 * n_a, m, T_x)``.
    parts : dict[str, np.ndarray]
        The two direction sequences, ``{"a_forward": ..., "a_backward": ...}``,
        each ``(n_a, m, T_x)`` and both in forward time order.
    """
    a_forward, _, _ = rnn_forward(x, parameters_forward, a0_forward)

    # Run the second recurrence over the reversed sequence, then realign its
    # activations back to forward time order so step t of both directions lines
    # up before concatenation.
    a_backward_reversed, _, _ = rnn_forward(
        x[:, :, ::-1], parameters_backward, a0_backward
    )
    a_backward = a_backward_reversed[:, :, ::-1]

    a_bidirectional = np.concatenate([a_forward, a_backward], axis=0)
    return a_bidirectional, {"a_forward": a_forward, "a_backward": a_backward}

clip_gradients

clip_gradients(gradients: dict[str, ndarray], max_norm: float) -> dict[str, np.ndarray]

Global-norm gradient clipping, the exploding-gradient remedy.

Computes the single L2 norm of all gradients viewed as one long vector and, if it exceeds max_norm, rescales every gradient by the same factor. This preserves the gradient direction while capping its magnitude, exactly as prescribed by Pascanu, Mikolov & Bengio (2013).

Parameters:

Name	Type	Description	Default
gradients	dict[str, ndarray]	Gradients to clip, typically the parameter gradients dWax, dWaa, dWya, dba, dby.	required
max_norm	float	The maximum allowed global L2 norm.	required

Returns:

Type	Description
dict[str, ndarray]	A new dictionary of clipped gradients. The input is not modified.

Source code in src/dlhub/nn/sequence/rnn.py

def clip_gradients(
    gradients: dict[str, np.ndarray], max_norm: float
) -> dict[str, np.ndarray]:
    """
    Global-norm gradient clipping, the exploding-gradient remedy.

    Computes the single L2 norm of all gradients viewed as one long vector and,
    if it exceeds ``max_norm``, rescales every gradient by the same factor. This
    preserves the gradient *direction* while capping its magnitude, exactly as
    prescribed by Pascanu, Mikolov & Bengio (2013).

    Parameters
    ----------
    gradients : dict[str, np.ndarray]
        Gradients to clip, typically the parameter gradients ``dWax``,
        ``dWaa``, ``dWya``, ``dba``, ``dby``.
    max_norm : float
        The maximum allowed global L2 norm.

    Returns
    -------
    dict[str, np.ndarray]
        A new dictionary of clipped gradients. The input is not modified.
    """
    total_norm = np.sqrt(sum(np.sum(g**2) for g in gradients.values()))

    if total_norm > max_norm:
        scale = max_norm / total_norm
        return {key: g * scale for key, g in gradients.items()}

    return {key: g.copy() for key, g in gradients.items()}

deep_rnn_forward

deep_rnn_forward(x: ndarray, parameters_per_layer: list[dict[str, ndarray]], a0_per_layer: list[ndarray] | None = None) -> tuple[list[np.ndarray], np.ndarray]

Deep (stacked) RNN forward pass, as a composition of the vanilla cell.

Each layer is itself a full RNN over the sequence produced by the layer below: layer l consumes the activation sequence a^{[l-1]} in place of the input x. There is no new cell mathematics -- this is :func:rnn_forward applied once per layer. Only the top layer's read-out is a model output; the intermediate read-outs computed by the reused forward pass are not used.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence to the first layer, shape (n_x, m, T_x).	required
parameters_per_layer	list[dict[str, ndarray]]	One weight set per layer, bottom to top. Layer l's Wax must accept the activation width of the layer below (n_x for layer 0).	required
a0_per_layer	list[ndarray] or None	Per-layer initial hidden states. Defaults to zero for every layer.	None

Returns:

Name	Type	Description
activations	list[ndarray]	Each layer's hidden-state sequence, bottom to top, each (n_a^{[l]}, m, T_x).
y_pred_top	ndarray	The top layer's output distributions, shape (n_y, m, T_x).

Raises:

Type	Description
ValueError	If parameters_per_layer is empty.

Source code in src/dlhub/nn/sequence/rnn.py

def deep_rnn_forward(
    x: np.ndarray,
    parameters_per_layer: list[dict[str, np.ndarray]],
    a0_per_layer: list[np.ndarray] | None = None,
) -> tuple[list[np.ndarray], np.ndarray]:
    """
    Deep (stacked) RNN forward pass, as a composition of the vanilla cell.

    Each layer is itself a full RNN over the sequence produced by the layer
    below: layer ``l`` consumes the activation sequence ``a^{[l-1]}`` in place of
    the input ``x``. There is no new cell mathematics -- this is
    :func:`rnn_forward` applied once per layer. Only the top layer's read-out is
    a model output; the intermediate read-outs computed by the reused forward
    pass are not used.

    Parameters
    ----------
    x : np.ndarray
        Input sequence to the first layer, shape ``(n_x, m, T_x)``.
    parameters_per_layer : list[dict[str, np.ndarray]]
        One weight set per layer, bottom to top. Layer ``l``'s ``Wax`` must
        accept the activation width of the layer below (``n_x`` for layer 0).
    a0_per_layer : list[np.ndarray] or None, optional
        Per-layer initial hidden states. Defaults to zero for every layer.

    Returns
    -------
    activations : list[np.ndarray]
        Each layer's hidden-state sequence, bottom to top, each
        ``(n_a^{[l]}, m, T_x)``.
    y_pred_top : np.ndarray
        The top layer's output distributions, shape ``(n_y, m, T_x)``.

    Raises
    ------
    ValueError
        If ``parameters_per_layer`` is empty.
    """
    if not parameters_per_layer:
        raise ValueError("a deep RNN needs at least one layer")

    if a0_per_layer is None:
        a0_per_layer = [None] * len(parameters_per_layer)

    activations: list[np.ndarray] = []
    layer_input = x
    y_pred_top = None
    for layer_params, a0 in zip(parameters_per_layer, a0_per_layer, strict=True):
        a_layer, y_pred_top, _ = rnn_forward(layer_input, layer_params, a0)
        activations.append(a_layer)
        layer_input = a_layer  # the layer above reads these activations

    return activations, y_pred_top

make_fixture

make_fixture() -> dict

Build the deterministic fixture every implementation and test shares.

Draws are recorded in exactly the order the topic dossier pins, under np.random.seed(1) and float64 (the default of randn), so the seven parameter arrays reproduce bit-for-bit across implementations. The one-hot targets are drawn after those seven so that adding them cannot disturb the arrays a forward-parity check compares.

Returns:

Type	Description
dict	{"x", "a0", "parameters", "y", "dims"} where parameters is the dict {"Wax", "Waa", "Wya", "ba", "by"}, y is one-hot of shape (n_y, m, T_x), and dims records n_x, n_a, n_y, m, T_x.

Source code in src/dlhub/nn/sequence/rnn.py

def make_fixture() -> dict:
    """
    Build the deterministic fixture every implementation and test shares.

    Draws are recorded in exactly the order the topic dossier pins, under
    ``np.random.seed(1)`` and ``float64`` (the default of ``randn``), so the
    seven parameter arrays reproduce bit-for-bit across implementations. The
    one-hot targets are drawn *after* those seven so that adding them cannot
    disturb the arrays a forward-parity check compares.

    Returns
    -------
    dict
        ``{"x", "a0", "parameters", "y", "dims"}`` where ``parameters`` is the
        dict ``{"Wax", "Waa", "Wya", "ba", "by"}``, ``y`` is one-hot of shape
        ``(n_y, m, T_x)``, and ``dims`` records ``n_x, n_a, n_y, m, T_x``.
    """
    np.random.seed(1)
    n_x, n_a, n_y, m, T_x = 3, 5, 2, 10, 4

    x = np.random.randn(n_x, m, T_x)  # 1
    a0 = np.random.randn(n_a, m)  # 2
    Wax = np.random.randn(n_a, n_x)  # 3
    Waa = np.random.randn(n_a, n_a)  # 4
    Wya = np.random.randn(n_y, n_a)  # 5
    ba = np.random.randn(n_a, 1)  # 6
    by = np.random.randn(n_y, 1)  # 7

    # Targets, drawn after the seven parameter arrays above.
    labels = np.random.randint(0, n_y, size=(m, T_x))
    y = np.zeros((n_y, m, T_x))
    for i in range(m):
        for t in range(T_x):
            y[labels[i, t], i, t] = 1.0  # one-hot encode the class at (i, t)

    parameters = {"Wax": Wax, "Waa": Waa, "Wya": Wya, "ba": ba, "by": by}
    dims = {"n_x": n_x, "n_a": n_a, "n_y": n_y, "m": m, "T_x": T_x}
    return {"x": x, "a0": a0, "parameters": parameters, "y": y, "dims": dims}

rnn_backward

rnn_backward(y: ndarray, caches: list[tuple], parameters: dict[str, ndarray]) -> dict[str, np.ndarray]

Backpropagation through time (BPTT).

Walks the unrolled graph from t = T_x back to t = 1. The shared weights receive accumulated gradients over every timestep, and the hidden-state gradient is threaded backward so that each step adds its recurrence contribution (da_prev) to the next step's incoming gradient.

Parameters:

Name	Type	Description	Default
y	ndarray	One-hot targets, shape (n_y, m, T_x).	required
caches	list[tuple]	Per-timestep caches from :func:rnn_forward.	required
parameters	dict[str, ndarray]	The shared weights and biases (see :func:rnn_cell_forward).	required

Returns:

Type	Description
dict[str, ndarray]	Accumulated gradients dWax, dWaa, dWya, dba, dby; the gradient da0 w.r.t. the initial hidden state; and dx, the gradient w.r.t. the whole input sequence, shape (n_x, m, T_x).

Source code in src/dlhub/nn/sequence/rnn.py

def rnn_backward(
    y: np.ndarray, caches: list[tuple], parameters: dict[str, np.ndarray]
) -> dict[str, np.ndarray]:
    """
    Backpropagation through time (BPTT).

    Walks the unrolled graph from ``t = T_x`` back to ``t = 1``. The shared
    weights receive **accumulated** gradients over every timestep, and the
    hidden-state gradient is threaded backward so that each step adds its
    recurrence contribution (``da_prev``) to the next step's incoming gradient.

    Parameters
    ----------
    y : np.ndarray
        One-hot targets, shape ``(n_y, m, T_x)``.
    caches : list[tuple]
        Per-timestep caches from :func:`rnn_forward`.
    parameters : dict[str, np.ndarray]
        The shared weights and biases (see :func:`rnn_cell_forward`).

    Returns
    -------
    dict[str, np.ndarray]
        Accumulated gradients ``dWax``, ``dWaa``, ``dWya``, ``dba``, ``dby``;
        the gradient ``da0`` w.r.t. the initial hidden state; and ``dx``, the
        gradient w.r.t. the whole input sequence, shape ``(n_x, m, T_x)``.
    """
    T_x = len(caches)
    _, m, _ = y.shape
    n_x = caches[0][2].shape[0]
    n_a = parameters["Waa"].shape[0]

    dWax = np.zeros_like(parameters["Wax"])
    dWaa = np.zeros_like(parameters["Waa"])
    dWya = np.zeros_like(parameters["Wya"])
    dba = np.zeros_like(parameters["ba"])
    dby = np.zeros_like(parameters["by"])
    dx = np.zeros((n_x, m, T_x))

    # The recurrence contribution into the current step, carried back one step
    # at a time. Zero at the final timestep, which has no successor.
    da_next = np.zeros((n_a, m))

    for t in reversed(range(T_x)):
        grads_t = rnn_cell_backward(y[:, :, t], da_next, caches[t], parameters)

        # Shared weights accumulate across all timesteps.
        dWax += grads_t["dWax"]
        dWaa += grads_t["dWaa"]
        dWya += grads_t["dWya"]
        dba += grads_t["dba"]
        dby += grads_t["dby"]

        dx[:, :, t] = grads_t["dxt"]
        da_next = grads_t["da_prev"]  # hand the recurrence back to t-1

    # After the loop, the recurrence gradient has reached a^{<0>}.
    da0 = da_next

    return {
        "dWax": dWax,
        "dWaa": dWaa,
        "dWya": dWya,
        "dba": dba,
        "dby": dby,
        "da0": da0,
        "dx": dx,
    }

rnn_cell_backward

rnn_cell_backward(y_t: ndarray, da_next: ndarray, cache: tuple, parameters: dict[str, ndarray]) -> dict[str, np.ndarray]

Backward pass for a single timestep.

Combines the two gradient sources that reach a^{<t>} -- the output at this step and the recurrence carried back from t+1 (da_next) -- and propagates them through the softmax read-out and the tanh recurrence.

Parameters:

Name	Type	Description	Default
y_t	ndarray	One-hot target at this timestep, shape (n_y, m).	required
da_next	ndarray	Gradient of the loss w.r.t. this step's hidden state arriving from the recurrence at t+1, shape (n_a, m). Zero for the final step.	required
cache	tuple	(a_next, a_prev, x_t, y_pred_t) from :func:rnn_cell_forward.	required
parameters	dict[str, ndarray]	The shared weights and biases (see :func:rnn_cell_forward).	required

Returns:

Type	Description
dict[str, ndarray]	This step's gradient contributions: dWax, dWaa, dWya, dba, dby, together with dxt (into the input) and da_prev (the recurrence handed back to t-1).

Source code in src/dlhub/nn/sequence/rnn.py

def rnn_cell_backward(
    y_t: np.ndarray,
    da_next: np.ndarray,
    cache: tuple,
    parameters: dict[str, np.ndarray],
) -> dict[str, np.ndarray]:
    """
    Backward pass for a single timestep.

    Combines the two gradient sources that reach ``a^{<t>}`` -- the output at
    this step and the recurrence carried back from ``t+1`` (``da_next``) -- and
    propagates them through the softmax read-out and the tanh recurrence.

    Parameters
    ----------
    y_t : np.ndarray
        One-hot target at this timestep, shape ``(n_y, m)``.
    da_next : np.ndarray
        Gradient of the loss w.r.t. this step's hidden state arriving from the
        recurrence at ``t+1``, shape ``(n_a, m)``. Zero for the final step.
    cache : tuple
        ``(a_next, a_prev, x_t, y_pred_t)`` from :func:`rnn_cell_forward`.
    parameters : dict[str, np.ndarray]
        The shared weights and biases (see :func:`rnn_cell_forward`).

    Returns
    -------
    dict[str, np.ndarray]
        This step's gradient contributions: ``dWax``, ``dWaa``, ``dWya``,
        ``dba``, ``dby``, together with ``dxt`` (into the input) and ``da_prev``
        (the recurrence handed back to ``t-1``).
    """
    a_next, a_prev, x_t, y_pred_t = cache
    Wax, Waa, Wya = parameters["Wax"], parameters["Waa"], parameters["Wya"]
    m = x_t.shape[1]

    # --- Output path: softmax cross-entropy -------------------------------
    # For softmax + cross-entropy the logit gradient collapses to (y_hat - y),
    # scaled by the 1/m batch mean used in compute_loss.
    dz_y = (y_pred_t - y_t) / m  # (n_y, m)
    dWya = dz_y @ a_next.T  # (n_y, n_a)
    dby = np.sum(dz_y, axis=1, keepdims=True)  # (n_y, 1)
    da_from_output = Wya.T @ dz_y  # (n_a, m)

    # --- Merge the two gradient sources at a^{<t>} ------------------------
    da_t = da_from_output + da_next  # (n_a, m)

    # --- Recurrence path: tanh backward -----------------------------------
    # d/dz tanh(z) = 1 - tanh(z)^2 = 1 - (a^{<t>})^2, read off the activation.
    dtanh = (1 - a_next**2) * da_t  # (n_a, m)
    dba = np.sum(dtanh, axis=1, keepdims=True)  # (n_a, 1)
    dWax = dtanh @ x_t.T  # (n_a, n_x)  -- input-path weight gradient
    dWaa = dtanh @ a_prev.T  # (n_a, n_a)  -- recurrent-path weight gradient
    dxt = Wax.T @ dtanh  # (n_x, m)    -- gradient into the input
    da_prev = Waa.T @ dtanh  # (n_a, m)    -- gradient back to t-1

    return {
        "dWax": dWax,
        "dWaa": dWaa,
        "dWya": dWya,
        "dba": dba,
        "dby": dby,
        "dxt": dxt,
        "da_prev": da_prev,
    }

rnn_cell_forward

rnn_cell_forward(x_t: ndarray, a_prev: ndarray, parameters: dict[str, ndarray]) -> tuple[np.ndarray, np.ndarray, tuple]

One timestep of the forward recurrence.

Computes the hidden-state update and the softmax read-out for a single step::

a^{<t>}     = tanh(W_aa a^{<t-1>} + W_ax x^{<t>} + b_a)
y_hat^{<t>} = softmax(W_ya a^{<t>} + b_y)

Parameters:

Name	Type	Description	Default
x_t	ndarray	Input at this timestep, shape (n_x, m).	required
a_prev	ndarray	Hidden state carried in from the previous step, a^{<t-1>}, shape (n_a, m).	required
parameters	dict[str, ndarray]	The shared weights and biases: Wax (n_a, n_x), Waa (n_a, n_a), Wya (n_y, n_a), ba (n_a, 1), by (n_y, 1).	required

Returns:

Name	Type	Description
a_next	ndarray	Updated hidden state a^{<t>}, shape (n_a, m).
y_pred_t	ndarray	Output distribution y_hat^{<t>}, shape (n_y, m).
cache	tuple	(a_next, a_prev, x_t, y_pred_t) -- everything the backward step for this timestep needs.

Source code in src/dlhub/nn/sequence/rnn.py

def rnn_cell_forward(
    x_t: np.ndarray, a_prev: np.ndarray, parameters: dict[str, np.ndarray]
) -> tuple[np.ndarray, np.ndarray, tuple]:
    """
    One timestep of the forward recurrence.

    Computes the hidden-state update and the softmax read-out for a single step::

        a^{<t>}     = tanh(W_aa a^{<t-1>} + W_ax x^{<t>} + b_a)
        y_hat^{<t>} = softmax(W_ya a^{<t>} + b_y)

    Parameters
    ----------
    x_t : np.ndarray
        Input at this timestep, shape ``(n_x, m)``.
    a_prev : np.ndarray
        Hidden state carried in from the previous step, ``a^{<t-1>}``, shape
        ``(n_a, m)``.
    parameters : dict[str, np.ndarray]
        The shared weights and biases: ``Wax`` ``(n_a, n_x)``, ``Waa``
        ``(n_a, n_a)``, ``Wya`` ``(n_y, n_a)``, ``ba`` ``(n_a, 1)``, ``by``
        ``(n_y, 1)``.

    Returns
    -------
    a_next : np.ndarray
        Updated hidden state ``a^{<t>}``, shape ``(n_a, m)``.
    y_pred_t : np.ndarray
        Output distribution ``y_hat^{<t>}``, shape ``(n_y, m)``.
    cache : tuple
        ``(a_next, a_prev, x_t, y_pred_t)`` -- everything the backward step for
        this timestep needs.
    """
    Wax, Waa, Wya = parameters["Wax"], parameters["Waa"], parameters["Wya"]
    ba, by = parameters["ba"], parameters["by"]

    # Hidden-state update: the recurrent term W_aa a^{<t-1>} and the input term
    # W_ax x^{<t>} are added, then squashed by tanh.
    a_next = np.tanh(Waa @ a_prev + Wax @ x_t + ba)

    # Read-out: an affine map of the hidden state through softmax.
    y_pred_t = softmax(Wya @ a_next + by)

    cache = (a_next, a_prev, x_t, y_pred_t)
    return a_next, y_pred_t, cache

rnn_forward

rnn_forward(x: ndarray, parameters: dict[str, ndarray], a0: ndarray | None = None) -> tuple[np.ndarray, np.ndarray, list[tuple]]

Forward propagation through time across a whole sequence.

Unrolls :func:rnn_cell_forward over T_x timesteps, threading the hidden state from one step to the next.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence of shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	The shared weights and biases (see :func:rnn_cell_forward).	required
a0	ndarray or None	Initial hidden state a^{<0>} of shape (n_a, m). Defaults to the zero vector, matching the standard convention a^{<0>} = 0.	None

Returns:

Name	Type	Description
a	ndarray	All hidden states stacked over time, shape (n_a, m, T_x).
y_pred	ndarray	All output distributions, shape (n_y, m, T_x).
caches	list[tuple]	Per-timestep caches, in order, for :func:rnn_backward.

Raises:

Type	Description
ValueError	If x is not three-dimensional, or if a supplied a0 does not have shape (n_a, m).

Source code in src/dlhub/nn/sequence/rnn.py

def rnn_forward(
    x: np.ndarray,
    parameters: dict[str, np.ndarray],
    a0: np.ndarray | None = None,
) -> tuple[np.ndarray, np.ndarray, list[tuple]]:
    """
    Forward propagation through time across a whole sequence.

    Unrolls :func:`rnn_cell_forward` over ``T_x`` timesteps, threading the
    hidden state from one step to the next.

    Parameters
    ----------
    x : np.ndarray
        Input sequence of shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        The shared weights and biases (see :func:`rnn_cell_forward`).
    a0 : np.ndarray or None, optional
        Initial hidden state ``a^{<0>}`` of shape ``(n_a, m)``. Defaults to the
        zero vector, matching the standard convention ``a^{<0>} = 0``.

    Returns
    -------
    a : np.ndarray
        All hidden states stacked over time, shape ``(n_a, m, T_x)``.
    y_pred : np.ndarray
        All output distributions, shape ``(n_y, m, T_x)``.
    caches : list[tuple]
        Per-timestep caches, in order, for :func:`rnn_backward`.

    Raises
    ------
    ValueError
        If ``x`` is not three-dimensional, or if a supplied ``a0`` does not have
        shape ``(n_a, m)``.
    """
    if x.ndim != 3:
        raise ValueError(f"x must have shape (n_x, m, T_x); got ndim {x.ndim}")

    n_x, m, T_x = x.shape
    n_a = parameters["Waa"].shape[0]
    n_y = parameters["Wya"].shape[0]

    if a0 is None:
        a0 = np.zeros((n_a, m))
    elif a0.shape != (n_a, m):
        raise ValueError(f"a0 must have shape {(n_a, m)}; got {a0.shape}")

    a = np.zeros((n_a, m, T_x))
    y_pred = np.zeros((n_y, m, T_x))
    caches: list[tuple] = []

    a_prev = a0
    for t in range(T_x):
        a_next, y_pred_t, cache = rnn_cell_forward(x[:, :, t], a_prev, parameters)
        a[:, :, t] = a_next
        y_pred[:, :, t] = y_pred_t
        caches.append(cache)
        a_prev = a_next  # carry the state forward to t+1

    return a, y_pred, caches

update_parameters

update_parameters(parameters: dict[str, ndarray], gradients: dict[str, ndarray], learning_rate: float) -> dict[str, np.ndarray]

One step of gradient descent on the shared weights.

Applies theta <- theta - learning_rate * dtheta to each of Wax, Waa, Wya, ba, by.

Parameters:

Name	Type	Description	Default
parameters	dict[str, ndarray]	The current weights and biases.	required
gradients	dict[str, ndarray]	Their gradients, keyed dWax, dWaa, dWya, dba, dby.	required
learning_rate	float	The step size.	required

Returns:

Type	Description
dict[str, ndarray]	A new dictionary of updated parameters. The input is not modified.

Source code in src/dlhub/nn/sequence/rnn.py

def update_parameters(
    parameters: dict[str, np.ndarray],
    gradients: dict[str, np.ndarray],
    learning_rate: float,
) -> dict[str, np.ndarray]:
    """
    One step of gradient descent on the shared weights.

    Applies ``theta <- theta - learning_rate * dtheta`` to each of ``Wax``,
    ``Waa``, ``Wya``, ``ba``, ``by``.

    Parameters
    ----------
    parameters : dict[str, np.ndarray]
        The current weights and biases.
    gradients : dict[str, np.ndarray]
        Their gradients, keyed ``dWax``, ``dWaa``, ``dWya``, ``dba``, ``dby``.
    learning_rate : float
        The step size.

    Returns
    -------
    dict[str, np.ndarray]
        A new dictionary of updated parameters. The input is not modified.
    """
    return {
        key: parameters[key] - learning_rate * gradients[f"d{key}"]
        for key in ("Wax", "Waa", "Wya", "ba", "by")
    }

gru

Gated Recurrent Unit (GRU)

A from-scratch, teaching reference for the GRU: the two gates, the candidate activation, the blend that carries state forward, and the hand-derived backpropagation through time. This module is the explanation the accompanying documents point at, so the forward pass appears in the shapes the topic declares and every gradient is computed term by term rather than fused into one expression.

Notation extends the vanilla RNN reference in :mod:dlhub.nn.sequence.rnn -- same a^{<t>}, same shape conventions, same read-out and loss -- with the gate vectors written as Andrew Ng's Course 5 writes them. For the GRU the hidden state and the cell state are the same object, a^{<t>} = c^{<t>}, so there is no separate c to thread. For t = 1 ... T_x::

Gamma_r^{<t>} = sigma(W_r [a^{<t-1>}; x^{<t>}] + b_r)
Gamma_u^{<t>} = sigma(W_u [a^{<t-1>}; x^{<t>}] + b_u)
a_tilde^{<t>} = tanh(W_c [Gamma_r^{<t>} * a^{<t-1>}; x^{<t>}] + b_c)
a^{<t>}       = Gamma_u^{<t>} * a_tilde^{<t>} + (1 - Gamma_u^{<t>}) * a^{<t-1>}
y_hat^{<t>}   = softmax(W_y a^{<t>} + b_y)

(* is elementwise.) The update gate Gamma_u interpolates between keeping the previous state and taking the candidate; the reset gate Gamma_r decides how much of the previous state the candidate is allowed to see. The (1 - Gamma_u) term is the GRU's answer to the vanishing gradient: when the gate is near zero the state is copied forward, and its gradient with it, instead of being pushed through another tanh and another weight matrix. The vanilla RNN in this package has no such path, which is the comparison the topic exists to make. The LSTM's version of the same idea, with a separate cell state, is in :mod:dlhub.nn.sequence.lstm.

References

• Cho, K., van Merrienboer, B., Gulcehre, C., Bahdanau, D., Bougares, F., Schwenk, H., & Bengio, Y. (2014). Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation. EMNLP. https://arxiv.org/abs/1406.1078
• Chung, J., Gulcehre, C., Cho, K., & Bengio, Y. (2014). Empirical Evaluation of Gated Recurrent Neural Networks on Sequence Modeling. https://arxiv.org/abs/1412.3555
• Goodfellow, I., Bengio, Y., & Courville, A. (2016). Deep Learning, Ch. 10. MIT Press. https://www.deeplearningbook.org/
• Ng, A. (2018). Deep Learning Specialization, Course 5 (Sequence Models), Week 1. https://www.coursera.org/specializations/deep-learning

Author

Deep Learning Reference Hub

License

MIT

Notes

• The reset gate is applied before the hidden transform. The candidate reads [Gamma_r * a^{<t-1>}; x^{<t>}], so the gate acts on the state inside the concatenation W_c multiplies. This is the Cho/Ng form and it is canonical for this hub. Several frameworks implement the reset after the recurrent transform instead -- tanh(W_x x + r * (W_h a + b_h)), which is what torch.nn.GRU and tf.keras.layers.GRU(reset_after=True) compute. The two are not algebraically equal; the same layer name denotes two different functions across the ecosystem. The hub's TensorFlow port reproduces this module with reset_after=False.
• Stable sigmoid. Gates use :func:dlhub.nn.sequence._common.sigmoid, which never exponentiates a positive number; see that module's Notes. The softmax read-out and the loss are imported from :mod:dlhub.nn.sequence.rnn, the canonical home of the max-subtraction stability shift, rather than restated.
• Derivatives are read off the cached activations. sigma'(z) = Gamma (1 - Gamma) and tanh'(z) = 1 - a_tilde^2 both reuse the value the forward pass already computed, so the backward pass evaluates no new sigmoid and no new tanh -- the same trick the vanilla RNN uses for its single tanh.
• Concatenated weight form. One matrix per gate acts on [a^{<t-1>}; x^{<t>}], so W_g is (n_a, n_a + n_x). The vanilla RNN deliberately keeps W_aa and W_ax separate to expose the two BPTT paths; that lesson is taught there, and repeating it across three gates is clutter. The two halves are still nameable: :func:split_gate_matrix cuts any W_g or dW_g at column n_a into its recurrent and input parts, which is what :func:main reports.
• The cache is a dict, not a tuple. The vanilla RNN's four-element positional cache is readable; a gated cell's is not, so every cached array is named.

gru_cell_forward

gru_cell_forward(x_t: ndarray, a_prev: ndarray, parameters: dict[str, ndarray]) -> tuple[np.ndarray, np.ndarray, dict[str, np.ndarray]]

One timestep of the GRU forward recurrence.

Computes the two gates, the candidate activation, the blended state, and the softmax read-out for a single step, in that order.

Parameters:

Name	Type	Description	Default
x_t	ndarray	Input at this timestep, shape (n_x, m).	required
a_prev	ndarray	Hidden state carried in from the previous step, a^{<t-1>}, shape (n_a, m).	required
parameters	dict[str, ndarray]	The shared weights and biases: Wr, Wu, Wc each (n_a, n_a + n_x); br, bu, bc each (n_a, 1); Wy (n_y, n_a); by (n_y, 1).	required

Returns:

Name	Type	Description
a_next	ndarray	Updated hidden state a^{<t>}, shape (n_a, m).
y_pred_t	ndarray	Output distribution y_hat^{<t>}, shape (n_y, m).
cache	dict[str, ndarray]	Everything this step's backward pass needs, each array named.

Source code in src/dlhub/nn/sequence/gru.py

def gru_cell_forward(
    x_t: np.ndarray, a_prev: np.ndarray, parameters: dict[str, np.ndarray]
) -> tuple[np.ndarray, np.ndarray, dict[str, np.ndarray]]:
    """
    One timestep of the GRU forward recurrence.

    Computes the two gates, the candidate activation, the blended state, and the
    softmax read-out for a single step, in that order.

    Parameters
    ----------
    x_t : np.ndarray
        Input at this timestep, shape ``(n_x, m)``.
    a_prev : np.ndarray
        Hidden state carried in from the previous step, ``a^{<t-1>}``, shape
        ``(n_a, m)``.
    parameters : dict[str, np.ndarray]
        The shared weights and biases: ``Wr``, ``Wu``, ``Wc`` each
        ``(n_a, n_a + n_x)``; ``br``, ``bu``, ``bc`` each ``(n_a, 1)``; ``Wy``
        ``(n_y, n_a)``; ``by`` ``(n_y, 1)``.

    Returns
    -------
    a_next : np.ndarray
        Updated hidden state ``a^{<t>}``, shape ``(n_a, m)``.
    y_pred_t : np.ndarray
        Output distribution ``y_hat^{<t>}``, shape ``(n_y, m)``.
    cache : dict[str, np.ndarray]
        Everything this step's backward pass needs, each array named.
    """
    Wr, Wu, Wc = parameters["Wr"], parameters["Wu"], parameters["Wc"]
    br, bu, bc = parameters["br"], parameters["bu"], parameters["bc"]
    Wy, by = parameters["Wy"], parameters["by"]

    # The concatenated form: one stacked vector [a^{<t-1>}; x^{<t>}], shape
    # (n_a + n_x, m), that every gate weight multiplies.
    concat = np.concatenate([a_prev, x_t], axis=0)

    # --- Gates ------------------------------------------------------------
    gamma_r = sigmoid(Wr @ concat + br)  # reset: how much state the candidate sees
    gamma_u = sigmoid(Wu @ concat + bu)  # update: keep the old state, or take the new

    # --- Candidate --------------------------------------------------------
    # The reset gate is applied *before* the hidden transform: it scales
    # a^{<t-1>} inside the concatenation W_c acts on. See the module Notes.
    concat_reset = np.concatenate([gamma_r * a_prev, x_t], axis=0)
    a_tilde = np.tanh(Wc @ concat_reset + bc)

    # --- Blend ------------------------------------------------------------
    # A convex combination, elementwise: gate near 1 takes the candidate, gate
    # near 0 copies the previous state forward unchanged.
    a_next = gamma_u * a_tilde + (1 - gamma_u) * a_prev

    # --- Read-out ---------------------------------------------------------
    y_pred_t = softmax(Wy @ a_next + by)

    cache = {
        "a_next": a_next,
        "a_prev": a_prev,
        "gamma_r": gamma_r,
        "gamma_u": gamma_u,
        "a_tilde": a_tilde,
        "concat": concat,
        "concat_reset": concat_reset,
        "x_t": x_t,
        "y_pred_t": y_pred_t,
    }
    return a_next, y_pred_t, cache

gru_forward

gru_forward(x: ndarray, parameters: dict[str, ndarray], a0: ndarray | None = None) -> tuple[np.ndarray, np.ndarray, list[dict[str, np.ndarray]]]

Forward propagation through time across a whole sequence.

Unrolls :func:gru_cell_forward over T_x timesteps, threading the hidden state from one step to the next.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence of shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	The shared weights and biases (see :func:gru_cell_forward).	required
a0	ndarray or None	Initial hidden state a^{<0>} of shape (n_a, m). Defaults to the zero vector, matching the standard convention a^{<0>} = 0.	None

Returns:

Name	Type	Description
a	ndarray	All hidden states stacked over time, shape (n_a, m, T_x).
y_pred	ndarray	All output distributions, shape (n_y, m, T_x).
caches	list[dict[str, ndarray]]	Per-timestep caches, in order, for :func:gru_backward.

Raises:

Type	Description
ValueError	If x is not three-dimensional, or if a supplied a0 does not have shape (n_a, m).

Source code in src/dlhub/nn/sequence/gru.py

def gru_forward(
    x: np.ndarray,
    parameters: dict[str, np.ndarray],
    a0: np.ndarray | None = None,
) -> tuple[np.ndarray, np.ndarray, list[dict[str, np.ndarray]]]:
    """
    Forward propagation through time across a whole sequence.

    Unrolls :func:`gru_cell_forward` over ``T_x`` timesteps, threading the hidden
    state from one step to the next.

    Parameters
    ----------
    x : np.ndarray
        Input sequence of shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        The shared weights and biases (see :func:`gru_cell_forward`).
    a0 : np.ndarray or None, optional
        Initial hidden state ``a^{<0>}`` of shape ``(n_a, m)``. Defaults to the
        zero vector, matching the standard convention ``a^{<0>} = 0``.

    Returns
    -------
    a : np.ndarray
        All hidden states stacked over time, shape ``(n_a, m, T_x)``.
    y_pred : np.ndarray
        All output distributions, shape ``(n_y, m, T_x)``.
    caches : list[dict[str, np.ndarray]]
        Per-timestep caches, in order, for :func:`gru_backward`.

    Raises
    ------
    ValueError
        If ``x`` is not three-dimensional, or if a supplied ``a0`` does not have
        shape ``(n_a, m)``.
    """
    if x.ndim != 3:
        raise ValueError(f"x must have shape (n_x, m, T_x); got ndim {x.ndim}")

    n_x, m, T_x = x.shape
    n_a = parameters["Wu"].shape[0]
    n_y = parameters["Wy"].shape[0]

    if a0 is None:
        a0 = np.zeros((n_a, m))
    elif a0.shape != (n_a, m):
        raise ValueError(f"a0 must have shape {(n_a, m)}; got {a0.shape}")

    a = np.zeros((n_a, m, T_x))
    y_pred = np.zeros((n_y, m, T_x))
    caches: list[dict[str, np.ndarray]] = []

    a_prev = a0
    for t in range(T_x):
        a_next, y_pred_t, cache = gru_cell_forward(x[:, :, t], a_prev, parameters)
        a[:, :, t] = a_next
        y_pred[:, :, t] = y_pred_t
        caches.append(cache)
        a_prev = a_next  # carry the state forward to t+1

    return a, y_pred, caches

gru_cell_backward

gru_cell_backward(y_t: ndarray, da_next: ndarray, cache: dict[str, ndarray], parameters: dict[str, ndarray]) -> dict[str, np.ndarray]

Backward pass for a single GRU timestep.

Walks the forward pass backwards, one equation at a time. Three separate paths deposit gradient into a^{<t-1>}, and the whole point of the cell is that the first of them is a plain elementwise factor:

1. the blend, through (1 - Gamma_u) -- no weight matrix, no saturating nonlinearity, so gradient crosses the step essentially undamped when the update gate is near zero;
2. the candidate, through W_c and then the reset gate;
3. the gates themselves, through W_r and W_u.

Parameters:

Name	Type	Description	Default
y_t	ndarray	One-hot target at this timestep, shape (n_y, m).	required
da_next	ndarray	Gradient of the loss w.r.t. this step's hidden state arriving from the recurrence at t+1, shape (n_a, m). Zero for the final step.	required
cache	dict[str, ndarray]	This step's cache from :func:gru_cell_forward.	required
parameters	dict[str, ndarray]	The shared weights and biases (see :func:gru_cell_forward).	required

Returns:

Type	Description
dict[str, ndarray]	This step's gradient contributions: dWr, dWu, dWc, dbr, dbu, dbc, dWy, dby, together with dxt (into the input) and da_prev (the recurrence handed back to t-1).

Source code in src/dlhub/nn/sequence/gru.py

def gru_cell_backward(
    y_t: np.ndarray,
    da_next: np.ndarray,
    cache: dict[str, np.ndarray],
    parameters: dict[str, np.ndarray],
) -> dict[str, np.ndarray]:
    """
    Backward pass for a single GRU timestep.

    Walks the forward pass backwards, one equation at a time. Three separate
    paths deposit gradient into ``a^{<t-1>}``, and the whole point of the cell is
    that the first of them is a plain elementwise factor:

    1. the **blend**, through ``(1 - Gamma_u)`` -- no weight matrix, no
       saturating nonlinearity, so gradient crosses the step essentially undamped
       when the update gate is near zero;
    2. the **candidate**, through ``W_c`` and then the reset gate;
    3. the **gates** themselves, through ``W_r`` and ``W_u``.

    Parameters
    ----------
    y_t : np.ndarray
        One-hot target at this timestep, shape ``(n_y, m)``.
    da_next : np.ndarray
        Gradient of the loss w.r.t. this step's hidden state arriving from the
        recurrence at ``t+1``, shape ``(n_a, m)``. Zero for the final step.
    cache : dict[str, np.ndarray]
        This step's cache from :func:`gru_cell_forward`.
    parameters : dict[str, np.ndarray]
        The shared weights and biases (see :func:`gru_cell_forward`).

    Returns
    -------
    dict[str, np.ndarray]
        This step's gradient contributions: ``dWr``, ``dWu``, ``dWc``, ``dbr``,
        ``dbu``, ``dbc``, ``dWy``, ``dby``, together with ``dxt`` (into the
        input) and ``da_prev`` (the recurrence handed back to ``t-1``).
    """
    a_next, a_prev = cache["a_next"], cache["a_prev"]
    gamma_r, gamma_u = cache["gamma_r"], cache["gamma_u"]
    a_tilde = cache["a_tilde"]
    concat, concat_reset = cache["concat"], cache["concat_reset"]
    y_pred_t = cache["y_pred_t"]

    Wr, Wu, Wc, Wy = (
        parameters["Wr"],
        parameters["Wu"],
        parameters["Wc"],
        parameters["Wy"],
    )
    n_a, m = a_next.shape

    # --- Output path: softmax cross-entropy -------------------------------
    # For softmax + cross-entropy the logit gradient collapses to (y_hat - y),
    # scaled by the 1/m batch mean used in compute_loss.
    dz_y = (y_pred_t - y_t) / m  # (n_y, m)
    dWy = dz_y @ a_next.T  # (n_y, n_a)
    dby = np.sum(dz_y, axis=1, keepdims=True)  # (n_y, 1)
    da_from_output = Wy.T @ dz_y  # (n_a, m)

    # --- Merge the two gradient sources at a^{<t>} ------------------------
    da_t = da_from_output + da_next  # (n_a, m)

    # --- Blend: a^{<t>} = Gamma_u * a_tilde + (1 - Gamma_u) * a^{<t-1>} ---
    # Differentiate the convex combination in its three arguments.
    dgamma_u = da_t * (a_tilde - a_prev)  # (n_a, m)
    da_tilde = da_t * gamma_u  # (n_a, m)
    da_prev = da_t * (1 - gamma_u)  # path 1: the undamped carry

    # --- Candidate: a_tilde = tanh(W_c [Gamma_r * a^{<t-1>}; x^{<t>}] + b_c)
    dz_c = da_tilde * (1 - a_tilde**2)  # tanh', read off the activation
    dWc = dz_c @ concat_reset.T  # (n_a, n_a + n_x)
    dbc = np.sum(dz_c, axis=1, keepdims=True)  # (n_a, 1)
    dconcat_reset = Wc.T @ dz_c  # (n_a + n_x, m)

    # The top n_a rows are the gradient w.r.t. the *reset* state, which the
    # product rule then splits between the gate and the state itself.
    d_reset_state = dconcat_reset[:n_a, :]  # d/d(Gamma_r * a^{<t-1>})
    dgamma_r = d_reset_state * a_prev
    da_prev = da_prev + d_reset_state * gamma_r  # path 2: through the candidate
    dxt = dconcat_reset[n_a:, :]  # (n_x, m)

    # --- Gates: sigma'(z) = Gamma (1 - Gamma), read off the cached gate ----
    dz_r = dgamma_r * gamma_r * (1 - gamma_r)
    dWr = dz_r @ concat.T
    dbr = np.sum(dz_r, axis=1, keepdims=True)
    dconcat_r = Wr.T @ dz_r

    dz_u = dgamma_u * gamma_u * (1 - gamma_u)
    dWu = dz_u @ concat.T
    dbu = np.sum(dz_u, axis=1, keepdims=True)
    dconcat_u = Wu.T @ dz_u

    # Both gates read the same stacked vector, so each contributes to both the
    # recurrent gradient and the input gradient. Path 3.
    da_prev = da_prev + dconcat_r[:n_a, :] + dconcat_u[:n_a, :]
    dxt = dxt + dconcat_r[n_a:, :] + dconcat_u[n_a:, :]

    return {
        "dWr": dWr,
        "dWu": dWu,
        "dWc": dWc,
        "dbr": dbr,
        "dbu": dbu,
        "dbc": dbc,
        "dWy": dWy,
        "dby": dby,
        "dxt": dxt,
        "da_prev": da_prev,
    }

gru_backward

gru_backward(y: ndarray, caches: list[dict[str, ndarray]], parameters: dict[str, ndarray]) -> dict[str, np.ndarray]

Backpropagation through time (BPTT) for the GRU.

Walks the unrolled graph from t = T_x back to t = 1. Every gate weight is shared across time, so each receives an accumulated gradient summed over all timesteps, and the hidden-state gradient is threaded backward so each step adds its recurrence contribution (da_prev) to the next step's incoming gradient.

Parameters:

Name	Type	Description	Default
y	ndarray	One-hot targets, shape (n_y, m, T_x).	required
caches	list[dict[str, ndarray]]	Per-timestep caches from :func:gru_forward.	required
parameters	dict[str, ndarray]	The shared weights and biases (see :func:gru_cell_forward).	required

Returns:

Type	Description
dict[str, ndarray]	Accumulated gradients dWr, dWu, dWc, dbr, dbu, dbc, dWy, dby; the gradient da0 w.r.t. the initial hidden state; and dx, the gradient w.r.t. the whole input sequence, shape (n_x, m, T_x).

Source code in src/dlhub/nn/sequence/gru.py

def gru_backward(
    y: np.ndarray,
    caches: list[dict[str, np.ndarray]],
    parameters: dict[str, np.ndarray],
) -> dict[str, np.ndarray]:
    """
    Backpropagation through time (BPTT) for the GRU.

    Walks the unrolled graph from ``t = T_x`` back to ``t = 1``. Every gate
    weight is shared across time, so each receives an **accumulated** gradient
    summed over all timesteps, and the hidden-state gradient is threaded
    backward so each step adds its recurrence contribution (``da_prev``) to the
    next step's incoming gradient.

    Parameters
    ----------
    y : np.ndarray
        One-hot targets, shape ``(n_y, m, T_x)``.
    caches : list[dict[str, np.ndarray]]
        Per-timestep caches from :func:`gru_forward`.
    parameters : dict[str, np.ndarray]
        The shared weights and biases (see :func:`gru_cell_forward`).

    Returns
    -------
    dict[str, np.ndarray]
        Accumulated gradients ``dWr``, ``dWu``, ``dWc``, ``dbr``, ``dbu``,
        ``dbc``, ``dWy``, ``dby``; the gradient ``da0`` w.r.t. the initial hidden
        state; and ``dx``, the gradient w.r.t. the whole input sequence, shape
        ``(n_x, m, T_x)``.
    """
    T_x = len(caches)
    _, m, _ = y.shape
    n_x = caches[0]["x_t"].shape[0]
    n_a = parameters["Wu"].shape[0]

    accumulated = {f"d{key}": np.zeros_like(parameters[key]) for key in PARAMETER_KEYS}
    dx = np.zeros((n_x, m, T_x))

    # The recurrence contribution into the current step, carried back one step
    # at a time. Zero at the final timestep, which has no successor.
    da_next = np.zeros((n_a, m))

    for t in reversed(range(T_x)):
        grads_t = gru_cell_backward(y[:, :, t], da_next, caches[t], parameters)

        # The shared weights accumulate: every timestep used the same matrix,
        # so every timestep adds a term to its gradient.
        for key in PARAMETER_KEYS:
            accumulated[f"d{key}"] += grads_t[f"d{key}"]

        dx[:, :, t] = grads_t["dxt"]
        da_next = grads_t["da_prev"]  # hand the recurrence back to t-1

    # After the loop, the recurrence gradient has reached a^{<0>}.
    return {**accumulated, "da0": da_next, "dx": dx}

make_fixture

make_fixture() -> dict

Build the deterministic fixture every GRU implementation and test shares.

Draws are recorded in exactly the order the topic dossier pins, under np.random.seed(1) and float64 (the default of randn), so the arrays reproduce bit-for-bit across implementations. The one-hot targets are drawn last so that adding them cannot disturb the arrays a forward-parity check compares. Dimensions match the vanilla RNN fixture, so the notebook can contrast the two cells on identical shapes.

Returns:

Type	Description
dict	{"x", "a0", "parameters", "y", "dims"} where parameters is the dict {"Wr", "Wu", "Wc", "Wy", "br", "bu", "bc", "by"}, y is one-hot of shape (n_y, m, T_x), and dims records n_x, n_a, n_y, m, T_x.

Source code in src/dlhub/nn/sequence/gru.py

def make_fixture() -> dict:
    """
    Build the deterministic fixture every GRU implementation and test shares.

    Draws are recorded in exactly the order the topic dossier pins, under
    ``np.random.seed(1)`` and ``float64`` (the default of ``randn``), so the
    arrays reproduce bit-for-bit across implementations. The one-hot targets are
    drawn *last* so that adding them cannot disturb the arrays a forward-parity
    check compares. Dimensions match the vanilla RNN fixture, so the notebook can
    contrast the two cells on identical shapes.

    Returns
    -------
    dict
        ``{"x", "a0", "parameters", "y", "dims"}`` where ``parameters`` is the
        dict ``{"Wr", "Wu", "Wc", "Wy", "br", "bu", "bc", "by"}``, ``y`` is
        one-hot of shape ``(n_y, m, T_x)``, and ``dims`` records
        ``n_x, n_a, n_y, m, T_x``.
    """
    np.random.seed(1)
    n_x, n_a, n_y, m, T_x = 3, 5, 2, 10, 4

    x = np.random.randn(n_x, m, T_x)  # 1
    a0 = np.random.randn(n_a, m)  # 2
    Wr = np.random.randn(n_a, n_a + n_x)  # 3
    Wu = np.random.randn(n_a, n_a + n_x)  # 4
    Wc = np.random.randn(n_a, n_a + n_x)  # 5
    Wy = np.random.randn(n_y, n_a)  # 6
    br = np.random.randn(n_a, 1)  # 7
    bu = np.random.randn(n_a, 1)  # 8
    bc = np.random.randn(n_a, 1)  # 9
    by = np.random.randn(n_y, 1)  # 10

    # Targets, drawn after the ten parameter arrays above.
    labels = np.random.randint(0, n_y, size=(m, T_x))
    y = np.zeros((n_y, m, T_x))
    for i in range(m):
        for t in range(T_x):
            y[labels[i, t], i, t] = 1.0  # one-hot encode the class at (i, t)

    parameters = {
        "Wr": Wr,
        "Wu": Wu,
        "Wc": Wc,
        "Wy": Wy,
        "br": br,
        "bu": bu,
        "bc": bc,
        "by": by,
    }
    dims = {"n_x": n_x, "n_a": n_a, "n_y": n_y, "m": m, "T_x": T_x}
    return {"x": x, "a0": a0, "parameters": parameters, "y": y, "dims": dims}

main

main() -> None

Run the shared fixture end to end and print shapes, loss, and grad norms.

Source code in src/dlhub/nn/sequence/gru.py

def main() -> None:
    """Run the shared fixture end to end and print shapes, loss, and grad norms."""
    fixture = make_fixture()
    x, a0, parameters, y = (
        fixture["x"],
        fixture["a0"],
        fixture["parameters"],
        fixture["y"],
    )
    n_a = fixture["dims"]["n_a"]

    a, y_pred, caches = gru_forward(x, parameters, a0)
    loss = compute_loss(y_pred, y)
    gradients = gru_backward(y, caches, parameters)

    print(f"hidden states a: {a.shape}")
    print(f"outputs y_pred:  {y_pred.shape}")
    print(f"total loss:      {loss:.6f}")
    for key in ("dWr", "dWu", "dWc", "dWy", "dbr", "dbu", "dbc", "dby", "da0"):
        print(f"||{key}|| = {np.linalg.norm(gradients[key]):.6f}")

    # The concatenated gate gradient still names its two halves: the recurrent
    # path dW_ga and the input path dW_gx, split at column n_a.
    for key in ("dWr", "dWu", "dWc"):
        recurrent, input_ = split_gate_matrix(gradients[key], n_a)
        print(
            f"{key} = [recurrent | input] -> "
            f"||{key}a|| = {np.linalg.norm(recurrent):.6f}, "
            f"||{key}x|| = {np.linalg.norm(input_):.6f}"
        )

lstm

Long Short-Term Memory (LSTM)

A from-scratch, teaching reference for the LSTM: the three gates, the cell state, the additive path that carries gradient across time, and the hand-derived backpropagation through time. This module is the explanation the accompanying documents point at, so the forward pass appears in the shapes the topic declares and every gradient is computed term by term rather than fused into one expression.

Notation extends the vanilla RNN reference in :mod:dlhub.nn.sequence.rnn -- same a^{<t>}, same shape conventions, same read-out and loss -- by adding a separate cell state c^{<t>} and the gate vectors, written as Andrew Ng's Course 5 writes them. For t = 1 ... T_x::

Gamma_f^{<t>} = sigma(W_f [a^{<t-1>}; x^{<t>}] + b_f)
Gamma_u^{<t>} = sigma(W_u [a^{<t-1>}; x^{<t>}] + b_u)
c_tilde^{<t>} = tanh (W_c [a^{<t-1>}; x^{<t>}] + b_c)
Gamma_o^{<t>} = sigma(W_o [a^{<t-1>}; x^{<t>}] + b_o)
c^{<t>}       = Gamma_u^{<t>} * c_tilde^{<t>} + Gamma_f^{<t>} * c^{<t-1>}
a^{<t>}       = Gamma_o^{<t>} * tanh(c^{<t>})
y_hat^{<t>}   = softmax(W_y a^{<t>} + b_y)

(* is elementwise.) Gates are computed and reported in the order forget, update, candidate, output throughout this module. The update gate is Hochreiter & Schmidhuber's input gate i; Ng writes it Gamma_u, which this hub follows so that its correspondence with the GRU's update gate is visible at a glance.

The cell-state line is the reason the architecture exists. c^{<t>} is built by addition, and holding the gates fixed its derivative with respect to c^{<t-1>} is the elementwise factor Gamma_f^{<t>} -- not a repeated multiplication by W_aa^T through a tanh derivative, as in the vanilla RNN, whose product decays geometrically. A forget gate near 1 therefore transports gradient across many steps essentially undamped. The GRU in :mod:dlhub.nn.sequence.gru achieves the same thing with one gate fewer and no separate cell state.

References

• Hochreiter, S., & Schmidhuber, J. (1997). Long Short-Term Memory. Neural Computation, 9(8), 1735-1780.
• Gers, F. A., Schmidhuber, J., & Cummins, F. (2000). Learning to Forget: Continual Prediction with LSTM. Neural Computation, 12(10), 2451-2471. (The forget gate, which the 1997 cell lacked and every modern LSTM has.)
• Goodfellow, I., Bengio, Y., & Courville, A. (2016). Deep Learning, Ch. 10. MIT Press. https://www.deeplearningbook.org/
• Ng, A. (2018). Deep Learning Specialization, Course 5 (Sequence Models), Week 1. https://www.coursera.org/specializations/deep-learning

Author

Deep Learning Reference Hub

License

MIT

Notes

• Stable sigmoid. Gates use :func:dlhub.nn.sequence._common.sigmoid, which never exponentiates a positive number; see that module's Notes. The softmax read-out and the loss are imported from :mod:dlhub.nn.sequence.rnn, the canonical home of the max-subtraction stability shift, rather than restated.
• Derivatives are read off the cached activations. sigma'(z) = Gamma (1 - Gamma) and tanh'(z) = 1 - tanh(z)^2 both reuse values the forward pass already computed, so the backward pass evaluates no new sigmoid and no new tanh. tanh(c^{<t>}) is cached for the same reason: it appears in the output-gate gradient and again in the cell-state gradient.
• Concatenated weight form. One matrix per gate acts on [a^{<t-1>}; x^{<t>}], so W_g is (n_a, n_a + n_x). The vanilla RNN deliberately keeps W_aa and W_ax separate to expose the two BPTT paths; that lesson is taught there, and repeating it across four gates is clutter. The two halves are still nameable: :func:split_gate_matrix cuts any W_g or dW_g at column n_a into its recurrent and input parts, which is what :func:main reports.
• The cache is a dict, not a tuple. The vanilla RNN's four-element positional cache is readable; a gated cell's eleven-element one is not, so every cached array is named.
• No epsilon anywhere. Nothing in the cell divides by a learned quantity, and the stable softmax output is strictly positive, so the cross-entropy log needs no guard. An epsilon added "for safety" would shift the loss and break the finite-difference gradient check at the topic's 1e-7 tolerance.

lstm_cell_forward

lstm_cell_forward(x_t: ndarray, a_prev: ndarray, c_prev: ndarray, parameters: dict[str, ndarray]) -> tuple[np.ndarray, np.ndarray, np.ndarray, dict[str, np.ndarray]]

One timestep of the LSTM forward recurrence.

Computes the four gate quantities in the topic's order (forget, update, candidate, output), then the cell state, the hidden state, and the softmax read-out.

Parameters:

Name	Type	Description	Default
x_t	ndarray	Input at this timestep, shape (n_x, m).	required
a_prev	ndarray	Hidden state carried in from the previous step, a^{<t-1>}, shape (n_a, m).	required
c_prev	ndarray	Cell state carried in from the previous step, c^{<t-1>}, shape (n_a, m).	required
parameters	dict[str, ndarray]	The shared weights and biases: Wf, Wu, Wc, Wo each (n_a, n_a + n_x); bf, bu, bc, bo each (n_a, 1); Wy (n_y, n_a); by (n_y, 1).	required

Returns:

Name	Type	Description
a_next	ndarray	Updated hidden state a^{<t>}, shape (n_a, m).
c_next	ndarray	Updated cell state c^{<t>}, shape (n_a, m).
y_pred_t	ndarray	Output distribution y_hat^{<t>}, shape (n_y, m).
cache	dict[str, ndarray]	Everything this step's backward pass needs, each array named.

Source code in src/dlhub/nn/sequence/lstm.py

def lstm_cell_forward(
    x_t: np.ndarray,
    a_prev: np.ndarray,
    c_prev: np.ndarray,
    parameters: dict[str, np.ndarray],
) -> tuple[np.ndarray, np.ndarray, np.ndarray, dict[str, np.ndarray]]:
    """
    One timestep of the LSTM forward recurrence.

    Computes the four gate quantities in the topic's order (forget, update,
    candidate, output), then the cell state, the hidden state, and the softmax
    read-out.

    Parameters
    ----------
    x_t : np.ndarray
        Input at this timestep, shape ``(n_x, m)``.
    a_prev : np.ndarray
        Hidden state carried in from the previous step, ``a^{<t-1>}``, shape
        ``(n_a, m)``.
    c_prev : np.ndarray
        Cell state carried in from the previous step, ``c^{<t-1>}``, shape
        ``(n_a, m)``.
    parameters : dict[str, np.ndarray]
        The shared weights and biases: ``Wf``, ``Wu``, ``Wc``, ``Wo`` each
        ``(n_a, n_a + n_x)``; ``bf``, ``bu``, ``bc``, ``bo`` each ``(n_a, 1)``;
        ``Wy`` ``(n_y, n_a)``; ``by`` ``(n_y, 1)``.

    Returns
    -------
    a_next : np.ndarray
        Updated hidden state ``a^{<t>}``, shape ``(n_a, m)``.
    c_next : np.ndarray
        Updated cell state ``c^{<t>}``, shape ``(n_a, m)``.
    y_pred_t : np.ndarray
        Output distribution ``y_hat^{<t>}``, shape ``(n_y, m)``.
    cache : dict[str, np.ndarray]
        Everything this step's backward pass needs, each array named.
    """
    Wf, Wu, Wc, Wo = (
        parameters["Wf"],
        parameters["Wu"],
        parameters["Wc"],
        parameters["Wo"],
    )
    bf, bu, bc, bo = (
        parameters["bf"],
        parameters["bu"],
        parameters["bc"],
        parameters["bo"],
    )
    Wy, by = parameters["Wy"], parameters["by"]

    # The concatenated form: one stacked vector [a^{<t-1>}; x^{<t>}], shape
    # (n_a + n_x, m), that all four gate weights multiply. Unlike the GRU, no
    # gate is applied inside this concatenation -- the LSTM has no reset gate.
    concat = np.concatenate([a_prev, x_t], axis=0)

    # --- The four gate quantities, in the topic's order -------------------
    gamma_f = sigmoid(Wf @ concat + bf)  # forget: how much of c^{<t-1>} survives
    gamma_u = sigmoid(Wu @ concat + bu)  # update ("input"): how much candidate enters
    c_tilde = np.tanh(Wc @ concat + bc)  # candidate: what could be written
    gamma_o = sigmoid(Wo @ concat + bo)  # output: how much of the cell is exposed

    # --- Cell state: the additive path ------------------------------------
    # Two gated terms are *added*. Nothing multiplies c^{<t-1>} but an
    # elementwise gate, which is the whole architectural point (module Notes).
    c_next = gamma_u * c_tilde + gamma_f * c_prev

    # --- Hidden state: a squashed, gated view of the cell -----------------
    tanh_c_next = np.tanh(c_next)
    a_next = gamma_o * tanh_c_next

    # --- Read-out ---------------------------------------------------------
    y_pred_t = softmax(Wy @ a_next + by)

    cache = {
        "a_next": a_next,
        "c_next": c_next,
        "a_prev": a_prev,
        "c_prev": c_prev,
        "gamma_f": gamma_f,
        "gamma_u": gamma_u,
        "c_tilde": c_tilde,
        "gamma_o": gamma_o,
        "tanh_c_next": tanh_c_next,
        "concat": concat,
        "x_t": x_t,
        "y_pred_t": y_pred_t,
    }
    return a_next, c_next, y_pred_t, cache

lstm_forward

lstm_forward(x: ndarray, parameters: dict[str, ndarray], a0: ndarray | None = None, c0: ndarray | None = None) -> tuple[np.ndarray, np.ndarray, np.ndarray, list[dict[str, np.ndarray]]]

Forward propagation through time across a whole sequence.

Unrolls :func:lstm_cell_forward over T_x timesteps, threading two states from one step to the next: the hidden state a and the cell state c.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence of shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	The shared weights and biases (see :func:lstm_cell_forward).	required
a0	ndarray or None	Initial hidden state a^{<0>} of shape (n_a, m). Defaults to the zero vector.	None
c0	ndarray or None	Initial cell state c^{<0>} of shape (n_a, m). Defaults to the zero vector.	None

Returns:

Name	Type	Description
a	ndarray	All hidden states stacked over time, shape (n_a, m, T_x).
c	ndarray	All cell states stacked over time, shape (n_a, m, T_x).
y_pred	ndarray	All output distributions, shape (n_y, m, T_x).
caches	list[dict[str, ndarray]]	Per-timestep caches, in order, for :func:lstm_backward.

Raises:

Type	Description
ValueError	If x is not three-dimensional, or if a supplied a0 or c0 does not have shape (n_a, m).

Source code in src/dlhub/nn/sequence/lstm.py

def lstm_forward(
    x: np.ndarray,
    parameters: dict[str, np.ndarray],
    a0: np.ndarray | None = None,
    c0: np.ndarray | None = None,
) -> tuple[np.ndarray, np.ndarray, np.ndarray, list[dict[str, np.ndarray]]]:
    """
    Forward propagation through time across a whole sequence.

    Unrolls :func:`lstm_cell_forward` over ``T_x`` timesteps, threading **two**
    states from one step to the next: the hidden state ``a`` and the cell state
    ``c``.

    Parameters
    ----------
    x : np.ndarray
        Input sequence of shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        The shared weights and biases (see :func:`lstm_cell_forward`).
    a0 : np.ndarray or None, optional
        Initial hidden state ``a^{<0>}`` of shape ``(n_a, m)``. Defaults to the
        zero vector.
    c0 : np.ndarray or None, optional
        Initial cell state ``c^{<0>}`` of shape ``(n_a, m)``. Defaults to the
        zero vector.

    Returns
    -------
    a : np.ndarray
        All hidden states stacked over time, shape ``(n_a, m, T_x)``.
    c : np.ndarray
        All cell states stacked over time, shape ``(n_a, m, T_x)``.
    y_pred : np.ndarray
        All output distributions, shape ``(n_y, m, T_x)``.
    caches : list[dict[str, np.ndarray]]
        Per-timestep caches, in order, for :func:`lstm_backward`.

    Raises
    ------
    ValueError
        If ``x`` is not three-dimensional, or if a supplied ``a0`` or ``c0`` does
        not have shape ``(n_a, m)``.
    """
    if x.ndim != 3:
        raise ValueError(f"x must have shape (n_x, m, T_x); got ndim {x.ndim}")

    n_x, m, T_x = x.shape
    n_a = parameters["Wf"].shape[0]
    n_y = parameters["Wy"].shape[0]

    if a0 is None:
        a0 = np.zeros((n_a, m))
    elif a0.shape != (n_a, m):
        raise ValueError(f"a0 must have shape {(n_a, m)}; got {a0.shape}")

    if c0 is None:
        c0 = np.zeros((n_a, m))
    elif c0.shape != (n_a, m):
        raise ValueError(f"c0 must have shape {(n_a, m)}; got {c0.shape}")

    a = np.zeros((n_a, m, T_x))
    c = np.zeros((n_a, m, T_x))
    y_pred = np.zeros((n_y, m, T_x))
    caches: list[dict[str, np.ndarray]] = []

    a_prev, c_prev = a0, c0
    for t in range(T_x):
        a_next, c_next, y_pred_t, cache = lstm_cell_forward(
            x[:, :, t], a_prev, c_prev, parameters
        )
        a[:, :, t] = a_next
        c[:, :, t] = c_next
        y_pred[:, :, t] = y_pred_t
        caches.append(cache)
        a_prev, c_prev = a_next, c_next  # carry both states forward to t+1

    return a, c, y_pred, caches

lstm_cell_backward

lstm_cell_backward(y_t: ndarray, da_next: ndarray, dc_next: ndarray, cache: dict[str, ndarray], parameters: dict[str, ndarray]) -> dict[str, np.ndarray]

Backward pass for a single LSTM timestep.

Walks the forward pass backwards, one equation at a time. Two gradients arrive from the future, not one: da_next along the hidden state and dc_next along the cell state. They meet at c^{<t>}, because the hidden state is a function of the cell state at the same step.

The line worth reading twice is dc_prev = dc_t * Gamma_f. That is the whole thesis of the architecture as one statement: the gradient crossing from t to t-1 along the cell path is scaled elementwise by the forget gate, with no weight matrix and no saturating nonlinearity in the way.

Parameters:

Name	Type	Description	Default
y_t	ndarray	One-hot target at this timestep, shape (n_y, m).	required
da_next	ndarray	Gradient w.r.t. this step's hidden state arriving from the recurrence at t+1, shape (n_a, m). Zero for the final step.	required
dc_next	ndarray	Gradient w.r.t. this step's cell state arriving from the recurrence at t+1, shape (n_a, m). Zero for the final step.	required
cache	dict[str, ndarray]	This step's cache from :func:lstm_cell_forward.	required
parameters	dict[str, ndarray]	The shared weights and biases (see :func:lstm_cell_forward).	required

Returns:

Type	Description
dict[str, ndarray]	This step's gradient contributions: dWf, dWu, dWc, dWo, dbf, dbu, dbc, dbo, dWy, dby, together with dxt (into the input) and da_prev, dc_prev (the two recurrences handed back to t-1).

Source code in src/dlhub/nn/sequence/lstm.py

def lstm_cell_backward(
    y_t: np.ndarray,
    da_next: np.ndarray,
    dc_next: np.ndarray,
    cache: dict[str, np.ndarray],
    parameters: dict[str, np.ndarray],
) -> dict[str, np.ndarray]:
    """
    Backward pass for a single LSTM timestep.

    Walks the forward pass backwards, one equation at a time. Two gradients
    arrive from the future, not one: ``da_next`` along the hidden state and
    ``dc_next`` along the cell state. They meet at ``c^{<t>}``, because the
    hidden state is a function of the cell state at the same step.

    The line worth reading twice is ``dc_prev = dc_t * Gamma_f``. That is the
    whole thesis of the architecture as one statement: the gradient crossing
    from ``t`` to ``t-1`` along the cell path is scaled elementwise by the forget
    gate, with no weight matrix and no saturating nonlinearity in the way.

    Parameters
    ----------
    y_t : np.ndarray
        One-hot target at this timestep, shape ``(n_y, m)``.
    da_next : np.ndarray
        Gradient w.r.t. this step's hidden state arriving from the recurrence at
        ``t+1``, shape ``(n_a, m)``. Zero for the final step.
    dc_next : np.ndarray
        Gradient w.r.t. this step's cell state arriving from the recurrence at
        ``t+1``, shape ``(n_a, m)``. Zero for the final step.
    cache : dict[str, np.ndarray]
        This step's cache from :func:`lstm_cell_forward`.
    parameters : dict[str, np.ndarray]
        The shared weights and biases (see :func:`lstm_cell_forward`).

    Returns
    -------
    dict[str, np.ndarray]
        This step's gradient contributions: ``dWf``, ``dWu``, ``dWc``, ``dWo``,
        ``dbf``, ``dbu``, ``dbc``, ``dbo``, ``dWy``, ``dby``, together with
        ``dxt`` (into the input) and ``da_prev``, ``dc_prev`` (the two
        recurrences handed back to ``t-1``).
    """
    a_next, c_prev = cache["a_next"], cache["c_prev"]
    gamma_f, gamma_u = cache["gamma_f"], cache["gamma_u"]
    c_tilde, gamma_o = cache["c_tilde"], cache["gamma_o"]
    tanh_c_next, concat = cache["tanh_c_next"], cache["concat"]
    y_pred_t = cache["y_pred_t"]

    Wf, Wu, Wc, Wo, Wy = (
        parameters["Wf"],
        parameters["Wu"],
        parameters["Wc"],
        parameters["Wo"],
        parameters["Wy"],
    )
    n_a, m = a_next.shape

    # --- Output path: softmax cross-entropy -------------------------------
    # For softmax + cross-entropy the logit gradient collapses to (y_hat - y),
    # scaled by the 1/m batch mean used in compute_loss.
    dz_y = (y_pred_t - y_t) / m  # (n_y, m)
    dWy = dz_y @ a_next.T  # (n_y, n_a)
    dby = np.sum(dz_y, axis=1, keepdims=True)  # (n_y, 1)
    da_from_output = Wy.T @ dz_y  # (n_a, m)

    # --- Merge the two gradient sources at a^{<t>} ------------------------
    da_t = da_from_output + da_next  # (n_a, m)

    # --- Hidden state: a^{<t>} = Gamma_o * tanh(c^{<t>}) ------------------
    dgamma_o = da_t * tanh_c_next
    # The hidden state feeds the cell state's gradient: differentiating the
    # tanh gives 1 - tanh(c)^2, read off the cached tanh. This joins the
    # gradient that arrived along the cell path from t+1.
    dc_t = dc_next + da_t * gamma_o * (1 - tanh_c_next**2)  # (n_a, m)

    # --- Cell state: c^{<t>} = Gamma_u * c_tilde + Gamma_f * c^{<t-1>} ----
    # An addition of two products, so each factor's gradient is the other.
    dgamma_u = dc_t * c_tilde
    dc_tilde = dc_t * gamma_u
    dgamma_f = dc_t * c_prev
    dc_prev = dc_t * gamma_f  # the undamped carry: d c^{<t>} / d c^{<t-1>}

    # --- Pre-activations, in the topic's gate order -----------------------
    # sigma'(z) = Gamma (1 - Gamma) and tanh'(z) = 1 - c_tilde^2, both read off
    # the cached activation rather than recomputed.
    dz_f = dgamma_f * gamma_f * (1 - gamma_f)
    dz_u = dgamma_u * gamma_u * (1 - gamma_u)
    dz_c = dc_tilde * (1 - c_tilde**2)
    dz_o = dgamma_o * gamma_o * (1 - gamma_o)

    # --- Gate weights and biases ------------------------------------------
    # Each gate read the same stacked vector [a^{<t-1>}; x^{<t>}], so each gate
    # weight's gradient is its pre-activation gradient times that vector.
    dWf, dbf = dz_f @ concat.T, np.sum(dz_f, axis=1, keepdims=True)
    dWu, dbu = dz_u @ concat.T, np.sum(dz_u, axis=1, keepdims=True)
    dWc, dbc = dz_c @ concat.T, np.sum(dz_c, axis=1, keepdims=True)
    dWo, dbo = dz_o @ concat.T, np.sum(dz_o, axis=1, keepdims=True)

    # --- Back through the concatenation -----------------------------------
    # All four gates read the same vector, so their contributions add; the top
    # n_a rows are the recurrence into a^{<t-1>} and the rest is the input.
    dconcat = Wf.T @ dz_f + Wu.T @ dz_u + Wc.T @ dz_c + Wo.T @ dz_o
    da_prev = dconcat[:n_a, :]  # (n_a, m)
    dxt = dconcat[n_a:, :]  # (n_x, m)

    return {
        "dWf": dWf,
        "dWu": dWu,
        "dWc": dWc,
        "dWo": dWo,
        "dbf": dbf,
        "dbu": dbu,
        "dbc": dbc,
        "dbo": dbo,
        "dWy": dWy,
        "dby": dby,
        "dxt": dxt,
        "da_prev": da_prev,
        "dc_prev": dc_prev,
    }

lstm_backward

lstm_backward(y: ndarray, caches: list[dict[str, ndarray]], parameters: dict[str, ndarray]) -> dict[str, np.ndarray]

Backpropagation through time (BPTT) for the LSTM.

Walks the unrolled graph from t = T_x back to t = 1, threading two recurrent gradients backward -- one along the hidden state and one along the cell state. Every gate weight is shared across time, so each receives an accumulated gradient summed over all timesteps.

Parameters:

Name	Type	Description	Default
y	ndarray	One-hot targets, shape (n_y, m, T_x).	required
caches	list[dict[str, ndarray]]	Per-timestep caches from :func:lstm_forward.	required
parameters	dict[str, ndarray]	The shared weights and biases (see :func:lstm_cell_forward).	required

Returns:

Type	Description
dict[str, ndarray]	Accumulated gradients dWf, dWu, dWc, dWo, dbf, dbu, dbc, dbo, dWy, dby; the gradients da0 and dc0 w.r.t. the two initial states; and dx, the gradient w.r.t. the whole input sequence, shape (n_x, m, T_x).

Source code in src/dlhub/nn/sequence/lstm.py

def lstm_backward(
    y: np.ndarray,
    caches: list[dict[str, np.ndarray]],
    parameters: dict[str, np.ndarray],
) -> dict[str, np.ndarray]:
    """
    Backpropagation through time (BPTT) for the LSTM.

    Walks the unrolled graph from ``t = T_x`` back to ``t = 1``, threading
    **two** recurrent gradients backward -- one along the hidden state and one
    along the cell state. Every gate weight is shared across time, so each
    receives an accumulated gradient summed over all timesteps.

    Parameters
    ----------
    y : np.ndarray
        One-hot targets, shape ``(n_y, m, T_x)``.
    caches : list[dict[str, np.ndarray]]
        Per-timestep caches from :func:`lstm_forward`.
    parameters : dict[str, np.ndarray]
        The shared weights and biases (see :func:`lstm_cell_forward`).

    Returns
    -------
    dict[str, np.ndarray]
        Accumulated gradients ``dWf``, ``dWu``, ``dWc``, ``dWo``, ``dbf``,
        ``dbu``, ``dbc``, ``dbo``, ``dWy``, ``dby``; the gradients ``da0`` and
        ``dc0`` w.r.t. the two initial states; and ``dx``, the gradient w.r.t.
        the whole input sequence, shape ``(n_x, m, T_x)``.
    """
    T_x = len(caches)
    _, m, _ = y.shape
    n_x = caches[0]["x_t"].shape[0]
    n_a = parameters["Wf"].shape[0]

    accumulated = {f"d{key}": np.zeros_like(parameters[key]) for key in PARAMETER_KEYS}
    dx = np.zeros((n_x, m, T_x))

    # The two recurrence contributions into the current step, carried back one
    # step at a time. Both zero at the final timestep, which has no successor.
    da_next = np.zeros((n_a, m))
    dc_next = np.zeros((n_a, m))

    for t in reversed(range(T_x)):
        grads_t = lstm_cell_backward(
            y[:, :, t], da_next, dc_next, caches[t], parameters
        )

        # The shared weights accumulate: every timestep used the same matrix,
        # so every timestep adds a term to its gradient.
        for key in PARAMETER_KEYS:
            accumulated[f"d{key}"] += grads_t[f"d{key}"]

        dx[:, :, t] = grads_t["dxt"]
        da_next = grads_t["da_prev"]  # hand both recurrences back to t-1
        dc_next = grads_t["dc_prev"]

    # After the loop, the two recurrence gradients have reached the initial
    # states a^{<0>} and c^{<0>}.
    return {**accumulated, "da0": da_next, "dc0": dc_next, "dx": dx}

make_fixture

make_fixture() -> dict

Build the deterministic fixture every LSTM implementation and test shares.

Draws are recorded in exactly the order the topic dossier pins, under np.random.seed(1) and float64 (the default of randn), so the arrays reproduce bit-for-bit across implementations. The one-hot targets are drawn last so that adding them cannot disturb the arrays a forward-parity check compares. Dimensions match the vanilla RNN fixture, so the notebook can contrast the two cells on identical shapes.

Returns:

Type	Description
dict	{"x", "a0", "c0", "parameters", "y", "dims"} where parameters is the dict {"Wf", "Wu", "Wc", "Wo", "Wy", "bf", "bu", "bc", "bo", "by"}, y is one-hot of shape (n_y, m, T_x), and dims records n_x, n_a, n_y, m, T_x.

Source code in src/dlhub/nn/sequence/lstm.py

def make_fixture() -> dict:
    """
    Build the deterministic fixture every LSTM implementation and test shares.

    Draws are recorded in exactly the order the topic dossier pins, under
    ``np.random.seed(1)`` and ``float64`` (the default of ``randn``), so the
    arrays reproduce bit-for-bit across implementations. The one-hot targets are
    drawn *last* so that adding them cannot disturb the arrays a forward-parity
    check compares. Dimensions match the vanilla RNN fixture, so the notebook can
    contrast the two cells on identical shapes.

    Returns
    -------
    dict
        ``{"x", "a0", "c0", "parameters", "y", "dims"}`` where ``parameters`` is
        the dict ``{"Wf", "Wu", "Wc", "Wo", "Wy", "bf", "bu", "bc", "bo",
        "by"}``, ``y`` is one-hot of shape ``(n_y, m, T_x)``, and ``dims``
        records ``n_x, n_a, n_y, m, T_x``.
    """
    np.random.seed(1)
    n_x, n_a, n_y, m, T_x = 3, 5, 2, 10, 4

    x = np.random.randn(n_x, m, T_x)  # 1
    a0 = np.random.randn(n_a, m)  # 2
    c0 = np.random.randn(n_a, m)  # 3
    Wf = np.random.randn(n_a, n_a + n_x)  # 4
    Wu = np.random.randn(n_a, n_a + n_x)  # 5
    Wc = np.random.randn(n_a, n_a + n_x)  # 6
    Wo = np.random.randn(n_a, n_a + n_x)  # 7
    Wy = np.random.randn(n_y, n_a)  # 8
    bf = np.random.randn(n_a, 1)  # 9
    bu = np.random.randn(n_a, 1)  # 10
    bc = np.random.randn(n_a, 1)  # 11
    bo = np.random.randn(n_a, 1)  # 12
    by = np.random.randn(n_y, 1)  # 13

    # Targets, drawn after the thirteen arrays above.
    labels = np.random.randint(0, n_y, size=(m, T_x))
    y = np.zeros((n_y, m, T_x))
    for i in range(m):
        for t in range(T_x):
            y[labels[i, t], i, t] = 1.0  # one-hot encode the class at (i, t)

    parameters = {
        "Wf": Wf,
        "Wu": Wu,
        "Wc": Wc,
        "Wo": Wo,
        "Wy": Wy,
        "bf": bf,
        "bu": bu,
        "bc": bc,
        "bo": bo,
        "by": by,
    }
    dims = {"n_x": n_x, "n_a": n_a, "n_y": n_y, "m": m, "T_x": T_x}
    return {"x": x, "a0": a0, "c0": c0, "parameters": parameters, "y": y, "dims": dims}

main

main() -> None

Run the shared fixture end to end and print shapes, loss, and grad norms.

Source code in src/dlhub/nn/sequence/lstm.py

def main() -> None:
    """Run the shared fixture end to end and print shapes, loss, and grad norms."""
    fixture = make_fixture()
    x, a0, c0, parameters, y = (
        fixture["x"],
        fixture["a0"],
        fixture["c0"],
        fixture["parameters"],
        fixture["y"],
    )
    n_a = fixture["dims"]["n_a"]

    a, c, y_pred, caches = lstm_forward(x, parameters, a0, c0)
    loss = compute_loss(y_pred, y)
    gradients = lstm_backward(y, caches, parameters)

    print(f"hidden states a: {a.shape}")
    print(f"cell states c:   {c.shape}")
    print(f"outputs y_pred:  {y_pred.shape}")
    print(f"total loss:      {loss:.6f}")
    for key in ("dWf", "dWu", "dWc", "dWo", "dWy", "da0", "dc0"):
        print(f"||{key}|| = {np.linalg.norm(gradients[key]):.6f}")

    # The concatenated gate gradient still names its two halves: the recurrent
    # path dW_ga and the input path dW_gx, split at column n_a.
    for key in ("dWf", "dWu", "dWc", "dWo"):
        recurrent, input_ = split_gate_matrix(gradients[key], n_a)
        print(
            f"{key} = [recurrent | input] -> "
            f"||{key}a|| = {np.linalg.norm(recurrent):.6f}, "
            f"||{key}x|| = {np.linalg.norm(input_):.6f}"
        )

min_gated

Minimal Gated Recurrent Cells (minGRU and minLSTM)

A from-scratch, teaching reference for the two minimal gated cells of "Were RNNs All We Needed?" (Feng et al., 2024). Section numbers quoted below are that paper's, so any line here can be checked against it without reading it whole.

The reduction that makes these cells parallelisable is a single deletion: the gates stop depending on a^{<t-1>} and read x^{<t>} alone (Sec. 3.1.1, Sec. 3.2.1). For t = 1 ... T_x::

minGRU (Sec. 3.1.3)
    Gamma_u^{<t>} = sigma(W_u x^{<t>})
    a_tilde^{<t>} = W_c x^{<t>}
    a^{<t>} = (1 - Gamma_u^{<t>}) * a^{<t-1>} + Gamma_u^{<t>} * a_tilde^{<t>}

minLSTM (Sec. 3.2.4)
    Gamma_f^{<t>}  = sigma(W_f x^{<t>})
    Gamma_u^{<t>}  = sigma(W_u x^{<t>})
    a_tilde^{<t>}  = W_c x^{<t>}
    Gamma_f'^{<t>} = Gamma_f^{<t>} / (Gamma_f^{<t>} + Gamma_u^{<t>})
    Gamma_u'^{<t>} = Gamma_u^{<t>} / (Gamma_f^{<t>} + Gamma_u^{<t>})
    a^{<t>} = Gamma_f'^{<t>} * a^{<t-1>} + Gamma_u'^{<t>} * a_tilde^{<t>}

Neither cell has bias terms, and neither has a recurrent weight half: every weight here is (n_a, n_x). Both cells are then the same first-order linear recurrence (Sec. 2.3)::

a^{<t>} = alpha^{<t>} * a^{<t-1>} + beta^{<t>}

with alpha = 1 - Gamma_u and beta = Gamma_u * a_tilde for minGRU, and alpha = Gamma_f' and beta = Gamma_u' * a_tilde for minLSTM. Because alpha and beta depend on x alone, they are computable for every timestep at once, so the sequence resolves by an associative (parallel) scan rather than a serial loop. This module ships all three forms: :func:sequential_scan is the definition, :func:parallel_scan is the same function evaluated in a different association order, and :func:log_space_scan is the numerically stable version of the parallel one (Sec. B.1).

What the reduction throws away -- the three details a reader arriving from the classical cells is most likely to "fix" back:

• minGRU has no reset gate (Sec. 3.1.1). The candidate no longer reads the previous state, so there is nothing left to reset.
• minLSTM has no output gate (Sec. 3.2.3), and because a^{<t>} = c^{<t>} the separate cell state collapses with it. The two remaining gates are normalised to sum to one, which gives the state a time-independent scale.
• The candidate is a plain linear map: there is no tanh (Sec. 3.1.2). See :func:identity_candidate.

Parameter counts follow from the deletions: minGRU is O(2 d_h d_x) against the GRU's O(3 d_h (d_x + d_h)), and minLSTM O(3 d_h d_x) against the LSTM's O(4 d_h (d_x + d_h)).

Shapes, in the sequence topic's convention (time last)::

x(n_x, m, T_x)
W_f, W_u, W_c(n_a, n_x)
a0(n_a, m)
alpha, beta, a(n_a, m, T_x)

Scope: this module derives no BPTT. The classical GRU and LSTM references in this package carry the hand-derived, gradient-checked backward passes; what is new here is the reduction and the scan, and a fourth backward derivation would only repeat a lesson those two already teach.

References

• Feng, L., Tung, F., Ahmed, M. O., Bengio, Y., & Hajimirsadeghi, H. (2024). Were RNNs All We Needed? arXiv:2410.01201. https://arxiv.org/abs/2410.01201
• Heinsen, F. A. (2023). Efficient Parallelization of a Ubiquitous Sequential Computation. arXiv:2311.06281. https://arxiv.org/abs/2311.06281
• Cho, K., van Merrienboer, B., Gulcehre, C., Bahdanau, D., Bougares, F., Schwenk, H., & Bengio, Y. (2014). Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation. EMNLP. https://arxiv.org/abs/1406.1078
• Hochreiter, S., & Schmidhuber, J. (1997). Long Short-Term Memory. Neural Computation, 9(8), 1735-1780.

Author

Deep Learning Reference Hub

License

MIT

Notes

• Softplus carries the gates in log space. sigmoid is the shared, overflow-free helper the classical cells in this package use; softplus is added here, computed as log1p(exp(-|z|)) + max(z, 0) so it never exponentiates a positive number. The two are the same function seen twice, since sigma(z) = exp(-softplus(-z)), which is what makes the log-space gates a rearrangement rather than a second derivation: log Gamma_u = -softplus(-k) and log(1 - Gamma_u) = -softplus(k), with no gate ever formed and then logged.
• Why a log-space scan at all. Resolving the recurrence needs the product of alpha across a prefix, and every alpha lies strictly inside (0, 1), so that product decays exponentially with the prefix: over 512 steps of a nearly-closed gate it underflows to exactly zero even in float64. :func:parallel_scan never forms it as a bare number -- it composes the affine steps, so an underflowed decay merely zeroes a contribution that has genuinely vanished -- but any formulation that recovers beta by dividing through the running product divides by that zero, and in float32 the decay bottoms out far sooner. Sec. B.1 sidesteps the question by keeping every product as a sum of logs (Heinsen, 2023), where a 512-step decay is an ordinary negative number, and that is the form the paper ships. The price is the round trip through log and exp: measured here, the two scans agree to about 4e-15 on the fixture and 2e-12 over 512 steps, which is why the equivalence tolerance is 1e-10 rather than machine epsilon.
• The log-space form needs a positive state. Logs of beta and of the initial state only exist when both are non-negative, which is why Sec. B.2.1 replaces the candidate's identity with :func:g, a map that is strictly positive everywhere and still linear (slope 1) on x >= 0. The two candidates give different cells, so an equivalence check compares the log-space scan against the same candidate, not against the linear one.
• No epsilon in minLSTM's normaliser. Gamma_f + Gamma_u is a sum of two strictly positive sigmoids, so the division in :func:min_lstm_coefficients is safe without a guard. It underflows only when both pre-activations fall below about -745, where both gates round to zero -- precisely the regime :func:min_lstm_log_coefficients handles exactly, by taking the normaliser with logaddexp instead of a division.
• No power-of-two assumption. Both :func:parallel_scan and :func:logcumsumexp double their stride with while stride < T_x, so a sequence of any length gets the ceil(log2 T_x) rounds it needs. The fixture's T_x = 7 is odd and not a power of two on purpose: a scan that quietly assumes padding to 8 passes on a length of 4 and fails here.

softplus

softplus(z: ndarray) -> np.ndarray

Stable log(1 + exp(z)).

Rewritten as log1p(exp(-|z|)) + max(z, 0), which is algebraically the same function but only ever exponentiates a non-positive number. See the module Notes.

Parameters:

Name	Type	Description	Default
z	ndarray	Pre-activation of any shape.	required

Returns:

Type	Description
ndarray	log(1 + exp(z)), elementwise, same shape as z.

Source code in src/dlhub/nn/sequence/min_gated.py

def softplus(z: np.ndarray) -> np.ndarray:
    """
    Stable ``log(1 + exp(z))``.

    Rewritten as ``log1p(exp(-|z|)) + max(z, 0)``, which is algebraically the
    same function but only ever exponentiates a non-positive number. See the
    module ``Notes``.

    Parameters
    ----------
    z : np.ndarray
        Pre-activation of any shape.

    Returns
    -------
    np.ndarray
        ``log(1 + exp(z))``, elementwise, same shape as ``z``.
    """
    return np.log1p(np.exp(-np.abs(z))) + np.maximum(z, 0.0)

identity_candidate

identity_candidate(k: ndarray) -> np.ndarray

The minimal cells' candidate activation: the identity (Sec. 3.1.2).

The classical GRU and LSTM squash their candidate with tanh. The minimal cells do not: the candidate is the plain linear map a_tilde^{<t>} = W_c x^{<t>}. This function exists so the deletion has a name and a docstring, rather than being an absence a later reader restores by accident.

Parameters:

Name	Type	Description	Default
k	ndarray	The candidate pre-activation W_c x.	required

Returns:

Type	Description
ndarray	k itself, unchanged.

Source code in src/dlhub/nn/sequence/min_gated.py

def identity_candidate(k: np.ndarray) -> np.ndarray:
    """
    The minimal cells' candidate activation: the identity (Sec. 3.1.2).

    The classical GRU and LSTM squash their candidate with ``tanh``. The
    minimal cells **do not**: the candidate is the plain linear map
    ``a_tilde^{<t>} = W_c x^{<t>}``. This function exists so the deletion has a
    name and a docstring, rather than being an absence a later reader restores
    by accident.

    Parameters
    ----------
    k : np.ndarray
        The candidate pre-activation ``W_c x``.

    Returns
    -------
    np.ndarray
        ``k`` itself, unchanged.
    """
    return k

g

g(k: ndarray) -> np.ndarray

The strictly positive candidate activation of Sec. B.2.1.

::

g(k) = k + 0.5      for k >= 0
     = sigma(k)     for k <  0

Continuous at zero (both branches give 0.5), positive everywhere, and still slope-1 linear on the positive half -- which is what lets the log-space scan take log(beta) at all.

Parameters:

Name	Type	Description	Default
k	ndarray	The candidate pre-activation W_c x.	required

Returns:

Type	Description
ndarray	Strictly positive values, same shape as k.

Source code in src/dlhub/nn/sequence/min_gated.py

def g(k: np.ndarray) -> np.ndarray:
    """
    The strictly positive candidate activation of Sec. B.2.1.

    ::

        g(k) = k + 0.5      for k >= 0
             = sigma(k)     for k <  0

    Continuous at zero (both branches give ``0.5``), positive everywhere, and
    still slope-1 linear on the positive half -- which is what lets the
    log-space scan take ``log(beta)`` at all.

    Parameters
    ----------
    k : np.ndarray
        The candidate pre-activation ``W_c x``.

    Returns
    -------
    np.ndarray
        Strictly positive values, same shape as ``k``.
    """
    return np.where(k >= 0.0, k + 0.5, sigmoid(k))

log_g

log_g(k: ndarray) -> np.ndarray

log(g(k)), computed without forming g(k) first (Sec. B.2.1).

::

log g(k) = log(k + 0.5)     for k >= 0
         = -softplus(-k)    for k <  0

The negative branch reuses the identity log sigma(k) = -softplus(-k), so no logarithm of an underflowed sigmoid is ever taken.

Parameters:

Name	Type	Description	Default
k	ndarray	The candidate pre-activation W_c x.	required

Returns:

Type	Description
ndarray	log(g(k)), same shape as k.

Source code in src/dlhub/nn/sequence/min_gated.py

def log_g(k: np.ndarray) -> np.ndarray:
    """
    ``log(g(k))``, computed without forming ``g(k)`` first (Sec. B.2.1).

    ::

        log g(k) = log(k + 0.5)     for k >= 0
                 = -softplus(-k)    for k <  0

    The negative branch reuses the identity ``log sigma(k) = -softplus(-k)``,
    so no logarithm of an underflowed sigmoid is ever taken.

    Parameters
    ----------
    k : np.ndarray
        The candidate pre-activation ``W_c x``.

    Returns
    -------
    np.ndarray
        ``log(g(k))``, same shape as ``k``.
    """
    # The inner `where` keeps `log` away from the negative branch's inputs, so
    # the unused half of the expression cannot emit an invalid-value warning.
    positive_branch = np.where(k >= 0.0, k + 0.5, 1.0)
    return np.where(k >= 0.0, np.log(positive_branch), -softplus(-k))

project_over_time

project_over_time(W: ndarray, x: ndarray) -> np.ndarray

Apply one weight matrix to every timestep in a single contraction.

Computes W x^{<t>} for all t, which is exactly the freedom the reduction buys: with no dependence on a^{<t-1>}, nothing in a gate has to wait for the step before it.

Parameters:

Name	Type	Description	Default
W	ndarray	Weight matrix of shape (n_a, n_x).	required
x	ndarray	Input sequence of shape (n_x, m, T_x).	required

Returns:

Type	Description
ndarray	Pre-activations of shape (n_a, m, T_x), whose slice at t equals W @ x[:, :, t].

Raises:

Type	Description
ValueError	If x is not three-dimensional.

Source code in src/dlhub/nn/sequence/min_gated.py

def project_over_time(W: np.ndarray, x: np.ndarray) -> np.ndarray:
    """
    Apply one weight matrix to every timestep in a single contraction.

    Computes ``W x^{<t>}`` for all ``t``, which is exactly the freedom the
    reduction buys: with no dependence on ``a^{<t-1>}``, nothing in a gate has
    to wait for the step before it.

    Parameters
    ----------
    W : np.ndarray
        Weight matrix of shape ``(n_a, n_x)``.
    x : np.ndarray
        Input sequence of shape ``(n_x, m, T_x)``.

    Returns
    -------
    np.ndarray
        Pre-activations of shape ``(n_a, m, T_x)``, whose slice at ``t`` equals
        ``W @ x[:, :, t]``.

    Raises
    ------
    ValueError
        If ``x`` is not three-dimensional.
    """
    if x.ndim != 3:
        raise ValueError(f"x must have shape (n_x, m, T_x); got ndim {x.ndim}")
    return np.tensordot(W, x, axes=([1], [0]))

min_gru_coefficients

min_gru_coefficients(x: ndarray, parameters: dict[str, ndarray], candidate: Callable[[ndarray], ndarray] = identity_candidate) -> tuple[np.ndarray, np.ndarray]

The minGRU recurrence coefficients (alpha, beta) (Sec. 3.1.3).

::

Gamma_u = sigma(W_u x)
a_tilde = candidate(W_c x)
alpha   = 1 - Gamma_u
beta    = Gamma_u * a_tilde

so that a^{<t>} = alpha^{<t>} * a^{<t-1>} + beta^{<t>} reproduces the cell's blend. There is no reset gate: the candidate never reads a^{<t-1>}.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	Wu and Wc, each (n_a, n_x).	required
candidate	Callable[[ndarray], ndarray]	Candidate activation applied to W_c x. Defaults to :func:identity_candidate, the paper's minimal form. Pass :func:g to match :func:log_space_scan's positivity requirement, or np.tanh to recover the classical GRU's candidate.	identity_candidate

Returns:

Name	Type	Description
alpha	ndarray	Per-step decay 1 - Gamma_u, shape (n_a, m, T_x).
beta	ndarray	Per-step injection Gamma_u * a_tilde, shape (n_a, m, T_x).

Source code in src/dlhub/nn/sequence/min_gated.py

def min_gru_coefficients(
    x: np.ndarray,
    parameters: dict[str, np.ndarray],
    candidate: Callable[[np.ndarray], np.ndarray] = identity_candidate,
) -> tuple[np.ndarray, np.ndarray]:
    """
    The minGRU recurrence coefficients ``(alpha, beta)`` (Sec. 3.1.3).

    ::

        Gamma_u = sigma(W_u x)
        a_tilde = candidate(W_c x)
        alpha   = 1 - Gamma_u
        beta    = Gamma_u * a_tilde

    so that ``a^{<t>} = alpha^{<t>} * a^{<t-1>} + beta^{<t>}`` reproduces the
    cell's blend. There is no reset gate: the candidate never reads
    ``a^{<t-1>}``.

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        ``Wu`` and ``Wc``, each ``(n_a, n_x)``.
    candidate : Callable[[np.ndarray], np.ndarray], optional
        Candidate activation applied to ``W_c x``. Defaults to
        :func:`identity_candidate`, the paper's minimal form. Pass :func:`g` to
        match :func:`log_space_scan`'s positivity requirement, or ``np.tanh``
        to recover the classical GRU's candidate.

    Returns
    -------
    alpha : np.ndarray
        Per-step decay ``1 - Gamma_u``, shape ``(n_a, m, T_x)``.
    beta : np.ndarray
        Per-step injection ``Gamma_u * a_tilde``, shape ``(n_a, m, T_x)``.
    """
    k_u = project_over_time(parameters["Wu"], x)  # (n_a, m, T_x)
    k_c = project_over_time(parameters["Wc"], x)  # (n_a, m, T_x)

    gamma_u = sigmoid(k_u)  # update gate
    a_tilde = candidate(k_c)  # candidate state -- linear by default

    alpha = 1.0 - gamma_u  # weight on a^{<t-1>}
    beta = gamma_u * a_tilde  # weight on the candidate
    return alpha, beta

min_lstm_coefficients

min_lstm_coefficients(x: ndarray, parameters: dict[str, ndarray], candidate: Callable[[ndarray], ndarray] = identity_candidate) -> tuple[np.ndarray, np.ndarray]

The minLSTM recurrence coefficients (alpha, beta) (Sec. 3.2.4).

::

Gamma_f  = sigma(W_f x)
Gamma_u  = sigma(W_u x)
a_tilde  = candidate(W_c x)
Gamma_f' = Gamma_f / (Gamma_f + Gamma_u)
Gamma_u' = Gamma_u / (Gamma_f + Gamma_u)
alpha    = Gamma_f'
beta     = Gamma_u' * a_tilde

The normalisation is what replaces the missing output gate: because Gamma_f' + Gamma_u' = 1 exactly, the state is a convex blend at every step and so keeps a time-independent scale.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	Wf, Wu, and Wc, each (n_a, n_x).	required
candidate	Callable[[ndarray], ndarray]	Candidate activation applied to W_c x. Defaults to :func:identity_candidate.	identity_candidate

Returns:

Name	Type	Description
alpha	ndarray	The normalised forget gate Gamma_f', shape (n_a, m, T_x).
beta	ndarray	Gamma_u' * a_tilde, shape (n_a, m, T_x).

Source code in src/dlhub/nn/sequence/min_gated.py

def min_lstm_coefficients(
    x: np.ndarray,
    parameters: dict[str, np.ndarray],
    candidate: Callable[[np.ndarray], np.ndarray] = identity_candidate,
) -> tuple[np.ndarray, np.ndarray]:
    """
    The minLSTM recurrence coefficients ``(alpha, beta)`` (Sec. 3.2.4).

    ::

        Gamma_f  = sigma(W_f x)
        Gamma_u  = sigma(W_u x)
        a_tilde  = candidate(W_c x)
        Gamma_f' = Gamma_f / (Gamma_f + Gamma_u)
        Gamma_u' = Gamma_u / (Gamma_f + Gamma_u)
        alpha    = Gamma_f'
        beta     = Gamma_u' * a_tilde

    The normalisation is what replaces the missing output gate: because
    ``Gamma_f' + Gamma_u' = 1`` exactly, the state is a convex blend at every
    step and so keeps a time-independent scale.

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        ``Wf``, ``Wu``, and ``Wc``, each ``(n_a, n_x)``.
    candidate : Callable[[np.ndarray], np.ndarray], optional
        Candidate activation applied to ``W_c x``. Defaults to
        :func:`identity_candidate`.

    Returns
    -------
    alpha : np.ndarray
        The normalised forget gate ``Gamma_f'``, shape ``(n_a, m, T_x)``.
    beta : np.ndarray
        ``Gamma_u' * a_tilde``, shape ``(n_a, m, T_x)``.
    """
    k_f = project_over_time(parameters["Wf"], x)  # (n_a, m, T_x)
    k_u = project_over_time(parameters["Wu"], x)  # (n_a, m, T_x)
    k_c = project_over_time(parameters["Wc"], x)  # (n_a, m, T_x)

    gamma_f = sigmoid(k_f)  # forget gate
    gamma_u = sigmoid(k_u)  # update (input) gate
    a_tilde = candidate(k_c)  # candidate state -- linear by default

    # Normalise the two gates so they sum to one. No epsilon: the denominator
    # is a sum of two strictly positive sigmoids (see the module Notes).
    normaliser = gamma_f + gamma_u
    gamma_f_prime = gamma_f / normaliser
    gamma_u_prime = gamma_u / normaliser

    alpha = gamma_f_prime
    beta = gamma_u_prime * a_tilde
    return alpha, beta

min_gru_log_coefficients

min_gru_log_coefficients(x: ndarray, parameters: dict[str, ndarray]) -> tuple[np.ndarray, np.ndarray]

The minGRU coefficients in log space (Sec. B.1), using :func:g.

Takes the gate logarithms straight from the pre-activation k_u = W_u x, never forming the gate itself::

log alpha = log(1 - Gamma_u) = -softplus(k_u)
log beta  = log Gamma_u + log g(k_c) = -softplus(-k_u) + log_g(k_c)

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	Wu and Wc, each (n_a, n_x).	required

Returns:

Name	Type	Description
log_alpha	ndarray	log(alpha), shape (n_a, m, T_x).
log_beta	ndarray	log(beta), shape (n_a, m, T_x).

Source code in src/dlhub/nn/sequence/min_gated.py

def min_gru_log_coefficients(
    x: np.ndarray, parameters: dict[str, np.ndarray]
) -> tuple[np.ndarray, np.ndarray]:
    """
    The minGRU coefficients in log space (Sec. B.1), using :func:`g`.

    Takes the gate logarithms straight from the pre-activation
    ``k_u = W_u x``, never forming the gate itself::

        log alpha = log(1 - Gamma_u) = -softplus(k_u)
        log beta  = log Gamma_u + log g(k_c) = -softplus(-k_u) + log_g(k_c)

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        ``Wu`` and ``Wc``, each ``(n_a, n_x)``.

    Returns
    -------
    log_alpha : np.ndarray
        ``log(alpha)``, shape ``(n_a, m, T_x)``.
    log_beta : np.ndarray
        ``log(beta)``, shape ``(n_a, m, T_x)``.
    """
    k_u = project_over_time(parameters["Wu"], x)
    k_c = project_over_time(parameters["Wc"], x)

    log_alpha = -softplus(k_u)  # log(1 - Gamma_u)
    log_beta = -softplus(-k_u) + log_g(k_c)  # log Gamma_u + log g(k_c)
    return log_alpha, log_beta

min_lstm_log_coefficients

min_lstm_log_coefficients(x: ndarray, parameters: dict[str, ndarray]) -> tuple[np.ndarray, np.ndarray]

The minLSTM coefficients in log space (Sec. B.1), using :func:g.

The gate normalisation becomes a subtraction::

log Gamma_f = -softplus(-k_f)
log Gamma_u = -softplus(-k_u)
log(Gamma_f + Gamma_u) = logaddexp(log Gamma_f, log Gamma_u)
log alpha = log Gamma_f - log(Gamma_f + Gamma_u)
log beta  = log Gamma_u - log(Gamma_f + Gamma_u) + log_g(k_c)

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	Wf, Wu, and Wc, each (n_a, n_x).	required

Returns:

Name	Type	Description
log_alpha	ndarray	log(alpha), shape (n_a, m, T_x).
log_beta	ndarray	log(beta), shape (n_a, m, T_x).

Source code in src/dlhub/nn/sequence/min_gated.py

def min_lstm_log_coefficients(
    x: np.ndarray, parameters: dict[str, np.ndarray]
) -> tuple[np.ndarray, np.ndarray]:
    """
    The minLSTM coefficients in log space (Sec. B.1), using :func:`g`.

    The gate normalisation becomes a subtraction::

        log Gamma_f = -softplus(-k_f)
        log Gamma_u = -softplus(-k_u)
        log(Gamma_f + Gamma_u) = logaddexp(log Gamma_f, log Gamma_u)
        log alpha = log Gamma_f - log(Gamma_f + Gamma_u)
        log beta  = log Gamma_u - log(Gamma_f + Gamma_u) + log_g(k_c)

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        ``Wf``, ``Wu``, and ``Wc``, each ``(n_a, n_x)``.

    Returns
    -------
    log_alpha : np.ndarray
        ``log(alpha)``, shape ``(n_a, m, T_x)``.
    log_beta : np.ndarray
        ``log(beta)``, shape ``(n_a, m, T_x)``.
    """
    k_f = project_over_time(parameters["Wf"], x)
    k_u = project_over_time(parameters["Wu"], x)
    k_c = project_over_time(parameters["Wc"], x)

    log_gamma_f = -softplus(-k_f)
    log_gamma_u = -softplus(-k_u)
    log_normaliser = np.logaddexp(log_gamma_f, log_gamma_u)

    log_alpha = log_gamma_f - log_normaliser
    log_beta = log_gamma_u - log_normaliser + log_g(k_c)
    return log_alpha, log_beta

sequential_scan

sequential_scan(alpha: ndarray, beta: ndarray, a0: ndarray | None = None) -> np.ndarray

Resolve a^{<t>} = alpha^{<t>} a^{<t-1>} + beta^{<t>} one step at a time.

This is the definition, written as the loop the recurrence describes. It is the reference the two scans below are checked against.

Parameters:

Name	Type	Description	Default
alpha	ndarray	Per-step coefficients, each (n_a, m, T_x).	required
beta	ndarray	Per-step coefficients, each (n_a, m, T_x).	required
a0	ndarray or None	Initial state a^{<0>}, shape (n_a, m). Defaults to zero.	None

Returns:

Type	Description
ndarray	The hidden states, shape (n_a, m, T_x).

Raises:

Type	Description
ValueError	If alpha and beta disagree in shape, or a0 is misshapen.

Source code in src/dlhub/nn/sequence/min_gated.py

def sequential_scan(
    alpha: np.ndarray, beta: np.ndarray, a0: np.ndarray | None = None
) -> np.ndarray:
    """
    Resolve ``a^{<t>} = alpha^{<t>} a^{<t-1>} + beta^{<t>}`` one step at a time.

    This is the definition, written as the loop the recurrence describes. It is
    the reference the two scans below are checked against.

    Parameters
    ----------
    alpha, beta : np.ndarray
        Per-step coefficients, each ``(n_a, m, T_x)``.
    a0 : np.ndarray or None, optional
        Initial state ``a^{<0>}``, shape ``(n_a, m)``. Defaults to zero.

    Returns
    -------
    np.ndarray
        The hidden states, shape ``(n_a, m, T_x)``.

    Raises
    ------
    ValueError
        If ``alpha`` and ``beta`` disagree in shape, or ``a0`` is misshapen.
    """
    if alpha.shape != beta.shape:
        raise ValueError(f"alpha {alpha.shape} and beta {beta.shape} must match")

    n_a, m, T_x = alpha.shape
    a_prev = _resolve_initial_state(a0, n_a, m)

    a = np.zeros((n_a, m, T_x))
    for t in range(T_x):
        a_t = alpha[:, :, t] * a_prev + beta[:, :, t]
        a[:, :, t] = a_t
        a_prev = a_t  # carry the state forward to t+1
    return a

parallel_scan

parallel_scan(alpha: ndarray, beta: ndarray, a0: ndarray | None = None) -> np.ndarray

Resolve the same recurrence by an associative scan, in plain space.

Each timestep is an affine map a -> alpha * a + beta, and composing two of them -- first (alpha_1, beta_1), then (alpha_2, beta_2) -- gives another affine map::

(alpha_1, beta_1) . (alpha_2, beta_2)
    = (alpha_2 * alpha_1,  alpha_2 * beta_1 + beta_2)

Composition is associative, so the prefix compositions can be built by doubling the stride (the Hillis-Steele inclusive scan) in ceil(log2 T_x) rounds instead of T_x serial steps. After the doubling, entry t holds the single affine map from a^{<0>} to a^{<t>}, which is then applied to the initial state.

Parameters:

Name	Type	Description	Default
alpha	ndarray	Per-step coefficients, each (n_a, m, T_x).	required
beta	ndarray	Per-step coefficients, each (n_a, m, T_x).	required
a0	ndarray or None	Initial state a^{<0>}, shape (n_a, m). Defaults to zero.	None

Returns:

Type	Description
ndarray	The hidden states, shape (n_a, m, T_x), equal to :func:sequential_scan's output up to floating-point association.

Raises:

Type	Description
ValueError	If alpha and beta disagree in shape, or a0 is misshapen.

Source code in src/dlhub/nn/sequence/min_gated.py

def parallel_scan(
    alpha: np.ndarray, beta: np.ndarray, a0: np.ndarray | None = None
) -> np.ndarray:
    """
    Resolve the same recurrence by an associative scan, in plain space.

    Each timestep is an affine map ``a -> alpha * a + beta``, and composing two
    of them -- first ``(alpha_1, beta_1)``, then ``(alpha_2, beta_2)`` -- gives
    another affine map::

        (alpha_1, beta_1) . (alpha_2, beta_2)
            = (alpha_2 * alpha_1,  alpha_2 * beta_1 + beta_2)

    Composition is associative, so the prefix compositions can be built by
    doubling the stride (the Hillis-Steele inclusive scan) in
    ``ceil(log2 T_x)`` rounds instead of ``T_x`` serial steps. After the
    doubling, entry ``t`` holds the single affine map from ``a^{<0>}`` to
    ``a^{<t>}``, which is then applied to the initial state.

    Parameters
    ----------
    alpha, beta : np.ndarray
        Per-step coefficients, each ``(n_a, m, T_x)``.
    a0 : np.ndarray or None, optional
        Initial state ``a^{<0>}``, shape ``(n_a, m)``. Defaults to zero.

    Returns
    -------
    np.ndarray
        The hidden states, shape ``(n_a, m, T_x)``, equal to
        :func:`sequential_scan`'s output up to floating-point association.

    Raises
    ------
    ValueError
        If ``alpha`` and ``beta`` disagree in shape, or ``a0`` is misshapen.
    """
    if alpha.shape != beta.shape:
        raise ValueError(f"alpha {alpha.shape} and beta {beta.shape} must match")

    n_a, m, T_x = alpha.shape
    a_start = _resolve_initial_state(a0, n_a, m)

    # prefix[t] is the composition of steps [t - span + 1 .. t]; it starts as
    # the single step t and its span doubles each round.
    prefix_alpha, prefix_beta = alpha.copy(), beta.copy()

    stride = 1
    while stride < T_x:
        # The earlier half of each pair: what steps [t - 2*stride + 1 .. t - stride]
        # already compose to. The later half is the entry being updated.
        left_alpha, left_beta = (
            prefix_alpha[:, :, :-stride],
            prefix_beta[:, :, :-stride],
        )
        right_alpha = prefix_alpha[:, :, stride:]
        right_beta = prefix_beta[:, :, stride:]

        composed_alpha = right_alpha * left_alpha
        composed_beta = right_alpha * left_beta + right_beta

        # Entries before `stride` already span all the history there is, so they
        # are carried through untouched.
        prefix_alpha = np.concatenate(
            [prefix_alpha[:, :, :stride], composed_alpha], axis=2
        )
        prefix_beta = np.concatenate(
            [prefix_beta[:, :, :stride], composed_beta], axis=2
        )
        stride *= 2

    return prefix_alpha * a_start[:, :, np.newaxis] + prefix_beta

logcumsumexp

logcumsumexp(z: ndarray) -> np.ndarray

Cumulative log(sum(exp(.))) along the last (time) axis.

Built with the same stride-doubling as :func:parallel_scan, with np.logaddexp -- itself associative -- in place of affine composition. NumPy has no built-in for this, and the naive log(cumsum(exp(z))) overflows exactly where a log-space scan is worth having.

Parameters:

Name	Type	Description	Default
z	ndarray	Log-domain values, shape (..., T).	required

Returns:

Type	Description
ndarray	out[..., t] = log(sum_{i <= t} exp(z[..., i])), same shape as z.

Source code in src/dlhub/nn/sequence/min_gated.py

def logcumsumexp(z: np.ndarray) -> np.ndarray:
    """
    Cumulative ``log(sum(exp(.)))`` along the last (time) axis.

    Built with the same stride-doubling as :func:`parallel_scan`, with
    ``np.logaddexp`` -- itself associative -- in place of affine composition.
    NumPy has no built-in for this, and the naive ``log(cumsum(exp(z)))``
    overflows exactly where a log-space scan is worth having.

    Parameters
    ----------
    z : np.ndarray
        Log-domain values, shape ``(..., T)``.

    Returns
    -------
    np.ndarray
        ``out[..., t] = log(sum_{i <= t} exp(z[..., i]))``, same shape as ``z``.
    """
    T = z.shape[-1]
    out = z.copy()

    stride = 1
    while stride < T:
        combined = np.logaddexp(out[..., stride:], out[..., :-stride])
        out = np.concatenate([out[..., :stride], combined], axis=-1)
        stride *= 2
    return out

log_space_scan

log_space_scan(log_alpha: ndarray, log_beta: ndarray, a0: ndarray | None = None) -> np.ndarray

Resolve the recurrence in log space -- the stable scan of Sec. B.1.

Writing A_t = sum_{i <= t} log alpha_i for the cumulative log-decay, the closed form of the recurrence is::

a^{<t>} = exp( A_t + log( sum_{j <= t} exp(log beta_j - A_j) ) )

with the initial state entering as the j = 0 term (coefficient one, hence log alpha_0 = 0). Every product has become a sum and every sum a logaddexp, so nothing underflows on the way. This is Heinsen's (2023) formulation, which the paper adopts.

Parameters:

Name	Type	Description	Default
log_alpha	ndarray	Logs of the per-step coefficients, each (n_a, m, T_x). Both require positive coefficients, which is why the candidate becomes :func:g.	required
log_beta	ndarray	Logs of the per-step coefficients, each (n_a, m, T_x). Both require positive coefficients, which is why the candidate becomes :func:g.	required
a0	ndarray or None	Initial state a^{<0>}, shape (n_a, m), and non-negative: the log of a negative state does not exist. Defaults to zero.	None

Returns:

Type	Description
ndarray	The hidden states, shape (n_a, m, T_x), all strictly positive.

Raises:

Type	Description
ValueError	If the two coefficient arrays disagree in shape, if a0 is misshapen, or if any entry of a0 is negative.

Source code in src/dlhub/nn/sequence/min_gated.py

def log_space_scan(
    log_alpha: np.ndarray, log_beta: np.ndarray, a0: np.ndarray | None = None
) -> np.ndarray:
    """
    Resolve the recurrence in log space -- the stable scan of Sec. B.1.

    Writing ``A_t = sum_{i <= t} log alpha_i`` for the cumulative log-decay, the
    closed form of the recurrence is::

        a^{<t>} = exp( A_t + log( sum_{j <= t} exp(log beta_j - A_j) ) )

    with the initial state entering as the ``j = 0`` term (coefficient one,
    hence ``log alpha_0 = 0``). Every product has become a sum and every sum a
    ``logaddexp``, so nothing underflows on the way. This is Heinsen's (2023)
    formulation, which the paper adopts.

    Parameters
    ----------
    log_alpha, log_beta : np.ndarray
        Logs of the per-step coefficients, each ``(n_a, m, T_x)``. Both require
        positive coefficients, which is why the candidate becomes :func:`g`.
    a0 : np.ndarray or None, optional
        Initial state ``a^{<0>}``, shape ``(n_a, m)``, and **non-negative**:
        the log of a negative state does not exist. Defaults to zero.

    Returns
    -------
    np.ndarray
        The hidden states, shape ``(n_a, m, T_x)``, all strictly positive.

    Raises
    ------
    ValueError
        If the two coefficient arrays disagree in shape, if ``a0`` is misshapen,
        or if any entry of ``a0`` is negative.
    """
    if log_alpha.shape != log_beta.shape:
        raise ValueError(
            f"log_alpha {log_alpha.shape} and log_beta {log_beta.shape} must match"
        )

    n_a, m, T_x = log_alpha.shape
    a_start = _resolve_initial_state(a0, n_a, m)
    if np.any(a_start < 0):
        raise ValueError("log_space_scan needs a non-negative a0; see module Notes")

    # Prepend the initial state as step 0: it is injected with coefficient one,
    # so its log-coefficient is zero and its log-injection is log(a0).
    zero_log_alpha = np.zeros((n_a, m, 1))
    with np.errstate(divide="ignore"):  # a0 == 0 is the default, and log 0 = -inf
        log_a_start = np.log(a_start)[:, :, np.newaxis]

    padded_log_alpha = np.concatenate([zero_log_alpha, log_alpha], axis=2)
    padded_log_beta = np.concatenate([log_a_start, log_beta], axis=2)

    cumulative_log_alpha = np.cumsum(padded_log_alpha, axis=2)  # A_t
    log_a = cumulative_log_alpha + logcumsumexp(padded_log_beta - cumulative_log_alpha)

    return np.exp(log_a[:, :, 1:])  # drop the prepended step 0

min_gru_forward_sequential

min_gru_forward_sequential(x: ndarray, parameters: dict[str, ndarray], a0: ndarray | None = None, candidate: Callable[[ndarray], ndarray] = identity_candidate) -> np.ndarray

Forward pass for minGRU, resolved by the serial recurrence.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	Wu and Wc, each (n_a, n_x).	required
a0	ndarray or None	Initial state, shape (n_a, m). Defaults to zero.	None
candidate	Callable[[ndarray], ndarray]	Candidate activation, default :func:identity_candidate.	identity_candidate

Returns:

Type	Description
ndarray	Hidden states a, shape (n_a, m, T_x).

Source code in src/dlhub/nn/sequence/min_gated.py

def min_gru_forward_sequential(
    x: np.ndarray,
    parameters: dict[str, np.ndarray],
    a0: np.ndarray | None = None,
    candidate: Callable[[np.ndarray], np.ndarray] = identity_candidate,
) -> np.ndarray:
    """
    Forward pass for minGRU, resolved by the serial recurrence.

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        ``Wu`` and ``Wc``, each ``(n_a, n_x)``.
    a0 : np.ndarray or None, optional
        Initial state, shape ``(n_a, m)``. Defaults to zero.
    candidate : Callable[[np.ndarray], np.ndarray], optional
        Candidate activation, default :func:`identity_candidate`.

    Returns
    -------
    np.ndarray
        Hidden states ``a``, shape ``(n_a, m, T_x)``.
    """
    alpha, beta = min_gru_coefficients(x, parameters, candidate)
    return sequential_scan(alpha, beta, a0)

min_gru_forward_parallel

min_gru_forward_parallel(x: ndarray, parameters: dict[str, ndarray], a0: ndarray | None = None, candidate: Callable[[ndarray], ndarray] = identity_candidate) -> np.ndarray

Forward pass for minGRU, resolved by the plain-space associative scan.

Same function as :func:min_gru_forward_sequential, evaluated in a different association order.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	Wu and Wc, each (n_a, n_x).	required
a0	ndarray or None	Initial state, shape (n_a, m). Defaults to zero.	None
candidate	Callable[[ndarray], ndarray]	Candidate activation, default :func:identity_candidate.	identity_candidate

Returns:

Type	Description
ndarray	Hidden states a, shape (n_a, m, T_x).

Source code in src/dlhub/nn/sequence/min_gated.py

def min_gru_forward_parallel(
    x: np.ndarray,
    parameters: dict[str, np.ndarray],
    a0: np.ndarray | None = None,
    candidate: Callable[[np.ndarray], np.ndarray] = identity_candidate,
) -> np.ndarray:
    """
    Forward pass for minGRU, resolved by the plain-space associative scan.

    Same function as :func:`min_gru_forward_sequential`, evaluated in a
    different association order.

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        ``Wu`` and ``Wc``, each ``(n_a, n_x)``.
    a0 : np.ndarray or None, optional
        Initial state, shape ``(n_a, m)``. Defaults to zero.
    candidate : Callable[[np.ndarray], np.ndarray], optional
        Candidate activation, default :func:`identity_candidate`.

    Returns
    -------
    np.ndarray
        Hidden states ``a``, shape ``(n_a, m, T_x)``.
    """
    alpha, beta = min_gru_coefficients(x, parameters, candidate)
    return parallel_scan(alpha, beta, a0)

min_gru_forward_log

min_gru_forward_log(x: ndarray, parameters: dict[str, ndarray], a0: ndarray | None = None) -> np.ndarray

Forward pass for minGRU through the log-space scan (Sec. B.1).

This is the positive-candidate cell: the candidate is :func:g, not the identity, because a log-space scan needs beta > 0. It therefore matches :func:min_gru_forward_sequential called with candidate=g, and not the default linear one.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	Wu and Wc, each (n_a, n_x).	required
a0	ndarray or None	Non-negative initial state, shape (n_a, m). Defaults to zero.	None

Returns:

Type	Description
ndarray	Hidden states a, shape (n_a, m, T_x), strictly positive.

Source code in src/dlhub/nn/sequence/min_gated.py

def min_gru_forward_log(
    x: np.ndarray, parameters: dict[str, np.ndarray], a0: np.ndarray | None = None
) -> np.ndarray:
    """
    Forward pass for minGRU through the log-space scan (Sec. B.1).

    This is the *positive-candidate* cell: the candidate is :func:`g`, not the
    identity, because a log-space scan needs ``beta > 0``. It therefore matches
    :func:`min_gru_forward_sequential` called with ``candidate=g``, and not the
    default linear one.

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        ``Wu`` and ``Wc``, each ``(n_a, n_x)``.
    a0 : np.ndarray or None, optional
        Non-negative initial state, shape ``(n_a, m)``. Defaults to zero.

    Returns
    -------
    np.ndarray
        Hidden states ``a``, shape ``(n_a, m, T_x)``, strictly positive.
    """
    log_alpha, log_beta = min_gru_log_coefficients(x, parameters)
    return log_space_scan(log_alpha, log_beta, a0)

min_lstm_forward_sequential

min_lstm_forward_sequential(x: ndarray, parameters: dict[str, ndarray], a0: ndarray | None = None, candidate: Callable[[ndarray], ndarray] = identity_candidate) -> np.ndarray

Forward pass for minLSTM, resolved by the serial recurrence.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	Wf, Wu, and Wc, each (n_a, n_x).	required
a0	ndarray or None	Initial state, shape (n_a, m). Defaults to zero.	None
candidate	Callable[[ndarray], ndarray]	Candidate activation, default :func:identity_candidate.	identity_candidate

Returns:

Type	Description
ndarray	Hidden states a, shape (n_a, m, T_x).

Source code in src/dlhub/nn/sequence/min_gated.py

def min_lstm_forward_sequential(
    x: np.ndarray,
    parameters: dict[str, np.ndarray],
    a0: np.ndarray | None = None,
    candidate: Callable[[np.ndarray], np.ndarray] = identity_candidate,
) -> np.ndarray:
    """
    Forward pass for minLSTM, resolved by the serial recurrence.

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        ``Wf``, ``Wu``, and ``Wc``, each ``(n_a, n_x)``.
    a0 : np.ndarray or None, optional
        Initial state, shape ``(n_a, m)``. Defaults to zero.
    candidate : Callable[[np.ndarray], np.ndarray], optional
        Candidate activation, default :func:`identity_candidate`.

    Returns
    -------
    np.ndarray
        Hidden states ``a``, shape ``(n_a, m, T_x)``.
    """
    alpha, beta = min_lstm_coefficients(x, parameters, candidate)
    return sequential_scan(alpha, beta, a0)

min_lstm_forward_parallel

min_lstm_forward_parallel(x: ndarray, parameters: dict[str, ndarray], a0: ndarray | None = None, candidate: Callable[[ndarray], ndarray] = identity_candidate) -> np.ndarray

Forward pass for minLSTM, resolved by the plain-space associative scan.

Same function as :func:min_lstm_forward_sequential, evaluated in a different association order.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	Wf, Wu, and Wc, each (n_a, n_x).	required
a0	ndarray or None	Initial state, shape (n_a, m). Defaults to zero.	None
candidate	Callable[[ndarray], ndarray]	Candidate activation, default :func:identity_candidate.	identity_candidate

Returns:

Type	Description
ndarray	Hidden states a, shape (n_a, m, T_x).

Source code in src/dlhub/nn/sequence/min_gated.py

def min_lstm_forward_parallel(
    x: np.ndarray,
    parameters: dict[str, np.ndarray],
    a0: np.ndarray | None = None,
    candidate: Callable[[np.ndarray], np.ndarray] = identity_candidate,
) -> np.ndarray:
    """
    Forward pass for minLSTM, resolved by the plain-space associative scan.

    Same function as :func:`min_lstm_forward_sequential`, evaluated in a
    different association order.

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        ``Wf``, ``Wu``, and ``Wc``, each ``(n_a, n_x)``.
    a0 : np.ndarray or None, optional
        Initial state, shape ``(n_a, m)``. Defaults to zero.
    candidate : Callable[[np.ndarray], np.ndarray], optional
        Candidate activation, default :func:`identity_candidate`.

    Returns
    -------
    np.ndarray
        Hidden states ``a``, shape ``(n_a, m, T_x)``.
    """
    alpha, beta = min_lstm_coefficients(x, parameters, candidate)
    return parallel_scan(alpha, beta, a0)

min_lstm_forward_log

min_lstm_forward_log(x: ndarray, parameters: dict[str, ndarray], a0: ndarray | None = None) -> np.ndarray

Forward pass for minLSTM through the log-space scan (Sec. B.1).

As with :func:min_gru_forward_log, the candidate is :func:g, so this matches :func:min_lstm_forward_sequential called with candidate=g.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	Wf, Wu, and Wc, each (n_a, n_x).	required
a0	ndarray or None	Non-negative initial state, shape (n_a, m). Defaults to zero.	None

Returns:

Type	Description
ndarray	Hidden states a, shape (n_a, m, T_x), strictly positive.

Source code in src/dlhub/nn/sequence/min_gated.py

def min_lstm_forward_log(
    x: np.ndarray, parameters: dict[str, np.ndarray], a0: np.ndarray | None = None
) -> np.ndarray:
    """
    Forward pass for minLSTM through the log-space scan (Sec. B.1).

    As with :func:`min_gru_forward_log`, the candidate is :func:`g`, so this
    matches :func:`min_lstm_forward_sequential` called with ``candidate=g``.

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        ``Wf``, ``Wu``, and ``Wc``, each ``(n_a, n_x)``.
    a0 : np.ndarray or None, optional
        Non-negative initial state, shape ``(n_a, m)``. Defaults to zero.

    Returns
    -------
    np.ndarray
        Hidden states ``a``, shape ``(n_a, m, T_x)``, strictly positive.
    """
    log_alpha, log_beta = min_lstm_log_coefficients(x, parameters)
    return log_space_scan(log_alpha, log_beta, a0)

make_fixture

make_fixture() -> dict

Build the deterministic fixture this module's tests and documents share.

Draws follow the order the topic dossier pins, under np.random.seed(1) and float64: x, a0, then minGRU's Wu, Wc and minLSTM's Wf, Wu, Wc. The dimensions match the sibling cell modules except for T_x = 7, which is deliberately odd and not a power of two -- a scan that assumes a padded length passes at T_x = 4 and fails here.

Returns:

Type	Description
dict	{"x", "a0", "min_gru", "min_lstm", "dims"}, where min_gru is {"Wu", "Wc"}, min_lstm is {"Wf", "Wu", "Wc"}, and dims records n_x, n_a, m, T_x.

Source code in src/dlhub/nn/sequence/min_gated.py

def make_fixture() -> dict:
    """
    Build the deterministic fixture this module's tests and documents share.

    Draws follow the order the topic dossier pins, under ``np.random.seed(1)``
    and ``float64``: ``x``, ``a0``, then minGRU's ``Wu``, ``Wc`` and minLSTM's
    ``Wf``, ``Wu``, ``Wc``. The dimensions match the sibling cell modules except
    for ``T_x = 7``, which is deliberately odd and not a power of two -- a scan
    that assumes a padded length passes at ``T_x = 4`` and fails here.

    Returns
    -------
    dict
        ``{"x", "a0", "min_gru", "min_lstm", "dims"}``, where ``min_gru`` is
        ``{"Wu", "Wc"}``, ``min_lstm`` is ``{"Wf", "Wu", "Wc"}``, and ``dims``
        records ``n_x, n_a, m, T_x``.
    """
    np.random.seed(1)
    n_x, n_a, m, T_x = 3, 5, 10, 7

    x = np.random.randn(n_x, m, T_x)  # 1
    a0 = np.random.randn(n_a, m)  # 2
    gru_Wu = np.random.randn(n_a, n_x)  # 3
    gru_Wc = np.random.randn(n_a, n_x)  # 4
    lstm_Wf = np.random.randn(n_a, n_x)  # 5
    lstm_Wu = np.random.randn(n_a, n_x)  # 6
    lstm_Wc = np.random.randn(n_a, n_x)  # 7

    dims = {"n_x": n_x, "n_a": n_a, "m": m, "T_x": T_x}
    return {
        "x": x,
        "a0": a0,
        "min_gru": {"Wu": gru_Wu, "Wc": gru_Wc},
        "min_lstm": {"Wf": lstm_Wf, "Wu": lstm_Wu, "Wc": lstm_Wc},
        "dims": dims,
    }

main

main() -> None

Run the fixture through all three forms and print the disagreements.

Source code in src/dlhub/nn/sequence/min_gated.py

def main() -> None:
    """Run the fixture through all three forms and print the disagreements."""
    fixture = make_fixture()
    x, a0, dims = fixture["x"], fixture["a0"], fixture["dims"]
    n_a, n_x = dims["n_a"], dims["n_x"]
    positive_a0 = np.abs(a0)  # the log-space scan needs a non-negative state

    cells = (
        (
            "minGRU",
            fixture["min_gru"],
            min_gru_forward_sequential,
            min_gru_forward_parallel,
            min_gru_forward_log,
            2 * n_a * n_x,
        ),
        (
            "minLSTM",
            fixture["min_lstm"],
            min_lstm_forward_sequential,
            min_lstm_forward_parallel,
            min_lstm_forward_log,
            3 * n_a * n_x,
        ),
    )

    print(f"x: {x.shape}   a0: {a0.shape}   T_x = {dims['T_x']} (odd, not 2^k)")
    for name, parameters, sequential, parallel, log_space, count in cells:
        a_sequential = sequential(x, parameters, a0)
        a_parallel = parallel(x, parameters, a0)
        a_log = log_space(x, parameters, positive_a0)
        a_sequential_g = sequential(x, parameters, positive_a0, candidate=g)

        print(f"\n{name}: a {a_sequential.shape}, {count} parameters")
        print(
            "  max |sequential - parallel| = "
            f"{np.max(np.abs(a_sequential - a_parallel)):.3e}"
        )
        print(
            "  max |sequential(g) - log space| = "
            f"{np.max(np.abs(a_sequential_g - a_log)):.3e}"
        )

rnn

Recurrent Neural Network (Vanilla RNN)

A from-scratch, teaching reference for the simple recurrent network: the recurrence relation, forward propagation through time, and backpropagation through time (BPTT). This module is the explanation the accompanying documents point at, so the forward pass appears in the shapes the topic declares and the gradients are computed term by term rather than collapsed into a single fused expression.

Notation follows Andrew Ng's Deep Learning Specialization, Course 5, Week 1, verbatim, so the sibling sequence topics (LSTM/GRU, language modeling) extend the same symbols. For t = 1 ... T_x::

a^{<t>}    = tanh(W_aa a^{<t-1>} + W_ax x^{<t>} + b_a)
y_hat^{<t>} = softmax(W_ya a^{<t>} + b_y)

with a^{<0>} the initial hidden state (the zero vector by default). The total loss is summed over timesteps, L = sum_t L^{<t>}, with each L^{<t>} the softmax cross-entropy of y_hat^{<t>} against the target y^{<t>}.

This implements the many-to-many, equal-length case (one output per input step). The other input/output cardinalities -- one-to-one, one-to-many, many-to-one, many-to-many with unequal lengths -- reuse this exact cell and differ only in which steps receive an input and which produce a read-out; they need no new cell mathematics. Bidirectional and deep (stacked) RNNs are provided below as compositions of this cell, shown as forward-pass shape walkthroughs rather than re-derived with their own BPTT. Gated cells (GRU, LSTM), the answer to the vanishing-gradient problem stated here, live in the lstm-and-gru topic.

References

• Elman, J. L. (1990). Finding Structure in Time. Cognitive Science, 14(2), 179-211.
• Rumelhart, D. E., Hinton, G. E., & Williams, R. J. (1986). Learning representations by back-propagating errors. Nature, 323, 533-536.
• Goodfellow, I., Bengio, Y., & Courville, A. (2016). Deep Learning, Ch. 10. MIT Press. https://www.deeplearningbook.org/
• Pascanu, R., Mikolov, T., & Bengio, Y. (2013). On the difficulty of training Recurrent Neural Networks. ICML. https://arxiv.org/abs/1211.5063
• Schuster, M., & Paliwal, K. K. (1997). Bidirectional Recurrent Neural Networks. IEEE Transactions on Signal Processing, 45(11), 2673-2681.
• Ng, A. (2018). Deep Learning Specialization, Course 5 (Sequence Models), Week 1. https://www.coursera.org/specializations/deep-learning

Author

Deep Learning Reference Hub

License

MIT

Notes

• Stable softmax. The textbook softmax exp(z) / sum(exp(z)) overflows for large logits. The read-out subtracts the per-column maximum before exponentiating, which is algebraically identical (the shift cancels in the ratio) but keeps every exponent <= 0. Because the shifted denominator is >= 1, the output is strictly positive, so the cross-entropy log is taken without any epsilon guard -- and the analytic gradient (y_hat - y) / m stays exact for the finite-difference check. The softmax and compute_loss implementations are the shared sequence-cell helpers in :mod:dlhub.nn.sequence._common, re-exported here -- the module whose lesson this is -- and from the package root, so every existing import path still resolves.
• The tanh backward uses 1 - tanh(z)^2 = 1 - (a^{<t>})^2: since a^{<t>} is already the tanh output, its derivative is read straight off the cached activation with no second tanh evaluation.
• Stacked weight equivalence. Ng also writes the recurrence with a single weight W_a = [W_aa | W_ax] acting on the concatenation [a^{<t-1>}; x^{<t>}]. This module keeps W_aa and W_ax separate so that the two weight-gradient accumulations -- the recurrent path dW_aa and the input path dW_ax -- appear as distinct terms, which is the BPTT lesson. The two forms are numerically identical, and PyTorch's nn.RNN likewise keeps weight_hh and weight_ih separate.

rnn_cell_forward

rnn_cell_forward(x_t: ndarray, a_prev: ndarray, parameters: dict[str, ndarray]) -> tuple[np.ndarray, np.ndarray, tuple]

One timestep of the forward recurrence.

Computes the hidden-state update and the softmax read-out for a single step::

a^{<t>}     = tanh(W_aa a^{<t-1>} + W_ax x^{<t>} + b_a)
y_hat^{<t>} = softmax(W_ya a^{<t>} + b_y)

Parameters:

Name	Type	Description	Default
x_t	ndarray	Input at this timestep, shape (n_x, m).	required
a_prev	ndarray	Hidden state carried in from the previous step, a^{<t-1>}, shape (n_a, m).	required
parameters	dict[str, ndarray]	The shared weights and biases: Wax (n_a, n_x), Waa (n_a, n_a), Wya (n_y, n_a), ba (n_a, 1), by (n_y, 1).	required

Returns:

Name	Type	Description
a_next	ndarray	Updated hidden state a^{<t>}, shape (n_a, m).
y_pred_t	ndarray	Output distribution y_hat^{<t>}, shape (n_y, m).
cache	tuple	(a_next, a_prev, x_t, y_pred_t) -- everything the backward step for this timestep needs.

Source code in src/dlhub/nn/sequence/rnn.py

def rnn_cell_forward(
    x_t: np.ndarray, a_prev: np.ndarray, parameters: dict[str, np.ndarray]
) -> tuple[np.ndarray, np.ndarray, tuple]:
    """
    One timestep of the forward recurrence.

    Computes the hidden-state update and the softmax read-out for a single step::

        a^{<t>}     = tanh(W_aa a^{<t-1>} + W_ax x^{<t>} + b_a)
        y_hat^{<t>} = softmax(W_ya a^{<t>} + b_y)

    Parameters
    ----------
    x_t : np.ndarray
        Input at this timestep, shape ``(n_x, m)``.
    a_prev : np.ndarray
        Hidden state carried in from the previous step, ``a^{<t-1>}``, shape
        ``(n_a, m)``.
    parameters : dict[str, np.ndarray]
        The shared weights and biases: ``Wax`` ``(n_a, n_x)``, ``Waa``
        ``(n_a, n_a)``, ``Wya`` ``(n_y, n_a)``, ``ba`` ``(n_a, 1)``, ``by``
        ``(n_y, 1)``.

    Returns
    -------
    a_next : np.ndarray
        Updated hidden state ``a^{<t>}``, shape ``(n_a, m)``.
    y_pred_t : np.ndarray
        Output distribution ``y_hat^{<t>}``, shape ``(n_y, m)``.
    cache : tuple
        ``(a_next, a_prev, x_t, y_pred_t)`` -- everything the backward step for
        this timestep needs.
    """
    Wax, Waa, Wya = parameters["Wax"], parameters["Waa"], parameters["Wya"]
    ba, by = parameters["ba"], parameters["by"]

    # Hidden-state update: the recurrent term W_aa a^{<t-1>} and the input term
    # W_ax x^{<t>} are added, then squashed by tanh.
    a_next = np.tanh(Waa @ a_prev + Wax @ x_t + ba)

    # Read-out: an affine map of the hidden state through softmax.
    y_pred_t = softmax(Wya @ a_next + by)

    cache = (a_next, a_prev, x_t, y_pred_t)
    return a_next, y_pred_t, cache

rnn_forward

rnn_forward(x: ndarray, parameters: dict[str, ndarray], a0: ndarray | None = None) -> tuple[np.ndarray, np.ndarray, list[tuple]]

Forward propagation through time across a whole sequence.

Unrolls :func:rnn_cell_forward over T_x timesteps, threading the hidden state from one step to the next.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence of shape (n_x, m, T_x).	required
parameters	dict[str, ndarray]	The shared weights and biases (see :func:rnn_cell_forward).	required
a0	ndarray or None	Initial hidden state a^{<0>} of shape (n_a, m). Defaults to the zero vector, matching the standard convention a^{<0>} = 0.	None

Returns:

Name	Type	Description
a	ndarray	All hidden states stacked over time, shape (n_a, m, T_x).
y_pred	ndarray	All output distributions, shape (n_y, m, T_x).
caches	list[tuple]	Per-timestep caches, in order, for :func:rnn_backward.

Raises:

Type	Description
ValueError	If x is not three-dimensional, or if a supplied a0 does not have shape (n_a, m).

Source code in src/dlhub/nn/sequence/rnn.py

def rnn_forward(
    x: np.ndarray,
    parameters: dict[str, np.ndarray],
    a0: np.ndarray | None = None,
) -> tuple[np.ndarray, np.ndarray, list[tuple]]:
    """
    Forward propagation through time across a whole sequence.

    Unrolls :func:`rnn_cell_forward` over ``T_x`` timesteps, threading the
    hidden state from one step to the next.

    Parameters
    ----------
    x : np.ndarray
        Input sequence of shape ``(n_x, m, T_x)``.
    parameters : dict[str, np.ndarray]
        The shared weights and biases (see :func:`rnn_cell_forward`).
    a0 : np.ndarray or None, optional
        Initial hidden state ``a^{<0>}`` of shape ``(n_a, m)``. Defaults to the
        zero vector, matching the standard convention ``a^{<0>} = 0``.

    Returns
    -------
    a : np.ndarray
        All hidden states stacked over time, shape ``(n_a, m, T_x)``.
    y_pred : np.ndarray
        All output distributions, shape ``(n_y, m, T_x)``.
    caches : list[tuple]
        Per-timestep caches, in order, for :func:`rnn_backward`.

    Raises
    ------
    ValueError
        If ``x`` is not three-dimensional, or if a supplied ``a0`` does not have
        shape ``(n_a, m)``.
    """
    if x.ndim != 3:
        raise ValueError(f"x must have shape (n_x, m, T_x); got ndim {x.ndim}")

    n_x, m, T_x = x.shape
    n_a = parameters["Waa"].shape[0]
    n_y = parameters["Wya"].shape[0]

    if a0 is None:
        a0 = np.zeros((n_a, m))
    elif a0.shape != (n_a, m):
        raise ValueError(f"a0 must have shape {(n_a, m)}; got {a0.shape}")

    a = np.zeros((n_a, m, T_x))
    y_pred = np.zeros((n_y, m, T_x))
    caches: list[tuple] = []

    a_prev = a0
    for t in range(T_x):
        a_next, y_pred_t, cache = rnn_cell_forward(x[:, :, t], a_prev, parameters)
        a[:, :, t] = a_next
        y_pred[:, :, t] = y_pred_t
        caches.append(cache)
        a_prev = a_next  # carry the state forward to t+1

    return a, y_pred, caches

rnn_cell_backward

rnn_cell_backward(y_t: ndarray, da_next: ndarray, cache: tuple, parameters: dict[str, ndarray]) -> dict[str, np.ndarray]

Backward pass for a single timestep.

Combines the two gradient sources that reach a^{<t>} -- the output at this step and the recurrence carried back from t+1 (da_next) -- and propagates them through the softmax read-out and the tanh recurrence.

Parameters:

Name	Type	Description	Default
y_t	ndarray	One-hot target at this timestep, shape (n_y, m).	required
da_next	ndarray	Gradient of the loss w.r.t. this step's hidden state arriving from the recurrence at t+1, shape (n_a, m). Zero for the final step.	required
cache	tuple	(a_next, a_prev, x_t, y_pred_t) from :func:rnn_cell_forward.	required
parameters	dict[str, ndarray]	The shared weights and biases (see :func:rnn_cell_forward).	required

Returns:

Type	Description
dict[str, ndarray]	This step's gradient contributions: dWax, dWaa, dWya, dba, dby, together with dxt (into the input) and da_prev (the recurrence handed back to t-1).

Source code in src/dlhub/nn/sequence/rnn.py

def rnn_cell_backward(
    y_t: np.ndarray,
    da_next: np.ndarray,
    cache: tuple,
    parameters: dict[str, np.ndarray],
) -> dict[str, np.ndarray]:
    """
    Backward pass for a single timestep.

    Combines the two gradient sources that reach ``a^{<t>}`` -- the output at
    this step and the recurrence carried back from ``t+1`` (``da_next``) -- and
    propagates them through the softmax read-out and the tanh recurrence.

    Parameters
    ----------
    y_t : np.ndarray
        One-hot target at this timestep, shape ``(n_y, m)``.
    da_next : np.ndarray
        Gradient of the loss w.r.t. this step's hidden state arriving from the
        recurrence at ``t+1``, shape ``(n_a, m)``. Zero for the final step.
    cache : tuple
        ``(a_next, a_prev, x_t, y_pred_t)`` from :func:`rnn_cell_forward`.
    parameters : dict[str, np.ndarray]
        The shared weights and biases (see :func:`rnn_cell_forward`).

    Returns
    -------
    dict[str, np.ndarray]
        This step's gradient contributions: ``dWax``, ``dWaa``, ``dWya``,
        ``dba``, ``dby``, together with ``dxt`` (into the input) and ``da_prev``
        (the recurrence handed back to ``t-1``).
    """
    a_next, a_prev, x_t, y_pred_t = cache
    Wax, Waa, Wya = parameters["Wax"], parameters["Waa"], parameters["Wya"]
    m = x_t.shape[1]

    # --- Output path: softmax cross-entropy -------------------------------
    # For softmax + cross-entropy the logit gradient collapses to (y_hat - y),
    # scaled by the 1/m batch mean used in compute_loss.
    dz_y = (y_pred_t - y_t) / m  # (n_y, m)
    dWya = dz_y @ a_next.T  # (n_y, n_a)
    dby = np.sum(dz_y, axis=1, keepdims=True)  # (n_y, 1)
    da_from_output = Wya.T @ dz_y  # (n_a, m)

    # --- Merge the two gradient sources at a^{<t>} ------------------------
    da_t = da_from_output + da_next  # (n_a, m)

    # --- Recurrence path: tanh backward -----------------------------------
    # d/dz tanh(z) = 1 - tanh(z)^2 = 1 - (a^{<t>})^2, read off the activation.
    dtanh = (1 - a_next**2) * da_t  # (n_a, m)
    dba = np.sum(dtanh, axis=1, keepdims=True)  # (n_a, 1)
    dWax = dtanh @ x_t.T  # (n_a, n_x)  -- input-path weight gradient
    dWaa = dtanh @ a_prev.T  # (n_a, n_a)  -- recurrent-path weight gradient
    dxt = Wax.T @ dtanh  # (n_x, m)    -- gradient into the input
    da_prev = Waa.T @ dtanh  # (n_a, m)    -- gradient back to t-1

    return {
        "dWax": dWax,
        "dWaa": dWaa,
        "dWya": dWya,
        "dba": dba,
        "dby": dby,
        "dxt": dxt,
        "da_prev": da_prev,
    }

rnn_backward

rnn_backward(y: ndarray, caches: list[tuple], parameters: dict[str, ndarray]) -> dict[str, np.ndarray]

Backpropagation through time (BPTT).

Walks the unrolled graph from t = T_x back to t = 1. The shared weights receive accumulated gradients over every timestep, and the hidden-state gradient is threaded backward so that each step adds its recurrence contribution (da_prev) to the next step's incoming gradient.

Parameters:

Name	Type	Description	Default
y	ndarray	One-hot targets, shape (n_y, m, T_x).	required
caches	list[tuple]	Per-timestep caches from :func:rnn_forward.	required
parameters	dict[str, ndarray]	The shared weights and biases (see :func:rnn_cell_forward).	required

Returns:

Type	Description
dict[str, ndarray]	Accumulated gradients dWax, dWaa, dWya, dba, dby; the gradient da0 w.r.t. the initial hidden state; and dx, the gradient w.r.t. the whole input sequence, shape (n_x, m, T_x).

Source code in src/dlhub/nn/sequence/rnn.py

def rnn_backward(
    y: np.ndarray, caches: list[tuple], parameters: dict[str, np.ndarray]
) -> dict[str, np.ndarray]:
    """
    Backpropagation through time (BPTT).

    Walks the unrolled graph from ``t = T_x`` back to ``t = 1``. The shared
    weights receive **accumulated** gradients over every timestep, and the
    hidden-state gradient is threaded backward so that each step adds its
    recurrence contribution (``da_prev``) to the next step's incoming gradient.

    Parameters
    ----------
    y : np.ndarray
        One-hot targets, shape ``(n_y, m, T_x)``.
    caches : list[tuple]
        Per-timestep caches from :func:`rnn_forward`.
    parameters : dict[str, np.ndarray]
        The shared weights and biases (see :func:`rnn_cell_forward`).

    Returns
    -------
    dict[str, np.ndarray]
        Accumulated gradients ``dWax``, ``dWaa``, ``dWya``, ``dba``, ``dby``;
        the gradient ``da0`` w.r.t. the initial hidden state; and ``dx``, the
        gradient w.r.t. the whole input sequence, shape ``(n_x, m, T_x)``.
    """
    T_x = len(caches)
    _, m, _ = y.shape
    n_x = caches[0][2].shape[0]
    n_a = parameters["Waa"].shape[0]

    dWax = np.zeros_like(parameters["Wax"])
    dWaa = np.zeros_like(parameters["Waa"])
    dWya = np.zeros_like(parameters["Wya"])
    dba = np.zeros_like(parameters["ba"])
    dby = np.zeros_like(parameters["by"])
    dx = np.zeros((n_x, m, T_x))

    # The recurrence contribution into the current step, carried back one step
    # at a time. Zero at the final timestep, which has no successor.
    da_next = np.zeros((n_a, m))

    for t in reversed(range(T_x)):
        grads_t = rnn_cell_backward(y[:, :, t], da_next, caches[t], parameters)

        # Shared weights accumulate across all timesteps.
        dWax += grads_t["dWax"]
        dWaa += grads_t["dWaa"]
        dWya += grads_t["dWya"]
        dba += grads_t["dba"]
        dby += grads_t["dby"]

        dx[:, :, t] = grads_t["dxt"]
        da_next = grads_t["da_prev"]  # hand the recurrence back to t-1

    # After the loop, the recurrence gradient has reached a^{<0>}.
    da0 = da_next

    return {
        "dWax": dWax,
        "dWaa": dWaa,
        "dWya": dWya,
        "dba": dba,
        "dby": dby,
        "da0": da0,
        "dx": dx,
    }

clip_gradients

clip_gradients(gradients: dict[str, ndarray], max_norm: float) -> dict[str, np.ndarray]

Global-norm gradient clipping, the exploding-gradient remedy.

Computes the single L2 norm of all gradients viewed as one long vector and, if it exceeds max_norm, rescales every gradient by the same factor. This preserves the gradient direction while capping its magnitude, exactly as prescribed by Pascanu, Mikolov & Bengio (2013).

Parameters:

Name	Type	Description	Default
gradients	dict[str, ndarray]	Gradients to clip, typically the parameter gradients dWax, dWaa, dWya, dba, dby.	required
max_norm	float	The maximum allowed global L2 norm.	required

Returns:

Type	Description
dict[str, ndarray]	A new dictionary of clipped gradients. The input is not modified.

Source code in src/dlhub/nn/sequence/rnn.py

def clip_gradients(
    gradients: dict[str, np.ndarray], max_norm: float
) -> dict[str, np.ndarray]:
    """
    Global-norm gradient clipping, the exploding-gradient remedy.

    Computes the single L2 norm of all gradients viewed as one long vector and,
    if it exceeds ``max_norm``, rescales every gradient by the same factor. This
    preserves the gradient *direction* while capping its magnitude, exactly as
    prescribed by Pascanu, Mikolov & Bengio (2013).

    Parameters
    ----------
    gradients : dict[str, np.ndarray]
        Gradients to clip, typically the parameter gradients ``dWax``,
        ``dWaa``, ``dWya``, ``dba``, ``dby``.
    max_norm : float
        The maximum allowed global L2 norm.

    Returns
    -------
    dict[str, np.ndarray]
        A new dictionary of clipped gradients. The input is not modified.
    """
    total_norm = np.sqrt(sum(np.sum(g**2) for g in gradients.values()))

    if total_norm > max_norm:
        scale = max_norm / total_norm
        return {key: g * scale for key, g in gradients.items()}

    return {key: g.copy() for key, g in gradients.items()}

update_parameters

update_parameters(parameters: dict[str, ndarray], gradients: dict[str, ndarray], learning_rate: float) -> dict[str, np.ndarray]

One step of gradient descent on the shared weights.

Applies theta <- theta - learning_rate * dtheta to each of Wax, Waa, Wya, ba, by.

Parameters:

Name	Type	Description	Default
parameters	dict[str, ndarray]	The current weights and biases.	required
gradients	dict[str, ndarray]	Their gradients, keyed dWax, dWaa, dWya, dba, dby.	required
learning_rate	float	The step size.	required

Returns:

Type	Description
dict[str, ndarray]	A new dictionary of updated parameters. The input is not modified.

Source code in src/dlhub/nn/sequence/rnn.py

def update_parameters(
    parameters: dict[str, np.ndarray],
    gradients: dict[str, np.ndarray],
    learning_rate: float,
) -> dict[str, np.ndarray]:
    """
    One step of gradient descent on the shared weights.

    Applies ``theta <- theta - learning_rate * dtheta`` to each of ``Wax``,
    ``Waa``, ``Wya``, ``ba``, ``by``.

    Parameters
    ----------
    parameters : dict[str, np.ndarray]
        The current weights and biases.
    gradients : dict[str, np.ndarray]
        Their gradients, keyed ``dWax``, ``dWaa``, ``dWya``, ``dba``, ``dby``.
    learning_rate : float
        The step size.

    Returns
    -------
    dict[str, np.ndarray]
        A new dictionary of updated parameters. The input is not modified.
    """
    return {
        key: parameters[key] - learning_rate * gradients[f"d{key}"]
        for key in ("Wax", "Waa", "Wya", "ba", "by")
    }

bidirectional_rnn_forward

bidirectional_rnn_forward(x: ndarray, parameters_forward: dict[str, ndarray], parameters_backward: dict[str, ndarray], a0_forward: ndarray | None = None, a0_backward: ndarray | None = None) -> tuple[np.ndarray, dict[str, np.ndarray]]

Bidirectional RNN forward pass, as a composition of the vanilla cell.

Runs two independent recurrences over the sequence -- one left-to-right (a->) and one right-to-left (a<-) -- and concatenates their hidden states at each timestep as [a->^{<t>}; a<-^{<t>}]. There is no new cell mathematics: this is :func:rnn_forward applied twice, once to the time-reversed input. A bidirectional model attaches its own output layer, mapping the 2 n_a-dimensional concatenation to n_y; the per-direction read-outs computed by the reused forward pass are not used here.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence, shape (n_x, m, T_x).	required
parameters_forward	dict[str, ndarray]	Independent weight sets for the two directions.	required
parameters_backward	dict[str, ndarray]	Independent weight sets for the two directions.	required
a0_forward	ndarray or None	Initial hidden states for each direction, each shape (n_a, m). Default to zero.	None
a0_backward	ndarray or None	Initial hidden states for each direction, each shape (n_a, m). Default to zero.	None

Returns:

Name	Type	Description
a_bidirectional	ndarray	Concatenated hidden states, shape (2 * n_a, m, T_x).
parts	dict[str, ndarray]	The two direction sequences, {"a_forward": ..., "a_backward": ...}, each (n_a, m, T_x) and both in forward time order.

Source code in src/dlhub/nn/sequence/rnn.py

def bidirectional_rnn_forward(
    x: np.ndarray,
    parameters_forward: dict[str, np.ndarray],
    parameters_backward: dict[str, np.ndarray],
    a0_forward: np.ndarray | None = None,
    a0_backward: np.ndarray | None = None,
) -> tuple[np.ndarray, dict[str, np.ndarray]]:
    """
    Bidirectional RNN forward pass, as a composition of the vanilla cell.

    Runs two independent recurrences over the sequence -- one left-to-right
    (``a->``) and one right-to-left (``a<-``) -- and concatenates their hidden
    states at each timestep as ``[a->^{<t>}; a<-^{<t>}]``. There is no new cell
    mathematics: this is :func:`rnn_forward` applied twice, once to the
    time-reversed input. A bidirectional model attaches its own output layer,
    mapping the ``2 n_a``-dimensional concatenation to ``n_y``; the per-direction
    read-outs computed by the reused forward pass are not used here.

    Parameters
    ----------
    x : np.ndarray
        Input sequence, shape ``(n_x, m, T_x)``.
    parameters_forward, parameters_backward : dict[str, np.ndarray]
        Independent weight sets for the two directions.
    a0_forward, a0_backward : np.ndarray or None, optional
        Initial hidden states for each direction, each shape ``(n_a, m)``.
        Default to zero.

    Returns
    -------
    a_bidirectional : np.ndarray
        Concatenated hidden states, shape ``(2 * n_a, m, T_x)``.
    parts : dict[str, np.ndarray]
        The two direction sequences, ``{"a_forward": ..., "a_backward": ...}``,
        each ``(n_a, m, T_x)`` and both in forward time order.
    """
    a_forward, _, _ = rnn_forward(x, parameters_forward, a0_forward)

    # Run the second recurrence over the reversed sequence, then realign its
    # activations back to forward time order so step t of both directions lines
    # up before concatenation.
    a_backward_reversed, _, _ = rnn_forward(
        x[:, :, ::-1], parameters_backward, a0_backward
    )
    a_backward = a_backward_reversed[:, :, ::-1]

    a_bidirectional = np.concatenate([a_forward, a_backward], axis=0)
    return a_bidirectional, {"a_forward": a_forward, "a_backward": a_backward}

deep_rnn_forward

deep_rnn_forward(x: ndarray, parameters_per_layer: list[dict[str, ndarray]], a0_per_layer: list[ndarray] | None = None) -> tuple[list[np.ndarray], np.ndarray]

Deep (stacked) RNN forward pass, as a composition of the vanilla cell.

Each layer is itself a full RNN over the sequence produced by the layer below: layer l consumes the activation sequence a^{[l-1]} in place of the input x. There is no new cell mathematics -- this is :func:rnn_forward applied once per layer. Only the top layer's read-out is a model output; the intermediate read-outs computed by the reused forward pass are not used.

Parameters:

Name	Type	Description	Default
x	ndarray	Input sequence to the first layer, shape (n_x, m, T_x).	required
parameters_per_layer	list[dict[str, ndarray]]	One weight set per layer, bottom to top. Layer l's Wax must accept the activation width of the layer below (n_x for layer 0).	required
a0_per_layer	list[ndarray] or None	Per-layer initial hidden states. Defaults to zero for every layer.	None

Returns:

Name	Type	Description
activations	list[ndarray]	Each layer's hidden-state sequence, bottom to top, each (n_a^{[l]}, m, T_x).
y_pred_top	ndarray	The top layer's output distributions, shape (n_y, m, T_x).

Raises:

Type	Description
ValueError	If parameters_per_layer is empty.

Source code in src/dlhub/nn/sequence/rnn.py

def deep_rnn_forward(
    x: np.ndarray,
    parameters_per_layer: list[dict[str, np.ndarray]],
    a0_per_layer: list[np.ndarray] | None = None,
) -> tuple[list[np.ndarray], np.ndarray]:
    """
    Deep (stacked) RNN forward pass, as a composition of the vanilla cell.

    Each layer is itself a full RNN over the sequence produced by the layer
    below: layer ``l`` consumes the activation sequence ``a^{[l-1]}`` in place of
    the input ``x``. There is no new cell mathematics -- this is
    :func:`rnn_forward` applied once per layer. Only the top layer's read-out is
    a model output; the intermediate read-outs computed by the reused forward
    pass are not used.

    Parameters
    ----------
    x : np.ndarray
        Input sequence to the first layer, shape ``(n_x, m, T_x)``.
    parameters_per_layer : list[dict[str, np.ndarray]]
        One weight set per layer, bottom to top. Layer ``l``'s ``Wax`` must
        accept the activation width of the layer below (``n_x`` for layer 0).
    a0_per_layer : list[np.ndarray] or None, optional
        Per-layer initial hidden states. Defaults to zero for every layer.

    Returns
    -------
    activations : list[np.ndarray]
        Each layer's hidden-state sequence, bottom to top, each
        ``(n_a^{[l]}, m, T_x)``.
    y_pred_top : np.ndarray
        The top layer's output distributions, shape ``(n_y, m, T_x)``.

    Raises
    ------
    ValueError
        If ``parameters_per_layer`` is empty.
    """
    if not parameters_per_layer:
        raise ValueError("a deep RNN needs at least one layer")

    if a0_per_layer is None:
        a0_per_layer = [None] * len(parameters_per_layer)

    activations: list[np.ndarray] = []
    layer_input = x
    y_pred_top = None
    for layer_params, a0 in zip(parameters_per_layer, a0_per_layer, strict=True):
        a_layer, y_pred_top, _ = rnn_forward(layer_input, layer_params, a0)
        activations.append(a_layer)
        layer_input = a_layer  # the layer above reads these activations

    return activations, y_pred_top

make_fixture

make_fixture() -> dict

Build the deterministic fixture every implementation and test shares.

Draws are recorded in exactly the order the topic dossier pins, under np.random.seed(1) and float64 (the default of randn), so the seven parameter arrays reproduce bit-for-bit across implementations. The one-hot targets are drawn after those seven so that adding them cannot disturb the arrays a forward-parity check compares.

Returns:

Type	Description
dict	{"x", "a0", "parameters", "y", "dims"} where parameters is the dict {"Wax", "Waa", "Wya", "ba", "by"}, y is one-hot of shape (n_y, m, T_x), and dims records n_x, n_a, n_y, m, T_x.

Source code in src/dlhub/nn/sequence/rnn.py

def make_fixture() -> dict:
    """
    Build the deterministic fixture every implementation and test shares.

    Draws are recorded in exactly the order the topic dossier pins, under
    ``np.random.seed(1)`` and ``float64`` (the default of ``randn``), so the
    seven parameter arrays reproduce bit-for-bit across implementations. The
    one-hot targets are drawn *after* those seven so that adding them cannot
    disturb the arrays a forward-parity check compares.

    Returns
    -------
    dict
        ``{"x", "a0", "parameters", "y", "dims"}`` where ``parameters`` is the
        dict ``{"Wax", "Waa", "Wya", "ba", "by"}``, ``y`` is one-hot of shape
        ``(n_y, m, T_x)``, and ``dims`` records ``n_x, n_a, n_y, m, T_x``.
    """
    np.random.seed(1)
    n_x, n_a, n_y, m, T_x = 3, 5, 2, 10, 4

    x = np.random.randn(n_x, m, T_x)  # 1
    a0 = np.random.randn(n_a, m)  # 2
    Wax = np.random.randn(n_a, n_x)  # 3
    Waa = np.random.randn(n_a, n_a)  # 4
    Wya = np.random.randn(n_y, n_a)  # 5
    ba = np.random.randn(n_a, 1)  # 6
    by = np.random.randn(n_y, 1)  # 7

    # Targets, drawn after the seven parameter arrays above.
    labels = np.random.randint(0, n_y, size=(m, T_x))
    y = np.zeros((n_y, m, T_x))
    for i in range(m):
        for t in range(T_x):
            y[labels[i, t], i, t] = 1.0  # one-hot encode the class at (i, t)

    parameters = {"Wax": Wax, "Waa": Waa, "Wya": Wya, "ba": ba, "by": by}
    dims = {"n_x": n_x, "n_a": n_a, "n_y": n_y, "m": m, "T_x": T_x}
    return {"x": x, "a0": a0, "parameters": parameters, "y": y, "dims": dims}

main

main() -> None

Run the shared fixture end to end and print shapes, loss, and grad norms.

Source code in src/dlhub/nn/sequence/rnn.py

def main() -> None:
    """Run the shared fixture end to end and print shapes, loss, and grad norms."""
    fixture = make_fixture()
    x, a0, parameters, y = (
        fixture["x"],
        fixture["a0"],
        fixture["parameters"],
        fixture["y"],
    )

    a, y_pred, caches = rnn_forward(x, parameters, a0)
    loss = compute_loss(y_pred, y)
    gradients = rnn_backward(y, caches, parameters)

    print(f"hidden states a: {a.shape}")
    print(f"outputs y_pred:  {y_pred.shape}")
    print(f"total loss:      {loss:.6f}")
    for key in ("dWax", "dWaa", "dWya", "dba", "dby", "da0"):
        print(f"||{key}|| = {np.linalg.norm(gradients[key]):.6f}")