Tuning

Search strategies for choosing hyperparameters, from random sampling through model-based search to the multi-fidelity methods that spend their budget unevenly on purpose.

One entry point per method, plus a dispatcher that takes a method name. The per-method functions take that method's own search-space format and return its own result type; the dispatcher takes the framework's and returns an ExperimentResult.

dlhub.tuning

Hyperparameter Tuning

Search strategies for choosing hyperparameters, from random sampling through model-based search to the multi-fidelity methods that spend their budget unevenly on purpose.

Author

Deep Learning Reference Hub

License

MIT

BayesianOptimizationResult dataclass

Container for Bayesian optimization results.

Attributes:

Name	Type	Description
best_params	dict	Best hyperparameter configuration found
best_score	float	Best objective function value achieved
history	list	History of all evaluations
convergence_data	dict	Convergence statistics and diagnostics

Source code in src/dlhub/tuning/bayesian.py

@dataclass
class BayesianOptimizationResult:
    """
    Container for Bayesian optimization results.

    Attributes
    ----------
    best_params : dict
        Best hyperparameter configuration found
    best_score : float
        Best objective function value achieved
    history : list
        History of all evaluations
    convergence_data : dict
        Convergence statistics and diagnostics
    """

    best_params: dict[str, Any]
    best_score: float
    history: list[tuple[dict[str, Any], float]]
    convergence_data: dict[str, Any]

BayesianOptimizer

Bayesian Optimization using Gaussian Process surrogate models.

This implementation uses Expected Improvement as the acquisition function to balance exploration and exploitation in hyperparameter search.

Parameters:

Name	Type	Description	Default
objective_function	callable	Function to optimize. Should take hyperparameter dict and return float	required
search_space	dict	Dictionary defining search space for each hyperparameter. Format: {'param_name': (min_val, max_val)} for continuous parameters	required
acquisition	str	Acquisition function ('ei' for Expected Improvement, 'ucb' for UCB)	'ei'
kappa	float	Exploration parameter for UCB (ignored if acquisition='ei')	2.576
xi	float	Exploration parameter for Expected Improvement	0.01
n_initial	int	Number of random initial evaluations	5
random_state	int	Random seed for reproducibility	None

Source code in src/dlhub/tuning/bayesian.py

class BayesianOptimizer:
    """
    Bayesian Optimization using Gaussian Process surrogate models.

    This implementation uses Expected Improvement as the acquisition function
    to balance exploration and exploitation in hyperparameter search.

    Parameters
    ----------
    objective_function : callable
        Function to optimize. Should take hyperparameter dict and return float
    search_space : dict
        Dictionary defining search space for each hyperparameter.
        Format: {'param_name': (min_val, max_val)} for continuous parameters
    acquisition : str, default='ei'
        Acquisition function ('ei' for Expected Improvement, 'ucb' for UCB)
    kappa : float, default=2.576
        Exploration parameter for UCB (ignored if acquisition='ei')
    xi : float, default=0.01
        Exploration parameter for Expected Improvement
    n_initial : int, default=5
        Number of random initial evaluations
    random_state : int, optional
        Random seed for reproducibility
    """

    def __init__(
        self,
        objective_function: Callable[[dict], float],
        search_space: dict[str, tuple[float, float]],
        acquisition: str = "ei",
        kappa: float = 2.576,
        xi: float = 0.01,
        n_initial: int = 5,
        random_state: int | None = None,
    ):

        self.objective_function = objective_function
        self.search_space = search_space
        self.acquisition = acquisition.lower()
        self.kappa = kappa
        self.xi = xi
        self.n_initial = n_initial

        # Checked here rather than where the acquisition is first evaluated,
        # which is after the initial design has run. Each of those evaluations
        # is a full training run, so a misspelled name must not cost them.
        if self.acquisition not in ("ei", "ucb"):
            raise ValueError(
                f"Unknown acquisition function: {acquisition!r}. Use 'ei' or 'ucb'."
            )

        if random_state is not None:
            np.random.seed(random_state)

        # Initialize internal state
        self.param_names = list(search_space.keys())
        self.bounds = np.array([search_space[name] for name in self.param_names])
        self.gp = GaussianProcess()
        self.X_observed = []
        self.y_observed = []
        self.history = []
        self.best_score = -np.inf
        self.best_params = None

    def _normalize_params(self, X: np.ndarray) -> np.ndarray:
        """Normalize parameters to [0, 1] range."""
        return (X - self.bounds[:, 0]) / (self.bounds[:, 1] - self.bounds[:, 0])

    def _denormalize_params(self, X_norm: np.ndarray) -> np.ndarray:
        """Denormalize parameters from [0, 1] to original range."""
        return X_norm * (self.bounds[:, 1] - self.bounds[:, 0]) + self.bounds[:, 0]

    def _array_to_dict(self, X: np.ndarray) -> dict[str, float]:
        """Convert parameter array to dictionary."""
        return {name: float(val) for name, val in zip(self.param_names, X)}

    def _expected_improvement(self, X: np.ndarray) -> np.ndarray:
        """
        Compute Expected Improvement acquisition function.

        Parameters
        ----------
        X : np.ndarray, shape (n_points, n_params)
            Normalized parameter points to evaluate

        Returns
        -------
        np.ndarray, shape (n_points,)
            Expected improvement values
        """
        if len(self.X_observed) == 0:
            return np.ones(len(X))

        mean, std = self.gp.predict(X)

        f_max = max(self.y_observed)

        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            z = (mean - f_max - self.xi) / std
            ei = (mean - f_max - self.xi) * norm.cdf(z) + std * norm.pdf(z)
            ei[std == 0.0] = 0.0

        return ei

    def _upper_confidence_bound(self, X: np.ndarray) -> np.ndarray:
        """
        Compute Upper Confidence Bound acquisition function.

        Parameters
        ----------
        X : np.ndarray, shape (n_points, n_params)
            Normalized parameter points to evaluate

        Returns
        -------
        np.ndarray, shape (n_points,)
            Upper confidence bound values
        """
        if len(self.X_observed) == 0:
            return np.ones(len(X))

        mean, std = self.gp.predict(X)
        return mean + self.kappa * std

    def _acquisition_function(self, X: np.ndarray) -> np.ndarray:
        """Evaluate the chosen acquisition function."""
        if self.acquisition == "ei":
            return self._expected_improvement(X)
        elif self.acquisition == "ucb":
            return self._upper_confidence_bound(X)
        else:
            raise ValueError(f"Unknown acquisition function: {self.acquisition}")

    def _optimize_acquisition(self) -> np.ndarray:
        """
        Find the point that maximizes the acquisition function.

        Returns
        -------
        np.ndarray, shape (n_params,)
            Normalized parameters that maximize acquisition function
        """

        # Objective to minimize (negative acquisition)
        def objective(x):
            return -self._acquisition_function(x.reshape(1, -1))[0]

        # Try multiple random starting points
        n_restarts = 10
        best_x = None
        best_val = np.inf

        for _ in range(n_restarts):
            x0 = np.random.uniform(0, 1, len(self.param_names))

            try:
                result = minimize(
                    objective,
                    x0,
                    bounds=[(0, 1)] * len(self.param_names),
                    method="L-BFGS-B",
                )

                if result.fun < best_val:
                    best_val = result.fun
                    best_x = result.x
            except Exception:
                continue

        if best_x is None:
            best_x = np.random.uniform(0, 1, len(self.param_names))

        return best_x

    def _evaluate_objective(self, params: dict[str, float]) -> float:
        """
        Evaluate objective function and handle exceptions.

        Parameters
        ----------
        params : dict
            Hyperparameter configuration

        Returns
        -------
        float
            Objective function value (np.nan if evaluation failed)
        """
        try:
            score = self.objective_function(params)
            if np.isnan(score) or np.isinf(score):
                return np.nan
            return float(score)
        except Exception as e:
            warnings.warn(f"Objective evaluation failed: {e}")
            return np.nan

    def optimize(
        self, n_iterations: int = 20, verbose: int = 1
    ) -> BayesianOptimizationResult:
        """
        Run Bayesian optimization.

        Parameters
        ----------
        n_iterations : int, default=20
            Maximum number of optimization iterations
        verbose : int, default=1
            Reporting level: 0 is silent, 1 prints phases and new bests, 2 adds
            every evaluation and the final configuration.

        Returns
        -------
        BayesianOptimizationResult
            Optimization results including best parameters and history
        """
        if verbose >= 1:
            print("Starting Bayesian Optimization...")
            print(f"Search space: {self.search_space}")
            print(f"Acquisition function: {self.acquisition}")

        if verbose >= 1:
            print(f"\nPhase 1: Random initialization ({self.n_initial} points)")

        for i in range(self.n_initial):
            X_raw = np.random.uniform(self.bounds[:, 0], self.bounds[:, 1])
            params = self._array_to_dict(X_raw)

            score = self._evaluate_objective(params)

            if not np.isnan(score):
                X_norm = self._normalize_params(X_raw)
                self.X_observed.append(X_norm)
                self.y_observed.append(score)
                self.history.append((params.copy(), score))

                if score > self.best_score:
                    self.best_score = score
                    self.best_params = params.copy()

                if verbose >= 1:
                    print(f"  {i + 1}/{self.n_initial}: Score = {score:.4f}")

        if len(self.X_observed) == 0:
            raise RuntimeError("All initial evaluations failed")

        if verbose >= 1:
            print(f"\nPhase 2: Bayesian optimization ({n_iterations} iterations)")

        for iteration in range(n_iterations):
            X_train = np.array(self.X_observed)
            y_train = np.array(self.y_observed)
            self.gp.fit(X_train, y_train)

            X_next_norm = self._optimize_acquisition()
            X_next_raw = self._denormalize_params(X_next_norm)
            params_next = self._array_to_dict(X_next_raw)

            score = self._evaluate_objective(params_next)

            if not np.isnan(score):
                self.X_observed.append(X_next_norm)
                self.y_observed.append(score)
                self.history.append((params_next.copy(), score))

                if score > self.best_score:
                    self.best_score = score
                    self.best_params = params_next.copy()

                    if verbose >= 1:
                        print(
                            f"  Iter {iteration + 1}: Score = {score:.4f} (NEW BEST!)"
                        )
                else:
                    if verbose >= 2:
                        print(f"  Iter {iteration + 1}: Score = {score:.4f}")
            else:
                if verbose >= 1:
                    print(f"  Iter {iteration + 1}: Evaluation failed")

        successful_initial = [s for _, s in self.history[: self.n_initial]]
        convergence_data = {
            "n_evaluations": len(self.history),
            "n_failed": n_iterations + self.n_initial - len(self.history),
            # The initial design is random, so its mean is what an equal budget
            # of random search would have averaged. Guarded because every
            # initial point can fail while later ones succeed, and a mean over
            # an empty list is a nan that propagates into the whole summary.
            "improvement_over_random": (
                self.best_score - float(np.mean(successful_initial))
                if successful_initial
                else 0.0
            ),
            "scores": [score for _, score in self.history],
        }

        if verbose >= 1:
            print("\nOptimization completed!")
            print(f"Best score: {self.best_score:.4f}")
        if verbose >= 2:
            print(f"Best parameters: {self.best_params}")
            print(f"Total evaluations: {len(self.history)}")

        return BayesianOptimizationResult(
            best_params=self.best_params,  # type: ignore
            best_score=self.best_score,
            history=self.history,
            convergence_data=convergence_data,
        )

optimize

optimize(n_iterations: int = 20, verbose: int = 1) -> BayesianOptimizationResult

Run Bayesian optimization.

Parameters:

Name	Type	Description	Default
n_iterations	int	Maximum number of optimization iterations	20
verbose	int	Reporting level: 0 is silent, 1 prints phases and new bests, 2 adds every evaluation and the final configuration.	1

Returns:

Type	Description
BayesianOptimizationResult	Optimization results including best parameters and history

Source code in src/dlhub/tuning/bayesian.py

def optimize(
    self, n_iterations: int = 20, verbose: int = 1
) -> BayesianOptimizationResult:
    """
    Run Bayesian optimization.

    Parameters
    ----------
    n_iterations : int, default=20
        Maximum number of optimization iterations
    verbose : int, default=1
        Reporting level: 0 is silent, 1 prints phases and new bests, 2 adds
        every evaluation and the final configuration.

    Returns
    -------
    BayesianOptimizationResult
        Optimization results including best parameters and history
    """
    if verbose >= 1:
        print("Starting Bayesian Optimization...")
        print(f"Search space: {self.search_space}")
        print(f"Acquisition function: {self.acquisition}")

    if verbose >= 1:
        print(f"\nPhase 1: Random initialization ({self.n_initial} points)")

    for i in range(self.n_initial):
        X_raw = np.random.uniform(self.bounds[:, 0], self.bounds[:, 1])
        params = self._array_to_dict(X_raw)

        score = self._evaluate_objective(params)

        if not np.isnan(score):
            X_norm = self._normalize_params(X_raw)
            self.X_observed.append(X_norm)
            self.y_observed.append(score)
            self.history.append((params.copy(), score))

            if score > self.best_score:
                self.best_score = score
                self.best_params = params.copy()

            if verbose >= 1:
                print(f"  {i + 1}/{self.n_initial}: Score = {score:.4f}")

    if len(self.X_observed) == 0:
        raise RuntimeError("All initial evaluations failed")

    if verbose >= 1:
        print(f"\nPhase 2: Bayesian optimization ({n_iterations} iterations)")

    for iteration in range(n_iterations):
        X_train = np.array(self.X_observed)
        y_train = np.array(self.y_observed)
        self.gp.fit(X_train, y_train)

        X_next_norm = self._optimize_acquisition()
        X_next_raw = self._denormalize_params(X_next_norm)
        params_next = self._array_to_dict(X_next_raw)

        score = self._evaluate_objective(params_next)

        if not np.isnan(score):
            self.X_observed.append(X_next_norm)
            self.y_observed.append(score)
            self.history.append((params_next.copy(), score))

            if score > self.best_score:
                self.best_score = score
                self.best_params = params_next.copy()

                if verbose >= 1:
                    print(
                        f"  Iter {iteration + 1}: Score = {score:.4f} (NEW BEST!)"
                    )
            else:
                if verbose >= 2:
                    print(f"  Iter {iteration + 1}: Score = {score:.4f}")
        else:
            if verbose >= 1:
                print(f"  Iter {iteration + 1}: Evaluation failed")

    successful_initial = [s for _, s in self.history[: self.n_initial]]
    convergence_data = {
        "n_evaluations": len(self.history),
        "n_failed": n_iterations + self.n_initial - len(self.history),
        # The initial design is random, so its mean is what an equal budget
        # of random search would have averaged. Guarded because every
        # initial point can fail while later ones succeed, and a mean over
        # an empty list is a nan that propagates into the whole summary.
        "improvement_over_random": (
            self.best_score - float(np.mean(successful_initial))
            if successful_initial
            else 0.0
        ),
        "scores": [score for _, score in self.history],
    }

    if verbose >= 1:
        print("\nOptimization completed!")
        print(f"Best score: {self.best_score:.4f}")
    if verbose >= 2:
        print(f"Best parameters: {self.best_params}")
        print(f"Total evaluations: {len(self.history)}")

    return BayesianOptimizationResult(
        best_params=self.best_params,  # type: ignore
        best_score=self.best_score,
        history=self.history,
        convergence_data=convergence_data,
    )

GaussianProcess

Simplified Gaussian Process for Bayesian Optimization.

Implements a GP with RBF kernel for modeling the objective function. This is a educational implementation - production code should use more robust libraries like GPy or scikit-learn.

Parameters:

Name	Type	Description	Default
kernel_lengthscale	float	Length scale parameter for RBF kernel	1.0
kernel_variance	float	Variance parameter for RBF kernel	1.0
noise_variance	float	Noise variance for numerical stability	1e-6

Source code in src/dlhub/tuning/bayesian.py

class GaussianProcess:
    """
    Simplified Gaussian Process for Bayesian Optimization.

    Implements a GP with RBF kernel for modeling the objective function.
    This is a educational implementation - production code should use
    more robust libraries like GPy or scikit-learn.

    Parameters
    ----------
    kernel_lengthscale : float, default=1.0
        Length scale parameter for RBF kernel
    kernel_variance : float, default=1.0
        Variance parameter for RBF kernel
    noise_variance : float, default=1e-6
        Noise variance for numerical stability
    """

    def __init__(
        self,
        kernel_lengthscale: float = 1.0,
        kernel_variance: float = 1.0,
        noise_variance: float = 1e-6,
    ):
        self.kernel_lengthscale = kernel_lengthscale
        self.kernel_variance = kernel_variance
        self.noise_variance = noise_variance
        self.X_train = None
        self.y_train = None
        self.K_inv = None

    def rbf_kernel(self, X1: np.ndarray, X2: np.ndarray) -> np.ndarray:
        """
        Compute RBF (Radial Basis Function) kernel matrix.

        Parameters
        ----------
        X1 : np.ndarray, shape (n1, d)
            First set of input points
        X2 : np.ndarray, shape (n2, d)
            Second set of input points

        Returns
        -------
        np.ndarray, shape (n1, n2)
            Kernel matrix K(X1, X2)
        """
        sq_dists = np.sum((X1[:, np.newaxis, :] - X2[np.newaxis, :, :]) ** 2, axis=2)

        return self.kernel_variance * np.exp(
            -0.5 * sq_dists / (self.kernel_lengthscale**2)
        )

    def fit(self, X: np.ndarray, y: np.ndarray) -> None:
        """
        Fit the Gaussian Process to training data.

        Parameters
        ----------
        X : np.ndarray, shape (n_samples, n_features)
            Training input points
        y : np.ndarray, shape (n_samples,)
            Training target values
        """
        self.X_train = X.copy()
        self.y_train = y.copy()

        # Compute kernel matrix and its inverse
        K = self.rbf_kernel(X, X)
        K += self.noise_variance * np.eye(len(X))

        try:
            self.K_inv = np.linalg.inv(K)
        except np.linalg.LinAlgError:
            warnings.warn("Kernel matrix is singular, using pseudo-inverse")
            self.K_inv = np.linalg.pinv(K)

    def predict(self, X: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
        """
        Make predictions with uncertainty estimates.

        Parameters
        ----------
        X : np.ndarray, shape (n_test, n_features)
            Test input points

        Returns
        -------
        mean : np.ndarray, shape (n_test,)
            Predicted mean values
        std : np.ndarray, shape (n_test,)
            Predicted standard deviations
        """
        if self.X_train is None:
            raise ValueError("GP must be fitted before making predictions")

        # Compute kernel matrices
        K_star = self.rbf_kernel(X, self.X_train)
        K_star_star = self.rbf_kernel(X, X)

        # Compute predictive mean
        mean = K_star @ self.K_inv @ self.y_train  # type: ignore

        var = np.diag(K_star_star) - np.diag(K_star @ self.K_inv @ K_star.T)
        var = np.maximum(var, 1e-10)
        std = np.sqrt(var)

        return mean, std

rbf_kernel

rbf_kernel(X1: ndarray, X2: ndarray) -> np.ndarray

Compute RBF (Radial Basis Function) kernel matrix.

Parameters:

Name	Type	Description	Default
X1	(ndarray, shape(n1, d))	First set of input points	required
X2	(ndarray, shape(n2, d))	Second set of input points	required

Returns:

Type	Description
(ndarray, shape(n1, n2))	Kernel matrix K(X1, X2)

Source code in src/dlhub/tuning/bayesian.py

def rbf_kernel(self, X1: np.ndarray, X2: np.ndarray) -> np.ndarray:
    """
    Compute RBF (Radial Basis Function) kernel matrix.

    Parameters
    ----------
    X1 : np.ndarray, shape (n1, d)
        First set of input points
    X2 : np.ndarray, shape (n2, d)
        Second set of input points

    Returns
    -------
    np.ndarray, shape (n1, n2)
        Kernel matrix K(X1, X2)
    """
    sq_dists = np.sum((X1[:, np.newaxis, :] - X2[np.newaxis, :, :]) ** 2, axis=2)

    return self.kernel_variance * np.exp(
        -0.5 * sq_dists / (self.kernel_lengthscale**2)
    )

fit

fit(X: ndarray, y: ndarray) -> None

Fit the Gaussian Process to training data.

Parameters:

Name	Type	Description	Default
X	(ndarray, shape(n_samples, n_features))	Training input points	required
y	(ndarray, shape(n_samples))	Training target values	required

Source code in src/dlhub/tuning/bayesian.py

def fit(self, X: np.ndarray, y: np.ndarray) -> None:
    """
    Fit the Gaussian Process to training data.

    Parameters
    ----------
    X : np.ndarray, shape (n_samples, n_features)
        Training input points
    y : np.ndarray, shape (n_samples,)
        Training target values
    """
    self.X_train = X.copy()
    self.y_train = y.copy()

    # Compute kernel matrix and its inverse
    K = self.rbf_kernel(X, X)
    K += self.noise_variance * np.eye(len(X))

    try:
        self.K_inv = np.linalg.inv(K)
    except np.linalg.LinAlgError:
        warnings.warn("Kernel matrix is singular, using pseudo-inverse")
        self.K_inv = np.linalg.pinv(K)

predict

predict(X: ndarray) -> tuple[np.ndarray, np.ndarray]

Make predictions with uncertainty estimates.

Parameters:

Name	Type	Description	Default
X	(ndarray, shape(n_test, n_features))	Test input points	required

Returns:

Name	Type	Description
mean	(ndarray, shape(n_test))	Predicted mean values
std	(ndarray, shape(n_test))	Predicted standard deviations

Source code in src/dlhub/tuning/bayesian.py

def predict(self, X: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
    """
    Make predictions with uncertainty estimates.

    Parameters
    ----------
    X : np.ndarray, shape (n_test, n_features)
        Test input points

    Returns
    -------
    mean : np.ndarray, shape (n_test,)
        Predicted mean values
    std : np.ndarray, shape (n_test,)
        Predicted standard deviations
    """
    if self.X_train is None:
        raise ValueError("GP must be fitted before making predictions")

    # Compute kernel matrices
    K_star = self.rbf_kernel(X, self.X_train)
    K_star_star = self.rbf_kernel(X, X)

    # Compute predictive mean
    mean = K_star @ self.K_inv @ self.y_train  # type: ignore

    var = np.diag(K_star_star) - np.diag(K_star @ self.K_inv @ K_star.T)
    var = np.maximum(var, 1e-10)
    std = np.sqrt(var)

    return mean, std

ExperimentConfig dataclass

Configuration for hyperparameter optimization experiment.

Attributes:

Name	Type	Description
experiment_name	str	Name of the experiment
optimization_method	OptimizationMethod	Optimization strategy to use
hyperparameters	list	List of HyperparameterConfig objects
objective_metric	str	Name of metric to optimize
maximize	bool, default=True	Whether to maximize the objective metric
n_trials	int, default=100	Number of trials to run
random_seed	int(optional)	Random seed for reproducibility
save_dir	str(optional)	Directory to save results
additional_config	dict(optional)	Additional method-specific configuration

Source code in src/dlhub/tuning/framework.py

@dataclass
class ExperimentConfig:
    """
    Configuration for hyperparameter optimization experiment.

    Attributes
    ----------
    experiment_name : str
        Name of the experiment
    optimization_method : OptimizationMethod
        Optimization strategy to use
    hyperparameters : list
        List of HyperparameterConfig objects
    objective_metric : str
        Name of metric to optimize
    maximize : bool, default=True
        Whether to maximize the objective metric
    n_trials : int, default=100
        Number of trials to run
    random_seed : int (optional)
        Random seed for reproducibility
    save_dir : str (optional)
        Directory to save results
    additional_config : dict (optional)
        Additional method-specific configuration
    """

    experiment_name: str
    optimization_method: OptimizationMethod
    hyperparameters: list[HyperparameterConfig]
    objective_metric: str
    maximize: bool = True
    n_trials: int = 100
    random_seed: int | None = None
    save_dir: str | None = None
    additional_config: dict[str, Any] = field(default_factory=dict)

ExperimentLogger

Logger for experiment results and metadata.

Parameters:

Name	Type	Description	Default
save_dir	str(optional)	Directory to save logs	required
experiment_name	str	Name of the experiment	required

Source code in src/dlhub/tuning/framework.py

class ExperimentLogger:
    """
    Logger for experiment results and metadata.

    Parameters
    ----------
    save_dir : str (optional)
        Directory to save logs
    experiment_name : str
        Name of the experiment
    """

    def __init__(self, save_dir: str | None, experiment_name: str):
        self.experiment_name = experiment_name
        self.save_dir = Path(save_dir) if save_dir else None

        if self.save_dir:
            self.save_dir.mkdir(parents=True, exist_ok=True)
            self.log_file = self.save_dir / f"{experiment_name}_log.jsonl"
        else:
            self.log_file = None

    def log_trial(self, trial_result: TrialResult) -> None:
        """
        Log a trial result.

        Parameters
        ----------
        trial_result : TrialResult
            Trial result to log
        """
        if self.log_file is None:
            return

        # Convert to dictionary
        trial_dict = {
            "trial_id": trial_result.trial_id,
            "hyperparams": trial_result.hyperparams,
            "metrics": trial_result.metrics,
            "training_time": trial_result.training_time,
            "status": trial_result.status,
            "metadata": trial_result.metadata,
            "timestamp": time.time(),
        }

        with open(self.log_file, "a") as f:
            f.write(json.dumps(trial_dict) + "\n")

    def save_results(self, result: ExperimentResult) -> None:
        """
        Save complete optimization results.

        Parameters
        ----------
        result : ExperimentResult
            Optimization results to save
        """
        if self.save_dir is None:
            return

        summary_file = self.save_dir / f"{self.experiment_name}_summary.json"
        summary = {
            "experiment_name": result.experiment_config.experiment_name,
            "optimization_method": result.experiment_config.optimization_method.value,
            "best_trial": {
                "hyperparams": result.best_trial.hyperparams,
                "metrics": result.best_trial.metrics,
                "trial_id": result.best_trial.trial_id,
            },
            "total_time": result.total_time,
            "n_trials": len(result.all_trials),
            "summary_statistics": result.summary_statistics,
        }

        with open(summary_file, "w") as f:
            json.dump(summary, f, indent=2)

    def load_results(self) -> list[TrialResult] | None:
        """
        Load trial results from log file.

        Returns
        -------
        list or None
            List of TrialResult objects, or None if no log exists
        """
        if self.log_file is None or not self.log_file.exists():
            return None

        trials = []
        with open(self.log_file) as f:
            for line in f:
                trial_dict = json.loads(line)
                trial = TrialResult(
                    trial_id=trial_dict["trial_id"],
                    hyperparams=trial_dict["hyperparams"],
                    metrics=trial_dict["metrics"],
                    training_time=trial_dict["training_time"],
                    status=trial_dict["status"],
                    metadata=trial_dict["metadata"],
                )
                trials.append(trial)

        return trials

log_trial

log_trial(trial_result: TrialResult) -> None

Log a trial result.

Parameters:

Name	Type	Description	Default
trial_result	TrialResult	Trial result to log	required

Source code in src/dlhub/tuning/framework.py

def log_trial(self, trial_result: TrialResult) -> None:
    """
    Log a trial result.

    Parameters
    ----------
    trial_result : TrialResult
        Trial result to log
    """
    if self.log_file is None:
        return

    # Convert to dictionary
    trial_dict = {
        "trial_id": trial_result.trial_id,
        "hyperparams": trial_result.hyperparams,
        "metrics": trial_result.metrics,
        "training_time": trial_result.training_time,
        "status": trial_result.status,
        "metadata": trial_result.metadata,
        "timestamp": time.time(),
    }

    with open(self.log_file, "a") as f:
        f.write(json.dumps(trial_dict) + "\n")

save_results

save_results(result: ExperimentResult) -> None

Save complete optimization results.

Parameters:

Name	Type	Description	Default
result	ExperimentResult	Optimization results to save	required

Source code in src/dlhub/tuning/framework.py

def save_results(self, result: ExperimentResult) -> None:
    """
    Save complete optimization results.

    Parameters
    ----------
    result : ExperimentResult
        Optimization results to save
    """
    if self.save_dir is None:
        return

    summary_file = self.save_dir / f"{self.experiment_name}_summary.json"
    summary = {
        "experiment_name": result.experiment_config.experiment_name,
        "optimization_method": result.experiment_config.optimization_method.value,
        "best_trial": {
            "hyperparams": result.best_trial.hyperparams,
            "metrics": result.best_trial.metrics,
            "trial_id": result.best_trial.trial_id,
        },
        "total_time": result.total_time,
        "n_trials": len(result.all_trials),
        "summary_statistics": result.summary_statistics,
    }

    with open(summary_file, "w") as f:
        json.dump(summary, f, indent=2)

load_results

load_results() -> list[TrialResult] | None

Load trial results from log file.

Returns:

Type	Description
list or None	List of TrialResult objects, or None if no log exists

Source code in src/dlhub/tuning/framework.py

def load_results(self) -> list[TrialResult] | None:
    """
    Load trial results from log file.

    Returns
    -------
    list or None
        List of TrialResult objects, or None if no log exists
    """
    if self.log_file is None or not self.log_file.exists():
        return None

    trials = []
    with open(self.log_file) as f:
        for line in f:
            trial_dict = json.loads(line)
            trial = TrialResult(
                trial_id=trial_dict["trial_id"],
                hyperparams=trial_dict["hyperparams"],
                metrics=trial_dict["metrics"],
                training_time=trial_dict["training_time"],
                status=trial_dict["status"],
                metadata=trial_dict["metadata"],
            )
            trials.append(trial)

    return trials

ExperimentResult dataclass

The outcome of one hyperparameter search: every trial, and the best of them.

Named for the :class:ExperimentConfig it answers. Not to be confused with :class:dlhub.optimizers.OptimizationRun, which traces a single descent.

Attributes:

Name	Type	Description
experiment_config	ExperimentConfig	Configuration used for the experiment
best_trial	TrialResult	Best performing trial
all_trials	list	All trial results
total_time	float	Total optimization time
summary_statistics	dict	Summary statistics and analysis

Source code in src/dlhub/tuning/framework.py

@dataclass
class ExperimentResult:
    """
    The outcome of one hyperparameter search: every trial, and the best of them.

    Named for the :class:`ExperimentConfig` it answers. Not to be confused with
    :class:`dlhub.optimizers.OptimizationRun`, which traces a single descent.

    Attributes
    ----------
    experiment_config : ExperimentConfig
        Configuration used for the experiment
    best_trial : TrialResult
        Best performing trial
    all_trials : list
        All trial results
    total_time : float
        Total optimization time
    summary_statistics : dict
        Summary statistics and analysis
    """

    experiment_config: ExperimentConfig
    best_trial: TrialResult
    all_trials: list[TrialResult]
    total_time: float
    summary_statistics: dict[str, Any] = field(default_factory=dict)

FunctionObjective

Bases: ObjectiveFunction

Wrapper for function-based objectives.

Parameters:

Name	Type	Description	Default
eval_function	callable	Function that takes hyperparams and returns metrics dict	required
metric_names	list	Names of metrics returned by eval_function	required

Source code in src/dlhub/tuning/framework.py

class FunctionObjective(ObjectiveFunction):
    """
    Wrapper for function-based objectives.

    Parameters
    ----------
    eval_function : callable
        Function that takes hyperparams and returns metrics dict
    metric_names : list
        Names of metrics returned by eval_function
    """

    def __init__(
        self, eval_function: Callable[[dict], dict[str, float]], metric_names: list[str]
    ):
        self.eval_function = eval_function
        self.metric_names = metric_names

    def evaluate(self, hyperparams: dict[str, Any]) -> dict[str, float]:
        """Evaluate using wrapped function."""
        return self.eval_function(hyperparams)

    def get_metric_names(self) -> list[str]:
        """Get metric names."""
        return self.metric_names

evaluate

evaluate(hyperparams: dict[str, Any]) -> dict[str, float]

Evaluate using wrapped function.

Source code in src/dlhub/tuning/framework.py

def evaluate(self, hyperparams: dict[str, Any]) -> dict[str, float]:
    """Evaluate using wrapped function."""
    return self.eval_function(hyperparams)

get_metric_names

get_metric_names() -> list[str]

Get metric names.

Source code in src/dlhub/tuning/framework.py

def get_metric_names(self) -> list[str]:
    """Get metric names."""
    return self.metric_names

HyperparameterConfig dataclass

Configuration for a single hyperparameter.

Attributes:

Name	Type	Description
name	str	Parameter name
type	str	Parameter type ('continuous', 'integer', 'categorical')
range	tuple or list	Valid range or choices for the parameter
scale	str, default='linear'	Scale for sampling ('linear', 'log')
default	Any(optional)	Default value for said parameter

Source code in src/dlhub/tuning/framework.py

@dataclass
class HyperparameterConfig:
    """
    Configuration for a single hyperparameter.

    Attributes
    ----------
    name : str
        Parameter name
    type : str
        Parameter type ('continuous', 'integer', 'categorical')
    range : tuple or list
        Valid range or choices for the parameter
    scale : str, default='linear'
        Scale for sampling ('linear', 'log')
    default : Any (optional)
        Default value for said parameter
    """

    name: str
    type: str
    range: tuple | list
    scale: str = "linear"
    default: Any = None

HyperparameterOptimizer

Main hyperparameter optimization framework.

Parameters:

Name	Type	Description	Default
config	ExperimentConfig	Experiment configuration	required
objective	ObjectiveFunction	Objective function to optimize	required

Source code in src/dlhub/tuning/framework.py

class HyperparameterOptimizer:
    """
    Main hyperparameter optimization framework.

    Parameters
    ----------
    config : ExperimentConfig
        Experiment configuration
    objective : ObjectiveFunction
        Objective function to optimize
    """

    def __init__(self, config: ExperimentConfig, objective: ObjectiveFunction):
        self.config = config
        self.objective = objective

        if config.random_seed is not None:
            np.random.seed(config.random_seed)

        self.sampler = HyperparameterSampler(config.hyperparameters, config.random_seed)

        self.logger = ExperimentLogger(config.save_dir, config.experiment_name)

        self.all_trials = []
        self.best_trial = None
        self.trial_counter = 0

    def _evaluate_trial(self, hyperparams: dict[str, Any]) -> TrialResult:
        """
        Evaluate a single trial.

        Parameters
        ----------
        hyperparams : dict
            Hyperparameter configuration

        Returns
        -------
        TrialResult
            Result of the trial
        """
        trial_id = self.trial_counter
        self.trial_counter += 1

        start_time = time.time()

        try:
            if not self.sampler.validate(hyperparams):
                raise ValueError("Invalid hyperparameter configuration")

            metrics = self.objective.evaluate(hyperparams)
            training_time = time.time() - start_time

            if self.config.objective_metric not in metrics:
                raise ValueError(
                    f"Objective metric '{self.config.objective_metric}' "
                    "not found in results"
                )

            result = TrialResult(
                trial_id=trial_id,
                hyperparams=hyperparams.copy(),
                metrics=metrics,
                training_time=training_time,
                status="success",
            )

        except Exception as e:
            training_time = time.time() - start_time
            warnings.warn(f"Trial {trial_id} failed: {e}")

            result = TrialResult(
                trial_id=trial_id,
                hyperparams=hyperparams.copy(),
                metrics={
                    self.config.objective_metric: -np.inf
                    if self.config.maximize
                    else np.inf
                },
                training_time=training_time,
                status="failed",
                metadata={"error": str(e)},
            )

        self.logger.log_trial(result)
        self.all_trials.append(result)

        self._update_best_trial(result)

        return result

    def _update_best_trial(self, trial: TrialResult) -> None:
        """Update best trial if current trial is better."""
        if trial.status != "success":
            return

        current_score = trial.metrics[self.config.objective_metric]

        if self.best_trial is None:
            self.best_trial = trial
        else:
            best_score = self.best_trial.metrics[self.config.objective_metric]

            if self.config.maximize:
                if current_score > best_score:
                    self.best_trial = trial
            else:
                if current_score < best_score:
                    self.best_trial = trial

    def optimize(self, verbose: bool = True) -> ExperimentResult:
        """
        Run hyperparameter optimization.

        Parameters
        ----------
        verbose : bool, default=True
            Whether to print progress

        Returns
        -------
        ExperimentResult
            Optimization results
        """
        if verbose:
            print(
                f"Starting Hyperparameter Optimization: {self.config.experiment_name}"
            )
            print(f"Method: {self.config.optimization_method.value}")
            print(f"Number of trials: {self.config.n_trials}")
            print(
                f"Objective: {'maximize' if self.config.maximize else 'minimize'} "
                f"{self.config.objective_metric}"
            )

        start_time = time.time()

        if self.config.optimization_method == OptimizationMethod.RANDOM_SEARCH:
            self._run_random_search(verbose)
        elif self.config.optimization_method == OptimizationMethod.GRID_SEARCH:
            self._run_grid_search(verbose)
        else:
            raise NotImplementedError(
                f"Method {self.config.optimization_method.value} not implemented "
                "in this simplified framework"
            )

        total_time = time.time() - start_time

        summary_stats = self._compute_summary_statistics()

        result = ExperimentResult(
            experiment_config=self.config,
            best_trial=self.best_trial,  # type: ignore
            all_trials=self.all_trials,
            total_time=total_time,
            summary_statistics=summary_stats,
        )

        self.logger.save_results(result)

        if verbose:
            print(f"\nOptimization completed in {total_time:.2f} seconds")
            print(
                f"Best {self.config.objective_metric}: "
                f"{self.best_trial.metrics[self.config.objective_metric]:.6f}"
            )  # type: ignore
            print(f"Best hyperparameters: {self.best_trial.hyperparams}")  # type: ignore

        return result

    def _run_random_search(self, verbose: bool) -> None:
        """Run random search optimization."""
        for i in range(self.config.n_trials):
            hyperparams = self.sampler.sample()
            trial = self._evaluate_trial(hyperparams)

            if verbose and (i + 1) % max(1, self.config.n_trials // 10) == 0:
                print(
                    f"Trial {i + 1}/{self.config.n_trials}: "
                    f"{self.config.objective_metric} = "
                    f"{trial.metrics.get(self.config.objective_metric, 'N/A')}"
                )

    def _run_grid_search(self, verbose: bool) -> None:
        """Run grid search optimization."""
        grid_configs = self._generate_grid()

        if verbose:
            print(f"Generated grid with {len(grid_configs)} configurations")

        for i, hyperparams in enumerate(grid_configs[: self.config.n_trials]):
            trial = self._evaluate_trial(hyperparams)

            if verbose and (i + 1) % max(1, len(grid_configs) // 10) == 0:
                print(
                    f"Trial {i + 1}/{min(len(grid_configs), self.config.n_trials)}: "
                    f"{self.config.objective_metric} = "
                    f"{trial.metrics.get(self.config.objective_metric, 'N/A')}"
                )

    def _generate_grid(self) -> list[dict[str, Any]]:
        """Generate grid of hyperparameter configurations."""
        from itertools import product

        param_grids = {}
        for config in self.config.hyperparameters:
            if config.type == "categorical":
                param_grids[config.name] = config.range
            elif config.type in ["continuous", "integer"]:
                n_points = 5  # Use 5 points per dimension
                low, high = config.range

                if config.scale == "log":
                    points = np.logspace(np.log10(low), np.log10(high), n_points)
                else:
                    points = np.linspace(low, high, n_points)

                if config.type == "integer":
                    points = np.unique(np.round(points).astype(int))

                param_grids[config.name] = points.tolist()

        keys = list(param_grids.keys())
        values = list(param_grids.values())

        grid_configs = []
        for combo in product(*values):
            config_dict = dict(zip(keys, combo))
            grid_configs.append(config_dict)

        return grid_configs

    def _compute_summary_statistics(self) -> dict[str, Any]:
        """Compute summary statistics from all trials."""
        successful_trials = [t for t in self.all_trials if t.status == "success"]

        if not successful_trials:
            return {}

        objective_scores = [
            t.metrics[self.config.objective_metric] for t in successful_trials
        ]

        training_times = [t.training_time for t in successful_trials]

        statistics = {
            "n_trials": len(self.all_trials),
            "n_successful": len(successful_trials),
            "n_failed": len(self.all_trials) - len(successful_trials),
            "objective_statistics": {
                "mean": float(np.mean(objective_scores)),
                "std": float(np.std(objective_scores)),
                "min": float(np.min(objective_scores)),
                "max": float(np.max(objective_scores)),
                "median": float(np.median(objective_scores)),
                "percentiles": {
                    "25th": float(np.percentile(objective_scores, 25)),
                    "75th": float(np.percentile(objective_scores, 75)),
                    "95th": float(np.percentile(objective_scores, 95)),
                },
            },
            "training_time_statistics": {
                "mean": float(np.mean(training_times)),
                "total": float(np.sum(training_times)),
                "min": float(np.min(training_times)),
                "max": float(np.max(training_times)),
            },
            "improvement_over_random": self._improvement_over_random(objective_scores),
        }

        return statistics

    def _improvement_over_random(self, objective_scores: list[float]) -> float:
        """
        Relative gain of the best trial over the first ten, which for random and
        grid search are as good a stand-in for chance as the run provides.

        Reported as 0.0 below ten trials, and when the baseline mean is zero:
        the relative gain over nothing is not a number, and a metric centred on
        zero is a legitimate thing to optimize.
        """
        if len(objective_scores) < 10:
            return 0.0

        baseline = float(np.mean(objective_scores[:10]))
        if baseline == 0.0:
            return 0.0

        best_score = self.best_trial.metrics[self.config.objective_metric]  # type: ignore
        return float((best_score - baseline) / abs(baseline))

optimize

optimize(verbose: bool = True) -> ExperimentResult

Run hyperparameter optimization.

Parameters:

Name	Type	Description	Default
verbose	bool	Whether to print progress	True

Returns:

Type	Description
ExperimentResult	Optimization results

Source code in src/dlhub/tuning/framework.py

def optimize(self, verbose: bool = True) -> ExperimentResult:
    """
    Run hyperparameter optimization.

    Parameters
    ----------
    verbose : bool, default=True
        Whether to print progress

    Returns
    -------
    ExperimentResult
        Optimization results
    """
    if verbose:
        print(
            f"Starting Hyperparameter Optimization: {self.config.experiment_name}"
        )
        print(f"Method: {self.config.optimization_method.value}")
        print(f"Number of trials: {self.config.n_trials}")
        print(
            f"Objective: {'maximize' if self.config.maximize else 'minimize'} "
            f"{self.config.objective_metric}"
        )

    start_time = time.time()

    if self.config.optimization_method == OptimizationMethod.RANDOM_SEARCH:
        self._run_random_search(verbose)
    elif self.config.optimization_method == OptimizationMethod.GRID_SEARCH:
        self._run_grid_search(verbose)
    else:
        raise NotImplementedError(
            f"Method {self.config.optimization_method.value} not implemented "
            "in this simplified framework"
        )

    total_time = time.time() - start_time

    summary_stats = self._compute_summary_statistics()

    result = ExperimentResult(
        experiment_config=self.config,
        best_trial=self.best_trial,  # type: ignore
        all_trials=self.all_trials,
        total_time=total_time,
        summary_statistics=summary_stats,
    )

    self.logger.save_results(result)

    if verbose:
        print(f"\nOptimization completed in {total_time:.2f} seconds")
        print(
            f"Best {self.config.objective_metric}: "
            f"{self.best_trial.metrics[self.config.objective_metric]:.6f}"
        )  # type: ignore
        print(f"Best hyperparameters: {self.best_trial.hyperparams}")  # type: ignore

    return result

HyperparameterSampler

Utility class for sampling hyperparameters from configurations.

Parameters:

Name	Type	Description	Default
hyperparameter_configs	list	List of HyperparameterConfig objects	required
random_state	int	Random seed for reproducibility	None

Source code in src/dlhub/tuning/framework.py

class HyperparameterSampler:
    """
    Utility class for sampling hyperparameters from configurations.

    Parameters
    ----------
    hyperparameter_configs : list
        List of HyperparameterConfig objects
    random_state : int, optional
        Random seed for reproducibility
    """

    def __init__(
        self,
        hyperparameter_configs: list[HyperparameterConfig],
        random_state: int | None = None,
    ):
        self.configs = hyperparameter_configs
        if random_state is not None:
            np.random.seed(random_state)

    def sample(self) -> dict[str, Any]:
        """
        Sample a hyperparameter configuration.

        Returns
        -------
        dict
            Sampled hyperparameter configuration
        """
        hyperparams = {}

        for config in self.configs:
            if config.type == "continuous":
                hyperparams[config.name] = self._sample_continuous(config)
            elif config.type == "integer":
                hyperparams[config.name] = self._sample_integer(config)
            elif config.type == "categorical":
                hyperparams[config.name] = self._sample_categorical(config)
            else:
                raise ValueError(f"Unknown parameter type: {config.type}")

        return hyperparams

    def _sample_continuous(self, config: HyperparameterConfig) -> float:
        """Sample continuous parameter."""
        low, high = config.range

        if config.scale == "log":
            return np.exp(np.random.uniform(np.log(low), np.log(high)))
        else:
            return np.random.uniform(low, high)

    def _sample_integer(self, config: HyperparameterConfig) -> int:
        """Sample integer parameter."""
        low, high = config.range

        if config.scale == "log":
            log_value = np.random.uniform(np.log(low), np.log(high))
            return int(np.round(np.exp(log_value)))
        else:
            return np.random.randint(low, high + 1)

    def _sample_categorical(self, config: HyperparameterConfig) -> Any:
        """Sample categorical parameter."""
        return np.random.choice(config.range)

    def validate(self, hyperparams: dict[str, Any]) -> bool:
        """
        Validate a hyperparameter configuration.

        Parameters
        ----------
        hyperparams : dict
            Hyperparameter configuration to validate

        Returns
        -------
        bool
            True if configuration is valid
        """
        for config in self.configs:
            if config.name not in hyperparams:
                return False

            value = hyperparams[config.name]

            if config.type in ["continuous", "integer"]:
                low, high = config.range
                if not (low <= value <= high):
                    return False
            elif config.type == "categorical":
                if value not in config.range:
                    return False

        return True

sample

sample() -> dict[str, Any]

Sample a hyperparameter configuration.

Returns:

Type	Description
dict	Sampled hyperparameter configuration

Source code in src/dlhub/tuning/framework.py

def sample(self) -> dict[str, Any]:
    """
    Sample a hyperparameter configuration.

    Returns
    -------
    dict
        Sampled hyperparameter configuration
    """
    hyperparams = {}

    for config in self.configs:
        if config.type == "continuous":
            hyperparams[config.name] = self._sample_continuous(config)
        elif config.type == "integer":
            hyperparams[config.name] = self._sample_integer(config)
        elif config.type == "categorical":
            hyperparams[config.name] = self._sample_categorical(config)
        else:
            raise ValueError(f"Unknown parameter type: {config.type}")

    return hyperparams

validate

validate(hyperparams: dict[str, Any]) -> bool

Validate a hyperparameter configuration.

Parameters:

Name	Type	Description	Default
hyperparams	dict	Hyperparameter configuration to validate	required

Returns:

Type	Description
bool	True if configuration is valid

Source code in src/dlhub/tuning/framework.py

def validate(self, hyperparams: dict[str, Any]) -> bool:
    """
    Validate a hyperparameter configuration.

    Parameters
    ----------
    hyperparams : dict
        Hyperparameter configuration to validate

    Returns
    -------
    bool
        True if configuration is valid
    """
    for config in self.configs:
        if config.name not in hyperparams:
            return False

        value = hyperparams[config.name]

        if config.type in ["continuous", "integer"]:
            low, high = config.range
            if not (low <= value <= high):
                return False
        elif config.type == "categorical":
            if value not in config.range:
                return False

    return True

ObjectiveFunction

Bases: ABC

Abstract base class for objective functions.

Defines the interface that objective functions must implement.

Source code in src/dlhub/tuning/framework.py

class ObjectiveFunction(ABC):
    """
    Abstract base class for objective functions.

    Defines the interface that objective functions must implement.
    """

    @abstractmethod
    def evaluate(self, hyperparams: dict[str, Any]) -> dict[str, float]:
        """
        Evaluate hyperparameters and return metrics.

        Parameters
        ----------
        hyperparams : dict
            Hyperparameter configuration

        Returns
        -------
        dict
            Dictionary of metric names to values
        """
        pass

    @abstractmethod
    def get_metric_names(self) -> list[str]:
        """
        Get names of all metrics returned by evaluate.

        Returns
        -------
        list
            List of metric names
        """
        pass

evaluate abstractmethod

evaluate(hyperparams: dict[str, Any]) -> dict[str, float]

Evaluate hyperparameters and return metrics.

Parameters:

Name	Type	Description	Default
hyperparams	dict	Hyperparameter configuration	required

Returns:

Type	Description
dict	Dictionary of metric names to values

Source code in src/dlhub/tuning/framework.py

@abstractmethod
def evaluate(self, hyperparams: dict[str, Any]) -> dict[str, float]:
    """
    Evaluate hyperparameters and return metrics.

    Parameters
    ----------
    hyperparams : dict
        Hyperparameter configuration

    Returns
    -------
    dict
        Dictionary of metric names to values
    """
    pass

get_metric_names abstractmethod

get_metric_names() -> list[str]

Get names of all metrics returned by evaluate.

Returns:

Type	Description
list	List of metric names

Source code in src/dlhub/tuning/framework.py

@abstractmethod
def get_metric_names(self) -> list[str]:
    """
    Get names of all metrics returned by evaluate.

    Returns
    -------
    list
        List of metric names
    """
    pass

OptimizationMethod

Bases: Enum

Enumeration of available optimization methods.

Source code in src/dlhub/tuning/framework.py

class OptimizationMethod(Enum):
    """Enumeration of available optimization methods."""

    RANDOM_SEARCH = "random_search"
    BAYESIAN = "bayesian"
    ASHA = "asha"
    PBT = "pbt"
    GRID_SEARCH = "grid_search"

TrialResult dataclass

Result from a single trial.

Attributes:

Name	Type	Description
trial_id	int	Unique trial identifier
hyperparams	dict	Hyperparameter configuration used
metrics	dict	All metrics recorded
training_time	float	Time taken for training
status	str	Trial status ('success', 'failed', 'cancelled')
metadata	dict	Additional trial information

Source code in src/dlhub/tuning/framework.py

@dataclass
class TrialResult:
    """
    Result from a single trial.

    Attributes
    ----------
    trial_id : int
        Unique trial identifier
    hyperparams : dict
        Hyperparameter configuration used
    metrics : dict
        All metrics recorded
    training_time : float
        Time taken for training
    status : str
        Trial status ('success', 'failed', 'cancelled')
    metadata : dict
        Additional trial information
    """

    trial_id: int
    hyperparams: dict[str, Any]
    metrics: dict[str, float]
    training_time: float
    status: str = "success"
    metadata: dict[str, Any] = field(default_factory=dict)

BaseTrainer

Bases: ABC

Abstract base class for training interface.

Defines the interface that training functions must implement to work with the learning rate finder.

Source code in src/dlhub/tuning/learning_rate_finder.py

class BaseTrainer(ABC):
    """
    Abstract base class for training interface.

    Defines the interface that training functions must implement
    to work with the learning rate finder.
    """

    @abstractmethod
    def train_batch(self, learning_rate: float) -> float:
        """
        Train one batch with given learning rate and return loss.

        Parameters
        ----------
        learning_rate : float
            Learning rate to use for this batch

        Returns
        -------
        float
            Loss value after training step
        """
        pass

    @abstractmethod
    def reset_model(self) -> None:
        """Reset model to initial state."""
        pass

train_batch abstractmethod

train_batch(learning_rate: float) -> float

Train one batch with given learning rate and return loss.

Parameters:

Name	Type	Description	Default
learning_rate	float	Learning rate to use for this batch	required

Returns:

Type	Description
float	Loss value after training step

Source code in src/dlhub/tuning/learning_rate_finder.py

@abstractmethod
def train_batch(self, learning_rate: float) -> float:
    """
    Train one batch with given learning rate and return loss.

    Parameters
    ----------
    learning_rate : float
        Learning rate to use for this batch

    Returns
    -------
    float
        Loss value after training step
    """
    pass

reset_model abstractmethod

reset_model() -> None

Reset model to initial state.

Source code in src/dlhub/tuning/learning_rate_finder.py

@abstractmethod
def reset_model(self) -> None:
    """Reset model to initial state."""
    pass

FunctionTrainer

Bases: BaseTrainer

Trainer wrapper for function-based training.

Wraps user-provided training and reset functions to conform to the BaseTrainer interface.

Parameters:

Name	Type	Description	Default
train_function	callable	Function that takes learning_rate and returns loss	required
reset_function	callable	Function to reset model state	required

Source code in src/dlhub/tuning/learning_rate_finder.py

class FunctionTrainer(BaseTrainer):
    """
    Trainer wrapper for function-based training.

    Wraps user-provided training and reset functions to conform
    to the BaseTrainer interface.

    Parameters
    ----------
    train_function : callable
        Function that takes learning_rate and returns loss
    reset_function : callable
        Function to reset model state
    """

    def __init__(
        self,
        train_function: Callable[[float], float],
        reset_function: Callable[[], None],
    ):
        self.train_function = train_function
        self.reset_function = reset_function

    def train_batch(self, learning_rate: float) -> float:
        """Train one batch with given learning rate."""
        return self.train_function(learning_rate)

    def reset_model(self) -> None:
        """Reset model to initial state."""
        self.reset_function()

train_batch

train_batch(learning_rate: float) -> float

Train one batch with given learning rate.

Source code in src/dlhub/tuning/learning_rate_finder.py

def train_batch(self, learning_rate: float) -> float:
    """Train one batch with given learning rate."""
    return self.train_function(learning_rate)

reset_model

reset_model() -> None

Reset model to initial state.

Source code in src/dlhub/tuning/learning_rate_finder.py

def reset_model(self) -> None:
    """Reset model to initial state."""
    self.reset_function()

LearningRateFinder

Learning Rate Finder for optimal learning rate discovery.

Implements the learning rate range test by training with exponentially increasing learning rates and analyzing the loss curve to suggest optimal learning rate ranges.

Parameters:

Name	Type	Description	Default
trainer	BaseTrainer	Training interface object	required
min_lr	float	Minimum learning rate to test	1e-7
max_lr	float	Maximum learning rate to test	10.0
num_iterations	int	Number of iterations to run the test	100
step_mode	str	How to step learning rate ('exp' for exponential, 'linear' for linear)	'exp'
smooth_beta	float	Smoothing factor for loss smoothing (exponential moving average)	0.98
divergence_threshold	float	Stop if loss > divergence_threshold * min_loss	4.0

Source code in src/dlhub/tuning/learning_rate_finder.py

class LearningRateFinder:
    """
    Learning Rate Finder for optimal learning rate discovery.

    Implements the learning rate range test by training with exponentially
    increasing learning rates and analyzing the loss curve to suggest
    optimal learning rate ranges.

    Parameters
    ----------
    trainer : BaseTrainer
        Training interface object
    min_lr : float, default=1e-7
        Minimum learning rate to test
    max_lr : float, default=10.0
        Maximum learning rate to test
    num_iterations : int, default=100
        Number of iterations to run the test
    step_mode : str, default='exp'
        How to step learning rate ('exp' for exponential, 'linear' for linear)
    smooth_beta : float, default=0.98
        Smoothing factor for loss smoothing (exponential moving average)
    divergence_threshold : float, default=4.0
        Stop if loss > divergence_threshold * min_loss
    """

    def __init__(
        self,
        trainer: BaseTrainer,
        min_lr: float = 1e-7,
        max_lr: float = 10.0,
        num_iterations: int = 100,
        step_mode: str = "exp",
        smooth_beta: float = 0.98,
        divergence_threshold: float = 4.0,
    ):

        self.trainer = trainer
        self.min_lr = min_lr
        self.max_lr = max_lr
        self.num_iterations = num_iterations
        self.step_mode = step_mode.lower()
        self.smooth_beta = smooth_beta
        self.divergence_threshold = divergence_threshold

        # Validated against the normalized value, not the raw argument: lowering
        # the case and then rejecting the un-lowered form makes the normalization
        # unreachable and turns "EXP" into an error.
        if min_lr >= max_lr:
            raise ValueError("min_lr must be less than max_lr")
        if not 0 < smooth_beta < 1:
            raise ValueError("smooth_beta must be between 0 and 1")
        if self.step_mode not in ["exp", "linear"]:
            raise ValueError("step_mode must be 'exp' or 'linear'")

    def _generate_learning_rates(self) -> np.ndarray:
        """
        Generate learning rate schedule.

        Returns
        -------
        np.ndarray
            Array of learning rates to test
        """
        if self.step_mode == "exp":
            return np.logspace(
                np.log10(self.min_lr), np.log10(self.max_lr), self.num_iterations
            )
        else:
            return np.linspace(self.min_lr, self.max_lr, self.num_iterations)

    def _smooth_losses(self, losses: np.ndarray) -> np.ndarray:
        """
        Apply exponential smoothing to losses.

        Parameters
        ----------
        losses : np.ndarray
            Raw loss values

        Returns
        -------
        np.ndarray
            Smoothed loss values
        """
        smoothed = np.zeros_like(losses)
        smoothed[0] = losses[0]

        for i in range(1, len(losses)):
            smoothed[i] = (
                self.smooth_beta * smoothed[i - 1] + (1 - self.smooth_beta) * losses[i]
            )

        return smoothed

    def _detect_divergence(self, losses: np.ndarray, iteration: int) -> bool:
        """
        Detect if training has diverged.

        Parameters
        ----------
        losses : np.ndarray
            Loss values so far
        iteration : int
            Current iteration

        Returns
        -------
        bool
            True if divergence detected
        """
        if iteration < 10:  # Need some history
            return False

        min_loss = np.min(losses[: iteration + 1])
        current_loss = losses[iteration]

        # Cast to a plain bool, as annotated. The comparison yields np.bool_,
        # which is falsy-correct but fails an `is False` identity check.
        return bool(current_loss > self.divergence_threshold * min_loss)

    def _analyze_results(
        self,
        learning_rates: np.ndarray,
        losses: np.ndarray,
        smoothed_losses: np.ndarray,
    ) -> dict[str, Any]:
        """
        Analyze learning rate finder results.

        Parameters
        ----------
        learning_rates : np.ndarray
            Learning rates tested
        losses : np.ndarray
            Raw loss values
        smoothed_losses : np.ndarray
            Smoothed loss values

        Returns
        -------
        dict
            Analysis results and metrics
        """
        analysis = {}

        analysis["min_loss"] = float(np.min(losses))
        analysis["max_loss"] = float(np.max(losses))
        analysis["min_loss_lr"] = float(learning_rates[np.argmin(losses)])

        gradients = np.gradient(smoothed_losses, np.log10(learning_rates))
        min_gradient_idx = np.argmin(gradients)
        analysis["min_gradient_lr"] = float(learning_rates[min_gradient_idx])
        analysis["min_gradient"] = float(gradients[min_gradient_idx])

        suggested_lr = analysis["min_gradient_lr"] / 10
        analysis["suggested_lr"] = float(suggested_lr)

        if len(losses) > 10:
            initial_loss = np.mean(losses[:5])
            min_loss = analysis["min_loss"]
            analysis["loss_reduction_ratio"] = (initial_loss - min_loss) / initial_loss
        else:
            analysis["loss_reduction_ratio"] = 0.0

        loss_variance = np.var(losses)
        analysis["loss_variance"] = float(loss_variance)
        analysis["coefficient_of_variation"] = float(
            np.sqrt(loss_variance) / np.mean(losses)
        )

        if len(losses) > 20:
            first_half_mean = np.mean(losses[: len(losses) // 2])
            second_half_mean = np.mean(losses[len(losses) // 2 :])
            analysis["convergence_ratio"] = second_half_mean / first_half_mean
        else:
            analysis["convergence_ratio"] = 1.0

        return analysis

    def find(self, verbose: bool = True) -> LearningRateFinderResult:
        """
        Run learning rate finder.

        Parameters
        ----------
        verbose : bool, default=True
            Whether to print progress information

        Returns
        -------
        LearningRateFinderResult
            Results of the learning rate finder
        """
        if verbose:
            print("Starting Learning Rate Finder...")
            print(f"Learning rate range: {self.min_lr:.2e} to {self.max_lr:.2e}")
            print(f"Number of iterations: {self.num_iterations}")
            print(f"Step mode: {self.step_mode}")

        learning_rates = self._generate_learning_rates()
        losses = []

        self.trainer.reset_model()

        for i, lr in enumerate(learning_rates):
            try:
                loss = self.trainer.train_batch(lr)

                if np.isnan(loss) or np.isinf(loss):
                    if verbose:
                        print(
                            f"Iteration {i + 1}: Learning rate {lr:.2e} - Invalid loss (nan/inf)"
                        )
                    break

                losses.append(float(loss))

                if self._detect_divergence(np.array(losses), i):
                    if verbose:
                        print(
                            f"Iteration {i + 1}: Learning rate {lr:.2e} - Training diverged"
                        )
                    break

                if verbose and (i + 1) % max(1, self.num_iterations // 10) == 0:
                    print(
                        f"Iteration {i + 1}/{self.num_iterations}: LR = {lr:.2e}, Loss = {loss:.6f}"
                    )

            except Exception as e:
                if verbose:
                    print(
                        f"Iteration {i + 1}: Learning rate {lr:.2e} - Training failed: {e}"
                    )
                break

        if len(losses) < 5:
            raise RuntimeError(
                "Learning rate finder failed - insufficient valid loss values"
            )

        learning_rates = learning_rates[: len(losses)]
        losses = np.array(losses)

        smoothed_losses = self._smooth_losses(losses)

        analysis = self._analyze_results(learning_rates, losses, smoothed_losses)

        if verbose:
            print("\nLearning Rate Finder completed!")
            print(f"Suggested learning rate: {analysis['suggested_lr']:.2e}")
            print(
                f"Learning rate with steepest gradient: {analysis['min_gradient_lr']:.2e}"
            )
            print(f"Minimum loss: {analysis['min_loss']:.6f}")
            print(f"Loss reduction ratio: {analysis['loss_reduction_ratio']:.2%}")

        return LearningRateFinderResult(
            learning_rates=learning_rates,
            losses=losses,
            smoothed_losses=smoothed_losses,
            suggested_lr=analysis["suggested_lr"],
            min_gradient_lr=analysis["min_gradient_lr"],
            analysis=analysis,
        )

    def plot_results(
        self,
        result: LearningRateFinderResult,
        figsize: tuple[int, int] = (12, 8),
        save_path: str | None = None,
    ) -> None:
        """
        Plot learning rate finder results.

        Parameters
        ----------
        result : LearningRateFinderResult
            Results from learning rate finder
        figsize : tuple, default=(12, 8)
            Figure size for the plot
        save_path : str, optional
            Path to save the plot
        """
        fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=figsize)

        # Plot 1: Learning rate vs Loss (log scale)
        ax1.semilogx(
            result.learning_rates, result.losses, "b-", alpha=0.7, label="Raw Loss"
        )
        ax1.semilogx(
            result.learning_rates,
            result.smoothed_losses,
            "r-",
            linewidth=2,
            label="Smoothed Loss",
        )
        ax1.axvline(
            result.suggested_lr,
            color="green",
            linestyle="--",
            alpha=0.8,
            label=f"Suggested LR: {result.suggested_lr:.2e}",
        )
        ax1.axvline(
            result.min_gradient_lr,
            color="orange",
            linestyle="--",
            alpha=0.8,
            label=f"Min Gradient LR: {result.min_gradient_lr:.2e}",
        )
        ax1.set_xlabel("Learning Rate")
        ax1.set_ylabel("Loss")
        ax1.set_title("Learning Rate vs Loss")
        ax1.legend()
        ax1.grid(True, alpha=0.3)

        # Plot 2: Learning rate vs Loss (linear scale, zoomed)
        # Focus on the interesting region around minimum
        min_loss_idx = np.argmin(result.smoothed_losses)
        start_idx = max(0, min_loss_idx - 20)
        end_idx = min(len(result.learning_rates), min_loss_idx + 20)

        ax2.plot(
            result.learning_rates[start_idx:end_idx],
            result.losses[start_idx:end_idx],
            "b-",
            alpha=0.7,
        )
        ax2.plot(
            result.learning_rates[start_idx:end_idx],
            result.smoothed_losses[start_idx:end_idx],
            "r-",
            linewidth=2,
        )
        ax2.axvline(result.suggested_lr, color="green", linestyle="--", alpha=0.8)
        ax2.axvline(result.min_gradient_lr, color="orange", linestyle="--", alpha=0.8)
        ax2.set_xlabel("Learning Rate")
        ax2.set_ylabel("Loss")
        ax2.set_title("Loss (Zoomed Region)")
        ax2.grid(True, alpha=0.3)

        # Plot 3: Loss gradient
        gradients = np.gradient(result.smoothed_losses, np.log10(result.learning_rates))
        ax3.semilogx(result.learning_rates, gradients, "purple", linewidth=2)
        ax3.axvline(result.min_gradient_lr, color="orange", linestyle="--", alpha=0.8)
        ax3.axhline(0, color="black", linestyle="-", alpha=0.3)
        ax3.set_xlabel("Learning Rate")
        ax3.set_ylabel("Loss Gradient")
        ax3.set_title("Loss Gradient vs Learning Rate")
        ax3.grid(True, alpha=0.3)

        # Plot 4: Statistics summary
        ax4.axis("off")
        stats_text = f"""
        Analysis Summary:

        Suggested Learning Rate: {result.suggested_lr:.2e}
        Min Gradient Learning Rate: {result.min_gradient_lr:.2e}

        Minimum Loss: {result.analysis["min_loss"]:.6f}
        Loss Reduction: {result.analysis["loss_reduction_ratio"]:.2%}

        Loss Variance: {result.analysis["loss_variance"]:.6f}
        Coefficient of Variation: {result.analysis["coefficient_of_variation"]:.3f}

        Convergence Ratio: {result.analysis["convergence_ratio"]:.3f}
        """
        ax4.text(
            0.05,
            0.95,
            stats_text,
            transform=ax4.transAxes,
            fontsize=10,
            verticalalignment="top",
            fontfamily="monospace",
            bbox=dict(boxstyle="round", facecolor="lightgray", alpha=0.8),
        )

        plt.tight_layout()

        if save_path:
            plt.savefig(save_path, dpi=300, bbox_inches="tight")
            print(f"Plot saved to {save_path}")

        plt.show()

find

find(verbose: bool = True) -> LearningRateFinderResult

Run learning rate finder.

Parameters:

Name	Type	Description	Default
verbose	bool	Whether to print progress information	True

Returns:

Type	Description
LearningRateFinderResult	Results of the learning rate finder

Source code in src/dlhub/tuning/learning_rate_finder.py

def find(self, verbose: bool = True) -> LearningRateFinderResult:
    """
    Run learning rate finder.

    Parameters
    ----------
    verbose : bool, default=True
        Whether to print progress information

    Returns
    -------
    LearningRateFinderResult
        Results of the learning rate finder
    """
    if verbose:
        print("Starting Learning Rate Finder...")
        print(f"Learning rate range: {self.min_lr:.2e} to {self.max_lr:.2e}")
        print(f"Number of iterations: {self.num_iterations}")
        print(f"Step mode: {self.step_mode}")

    learning_rates = self._generate_learning_rates()
    losses = []

    self.trainer.reset_model()

    for i, lr in enumerate(learning_rates):
        try:
            loss = self.trainer.train_batch(lr)

            if np.isnan(loss) or np.isinf(loss):
                if verbose:
                    print(
                        f"Iteration {i + 1}: Learning rate {lr:.2e} - Invalid loss (nan/inf)"
                    )
                break

            losses.append(float(loss))

            if self._detect_divergence(np.array(losses), i):
                if verbose:
                    print(
                        f"Iteration {i + 1}: Learning rate {lr:.2e} - Training diverged"
                    )
                break

            if verbose and (i + 1) % max(1, self.num_iterations // 10) == 0:
                print(
                    f"Iteration {i + 1}/{self.num_iterations}: LR = {lr:.2e}, Loss = {loss:.6f}"
                )

        except Exception as e:
            if verbose:
                print(
                    f"Iteration {i + 1}: Learning rate {lr:.2e} - Training failed: {e}"
                )
            break

    if len(losses) < 5:
        raise RuntimeError(
            "Learning rate finder failed - insufficient valid loss values"
        )

    learning_rates = learning_rates[: len(losses)]
    losses = np.array(losses)

    smoothed_losses = self._smooth_losses(losses)

    analysis = self._analyze_results(learning_rates, losses, smoothed_losses)

    if verbose:
        print("\nLearning Rate Finder completed!")
        print(f"Suggested learning rate: {analysis['suggested_lr']:.2e}")
        print(
            f"Learning rate with steepest gradient: {analysis['min_gradient_lr']:.2e}"
        )
        print(f"Minimum loss: {analysis['min_loss']:.6f}")
        print(f"Loss reduction ratio: {analysis['loss_reduction_ratio']:.2%}")

    return LearningRateFinderResult(
        learning_rates=learning_rates,
        losses=losses,
        smoothed_losses=smoothed_losses,
        suggested_lr=analysis["suggested_lr"],
        min_gradient_lr=analysis["min_gradient_lr"],
        analysis=analysis,
    )

plot_results

plot_results(result: LearningRateFinderResult, figsize: tuple[int, int] = (12, 8), save_path: str | None = None) -> None

Plot learning rate finder results.

Parameters:

Name	Type	Description	Default
result	LearningRateFinderResult	Results from learning rate finder	required
figsize	tuple	Figure size for the plot	(12, 8)
save_path	str	Path to save the plot	None

Source code in src/dlhub/tuning/learning_rate_finder.py

def plot_results(
    self,
    result: LearningRateFinderResult,
    figsize: tuple[int, int] = (12, 8),
    save_path: str | None = None,
) -> None:
    """
    Plot learning rate finder results.

    Parameters
    ----------
    result : LearningRateFinderResult
        Results from learning rate finder
    figsize : tuple, default=(12, 8)
        Figure size for the plot
    save_path : str, optional
        Path to save the plot
    """
    fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=figsize)

    # Plot 1: Learning rate vs Loss (log scale)
    ax1.semilogx(
        result.learning_rates, result.losses, "b-", alpha=0.7, label="Raw Loss"
    )
    ax1.semilogx(
        result.learning_rates,
        result.smoothed_losses,
        "r-",
        linewidth=2,
        label="Smoothed Loss",
    )
    ax1.axvline(
        result.suggested_lr,
        color="green",
        linestyle="--",
        alpha=0.8,
        label=f"Suggested LR: {result.suggested_lr:.2e}",
    )
    ax1.axvline(
        result.min_gradient_lr,
        color="orange",
        linestyle="--",
        alpha=0.8,
        label=f"Min Gradient LR: {result.min_gradient_lr:.2e}",
    )
    ax1.set_xlabel("Learning Rate")
    ax1.set_ylabel("Loss")
    ax1.set_title("Learning Rate vs Loss")
    ax1.legend()
    ax1.grid(True, alpha=0.3)

    # Plot 2: Learning rate vs Loss (linear scale, zoomed)
    # Focus on the interesting region around minimum
    min_loss_idx = np.argmin(result.smoothed_losses)
    start_idx = max(0, min_loss_idx - 20)
    end_idx = min(len(result.learning_rates), min_loss_idx + 20)

    ax2.plot(
        result.learning_rates[start_idx:end_idx],
        result.losses[start_idx:end_idx],
        "b-",
        alpha=0.7,
    )
    ax2.plot(
        result.learning_rates[start_idx:end_idx],
        result.smoothed_losses[start_idx:end_idx],
        "r-",
        linewidth=2,
    )
    ax2.axvline(result.suggested_lr, color="green", linestyle="--", alpha=0.8)
    ax2.axvline(result.min_gradient_lr, color="orange", linestyle="--", alpha=0.8)
    ax2.set_xlabel("Learning Rate")
    ax2.set_ylabel("Loss")
    ax2.set_title("Loss (Zoomed Region)")
    ax2.grid(True, alpha=0.3)

    # Plot 3: Loss gradient
    gradients = np.gradient(result.smoothed_losses, np.log10(result.learning_rates))
    ax3.semilogx(result.learning_rates, gradients, "purple", linewidth=2)
    ax3.axvline(result.min_gradient_lr, color="orange", linestyle="--", alpha=0.8)
    ax3.axhline(0, color="black", linestyle="-", alpha=0.3)
    ax3.set_xlabel("Learning Rate")
    ax3.set_ylabel("Loss Gradient")
    ax3.set_title("Loss Gradient vs Learning Rate")
    ax3.grid(True, alpha=0.3)

    # Plot 4: Statistics summary
    ax4.axis("off")
    stats_text = f"""
    Analysis Summary:

    Suggested Learning Rate: {result.suggested_lr:.2e}
    Min Gradient Learning Rate: {result.min_gradient_lr:.2e}

    Minimum Loss: {result.analysis["min_loss"]:.6f}
    Loss Reduction: {result.analysis["loss_reduction_ratio"]:.2%}

    Loss Variance: {result.analysis["loss_variance"]:.6f}
    Coefficient of Variation: {result.analysis["coefficient_of_variation"]:.3f}

    Convergence Ratio: {result.analysis["convergence_ratio"]:.3f}
    """
    ax4.text(
        0.05,
        0.95,
        stats_text,
        transform=ax4.transAxes,
        fontsize=10,
        verticalalignment="top",
        fontfamily="monospace",
        bbox=dict(boxstyle="round", facecolor="lightgray", alpha=0.8),
    )

    plt.tight_layout()

    if save_path:
        plt.savefig(save_path, dpi=300, bbox_inches="tight")
        print(f"Plot saved to {save_path}")

    plt.show()

LearningRateFinderResult dataclass

Container for learning rate finder results.

Attributes:

Name	Type	Description
learning_rates	ndarray	Array of learning rates tested
losses	ndarray	Corresponding loss values
smoothed_losses	ndarray	Smoothed loss values for trend analysis
suggested_lr	float	Suggested learning rate based on analysis
min_gradient_lr	float	Learning rate with steepest loss decrease
analysis	dict	Additional analysis metrics and diagnostics

Source code in src/dlhub/tuning/learning_rate_finder.py

@dataclass
class LearningRateFinderResult:
    """
    Container for learning rate finder results.

    Attributes
    ----------
    learning_rates : np.ndarray
        Array of learning rates tested
    losses : np.ndarray
        Corresponding loss values
    smoothed_losses : np.ndarray
        Smoothed loss values for trend analysis
    suggested_lr : float
        Suggested learning rate based on analysis
    min_gradient_lr : float
        Learning rate with steepest loss decrease
    analysis : dict
        Additional analysis metrics and diagnostics
    """

    learning_rates: np.ndarray
    losses: np.ndarray
    smoothed_losses: np.ndarray
    suggested_lr: float
    min_gradient_lr: float
    analysis: dict[str, Any]

ASHAOptimizer

Asynchronous Successive Halving Algorithm (ASHA) for multi-fidelity optimization.

ASHA efficiently allocates computational resources by starting many configurations at low fidelity and promoting the most promising ones to higher fidelities.

Parameters:

Name	Type	Description	Default
evaluator	FidelityEvaluator	Evaluator for hyperparameter configurations	required
reduction_factor	int	Factor by which to reduce number of configurations at each rung	3
min_budget	int	Minimum budget (fidelity) to start configurations	1
max_budget	int	Maximum budget (fidelity) for full evaluation	81
grace_period	int	Minimum budget before first promotion opportunity	1
max_concurrent	int	Maximum number of concurrent evaluations	4
random_state	int	Random seed for reproducibility	None

Source code in src/dlhub/tuning/multifidelity.py

class ASHAOptimizer:
    """
    Asynchronous Successive Halving Algorithm (ASHA) for multi-fidelity optimization.

    ASHA efficiently allocates computational resources by starting many configurations
    at low fidelity and promoting the most promising ones to higher fidelities.

    Parameters
    ----------
    evaluator : FidelityEvaluator
        Evaluator for hyperparameter configurations
    reduction_factor : int, default=3
        Factor by which to reduce number of configurations at each rung
    min_budget : int, default=1
        Minimum budget (fidelity) to start configurations
    max_budget : int, default=81
        Maximum budget (fidelity) for full evaluation
    grace_period : int, default=1
        Minimum budget before first promotion opportunity
    max_concurrent : int, default=4
        Maximum number of concurrent evaluations
    random_state : int, optional
        Random seed for reproducibility
    """

    def __init__(
        self,
        evaluator: FidelityEvaluator,
        reduction_factor: int = 3,
        min_budget: int = 1,
        max_budget: int = 81,
        grace_period: int = 1,
        max_concurrent: int = 4,
        random_state: int | None = None,
    ):

        self.evaluator = evaluator
        self.reduction_factor = reduction_factor
        self.min_budget = min_budget
        self.max_budget = max_budget
        self.grace_period = grace_period
        self.max_concurrent = max_concurrent

        if random_state is not None:
            np.random.seed(random_state)

        # Validate parameters
        eval_min, eval_max = evaluator.get_fidelity_range()
        if min_budget < eval_min or max_budget > eval_max:
            raise ValueError(
                f"Budget range [{min_budget}, {max_budget}] "
                f"outside evaluator range [{eval_min}, {eval_max}]"
            )

        # Initialize internal state
        self.config_counter = 0
        self.rungs = self._create_rungs()
        self.results_history = []
        self.active_evaluations = {}
        self.lock = threading.Lock()

        # Statistics tracking
        self.total_budget_used = 0
        self.best_result = None

    def _create_rungs(self) -> list[dict]:
        """
        Create ASHA rungs (fidelity levels and promotion thresholds).

        Returns
        -------
        list
            List of rung dictionaries with budget and promotion info
        """
        rungs = []
        current_budget = self.min_budget

        while current_budget <= self.max_budget:
            rung = {
                "budget": current_budget,
                "candidates": [],  # (config_id, score) tuples
                "promoted": set(),  # Set of promoted config_ids
                "n_required": 0,  # Number of configs needed for promotion
            }
            rungs.append(rung)
            current_budget *= self.reduction_factor

        # A rung promotes its top 1 / reduction_factor, so it needs at least
        # reduction_factor results before that fraction means anything. The
        # threshold is a property of the reduction factor, not of how many rungs
        # the ladder happens to have: deriving it from the rung count made a
        # taller ladder demand more results at the bottom and stall there.
        for rung in rungs[:-1]:  # The last rung promotes nowhere
            rung["n_required"] = self.reduction_factor

        return rungs

    def _get_rung_for_budget(self, budget: int) -> int | None:
        """Get rung index for given budget."""
        for i, rung in enumerate(self.rungs):
            if rung["budget"] == budget:
                return i
        return None

    def _add_result(self, result: CandidateResult) -> None:
        """Add result and check for promotions."""
        with self.lock:
            self.results_history.append(result)
            self.total_budget_used += result.fidelity

            if self.best_result is None or result.score > self.best_result.score:
                self.best_result = result

            rung_idx = self._get_rung_for_budget(result.fidelity)
            if rung_idx is not None:
                rung = self.rungs[rung_idx]
                rung["candidates"].append((result.config_id, result.score))

    def _get_next_config_to_evaluate(self) -> tuple[int, dict[str, Any], int] | None:
        """
        Claim the next promotion and return the work it implies.

        This is the only place the promotion rule lives, and calling it is not a
        query: the returned configuration is recorded as promoted so that
        concurrent workers cannot claim it twice. Callers must therefore use
        what they get rather than calling it to ask whether work exists.

        Returns
        -------
        tuple or None
            (config_id, hyperparams, fidelity) or None if no work available
        """
        with self.lock:
            for rung_idx in range(len(self.rungs) - 1):
                rung = self.rungs[rung_idx]
                next_rung = self.rungs[rung_idx + 1]

                if len(rung["candidates"]) >= rung["n_required"]:
                    sorted_candidates = sorted(
                        rung["candidates"], key=lambda x: x[1], reverse=True
                    )
                    n_promote = max(1, len(sorted_candidates) // self.reduction_factor)

                    for i in range(min(n_promote, len(sorted_candidates))):
                        config_id, score = sorted_candidates[i]

                        if config_id not in rung["promoted"]:
                            rung["promoted"].add(config_id)

                            hyperparams = None
                            for res in self.results_history:
                                if res.config_id == config_id:
                                    hyperparams = res.hyperparams
                                    break

                            if hyperparams is not None:
                                return config_id, hyperparams, next_rung["budget"]

            return None

    def _evaluate_config(
        self, config_id: int, hyperparams: dict[str, Any], fidelity: int
    ) -> CandidateResult:
        """Evaluate a single configuration."""
        start_time = time.time()

        try:
            score, metadata = self.evaluator.evaluate(hyperparams, fidelity)
            training_time = time.time() - start_time

            if np.isnan(score) or np.isinf(score):
                score = -np.inf

            return CandidateResult(
                config_id=config_id,
                hyperparams=hyperparams.copy(),
                fidelity=fidelity,
                score=float(score),
                training_time=training_time,
                metadata=metadata,
            )

        except Exception as e:
            training_time = time.time() - start_time
            warnings.warn(f"Evaluation failed for config {config_id}: {e}")

            return CandidateResult(
                config_id=config_id,
                hyperparams=hyperparams.copy(),
                fidelity=fidelity,
                score=-np.inf,
                training_time=training_time,
                metadata={"error": str(e)},
            )

    def suggest_initial_configurations(
        self, configurations: list[dict[str, Any]]
    ) -> None:
        """
        Add initial configurations to start evaluation.

        Parameters
        ----------
        configurations : list
            List of hyperparameter configurations to evaluate
        """
        with self.lock:
            for config in configurations:
                config_id = self.config_counter
                self.config_counter += 1
                # Queued at the lowest rung. Every configuration earns its way
                # up from here, which is what makes the search cheap.
                self.active_evaluations[config_id] = (config.copy(), self.min_budget)

    def optimize(
        self,
        initial_configurations: list[dict[str, Any]],
        max_iterations: int = 100,
        timeout: float | None = None,
        verbose: bool = True,
    ) -> MultiFidelityResult:
        """
        Run ASHA optimization.

        Parameters
        ----------
        initial_configurations : list
            Initial hyperparameter configurations to evaluate. Must be non-empty.
        max_iterations : int, default=100
            Maximum number of evaluations to perform. Counted at submission, and
            every submitted evaluation is recorded, so this bounds the results as
            well as the work -- at any `max_concurrent`.
        timeout : float, optional
            Maximum time in seconds (None for no timeout). Evaluations already
            running when the clock runs out are still awaited and recorded; the
            timeout stops new submissions, it does not cancel paid-for work.
        verbose : bool, default=True
            Whether to print progress information

        Returns
        -------
        MultiFidelityResult
            Optimization results. A run granted no budget -- `max_iterations=0`,
            or a `timeout` already elapsed -- records nothing and reports
            `best_config`, `best_score`, and `best_fidelity` as None.

        Raises
        ------
        ValueError
            If `initial_configurations` is empty. A search with no candidates has
            no answer to return, and an empty list is more often a search space
            that filtered down to nothing than a deliberate no-op -- so it is
            reported rather than absorbed into an empty result.
        """
        if not initial_configurations:
            raise ValueError(
                "initial_configurations is empty; ASHA needs at least one "
                "candidate to search"
            )

        if verbose:
            print("Starting ASHA Multi-Fidelity Optimization...")
            print(f"Reduction factor: {self.reduction_factor}")
            print(f"Budget range: [{self.min_budget}, {self.max_budget}]")
            print(f"Initial configurations: {len(initial_configurations)}")
            print(f"Max concurrent evaluations: {self.max_concurrent}")

        start_time = time.time()
        self.suggest_initial_configurations(initial_configurations)

        iteration = 0
        evaluations_completed = 0

        work_queue = []
        for config_id, (config, fidelity) in self.active_evaluations.items():
            work_queue.append((config_id, config, fidelity))
        self.active_evaluations.clear()

        def record(future, config_id: int) -> None:
            """Move one finished evaluation into the results, or warn."""
            nonlocal evaluations_completed

            try:
                result = future.result()
            except Exception as e:
                warnings.warn(f"Future failed for config {config_id}: {e}")
                return

            self._add_result(result)
            evaluations_completed += 1

            # `_add_result` has just run, so `best_result` is set; reading it into
            # a local says that to the type checker, and takes one attribute read
            # rather than two off an object other threads are writing.
            best = self.best_result
            if (
                verbose
                and best is not None
                and evaluations_completed % max(1, max_iterations // 20) == 0
            ):
                print(
                    f"Completed {evaluations_completed} evaluations - "
                    f"Best score: {best.score:.6f} "
                    f"(fidelity {best.fidelity})"
                )

        with ThreadPoolExecutor(max_workers=self.max_concurrent) as executor:
            active_futures = {}

            while iteration < max_iterations and (
                timeout is None or time.time() - start_time < timeout
            ):
                # `_get_next_config_to_evaluate` marks what it returns as
                # promoted, so it is called once per work item and its result is
                # kept. Calling it again as a loop condition would consume a
                # promotion and drop the configuration on the floor.
                while len(active_futures) < self.max_concurrent:
                    if work_queue:
                        config_id, hyperparams, fidelity = work_queue.pop(0)
                    else:
                        next_work = self._get_next_config_to_evaluate()
                        if next_work is None:
                            break
                        config_id, hyperparams, fidelity = next_work

                    # Submit evaluation
                    future = executor.submit(
                        self._evaluate_config, config_id, hyperparams, fidelity
                    )
                    active_futures[future] = (config_id, hyperparams, fidelity)
                    iteration += 1

                    if iteration >= max_iterations:
                        break

                if active_futures:
                    # The one-second bound is a poll interval, not a deadline: it
                    # returns control to the loop so `max_iterations` and
                    # `timeout` get re-checked while evaluations are still
                    # running. `as_completed` reports an elapsed poll by raising,
                    # and a tuner exists to run evaluations that take minutes, so
                    # letting that escape would abort every realistic run the
                    # moment no evaluation happened to finish within a second.
                    # Before 3.11 this is not the builtin `TimeoutError`, so it
                    # is caught under its own name rather than by coincidence.
                    completed_futures = []
                    try:
                        for future in as_completed(active_futures, timeout=1.0):
                            completed_futures.append(future)
                            break  # Process one at a time for responsiveness
                    except FuturesTimeoutError:
                        pass

                    for future in completed_futures:
                        config_id, _, _ = active_futures.pop(future)
                        record(future, config_id)

                # Break if no more work and no active evaluations
                if not active_futures and not work_queue:
                    # Requeued rather than discarded: this call promotes the
                    # configuration it returns, so dropping it would lose the
                    # promotion and end the run one rung short.
                    next_work = self._get_next_config_to_evaluate()
                    if next_work is None:
                        if verbose:
                            print("No more configurations to evaluate - stopping")
                        break
                    work_queue.append(next_work)

            # Whatever is still in flight when the loop exits has already been
            # submitted, so the pool will compute it whether or not anyone waits:
            # `ThreadPoolExecutor.__exit__` joins every worker. Recording it is
            # therefore free, and discarding it would under-report
            # `total_budget_used` by up to `max_concurrent - 1` evaluations --
            # flattering `budget_efficiency` with work the run really paid for.
            # This also keeps `max_iterations` meaning one thing at any worker
            # count: submissions, which now all become results.
            for future, (config_id, _, _) in list(active_futures.items()):
                record(future, config_id)
            active_futures.clear()

        total_time = time.time() - start_time

        statistics = self._compute_statistics()

        if verbose:
            print(f"\nOptimization completed in {total_time:.2f} seconds!")
            print(f"Total evaluations: {evaluations_completed}")
            print(f"Total budget used: {self.total_budget_used}")
            # `best_result` is None exactly when nothing was recorded, which the
            # budget limits make reachable without any caller error: the summary
            # says so rather than dereferencing it.
            if self.best_result is None:
                print("No evaluation completed - no best configuration to report")
            else:
                print(f"Best score: {self.best_result.score:.6f}")
                print(f"Best configuration: {self.best_result.hyperparams}")
                print(f"Best fidelity: {self.best_result.fidelity}")

        best = self.best_result

        return MultiFidelityResult(
            best_config=best.hyperparams if best is not None else None,
            best_score=best.score if best is not None else None,
            best_fidelity=best.fidelity if best is not None else None,
            all_results=self.results_history,
            total_time=total_time,
            total_budget_used=self.total_budget_used,
            statistics=statistics,
        )

    def _compute_statistics(self) -> dict[str, Any]:
        """Compute optimization statistics."""
        if not self.results_history:
            return {}

        fidelity_stats = defaultdict(list)
        for result in self.results_history:
            if result.score != -np.inf:
                fidelity_stats[result.fidelity].append(result.score)

        fidelity_analysis = {}
        for fidelity, scores in fidelity_stats.items():
            fidelity_analysis[fidelity] = {
                "n_evaluations": len(scores),
                "mean_score": np.mean(scores),
                "std_score": np.std(scores),
                "max_score": np.max(scores),
                "min_score": np.min(scores),
            }

        all_scores = [r.score for r in self.results_history if r.score != -np.inf]
        all_times = [r.training_time for r in self.results_history]

        # Reached only past the empty-history guard at the top of the method, so
        # `_add_result` has run and this is set. Testing it anyway is what lets
        # the `type: ignore` come off -- the guard is real, but a checker cannot
        # connect it to this attribute.
        best = self.best_result

        statistics = {
            "total_evaluations": len(self.results_history),
            "successful_evaluations": len(all_scores),
            "failed_evaluations": len(self.results_history) - len(all_scores),
            "mean_score": np.mean(all_scores) if all_scores else 0.0,
            "std_score": np.std(all_scores) if len(all_scores) > 1 else 0.0,
            "mean_training_time": np.mean(all_times),
            "total_training_time": np.sum(all_times),
            "fidelity_analysis": fidelity_analysis,
            "budget_efficiency": (
                best.score / self.total_budget_used
                if best is not None and self.total_budget_used > 0
                else 0.0
            ),
            "rungs_used": len([r for r in self.rungs if r["candidates"]]),
        }

        return statistics

suggest_initial_configurations

suggest_initial_configurations(configurations: list[dict[str, Any]]) -> None

Add initial configurations to start evaluation.

Parameters:

Name	Type	Description	Default
configurations	list	List of hyperparameter configurations to evaluate	required

Source code in src/dlhub/tuning/multifidelity.py

def suggest_initial_configurations(
    self, configurations: list[dict[str, Any]]
) -> None:
    """
    Add initial configurations to start evaluation.

    Parameters
    ----------
    configurations : list
        List of hyperparameter configurations to evaluate
    """
    with self.lock:
        for config in configurations:
            config_id = self.config_counter
            self.config_counter += 1
            # Queued at the lowest rung. Every configuration earns its way
            # up from here, which is what makes the search cheap.
            self.active_evaluations[config_id] = (config.copy(), self.min_budget)

optimize

optimize(initial_configurations: list[dict[str, Any]], max_iterations: int = 100, timeout: float | None = None, verbose: bool = True) -> MultiFidelityResult

Run ASHA optimization.

Parameters:

Name	Type	Description	Default
initial_configurations	list	Initial hyperparameter configurations to evaluate. Must be non-empty.	required
max_iterations	int	Maximum number of evaluations to perform. Counted at submission, and every submitted evaluation is recorded, so this bounds the results as well as the work -- at any max_concurrent.	100
timeout	float	Maximum time in seconds (None for no timeout). Evaluations already running when the clock runs out are still awaited and recorded; the timeout stops new submissions, it does not cancel paid-for work.	None
verbose	bool	Whether to print progress information	True

Returns:

Type	Description
MultiFidelityResult	Optimization results. A run granted no budget -- max_iterations=0, or a timeout already elapsed -- records nothing and reports best_config, best_score, and best_fidelity as None.

Raises:

Type	Description
ValueError	If initial_configurations is empty. A search with no candidates has no answer to return, and an empty list is more often a search space that filtered down to nothing than a deliberate no-op -- so it is reported rather than absorbed into an empty result.

Source code in src/dlhub/tuning/multifidelity.py

def optimize(
    self,
    initial_configurations: list[dict[str, Any]],
    max_iterations: int = 100,
    timeout: float | None = None,
    verbose: bool = True,
) -> MultiFidelityResult:
    """
    Run ASHA optimization.

    Parameters
    ----------
    initial_configurations : list
        Initial hyperparameter configurations to evaluate. Must be non-empty.
    max_iterations : int, default=100
        Maximum number of evaluations to perform. Counted at submission, and
        every submitted evaluation is recorded, so this bounds the results as
        well as the work -- at any `max_concurrent`.
    timeout : float, optional
        Maximum time in seconds (None for no timeout). Evaluations already
        running when the clock runs out are still awaited and recorded; the
        timeout stops new submissions, it does not cancel paid-for work.
    verbose : bool, default=True
        Whether to print progress information

    Returns
    -------
    MultiFidelityResult
        Optimization results. A run granted no budget -- `max_iterations=0`,
        or a `timeout` already elapsed -- records nothing and reports
        `best_config`, `best_score`, and `best_fidelity` as None.

    Raises
    ------
    ValueError
        If `initial_configurations` is empty. A search with no candidates has
        no answer to return, and an empty list is more often a search space
        that filtered down to nothing than a deliberate no-op -- so it is
        reported rather than absorbed into an empty result.
    """
    if not initial_configurations:
        raise ValueError(
            "initial_configurations is empty; ASHA needs at least one "
            "candidate to search"
        )

    if verbose:
        print("Starting ASHA Multi-Fidelity Optimization...")
        print(f"Reduction factor: {self.reduction_factor}")
        print(f"Budget range: [{self.min_budget}, {self.max_budget}]")
        print(f"Initial configurations: {len(initial_configurations)}")
        print(f"Max concurrent evaluations: {self.max_concurrent}")

    start_time = time.time()
    self.suggest_initial_configurations(initial_configurations)

    iteration = 0
    evaluations_completed = 0

    work_queue = []
    for config_id, (config, fidelity) in self.active_evaluations.items():
        work_queue.append((config_id, config, fidelity))
    self.active_evaluations.clear()

    def record(future, config_id: int) -> None:
        """Move one finished evaluation into the results, or warn."""
        nonlocal evaluations_completed

        try:
            result = future.result()
        except Exception as e:
            warnings.warn(f"Future failed for config {config_id}: {e}")
            return

        self._add_result(result)
        evaluations_completed += 1

        # `_add_result` has just run, so `best_result` is set; reading it into
        # a local says that to the type checker, and takes one attribute read
        # rather than two off an object other threads are writing.
        best = self.best_result
        if (
            verbose
            and best is not None
            and evaluations_completed % max(1, max_iterations // 20) == 0
        ):
            print(
                f"Completed {evaluations_completed} evaluations - "
                f"Best score: {best.score:.6f} "
                f"(fidelity {best.fidelity})"
            )

    with ThreadPoolExecutor(max_workers=self.max_concurrent) as executor:
        active_futures = {}

        while iteration < max_iterations and (
            timeout is None or time.time() - start_time < timeout
        ):
            # `_get_next_config_to_evaluate` marks what it returns as
            # promoted, so it is called once per work item and its result is
            # kept. Calling it again as a loop condition would consume a
            # promotion and drop the configuration on the floor.
            while len(active_futures) < self.max_concurrent:
                if work_queue:
                    config_id, hyperparams, fidelity = work_queue.pop(0)
                else:
                    next_work = self._get_next_config_to_evaluate()
                    if next_work is None:
                        break
                    config_id, hyperparams, fidelity = next_work

                # Submit evaluation
                future = executor.submit(
                    self._evaluate_config, config_id, hyperparams, fidelity
                )
                active_futures[future] = (config_id, hyperparams, fidelity)
                iteration += 1

                if iteration >= max_iterations:
                    break

            if active_futures:
                # The one-second bound is a poll interval, not a deadline: it
                # returns control to the loop so `max_iterations` and
                # `timeout` get re-checked while evaluations are still
                # running. `as_completed` reports an elapsed poll by raising,
                # and a tuner exists to run evaluations that take minutes, so
                # letting that escape would abort every realistic run the
                # moment no evaluation happened to finish within a second.
                # Before 3.11 this is not the builtin `TimeoutError`, so it
                # is caught under its own name rather than by coincidence.
                completed_futures = []
                try:
                    for future in as_completed(active_futures, timeout=1.0):
                        completed_futures.append(future)
                        break  # Process one at a time for responsiveness
                except FuturesTimeoutError:
                    pass

                for future in completed_futures:
                    config_id, _, _ = active_futures.pop(future)
                    record(future, config_id)

            # Break if no more work and no active evaluations
            if not active_futures and not work_queue:
                # Requeued rather than discarded: this call promotes the
                # configuration it returns, so dropping it would lose the
                # promotion and end the run one rung short.
                next_work = self._get_next_config_to_evaluate()
                if next_work is None:
                    if verbose:
                        print("No more configurations to evaluate - stopping")
                    break
                work_queue.append(next_work)

        # Whatever is still in flight when the loop exits has already been
        # submitted, so the pool will compute it whether or not anyone waits:
        # `ThreadPoolExecutor.__exit__` joins every worker. Recording it is
        # therefore free, and discarding it would under-report
        # `total_budget_used` by up to `max_concurrent - 1` evaluations --
        # flattering `budget_efficiency` with work the run really paid for.
        # This also keeps `max_iterations` meaning one thing at any worker
        # count: submissions, which now all become results.
        for future, (config_id, _, _) in list(active_futures.items()):
            record(future, config_id)
        active_futures.clear()

    total_time = time.time() - start_time

    statistics = self._compute_statistics()

    if verbose:
        print(f"\nOptimization completed in {total_time:.2f} seconds!")
        print(f"Total evaluations: {evaluations_completed}")
        print(f"Total budget used: {self.total_budget_used}")
        # `best_result` is None exactly when nothing was recorded, which the
        # budget limits make reachable without any caller error: the summary
        # says so rather than dereferencing it.
        if self.best_result is None:
            print("No evaluation completed - no best configuration to report")
        else:
            print(f"Best score: {self.best_result.score:.6f}")
            print(f"Best configuration: {self.best_result.hyperparams}")
            print(f"Best fidelity: {self.best_result.fidelity}")

    best = self.best_result

    return MultiFidelityResult(
        best_config=best.hyperparams if best is not None else None,
        best_score=best.score if best is not None else None,
        best_fidelity=best.fidelity if best is not None else None,
        all_results=self.results_history,
        total_time=total_time,
        total_budget_used=self.total_budget_used,
        statistics=statistics,
    )

CandidateResult dataclass

Result from evaluating a hyperparameter candidate.

Attributes:

Name	Type	Description
config_id	int	Unique identifier for the configuration
hyperparams	dict	Hyperparameter configuration
fidelity	int	Fidelity level used for evaluation
score	float	Performance score achieved
training_time	float	Time taken for training
metadata	dict	Additional metadata from training

Source code in src/dlhub/tuning/multifidelity.py

@dataclass
class CandidateResult:
    """
    Result from evaluating a hyperparameter candidate.

    Attributes
    ----------
    config_id : int
        Unique identifier for the configuration
    hyperparams : dict
        Hyperparameter configuration
    fidelity : int
        Fidelity level used for evaluation
    score : float
        Performance score achieved
    training_time : float
        Time taken for training
    metadata : dict
        Additional metadata from training
    """

    config_id: int
    hyperparams: dict[str, Any]
    fidelity: int
    score: float
    training_time: float
    metadata: dict[str, Any] = field(default_factory=dict)

FidelityConfig dataclass

Configuration for a fidelity level.

Attributes:

Name	Type	Description
name	str	Name of the fidelity level
budget	int	Budget/resource allocation for this fidelity
min_budget	int	Minimum budget required for this fidelity
max_budget	int	Maximum budget for this fidelity

Source code in src/dlhub/tuning/multifidelity.py

@dataclass
class FidelityConfig:
    """
    Configuration for a fidelity level.

    Attributes
    ----------
    name : str
        Name of the fidelity level
    budget : int
        Budget/resource allocation for this fidelity
    min_budget : int
        Minimum budget required for this fidelity
    max_budget : int
        Maximum budget for this fidelity
    """

    name: str
    budget: int
    min_budget: int = 1
    max_budget: int = 1000

FidelityEvaluator

Bases: ABC

Abstract base class for fidelity-aware evaluation.

Defines the interface for evaluating hyperparameter configurations at different fidelity levels.

Source code in src/dlhub/tuning/multifidelity.py

class FidelityEvaluator(ABC):
    """
    Abstract base class for fidelity-aware evaluation.

    Defines the interface for evaluating hyperparameter configurations
    at different fidelity levels.
    """

    @abstractmethod
    def evaluate(
        self, hyperparams: dict[str, Any], fidelity: int
    ) -> tuple[float, dict[str, Any]]:
        """
        Evaluate hyperparameters at given fidelity.

        Parameters
        ----------
        hyperparams : dict
            Hyperparameter configuration
        fidelity : int
            Fidelity level (e.g., training epochs, data size)

        Returns
        -------
        tuple
            (score, metadata) where score is performance and metadata contains
            additional information from training
        """
        pass

    @abstractmethod
    def get_fidelity_range(self) -> tuple[int, int]:
        """
        Get the valid fidelity range.

        Returns
        -------
        tuple
            (min_fidelity, max_fidelity)
        """
        pass

evaluate abstractmethod

evaluate(hyperparams: dict[str, Any], fidelity: int) -> tuple[float, dict[str, Any]]

Evaluate hyperparameters at given fidelity.

Parameters:

Name	Type	Description	Default
hyperparams	dict	Hyperparameter configuration	required
fidelity	int	Fidelity level (e.g., training epochs, data size)	required

Returns:

Type	Description
tuple	(score, metadata) where score is performance and metadata contains additional information from training

Source code in src/dlhub/tuning/multifidelity.py

@abstractmethod
def evaluate(
    self, hyperparams: dict[str, Any], fidelity: int
) -> tuple[float, dict[str, Any]]:
    """
    Evaluate hyperparameters at given fidelity.

    Parameters
    ----------
    hyperparams : dict
        Hyperparameter configuration
    fidelity : int
        Fidelity level (e.g., training epochs, data size)

    Returns
    -------
    tuple
        (score, metadata) where score is performance and metadata contains
        additional information from training
    """
    pass

get_fidelity_range abstractmethod

get_fidelity_range() -> tuple[int, int]

Get the valid fidelity range.

Returns:

Type	Description
tuple	(min_fidelity, max_fidelity)

Source code in src/dlhub/tuning/multifidelity.py

@abstractmethod
def get_fidelity_range(self) -> tuple[int, int]:
    """
    Get the valid fidelity range.

    Returns
    -------
    tuple
        (min_fidelity, max_fidelity)
    """
    pass

FunctionEvaluator

Bases: FidelityEvaluator

Function-based evaluator wrapper.

Wraps a user-provided evaluation function to conform to the FidelityEvaluator interface.

Parameters:

Name	Type	Description	Default
eval_function	callable	Function that takes (hyperparams, fidelity) and returns (score, metadata)	required
min_fidelity	int	Minimum fidelity level	1
max_fidelity	int	Maximum fidelity level	100

Source code in src/dlhub/tuning/multifidelity.py

class FunctionEvaluator(FidelityEvaluator):
    """
    Function-based evaluator wrapper.

    Wraps a user-provided evaluation function to conform to the
    FidelityEvaluator interface.

    Parameters
    ----------
    eval_function : callable
        Function that takes (hyperparams, fidelity) and returns (score, metadata)
    min_fidelity : int
        Minimum fidelity level
    max_fidelity : int
        Maximum fidelity level
    """

    def __init__(
        self, eval_function: Callable, min_fidelity: int = 1, max_fidelity: int = 100
    ):
        self.eval_function = eval_function
        self.min_fidelity = min_fidelity
        self.max_fidelity = max_fidelity

    def evaluate(
        self, hyperparams: dict[str, Any], fidelity: int
    ) -> tuple[float, dict[str, Any]]:
        """Evaluate using wrapped function."""
        return self.eval_function(hyperparams, fidelity)

    def get_fidelity_range(self) -> tuple[int, int]:
        """Get fidelity range."""
        return self.min_fidelity, self.max_fidelity

evaluate

evaluate(hyperparams: dict[str, Any], fidelity: int) -> tuple[float, dict[str, Any]]

Evaluate using wrapped function.

Source code in src/dlhub/tuning/multifidelity.py

def evaluate(
    self, hyperparams: dict[str, Any], fidelity: int
) -> tuple[float, dict[str, Any]]:
    """Evaluate using wrapped function."""
    return self.eval_function(hyperparams, fidelity)

get_fidelity_range

get_fidelity_range() -> tuple[int, int]

Get fidelity range.

Source code in src/dlhub/tuning/multifidelity.py

def get_fidelity_range(self) -> tuple[int, int]:
    """Get fidelity range."""
    return self.min_fidelity, self.max_fidelity

MultiFidelityResult dataclass

Results from multi-fidelity optimization.

Attributes:

Name	Type	Description
best_config	dict or None	Best hyperparameter configuration found, or None if the run recorded no results at all (see Notes)
best_score	float or None	Best score achieved, or None for a run with no results
best_fidelity	int or None	Fidelity level of best result, or None for a run with no results
all_results	list	All evaluation results
total_time	float	Total optimization time
total_budget_used	int	Total computational budget consumed
statistics	dict	Optimization statistics and analysis, empty for a run with no results

Notes

A run can legitimately record nothing: max_iterations=0 grants no budget, and a timeout that has already elapsed stops the first submission. Both return this dataclass with the three best_* fields set to None rather than raising, so the three are optional together -- either all are None or none are. all_results is empty and statistics is {} in exactly those runs, so if result.best_config is None and if not result.all_results are equivalent tests.

An empty initial_configurations is caller error rather than an empty run, and ASHAOptimizer.optimize rejects it with a ValueError.

Source code in src/dlhub/tuning/multifidelity.py

@dataclass
class MultiFidelityResult:
    """
    Results from multi-fidelity optimization.

    Attributes
    ----------
    best_config : dict or None
        Best hyperparameter configuration found, or None if the run recorded no
        results at all (see Notes)
    best_score : float or None
        Best score achieved, or None for a run with no results
    best_fidelity : int or None
        Fidelity level of best result, or None for a run with no results
    all_results : list
        All evaluation results
    total_time : float
        Total optimization time
    total_budget_used : int
        Total computational budget consumed
    statistics : dict
        Optimization statistics and analysis, empty for a run with no results

    Notes
    -----
    A run can legitimately record nothing: `max_iterations=0` grants no budget,
    and a `timeout` that has already elapsed stops the first submission. Both
    return this dataclass with the three `best_*` fields set to None rather than
    raising, so the three are optional together -- either all are None or none
    are. `all_results` is empty and `statistics` is `{}` in exactly those runs,
    so `if result.best_config is None` and `if not result.all_results` are
    equivalent tests.

    An empty `initial_configurations` is caller error rather than an empty run,
    and `ASHAOptimizer.optimize` rejects it with a ValueError.
    """

    best_config: dict[str, Any] | None
    best_score: float | None
    best_fidelity: int | None
    all_results: list[CandidateResult]
    total_time: float
    total_budget_used: int
    statistics: dict[str, Any] = field(default_factory=dict)

ChoicePerturbation

Bases: HyperparameterDistribution

Perturbation for categorical hyperparameters.

Parameters:

Name	Type	Description	Default
choices	list	List of possible values	required
change_probability	float	Probability of changing to a different value	0.3

Source code in src/dlhub/tuning/population_based.py

class ChoicePerturbation(HyperparameterDistribution):
    """
    Perturbation for categorical hyperparameters.

    Parameters
    ----------
    choices : list
        List of possible values
    change_probability : float, default=0.3
        Probability of changing to a different value
    """

    def __init__(self, choices: list[Any], change_probability: float = 0.3):
        self.choices = choices
        self.change_probability = change_probability

    def perturb(self, value: Any) -> Any:
        """Perturb categorical value."""
        if np.random.random() < self.change_probability:
            # Choose different value
            other_choices = [c for c in self.choices if c != value]
            if other_choices:
                return np.random.choice(other_choices)
        return value

    def resample(self) -> Any:
        """Resample from choices."""
        return np.random.choice(self.choices)

perturb

perturb(value: Any) -> Any

Perturb categorical value.

Source code in src/dlhub/tuning/population_based.py

def perturb(self, value: Any) -> Any:
    """Perturb categorical value."""
    if np.random.random() < self.change_probability:
        # Choose different value
        other_choices = [c for c in self.choices if c != value]
        if other_choices:
            return np.random.choice(other_choices)
    return value

resample

resample() -> Any

Resample from choices.

Source code in src/dlhub/tuning/population_based.py

def resample(self) -> Any:
    """Resample from choices."""
    return np.random.choice(self.choices)

FunctionWorker

Bases: WorkerInterface

Function-based worker implementation.

Wraps user-provided training functions to conform to WorkerInterface.

Parameters:

Name	Type	Description	Default
train_function	callable	Function that takes (hyperparams, steps) and returns (score, state)	required
save_function	callable	Function that returns current state	required
load_function	callable	Function that loads given state	required
reset_function	callable	Function that resets to initial state	required

Source code in src/dlhub/tuning/population_based.py

class FunctionWorker(WorkerInterface):
    """
    Function-based worker implementation.

    Wraps user-provided training functions to conform to WorkerInterface.

    Parameters
    ----------
    train_function : callable
        Function that takes (hyperparams, steps) and returns (score, state)
    save_function : callable
        Function that returns current state
    load_function : callable
        Function that loads given state
    reset_function : callable
        Function that resets to initial state
    """

    def __init__(
        self,
        train_function: Callable,
        save_function: Callable,
        load_function: Callable,
        reset_function: Callable,
    ):
        self.train_function = train_function
        self.save_function = save_function
        self.load_function = load_function
        self.reset_function = reset_function

    def train_step(
        self, hyperparams: dict[str, Any], steps: int = 1
    ) -> tuple[float, Any]:
        """Train using wrapped function."""
        return self.train_function(hyperparams, steps)

    def save_state(self) -> Any:
        """Save state using wrapped function."""
        return self.save_function()

    def load_state(self, state: Any) -> None:
        """Load state using wrapped function."""
        self.load_function(state)

    def reset(self) -> None:
        """Reset using wrapped function."""
        self.reset_function()

train_step

train_step(hyperparams: dict[str, Any], steps: int = 1) -> tuple[float, Any]

Train using wrapped function.

Source code in src/dlhub/tuning/population_based.py

def train_step(
    self, hyperparams: dict[str, Any], steps: int = 1
) -> tuple[float, Any]:
    """Train using wrapped function."""
    return self.train_function(hyperparams, steps)

save_state

save_state() -> Any

Save state using wrapped function.

Source code in src/dlhub/tuning/population_based.py

def save_state(self) -> Any:
    """Save state using wrapped function."""
    return self.save_function()

load_state

load_state(state: Any) -> None

Load state using wrapped function.

Source code in src/dlhub/tuning/population_based.py

def load_state(self, state: Any) -> None:
    """Load state using wrapped function."""
    self.load_function(state)

reset

reset() -> None

Reset using wrapped function.

Source code in src/dlhub/tuning/population_based.py

def reset(self) -> None:
    """Reset using wrapped function."""
    self.reset_function()

HyperparameterDistribution

Bases: ABC

Abstract class for hyperparameter distributions used in exploration.

Source code in src/dlhub/tuning/population_based.py

class HyperparameterDistribution(ABC):
    """Abstract class for hyperparameter distributions used in exploration."""

    @abstractmethod
    def perturb(self, value: Any) -> Any:
        """
        Perturb a hyperparameter value.

        Parameters
        ----------
        value : any
            Current hyperparameter value

        Returns
        -------
        any
            Perturbed hyperparameter value
        """
        pass

    @abstractmethod
    def resample(self) -> Any:
        """
        Resample a hyperparameter value from the distribution.

        Returns
        -------
        any
            New hyperparameter value
        """
        pass

perturb abstractmethod

perturb(value: Any) -> Any

Perturb a hyperparameter value.

Parameters:

Name	Type	Description	Default
value	any	Current hyperparameter value	required

Returns:

Type	Description
any	Perturbed hyperparameter value

Source code in src/dlhub/tuning/population_based.py

@abstractmethod
def perturb(self, value: Any) -> Any:
    """
    Perturb a hyperparameter value.

    Parameters
    ----------
    value : any
        Current hyperparameter value

    Returns
    -------
    any
        Perturbed hyperparameter value
    """
    pass

resample abstractmethod

resample() -> Any

Resample a hyperparameter value from the distribution.

Returns:

Type	Description
any	New hyperparameter value

Source code in src/dlhub/tuning/population_based.py

@abstractmethod
def resample(self) -> Any:
    """
    Resample a hyperparameter value from the distribution.

    Returns
    -------
    any
        New hyperparameter value
    """
    pass

LogUniformPerturbation

Bases: HyperparameterDistribution

Log-uniform perturbation for hyperparameters that vary over orders of magnitude.

Parameters:

Name	Type	Description	Default
factor_range	tuple	Range of multiplicative factors for perturbation	(0.8, 1.2)
bounds	tuple	(min, max) bounds for the hyperparameter	None

Source code in src/dlhub/tuning/population_based.py

class LogUniformPerturbation(HyperparameterDistribution):
    """
    Log-uniform perturbation for hyperparameters that vary over orders of magnitude.

    Parameters
    ----------
    factor_range : tuple, default=(0.8, 1.2)
        Range of multiplicative factors for perturbation
    bounds : tuple, optional
        (min, max) bounds for the hyperparameter
    """

    def __init__(
        self,
        factor_range: tuple[float, float] = (0.8, 1.2),
        bounds: tuple[float, float] | None = None,
    ):
        self.factor_range = factor_range
        self.bounds = bounds

    def perturb(self, value: float) -> float:
        """Perturb value by random multiplicative factor."""
        factor = np.random.uniform(*self.factor_range)
        new_value = value * factor

        if self.bounds is not None:
            new_value = np.clip(new_value, *self.bounds)

        return new_value

    def resample(self) -> float:
        """Resample from log-uniform distribution."""
        if self.bounds is None:
            raise ValueError("Bounds required for resampling")
        return np.exp(np.random.uniform(np.log(self.bounds[0]), np.log(self.bounds[1])))

perturb

perturb(value: float) -> float

Perturb value by random multiplicative factor.

Source code in src/dlhub/tuning/population_based.py

def perturb(self, value: float) -> float:
    """Perturb value by random multiplicative factor."""
    factor = np.random.uniform(*self.factor_range)
    new_value = value * factor

    if self.bounds is not None:
        new_value = np.clip(new_value, *self.bounds)

    return new_value

resample

resample() -> float

Resample from log-uniform distribution.

Source code in src/dlhub/tuning/population_based.py

def resample(self) -> float:
    """Resample from log-uniform distribution."""
    if self.bounds is None:
        raise ValueError("Bounds required for resampling")
    return np.exp(np.random.uniform(np.log(self.bounds[0]), np.log(self.bounds[1])))

PBTResult dataclass

Results from Population-Based Training.

Attributes:

Name	Type	Description
best_worker	WorkerState	Best performing worker at the end
final_population	list	Final state of all workers
population_history	list	History of population states over time
total_training_time	float	Total time spent training
total_steps	int	Total training steps across all workers
statistics	dict	Training statistics and analysis

Source code in src/dlhub/tuning/population_based.py

@dataclass
class PBTResult:
    """
    Results from Population-Based Training.

    Attributes
    ----------
    best_worker : WorkerState
        Best performing worker at the end
    final_population : list
        Final state of all workers
    population_history : list
        History of population states over time
    total_training_time : float
        Total time spent training
    total_steps : int
        Total training steps across all workers
    statistics : dict
        Training statistics and analysis
    """

    best_worker: WorkerState
    final_population: list[WorkerState]
    population_history: list[list[WorkerState]]
    total_training_time: float
    total_steps: int
    statistics: dict[str, Any] = field(default_factory=dict)

PopulationBasedTrainer

Population-Based Training optimizer.

Manages a population of workers, periodically evaluating performance and updating hyperparameters through exploitation and exploration.

Parameters:

Name	Type	Description	Default
worker_factory	callable	Factory function that creates new WorkerInterface instances	required
initial_hyperparams	list	Initial hyperparameter configurations for population	required
hyperparam_distributions	dict	Mapping from hyperparameter names to HyperparameterDistribution objects	required
population_size	int	Size of the population	10
eval_interval	int	Training steps between population evaluations	100
exploit_fraction	float	Fraction of worst performers to replace	0.2
explore_fraction	float	Fraction of hyperparameters to perturb during exploration	0.2
truncation_selection	bool	Whether to use truncation selection (replace worst with best)	True
random_state	int	Random seed for reproducibility	None

Source code in src/dlhub/tuning/population_based.py

class PopulationBasedTrainer:
    """
    Population-Based Training optimizer.

    Manages a population of workers, periodically evaluating performance
    and updating hyperparameters through exploitation and exploration.

    Parameters
    ----------
    worker_factory : callable
        Factory function that creates new WorkerInterface instances
    initial_hyperparams : list
        Initial hyperparameter configurations for population
    hyperparam_distributions : dict
        Mapping from hyperparameter names to HyperparameterDistribution objects
    population_size : int, default=10
        Size of the population
    eval_interval : int, default=100
        Training steps between population evaluations
    exploit_fraction : float, default=0.2
        Fraction of worst performers to replace
    explore_fraction : float, default=0.2
        Fraction of hyperparameters to perturb during exploration
    truncation_selection : bool, default=True
        Whether to use truncation selection (replace worst with best)
    random_state : int, optional
        Random seed for reproducibility
    """

    def __init__(
        self,
        worker_factory: Callable[[], WorkerInterface],
        initial_hyperparams: list[dict[str, Any]],
        hyperparam_distributions: dict[str, HyperparameterDistribution],
        population_size: int = 10,
        eval_interval: int = 100,
        exploit_fraction: float = 0.2,
        explore_fraction: float = 0.2,
        truncation_selection: bool = True,
        random_state: int | None = None,
    ):

        self.worker_factory = worker_factory
        self.initial_hyperparams = initial_hyperparams
        self.hyperparam_distributions = hyperparam_distributions
        self.population_size = population_size
        self.eval_interval = eval_interval
        self.exploit_fraction = exploit_fraction
        self.explore_fraction = explore_fraction
        self.truncation_selection = truncation_selection

        if random_state is not None:
            np.random.seed(random_state)

        # Initialize population
        self.population = []
        self.population_history = []
        self.total_steps = 0
        self.generation = 0

    def _initialize_population(self) -> None:
        """Initialize the population of workers."""
        self.population = []

        configs = self.initial_hyperparams.copy()
        while len(configs) < self.population_size:
            config = {}
            for param, dist in self.hyperparam_distributions.items():
                config[param] = dist.resample()
            configs.append(config)

        for i in range(self.population_size):
            config = configs[i % len(configs)]
            worker_state = WorkerState(
                worker_id=i,
                hyperparams=config.copy(),
                performance_history=[],
                training_step=0,
                model_state=None,
                metadata={"generation_created": 0},
            )
            self.population.append(worker_state)

    def _evaluate_population(self, workers: list[WorkerInterface]) -> list[float]:
        """
        Evaluate current performance of all workers.

        Parameters
        ----------
        workers : list
            List of worker instances

        Returns
        -------
        list
            Performance scores for each worker
        """
        scores = []
        for i, worker in enumerate(workers):
            try:
                # Train for evaluation interval
                score, model_state = worker.train_step(
                    self.population[i].hyperparams, self.eval_interval
                )

                # A diverged run reports nan, and nan sorts to the end of
                # np.argsort -- so the worker that just blew up would be read as
                # the population's best and have its hyperparameters copied into
                # everyone else. Treated as the worst possible score instead.
                if not np.isfinite(score):
                    score = -np.inf

                # Update worker state
                self.population[i].performance_history.append(score)
                self.population[i].training_step += self.eval_interval
                self.population[i].model_state = model_state

                scores.append(score)

            except Exception as e:
                warnings.warn(f"Worker {i} evaluation failed: {e}")
                scores.append(-np.inf)

        return scores

    def _exploit_and_explore(
        self, workers: list[WorkerInterface], scores: list[float]
    ) -> None:
        """
        Perform exploitation and exploration step.

        Parameters
        ----------
        workers : list
            List of worker instances
        scores : list
            Current performance scores
        """
        if len(scores) < 2:
            return

        sorted_indices = np.argsort(scores)

        # Truncation selection needs the two ends to be disjoint. Above half the
        # population they overlap, which puts a top performer in the list of
        # workers to overwrite: its state is replaced and then perturbed, so the
        # generation's best result is destroyed by the step meant to spread it.
        n_exploit = max(1, int(self.exploit_fraction * len(scores)))
        n_exploit = min(n_exploit, len(scores) // 2)

        worst_indices = sorted_indices[:n_exploit]
        best_indices = sorted_indices[-n_exploit:]

        for worst_idx in worst_indices:
            if self.truncation_selection:
                best_idx = np.random.choice(best_indices)

                self.population[worst_idx].hyperparams = self.population[
                    best_idx
                ].hyperparams.copy()

                if self.population[best_idx].model_state is not None:
                    workers[worst_idx].load_state(self.population[best_idx].model_state)
                    self.population[worst_idx].model_state = self.population[
                        best_idx
                    ].model_state

                self.population[worst_idx].performance_history = []
                self.population[worst_idx].metadata["generation_created"] = (
                    self.generation
                )

            self._perturb_hyperparams(worst_idx)

    def _perturb_hyperparams(self, worker_idx: int) -> None:
        """
        Perturb hyperparameters for exploration.

        Parameters
        ----------
        worker_idx : int
            Index of worker to perturb
        """
        hyperparams = self.population[worker_idx].hyperparams

        param_names = list(hyperparams.keys())
        n_perturb = max(1, int(self.explore_fraction * len(param_names)))
        params_to_perturb = np.random.choice(param_names, n_perturb, replace=False)

        for param in params_to_perturb:
            if param in self.hyperparam_distributions:
                dist = self.hyperparam_distributions[param]
                hyperparams[param] = dist.perturb(hyperparams[param])

    def train(
        self,
        max_steps: int = 10000,
        max_generations: int = 100,
        timeout: float | None = None,
        verbose: bool = True,
    ) -> PBTResult:
        """
        Run Population-Based Training.

        Parameters
        ----------
        max_steps : int, default=10000
            Maximum total training steps
        max_generations : int, default=100
            Maximum number of generations
        timeout : float, optional
            Maximum training time in seconds
        verbose : bool, default=True
            Whether to print progress information

        Returns
        -------
        PBTResult
            Training results including best worker and population history
        """
        if verbose:
            print("Starting Population-Based Training...")
            print(f"Population size: {self.population_size}")
            print(f"Evaluation interval: {self.eval_interval}")
            print(f"Exploit fraction: {self.exploit_fraction}")
            print(f"Explore fraction: {self.explore_fraction}")

        start_time = time.time()

        self._initialize_population()

        workers = [self.worker_factory() for _ in range(self.population_size)]

        for i, worker in enumerate(workers):
            worker.reset()

        best_score = -np.inf
        best_worker = None

        while (
            self.total_steps < max_steps
            and self.generation < max_generations
            and (timeout is None or time.time() - start_time < timeout)
        ):
            scores = self._evaluate_population(workers)
            self.total_steps += self.population_size * self.eval_interval

            valid_scores = [s for s in scores if s != -np.inf]
            if valid_scores:
                max_score_idx = np.argmax(scores)
                if scores[max_score_idx] > best_score:
                    best_score = scores[max_score_idx]
                    best_worker = copy.deepcopy(self.population[max_score_idx])

            population_snapshot = copy.deepcopy(self.population)
            self.population_history.append(population_snapshot)

            if verbose:
                if valid_scores:
                    mean_score = np.mean(valid_scores)
                    std_score = np.std(valid_scores) if len(valid_scores) > 1 else 0.0
                else:
                    mean_score = float("nan")
                    std_score = float("nan")

                print(
                    f"Generation {self.generation}: "
                    f"Best={best_score:.6f}, "
                    f"Mean={mean_score:.6f}±{std_score:.6f}, "
                    f"Steps={self.total_steps}"
                )

            self._exploit_and_explore(workers, scores)

            self.generation += 1

        total_time = time.time() - start_time

        if best_worker is None and self.population:
            best_overall_score = -np.inf
            for worker in self.population:
                if worker.performance_history:
                    worker_best = max(worker.performance_history)
                    if worker_best > best_overall_score:
                        best_overall_score = worker_best
                        best_worker = copy.deepcopy(worker)

        if best_worker is None and self.population:
            best_worker = copy.deepcopy(self.population[0])
            best_score = -np.inf

        final_scores = [
            (
                np.max(worker.performance_history)
                if worker.performance_history
                else -np.inf
            )
            for worker in self.population
        ]

        valid_final_scores = [s for s in final_scores if s != -np.inf]

        statistics = {
            "generations_completed": self.generation,
            "total_training_time": total_time,
            "final_population_mean": (
                np.mean(valid_final_scores) if valid_final_scores else float("nan")
            ),
            "final_population_std": (
                np.std(valid_final_scores) if len(valid_final_scores) > 1 else 0.0
            ),
            "best_score_progression": [
                max(
                    [
                        max(w.performance_history) if w.performance_history else -np.inf
                        for w in gen
                    ]
                )
                for gen in self.population_history
            ],
            "population_diversity": self._compute_diversity_metrics(),
            "convergence_generation": self._find_convergence_generation(),
        }

        if verbose:
            print(f"\nTraining completed in {total_time:.2f} seconds!")
            print(f"Generations: {self.generation}")
            print(f"Total steps: {self.total_steps}")
            print(f"Best score: {best_score:.6f}")
            if best_worker is not None:
                print(f"Best hyperparameters: {best_worker.hyperparams}")
            else:
                print("No valid workers found during training")

        return PBTResult(
            best_worker=best_worker,  # type: ignore
            final_population=self.population,
            population_history=self.population_history,
            total_training_time=total_time,
            total_steps=self.total_steps,
            statistics=statistics,
        )

    def _compute_diversity_metrics(self) -> dict[str, float]:
        """Compute population diversity metrics."""
        if not self.population_history:
            return {}

        diversity_over_time = []

        for generation in self.population_history:
            param_diversities = []

            for param_name in self.hyperparam_distributions.keys():
                values = []
                for worker in generation:
                    if param_name in worker.hyperparams:
                        val = worker.hyperparams[param_name]
                        if isinstance(val, (int, float)):
                            values.append(val)

                if len(values) > 1:
                    diversity = np.std(values) / (np.mean(values) + 1e-8)
                    param_diversities.append(diversity)

            if param_diversities:
                diversity_over_time.append(np.mean(param_diversities))

        return {
            "initial_diversity": diversity_over_time[0] if diversity_over_time else 0.0,
            "final_diversity": diversity_over_time[-1] if diversity_over_time else 0.0,
            "mean_diversity": (  # type: ignore
                np.mean(diversity_over_time) if diversity_over_time else 0.0
            ),
            "diversity_trend": (
                diversity_over_time[-1] - diversity_over_time[0]
                if len(diversity_over_time) > 1
                else 0.0
            ),
        }

    def _find_convergence_generation(self) -> int | None:
        """Find the generation where population converged."""
        if len(self.population_history) < 5:
            return None

        best_scores = []
        for generation in self.population_history:
            scores = [
                max(w.performance_history) if w.performance_history else -np.inf
                for w in generation
            ]
            best_scores.append(max(scores))

        improvement_threshold = 0.001
        window_size = 5

        for i in range(window_size, len(best_scores)):
            recent_improvement = (
                best_scores[i] - best_scores[i - window_size]
            ) / window_size
            if recent_improvement < improvement_threshold:
                return i

        return None

train

train(max_steps: int = 10000, max_generations: int = 100, timeout: float | None = None, verbose: bool = True) -> PBTResult

Run Population-Based Training.

Parameters:

Name	Type	Description	Default
max_steps	int	Maximum total training steps	10000
max_generations	int	Maximum number of generations	100
timeout	float	Maximum training time in seconds	None
verbose	bool	Whether to print progress information	True

Returns:

Type	Description
PBTResult	Training results including best worker and population history

Source code in src/dlhub/tuning/population_based.py

def train(
    self,
    max_steps: int = 10000,
    max_generations: int = 100,
    timeout: float | None = None,
    verbose: bool = True,
) -> PBTResult:
    """
    Run Population-Based Training.

    Parameters
    ----------
    max_steps : int, default=10000
        Maximum total training steps
    max_generations : int, default=100
        Maximum number of generations
    timeout : float, optional
        Maximum training time in seconds
    verbose : bool, default=True
        Whether to print progress information

    Returns
    -------
    PBTResult
        Training results including best worker and population history
    """
    if verbose:
        print("Starting Population-Based Training...")
        print(f"Population size: {self.population_size}")
        print(f"Evaluation interval: {self.eval_interval}")
        print(f"Exploit fraction: {self.exploit_fraction}")
        print(f"Explore fraction: {self.explore_fraction}")

    start_time = time.time()

    self._initialize_population()

    workers = [self.worker_factory() for _ in range(self.population_size)]

    for i, worker in enumerate(workers):
        worker.reset()

    best_score = -np.inf
    best_worker = None

    while (
        self.total_steps < max_steps
        and self.generation < max_generations
        and (timeout is None or time.time() - start_time < timeout)
    ):
        scores = self._evaluate_population(workers)
        self.total_steps += self.population_size * self.eval_interval

        valid_scores = [s for s in scores if s != -np.inf]
        if valid_scores:
            max_score_idx = np.argmax(scores)
            if scores[max_score_idx] > best_score:
                best_score = scores[max_score_idx]
                best_worker = copy.deepcopy(self.population[max_score_idx])

        population_snapshot = copy.deepcopy(self.population)
        self.population_history.append(population_snapshot)

        if verbose:
            if valid_scores:
                mean_score = np.mean(valid_scores)
                std_score = np.std(valid_scores) if len(valid_scores) > 1 else 0.0
            else:
                mean_score = float("nan")
                std_score = float("nan")

            print(
                f"Generation {self.generation}: "
                f"Best={best_score:.6f}, "
                f"Mean={mean_score:.6f}±{std_score:.6f}, "
                f"Steps={self.total_steps}"
            )

        self._exploit_and_explore(workers, scores)

        self.generation += 1

    total_time = time.time() - start_time

    if best_worker is None and self.population:
        best_overall_score = -np.inf
        for worker in self.population:
            if worker.performance_history:
                worker_best = max(worker.performance_history)
                if worker_best > best_overall_score:
                    best_overall_score = worker_best
                    best_worker = copy.deepcopy(worker)

    if best_worker is None and self.population:
        best_worker = copy.deepcopy(self.population[0])
        best_score = -np.inf

    final_scores = [
        (
            np.max(worker.performance_history)
            if worker.performance_history
            else -np.inf
        )
        for worker in self.population
    ]

    valid_final_scores = [s for s in final_scores if s != -np.inf]

    statistics = {
        "generations_completed": self.generation,
        "total_training_time": total_time,
        "final_population_mean": (
            np.mean(valid_final_scores) if valid_final_scores else float("nan")
        ),
        "final_population_std": (
            np.std(valid_final_scores) if len(valid_final_scores) > 1 else 0.0
        ),
        "best_score_progression": [
            max(
                [
                    max(w.performance_history) if w.performance_history else -np.inf
                    for w in gen
                ]
            )
            for gen in self.population_history
        ],
        "population_diversity": self._compute_diversity_metrics(),
        "convergence_generation": self._find_convergence_generation(),
    }

    if verbose:
        print(f"\nTraining completed in {total_time:.2f} seconds!")
        print(f"Generations: {self.generation}")
        print(f"Total steps: {self.total_steps}")
        print(f"Best score: {best_score:.6f}")
        if best_worker is not None:
            print(f"Best hyperparameters: {best_worker.hyperparams}")
        else:
            print("No valid workers found during training")

    return PBTResult(
        best_worker=best_worker,  # type: ignore
        final_population=self.population,
        population_history=self.population_history,
        total_training_time=total_time,
        total_steps=self.total_steps,
        statistics=statistics,
    )

UniformPerturbation

Bases: HyperparameterDistribution

Uniform perturbation for continuous hyperparameters.

Parameters:

Name	Type	Description	Default
noise_std	float	Standard deviation of Gaussian noise to add	0.1
bounds	tuple	(min, max) bounds for the hyperparameter	None

Source code in src/dlhub/tuning/population_based.py

class UniformPerturbation(HyperparameterDistribution):
    """
    Uniform perturbation for continuous hyperparameters.

    Parameters
    ----------
    noise_std : float, default=0.1
        Standard deviation of Gaussian noise to add
    bounds : tuple, optional
        (min, max) bounds for the hyperparameter
    """

    def __init__(
        self, noise_std: float = 0.1, bounds: tuple[float, float] | None = None
    ):
        self.noise_std = noise_std
        self.bounds = bounds

    def perturb(self, value: float) -> float:
        """Perturb value by adding Gaussian noise."""
        new_value = value + np.random.normal(0, self.noise_std)

        if self.bounds is not None:
            new_value = np.clip(new_value, *self.bounds)

        return new_value

    def resample(self) -> float:
        """Resample from uniform distribution."""
        if self.bounds is None:
            raise ValueError("Bounds required for resampling")
        return np.random.uniform(*self.bounds)

perturb

perturb(value: float) -> float

Perturb value by adding Gaussian noise.

Source code in src/dlhub/tuning/population_based.py

def perturb(self, value: float) -> float:
    """Perturb value by adding Gaussian noise."""
    new_value = value + np.random.normal(0, self.noise_std)

    if self.bounds is not None:
        new_value = np.clip(new_value, *self.bounds)

    return new_value

resample

resample() -> float

Resample from uniform distribution.

Source code in src/dlhub/tuning/population_based.py

def resample(self) -> float:
    """Resample from uniform distribution."""
    if self.bounds is None:
        raise ValueError("Bounds required for resampling")
    return np.random.uniform(*self.bounds)

WorkerInterface

Bases: ABC

Abstract interface for training workers in PBT.

Defines the methods that workers must implement to participate in population-based training.

Source code in src/dlhub/tuning/population_based.py

class WorkerInterface(ABC):
    """
    Abstract interface for training workers in PBT.

    Defines the methods that workers must implement to participate
    in population-based training.
    """

    @abstractmethod
    def train_step(
        self, hyperparams: dict[str, Any], steps: int = 1
    ) -> tuple[float, Any]:
        """
        Train for specified number of steps.

        Parameters
        ----------
        hyperparams : dict
            Current hyperparameter configuration
        steps : int, default=1
            Number of training steps to perform

        Returns
        -------
        tuple
            (performance_score, model_state)
        """
        pass

    @abstractmethod
    def save_state(self) -> Any:
        """
        Save current model state.

        Returns
        -------
        any
            Serializable model state
        """
        pass

    @abstractmethod
    def load_state(self, state: Any) -> None:
        """
        Load model state.

        Parameters
        ----------
        state : any
            Model state to load
        """
        pass

    @abstractmethod
    def reset(self) -> None:
        """Reset worker to initial state."""
        pass

train_step abstractmethod

train_step(hyperparams: dict[str, Any], steps: int = 1) -> tuple[float, Any]

Train for specified number of steps.

Parameters:

Name	Type	Description	Default
hyperparams	dict	Current hyperparameter configuration	required
steps	int	Number of training steps to perform	1

Returns:

Type	Description
tuple	(performance_score, model_state)

Source code in src/dlhub/tuning/population_based.py

@abstractmethod
def train_step(
    self, hyperparams: dict[str, Any], steps: int = 1
) -> tuple[float, Any]:
    """
    Train for specified number of steps.

    Parameters
    ----------
    hyperparams : dict
        Current hyperparameter configuration
    steps : int, default=1
        Number of training steps to perform

    Returns
    -------
    tuple
        (performance_score, model_state)
    """
    pass

save_state abstractmethod

save_state() -> Any

Save current model state.

Returns:

Type	Description
any	Serializable model state

Source code in src/dlhub/tuning/population_based.py

@abstractmethod
def save_state(self) -> Any:
    """
    Save current model state.

    Returns
    -------
    any
        Serializable model state
    """
    pass

load_state abstractmethod

load_state(state: Any) -> None

Load model state.

Parameters:

Name	Type	Description	Default
state	any	Model state to load	required

Source code in src/dlhub/tuning/population_based.py

@abstractmethod
def load_state(self, state: Any) -> None:
    """
    Load model state.

    Parameters
    ----------
    state : any
        Model state to load
    """
    pass

reset abstractmethod

reset() -> None

Reset worker to initial state.

Source code in src/dlhub/tuning/population_based.py

@abstractmethod
def reset(self) -> None:
    """Reset worker to initial state."""
    pass

WorkerState dataclass

State of a single worker in the population.

Attributes:

Name	Type	Description
worker_id	int	Unique identifier for the worker
hyperparams	dict	Current hyperparameter configuration
performance_history	list	History of performance scores
training_step	int	Current training step
model_state	any	Current model state (implementation dependent)
metadata	dict	Additional worker metadata

Source code in src/dlhub/tuning/population_based.py

@dataclass
class WorkerState:
    """
    State of a single worker in the population.

    Attributes
    ----------
    worker_id : int
        Unique identifier for the worker
    hyperparams : dict
        Current hyperparameter configuration
    performance_history : list
        History of performance scores
    training_step : int
        Current training step
    model_state : any
        Current model state (implementation dependent)
    metadata : dict
        Additional worker metadata
    """

    worker_id: int
    hyperparams: dict[str, Any]
    performance_history: list[float] = field(default_factory=list)
    training_step: int = 0
    model_state: Any = None
    metadata: dict[str, Any] = field(default_factory=dict)

ChoiceDistribution

Bases: ParameterDistribution

Categorical distribution for discrete choices.

Parameters:

Name	Type	Description	Default
choices	list	List of possible values to choose from	required
probabilities	list	Probability weights for each choice (uniform if None)	None

Source code in src/dlhub/tuning/random_search.py

class ChoiceDistribution(ParameterDistribution):
    """
    Categorical distribution for discrete choices.

    Parameters
    ----------
    choices : list
        List of possible values to choose from
    probabilities : list, optional
        Probability weights for each choice (uniform if None)
    """

    def __init__(self, choices: list[Any], probabilities: list[float] | None = None):
        self.choices = choices
        if probabilities is None:
            self.probabilities = [1.0 / len(choices)] * len(choices)
        else:
            if len(probabilities) != len(choices):
                raise ValueError("Probabilities must match number of choices")
            # Normalize probabilities
            total = sum(probabilities)
            self.probabilities = [p / total for p in probabilities]

    def sample(self) -> Any:
        """Sample from categorical distribution."""
        return np.random.choice(self.choices, p=self.probabilities)

    def __repr__(self) -> str:
        return f"ChoiceDistribution(choices={self.choices})"

sample

sample() -> Any

Sample from categorical distribution.

Source code in src/dlhub/tuning/random_search.py

def sample(self) -> Any:
    """Sample from categorical distribution."""
    return np.random.choice(self.choices, p=self.probabilities)

IntegerDistribution

Bases: ParameterDistribution

Discrete uniform distribution for integer parameters.

Parameters:

Name	Type	Description	Default
low	int	Lower bound (inclusive)	required
high	int	Upper bound (exclusive)	required

Source code in src/dlhub/tuning/random_search.py

class IntegerDistribution(ParameterDistribution):
    """
    Discrete uniform distribution for integer parameters.

    Parameters
    ----------
    low : int
        Lower bound (inclusive)
    high : int
        Upper bound (exclusive)
    """

    def __init__(self, low: int, high: int):
        self.low = low
        self.high = high
        self._dist = randint(low=low, high=high)

    def sample(self) -> int:
        """Sample from discrete uniform distribution."""
        return int(self._dist.rvs())

    def __repr__(self) -> str:
        return f"IntegerDistribution(low={self.low}, high={self.high})"

sample

sample() -> int

Sample from discrete uniform distribution.

Source code in src/dlhub/tuning/random_search.py

def sample(self) -> int:
    """Sample from discrete uniform distribution."""
    return int(self._dist.rvs())

LogUniformDistribution

Bases: ParameterDistribution

Log-uniform distribution for parameters that vary over orders of magnitude.

Particularly useful for learning rates, regularization parameters, etc.

Parameters:

Name	Type	Description	Default
low	float	Lower bound (must be positive)	required
high	float	Upper bound (must be positive)	required

Source code in src/dlhub/tuning/random_search.py

class LogUniformDistribution(ParameterDistribution):
    """
    Log-uniform distribution for parameters that vary over orders of magnitude.

    Particularly useful for learning rates, regularization parameters, etc.

    Parameters
    ----------
    low : float
        Lower bound (must be positive)
    high : float
        Upper bound (must be positive)
    """

    def __init__(self, low: float, high: float):
        if low <= 0 or high <= 0:
            raise ValueError("Log-uniform distribution requires positive bounds")
        self.low = low
        self.high = high
        self._dist = loguniform(a=low, b=high)

    def sample(self) -> float:
        """Sample from log-uniform distribution."""
        return self._dist.rvs()

    def __repr__(self) -> str:
        return f"LogUniformDistribution(low={self.low}, high={self.high})"

sample

sample() -> float

Sample from log-uniform distribution.

Source code in src/dlhub/tuning/random_search.py

def sample(self) -> float:
    """Sample from log-uniform distribution."""
    return self._dist.rvs()

ParameterDistribution

Base class for hyperparameter distributions.

Defines the interface for sampling hyperparameters from different probability distributions.

Source code in src/dlhub/tuning/random_search.py

class ParameterDistribution:
    """
    Base class for hyperparameter distributions.

    Defines the interface for sampling hyperparameters from different
    probability distributions.
    """

    def sample(self) -> Any:
        """Sample a value from the distribution."""
        raise NotImplementedError

    def __repr__(self) -> str:
        return f"{self.__class__.__name__}()"

sample

sample() -> Any

Sample a value from the distribution.

Source code in src/dlhub/tuning/random_search.py

def sample(self) -> Any:
    """Sample a value from the distribution."""
    raise NotImplementedError

PowerDistribution

Bases: ParameterDistribution

Power law distribution for parameters with non-uniform preferences.

Useful when smaller values are preferred (common in regularization).

Parameters:

Name	Type	Description	Default
low	float	Lower bound	required
high	float	Upper bound	required
power	float	Power parameter. Above 1 the mass concentrates near low, at 1 the distribution is uniform, and below 1 it concentrates near high.	2.0

Source code in src/dlhub/tuning/random_search.py

class PowerDistribution(ParameterDistribution):
    """
    Power law distribution for parameters with non-uniform preferences.

    Useful when smaller values are preferred (common in regularization).

    Parameters
    ----------
    low : float
        Lower bound
    high : float
        Upper bound
    power : float, default=2.0
        Power parameter. Above 1 the mass concentrates near `low`, at 1 the
        distribution is uniform, and below 1 it concentrates near `high`.
    """

    def __init__(self, low: float, high: float, power: float = 2.0):
        self.low = low
        self.high = high
        self.power = power

    def sample(self) -> float:
        """Sample from power distribution."""
        # u ** power, not u ** (1 / power). The latter is the standard
        # power-function distribution, whose mass moves toward `high` as the
        # exponent grows: the default power=2 would draw a mean of 2/3 of the
        # range, which is the opposite of what this class is documented to do
        # and useless for the weight-decay sweeps it exists for.
        u = np.random.random()
        return self.low + (self.high - self.low) * (u**self.power)

    def __repr__(self) -> str:
        return (
            f"PowerDistribution(low={self.low}, high={self.high}, power={self.power})"
        )

sample

sample() -> float

Sample from power distribution.

Source code in src/dlhub/tuning/random_search.py

def sample(self) -> float:
    """Sample from power distribution."""
    # u ** power, not u ** (1 / power). The latter is the standard
    # power-function distribution, whose mass moves toward `high` as the
    # exponent grows: the default power=2 would draw a mean of 2/3 of the
    # range, which is the opposite of what this class is documented to do
    # and useless for the weight-decay sweeps it exists for.
    u = np.random.random()
    return self.low + (self.high - self.low) * (u**self.power)

RandomSearchOptimizer

Random Search optimizer for hyperparameter tuning.

Implements efficient random sampling of hyperparameters with support for different probability distributions, parallel evaluation, and early stopping.

Parameters:

Name	Type	Description	Default
objective_function	callable	Function to optimize. Should take hyperparameter dict and return float	required
search_space	dict	Dictionary mapping parameter names to ParameterDistribution objects	required
n_iter	int	Number of parameter configurations to sample and evaluate	100
random_state	int(optional)	Random seed for reproducibility	None
n_jobs	int	Number of parallel jobs (-1 for all available cores)	1
early_stopping	bool	Whether to use early stopping based on improvement	False
patience	int	Number of iterations without improvement before stopping	10

Source code in src/dlhub/tuning/random_search.py

class RandomSearchOptimizer:
    """
    Random Search optimizer for hyperparameter tuning.

    Implements efficient random sampling of hyperparameters with support for
    different probability distributions, parallel evaluation, and early stopping.

    Parameters
    ----------
    objective_function : callable
        Function to optimize. Should take hyperparameter dict and return float
    search_space : dict
        Dictionary mapping parameter names to ParameterDistribution objects
    n_iter : int, default=100
        Number of parameter configurations to sample and evaluate
    random_state : int (optional)
        Random seed for reproducibility
    n_jobs : int, default=1
        Number of parallel jobs (-1 for all available cores)
    early_stopping : bool, default=False
        Whether to use early stopping based on improvement
    patience : int, default=10
        Number of iterations without improvement before stopping
    """

    def __init__(
        self,
        objective_function: Callable[[dict], float],
        search_space: dict[str, ParameterDistribution],
        n_iter: int = 100,
        random_state: int | None = None,
        n_jobs: int = 1,
        early_stopping: bool = False,
        patience: int = 10,
    ) -> None:

        self.objective_function = objective_function
        self.search_space = search_space
        self.n_iter = n_iter
        self.random_state = random_state
        self.n_jobs = n_jobs if n_jobs != -1 else mp.cpu_count()
        self.early_stopping = early_stopping
        self.patience = patience

        if random_state is not None:
            np.random.seed(random_state)

        self.results_history = []
        self.best_score = -np.inf
        self.best_params = None
        self.iterations_without_improvement = 0

    def sample_parameters(self) -> dict[str, Any]:
        """
        Sample a single parameter configuration from the search space.

        Returns
        -------
        dict
            Sampled hyperparameter configuration
        """
        params = {}
        for param_name, distribution in self.search_space.items():
            params[param_name] = distribution.sample()
        return params

    def sample_multiple_parameters(self, n_samples: int) -> list[dict[str, Any]]:
        """
        Sample multiple parameter configurations.

        Parameters
        ----------
        n_samples : int
            Number of configurations to sample

        Returns
        -------
        list
            List of parameter configurations
        """
        return [self.sample_parameters() for _ in range(n_samples)]

    def _evaluate_single(self, params: dict[str, Any]) -> tuple[dict[str, Any], float]:
        """
        Evaluate objective function for a single parameter configuration.

        Parameters
        ----------
        params : dict
            Parameter configuration to evaluate

        Returns
        -------
        tuple
            (parameters, score) tuple
        """
        try:
            score = self.objective_function(params)
            if np.isnan(score) or np.isinf(score):
                return params, -np.inf
            return params, float(score)
        except Exception as e:
            warnings.warn(f"Evaluation failed for {params}: {e}")
            return params, -np.inf

    def _evaluate_batch_sequential(
        self, param_list: list[dict[str, Any]]
    ) -> list[tuple[dict[str, Any], float]]:
        """Evaluate parameters sequentially."""
        results = []
        for params in param_list:
            result = self._evaluate_single(params)
            results.append(result)
        return results

    def _evaluate_batch_parallel(
        self, param_list: list[dict[str, Any]]
    ) -> list[tuple[dict[str, Any], float]]:
        """Evaluate parameters in parallel."""
        results = []
        with ProcessPoolExecutor(max_workers=self.n_jobs) as executor:
            # Submit all jobs
            future_to_params = {
                executor.submit(self._evaluate_single, params): params
                for params in param_list
            }

            # Collect results as they complete
            for future in as_completed(future_to_params):
                try:
                    result = future.result()
                    results.append(result)
                except Exception as e:
                    params = future_to_params[future]
                    warnings.warn(f"Parallel evaluation failed for {params}: {e}")
                    results.append((params, -np.inf))

        return results

    def _should_stop_early(self) -> bool:
        """Check if early stopping criteria are met."""
        if not self.early_stopping:
            return False
        return self.iterations_without_improvement >= self.patience

    def optimize(self, verbose: bool = True) -> RandomSearchResult:
        """
        Run random search optimization.

        Parameters
        ----------
        verbose : bool, default=True
            Whether to print progress information

        Returns
        -------
        RandomSearchResult
            Optimization results
        """
        if verbose:
            print("Starting Random Search Optimization...")
            print(f"Search space: {len(self.search_space)} parameters")
            print(f"Number of iterations: {self.n_iter}")
            print(f"Parallel jobs: {self.n_jobs}")
            if self.early_stopping:
                print(f"Early stopping: patience={self.patience}")

        start_time = time.time()
        all_params = []
        all_scores = []

        if self.n_jobs > 1:
            batch_size = min(self.n_jobs * 2, self.n_iter)
        else:
            batch_size = 10  # Process in small batches even for sequential

        iterations_completed = 0

        while iterations_completed < self.n_iter:
            remaining_iterations = self.n_iter - iterations_completed
            current_batch_size = min(batch_size, remaining_iterations)

            param_batch = self.sample_multiple_parameters(current_batch_size)

            if self.n_jobs > 1:
                batch_results = self._evaluate_batch_parallel(param_batch)
            else:
                batch_results = self._evaluate_batch_sequential(param_batch)

            improvement_found = False
            for params, score in batch_results:
                all_params.append(params)
                all_scores.append(score)

                if score > self.best_score:
                    self.best_score = score
                    self.best_params = params.copy()
                    improvement_found = True
                    self.iterations_without_improvement = 0

                    if verbose:
                        print(
                            f"Iteration {iterations_completed + 1}: "
                            f"New best score = {score:.6f}"
                        )
                else:
                    self.iterations_without_improvement += 1

                iterations_completed += 1

                if self._should_stop_early():
                    if verbose:
                        print(f"Early stopping at iteration {iterations_completed}")
                    break

            if self._should_stop_early():
                break

            # Progress update
            if verbose and iterations_completed % max(1, self.n_iter // 10) == 0:
                print(
                    f"Progress: {iterations_completed}/{self.n_iter} "
                    f"({100 * iterations_completed / self.n_iter:.1f}%) "
                    f"- Best score: {self.best_score:.6f}"
                )

        search_time = time.time() - start_time

        valid_scores = [s for s in all_scores if s != -np.inf]
        statistics = {
            "total_evaluations": len(all_scores),
            "successful_evaluations": len(valid_scores),
            "failed_evaluations": len(all_scores) - len(valid_scores),
            "mean_score": np.mean(valid_scores) if valid_scores else -np.inf,
            "std_score": np.std(valid_scores) if len(valid_scores) > 1 else 0.0,
            "score_percentiles": {
                "25th": np.percentile(valid_scores, 25) if valid_scores else -np.inf,
                "50th": np.percentile(valid_scores, 50) if valid_scores else -np.inf,
                "75th": np.percentile(valid_scores, 75) if valid_scores else -np.inf,
                "95th": np.percentile(valid_scores, 95) if valid_scores else -np.inf,
            },
            "improvement_over_random": (self.best_score - np.mean(valid_scores[:10]))
            if len(valid_scores) >= 10
            else 0.0,
            "early_stopped": self._should_stop_early(),
            "iterations_completed": iterations_completed,
        }

        if verbose:
            print(f"\nOptimization completed in {search_time:.2f} seconds!")
            print(f"Best score: {self.best_score:.6f}")
            print(f"Best parameters: {self.best_params}")
            print(f"Total evaluations: {statistics['total_evaluations']}")
            print(
                f"Success rate: {statistics['successful_evaluations'] / statistics['total_evaluations']:.2%}"
            )

        return RandomSearchResult(
            best_params=self.best_params,  # type: ignore
            best_score=self.best_score,
            all_params=all_params,
            all_scores=all_scores,
            search_time=search_time,
            statistics=statistics,
        )

sample_parameters

sample_parameters() -> dict[str, Any]

Sample a single parameter configuration from the search space.

Returns:

Type	Description
dict	Sampled hyperparameter configuration

Source code in src/dlhub/tuning/random_search.py

def sample_parameters(self) -> dict[str, Any]:
    """
    Sample a single parameter configuration from the search space.

    Returns
    -------
    dict
        Sampled hyperparameter configuration
    """
    params = {}
    for param_name, distribution in self.search_space.items():
        params[param_name] = distribution.sample()
    return params

sample_multiple_parameters

sample_multiple_parameters(n_samples: int) -> list[dict[str, Any]]

Sample multiple parameter configurations.

Parameters:

Name	Type	Description	Default
n_samples	int	Number of configurations to sample	required

Returns:

Type	Description
list	List of parameter configurations

Source code in src/dlhub/tuning/random_search.py

def sample_multiple_parameters(self, n_samples: int) -> list[dict[str, Any]]:
    """
    Sample multiple parameter configurations.

    Parameters
    ----------
    n_samples : int
        Number of configurations to sample

    Returns
    -------
    list
        List of parameter configurations
    """
    return [self.sample_parameters() for _ in range(n_samples)]

optimize

optimize(verbose: bool = True) -> RandomSearchResult

Run random search optimization.

Parameters:

Name	Type	Description	Default
verbose	bool	Whether to print progress information	True

Returns:

Type	Description
RandomSearchResult	Optimization results

Source code in src/dlhub/tuning/random_search.py

def optimize(self, verbose: bool = True) -> RandomSearchResult:
    """
    Run random search optimization.

    Parameters
    ----------
    verbose : bool, default=True
        Whether to print progress information

    Returns
    -------
    RandomSearchResult
        Optimization results
    """
    if verbose:
        print("Starting Random Search Optimization...")
        print(f"Search space: {len(self.search_space)} parameters")
        print(f"Number of iterations: {self.n_iter}")
        print(f"Parallel jobs: {self.n_jobs}")
        if self.early_stopping:
            print(f"Early stopping: patience={self.patience}")

    start_time = time.time()
    all_params = []
    all_scores = []

    if self.n_jobs > 1:
        batch_size = min(self.n_jobs * 2, self.n_iter)
    else:
        batch_size = 10  # Process in small batches even for sequential

    iterations_completed = 0

    while iterations_completed < self.n_iter:
        remaining_iterations = self.n_iter - iterations_completed
        current_batch_size = min(batch_size, remaining_iterations)

        param_batch = self.sample_multiple_parameters(current_batch_size)

        if self.n_jobs > 1:
            batch_results = self._evaluate_batch_parallel(param_batch)
        else:
            batch_results = self._evaluate_batch_sequential(param_batch)

        improvement_found = False
        for params, score in batch_results:
            all_params.append(params)
            all_scores.append(score)

            if score > self.best_score:
                self.best_score = score
                self.best_params = params.copy()
                improvement_found = True
                self.iterations_without_improvement = 0

                if verbose:
                    print(
                        f"Iteration {iterations_completed + 1}: "
                        f"New best score = {score:.6f}"
                    )
            else:
                self.iterations_without_improvement += 1

            iterations_completed += 1

            if self._should_stop_early():
                if verbose:
                    print(f"Early stopping at iteration {iterations_completed}")
                break

        if self._should_stop_early():
            break

        # Progress update
        if verbose and iterations_completed % max(1, self.n_iter // 10) == 0:
            print(
                f"Progress: {iterations_completed}/{self.n_iter} "
                f"({100 * iterations_completed / self.n_iter:.1f}%) "
                f"- Best score: {self.best_score:.6f}"
            )

    search_time = time.time() - start_time

    valid_scores = [s for s in all_scores if s != -np.inf]
    statistics = {
        "total_evaluations": len(all_scores),
        "successful_evaluations": len(valid_scores),
        "failed_evaluations": len(all_scores) - len(valid_scores),
        "mean_score": np.mean(valid_scores) if valid_scores else -np.inf,
        "std_score": np.std(valid_scores) if len(valid_scores) > 1 else 0.0,
        "score_percentiles": {
            "25th": np.percentile(valid_scores, 25) if valid_scores else -np.inf,
            "50th": np.percentile(valid_scores, 50) if valid_scores else -np.inf,
            "75th": np.percentile(valid_scores, 75) if valid_scores else -np.inf,
            "95th": np.percentile(valid_scores, 95) if valid_scores else -np.inf,
        },
        "improvement_over_random": (self.best_score - np.mean(valid_scores[:10]))
        if len(valid_scores) >= 10
        else 0.0,
        "early_stopped": self._should_stop_early(),
        "iterations_completed": iterations_completed,
    }

    if verbose:
        print(f"\nOptimization completed in {search_time:.2f} seconds!")
        print(f"Best score: {self.best_score:.6f}")
        print(f"Best parameters: {self.best_params}")
        print(f"Total evaluations: {statistics['total_evaluations']}")
        print(
            f"Success rate: {statistics['successful_evaluations'] / statistics['total_evaluations']:.2%}"
        )

    return RandomSearchResult(
        best_params=self.best_params,  # type: ignore
        best_score=self.best_score,
        all_params=all_params,
        all_scores=all_scores,
        search_time=search_time,
        statistics=statistics,
    )

RandomSearchResult dataclass

Container for random search optimization results.

Attributes:

Name	Type	Description
best_params	dict	Best hyperparameter configuration found
best_score	float	Best objective function value achieved
all_params	list	All parameter configurations evaluated
all_scores	list	All scores corresponding to parameter configurations
search_time	float	Total search time in seconds
statistics	dict	Search statistics and analysis

Source code in src/dlhub/tuning/random_search.py

@dataclass
class RandomSearchResult:
    """
    Container for random search optimization results.

    Attributes
    ----------
    best_params : dict
        Best hyperparameter configuration found
    best_score : float
        Best objective function value achieved
    all_params : list
        All parameter configurations evaluated
    all_scores : list
        All scores corresponding to parameter configurations
    search_time : float
        Total search time in seconds
    statistics : dict
        Search statistics and analysis
    """

    best_params: dict[str, Any]
    best_score: float
    all_params: list[dict[str, Any]]
    all_scores: list[float]
    search_time: float
    statistics: dict[str, Any] = field(default_factory=dict)

UniformDistribution

Bases: ParameterDistribution

Uniform distribution for continuous parameters.

Parameters:

Name	Type	Description	Default
low	float	Lower bound of the distribution	required
high	float	Upper bound of the distribution	required

Source code in src/dlhub/tuning/random_search.py

class UniformDistribution(ParameterDistribution):
    """
    Uniform distribution for continuous parameters.

    Parameters
    ----------
    low : float
        Lower bound of the distribution
    high : float
        Upper bound of the distribution
    """

    def __init__(self, low: float, high: float):
        self.low = low
        self.high = high
        self._dist = uniform(loc=low, scale=high - low)

    def sample(self) -> float:
        """Sample from uniform distribution."""
        return self._dist.rvs()

    def __repr__(self) -> str:
        return f"UniformDistribution(low={self.low}, high={self.high})"

sample

sample() -> float

Sample from uniform distribution.

Source code in src/dlhub/tuning/random_search.py

def sample(self) -> float:
    """Sample from uniform distribution."""
    return self._dist.rvs()

bayesian_optimize

bayesian_optimize(objective_function: Callable[[dict], float], search_space: dict[str, tuple[float, float]], n_iterations: int = 20, n_initial: int = 5, acquisition: str = 'ei', random_state: int | None = None, verbose: int = 1) -> BayesianOptimizationResult

Run a Bayesian search over continuous bounds, returning its own result type.

The method-specific entry point, alongside :func:~dlhub.tuning.random_search and :func:~dlhub.tuning.asha_optimize. To choose a method by name instead, call :func:~dlhub.tuning.optimize_hyperparameters, which takes the framework's richer search-space format and returns an ExperimentResult.

Takes a scalar objective and a {name: (low, high)} search space; every dimension is continuous, since the Gaussian process interpolates over them.

Parameters:

Name	Type	Description	Default
objective_function	callable	Function to optimize. Should accept hyperparameter dict and return float	required
search_space	dict	Search space definition: {'param_name': (min_val, max_val)}	required
n_iterations	int	Number of optimization iterations after initial random sampling	20
n_initial	int	Number of random initial points	5
acquisition	str	Acquisition function ('ei' or 'ucb')	'ei'
random_state	int	Random seed for reproducibility	None
verbose	int	Reporting level, as in :meth:BayesianOptimizer.optimize	1

Returns:

Type	Description
BayesianOptimizationResult	Optimization results

Examples:

>>> def objective(params):
...     # Simulate training a model and return validation accuracy
...     lr, wd = params["learning_rate"], params["weight_decay"]
...     # Dummy objective (replace with actual model training)
...     return -((lr - 0.001) ** 2) - (wd - 0.0001) ** 2 + np.random.normal(0, 0.01)
>>>
>>> search_space = {"learning_rate": (1e-5, 1e-1), "weight_decay": (1e-6, 1e-2)}
>>>
>>> result = bayesian_optimize(
...     objective, search_space, n_iterations=30, random_state=42
... )
>>> print(f"Best parameters: {result.best_params}")

Source code in src/dlhub/tuning/bayesian.py

def bayesian_optimize(
    objective_function: Callable[[dict], float],
    search_space: dict[str, tuple[float, float]],
    n_iterations: int = 20,
    n_initial: int = 5,
    acquisition: str = "ei",
    random_state: int | None = None,
    verbose: int = 1,
) -> BayesianOptimizationResult:
    """
    Run a Bayesian search over continuous bounds, returning its own result type.

    The method-specific entry point, alongside :func:`~dlhub.tuning.random_search`
    and :func:`~dlhub.tuning.asha_optimize`. To choose a method by name instead,
    call :func:`~dlhub.tuning.optimize_hyperparameters`, which takes the
    framework's richer search-space format and returns an ``ExperimentResult``.

    Takes a scalar objective and a ``{name: (low, high)}`` search space; every
    dimension is continuous, since the Gaussian process interpolates over them.

    Parameters
    ----------
    objective_function : callable
        Function to optimize. Should accept hyperparameter dict and return float
    search_space : dict
        Search space definition: {'param_name': (min_val, max_val)}
    n_iterations : int, default=20
        Number of optimization iterations after initial random sampling
    n_initial : int, default=5
        Number of random initial points
    acquisition : str, default='ei'
        Acquisition function ('ei' or 'ucb')
    random_state : int, optional
        Random seed for reproducibility
    verbose : int, default=1
        Reporting level, as in :meth:`BayesianOptimizer.optimize`

    Returns
    -------
    BayesianOptimizationResult
        Optimization results

    Examples
    --------
    >>> def objective(params):
    ...     # Simulate training a model and return validation accuracy
    ...     lr, wd = params["learning_rate"], params["weight_decay"]
    ...     # Dummy objective (replace with actual model training)
    ...     return -((lr - 0.001) ** 2) - (wd - 0.0001) ** 2 + np.random.normal(0, 0.01)
    >>>
    >>> search_space = {"learning_rate": (1e-5, 1e-1), "weight_decay": (1e-6, 1e-2)}
    >>>
    >>> result = bayesian_optimize(
    ...     objective, search_space, n_iterations=30, random_state=42
    ... )
    >>> print(f"Best parameters: {result.best_params}")
    """
    optimizer = BayesianOptimizer(
        objective_function=objective_function,
        search_space=search_space,
        acquisition=acquisition,
        n_initial=n_initial,
        random_state=random_state,
    )

    return optimizer.optimize(n_iterations=n_iterations, verbose=verbose)

optimize_hyperparameters

optimize_hyperparameters(objective_function: Callable[[dict], dict[str, float]], hyperparameters: list[dict[str, Any]], objective_metric: str, experiment_name: str = 'hyperparameter_optimization', optimization_method: str = 'random_search', n_trials: int = 100, maximize: bool = True, random_seed: int | None = None, save_dir: str | None = None, verbose: bool = True) -> ExperimentResult

Run a hyperparameter search by method name, returning an ExperimentResult.

The general dispatcher: optimization_method selects the strategy. It takes the framework's search-space format, which carries a type per hyperparameter and so covers categorical and integer dimensions as well as continuous ones. The method-specific entry points -- :func:~dlhub.tuning.bayesian_optimize, :func:~dlhub.tuning.random_search -- take their own formats and return their own result types.

Parameters:

Name	Type	Description	Default
objective_function	callable	Function that takes hyperparams dict and returns metrics dict	required
hyperparameters	list	List of hyperparameter definitions (dicts with 'name', 'type', 'range')	required
objective_metric	str	Name of metric to optimize	required
experiment_name	str	Name of the experiment	"hyperparameter_optimization"
optimization_method	str	Optimization method ('random_search' or 'grid_search')	"random_search"
n_trials	int	Number of trials	100
maximize	bool	Whether to maximize objective	True
random_seed	int(optional)	Random seed	None
save_dir	str(optional)	Directory to save results	None
verbose	bool	Whether to print progress	True

Returns:

Type	Description
ExperimentResult	Optimization results

Examples:

>>> def objective(hyperparams):
...     lr = hyperparams["learning_rate"]
...     wd = hyperparams["weight_decay"]
...     accuracy = 0.9 - (lr - 0.001) ** 2 - (wd - 0.0001) ** 2
...     return {"accuracy": accuracy, "loss": 1 - accuracy}
>>> hyperparams = [
...     {
...         "name": "learning_rate",
...         "type": "continuous",
...         "range": (1e-5, 1e-1),
...         "scale": "log",
...     },
...     {
...         "name": "weight_decay",
...         "type": "continuous",
...         "range": (1e-6, 1e-2),
...         "scale": "log",
...     },
... ]
>>> result = optimize_hyperparameters(
...     objective, hyperparams, "accuracy", n_trials=50, random_seed=42
... )

Source code in src/dlhub/tuning/framework.py

def optimize_hyperparameters(
    objective_function: Callable[[dict], dict[str, float]],
    hyperparameters: list[dict[str, Any]],
    objective_metric: str,
    experiment_name: str = "hyperparameter_optimization",
    optimization_method: str = "random_search",
    n_trials: int = 100,
    maximize: bool = True,
    random_seed: int | None = None,
    save_dir: str | None = None,
    verbose: bool = True,
) -> ExperimentResult:
    """
    Run a hyperparameter search by method name, returning an ``ExperimentResult``.

    The general dispatcher: ``optimization_method`` selects the strategy. It takes
    the framework's search-space format, which carries a type per hyperparameter
    and so covers categorical and integer dimensions as well as continuous ones.
    The method-specific entry points -- :func:`~dlhub.tuning.bayesian_optimize`,
    :func:`~dlhub.tuning.random_search` -- take their own formats and return their
    own result types.

    Parameters
    ----------
    objective_function : callable
        Function that takes hyperparams dict and returns metrics dict
    hyperparameters : list
        List of hyperparameter definitions (dicts with 'name', 'type', 'range')
    objective_metric : str
        Name of metric to optimize
    experiment_name : str, default="hyperparameter_optimization"
        Name of the experiment
    optimization_method : str, default="random_search"
        Optimization method ('random_search' or 'grid_search')
    n_trials : int, default=100
        Number of trials
    maximize : bool, default=True
        Whether to maximize objective
    random_seed : int (optional)
        Random seed
    save_dir : str (optional)
        Directory to save results
    verbose : bool, default=True
        Whether to print progress

    Returns
    -------
    ExperimentResult
        Optimization results

    Examples
    --------
    >>> def objective(hyperparams):
    ...     lr = hyperparams["learning_rate"]
    ...     wd = hyperparams["weight_decay"]
    ...     accuracy = 0.9 - (lr - 0.001) ** 2 - (wd - 0.0001) ** 2
    ...     return {"accuracy": accuracy, "loss": 1 - accuracy}
    >>> hyperparams = [
    ...     {
    ...         "name": "learning_rate",
    ...         "type": "continuous",
    ...         "range": (1e-5, 1e-1),
    ...         "scale": "log",
    ...     },
    ...     {
    ...         "name": "weight_decay",
    ...         "type": "continuous",
    ...         "range": (1e-6, 1e-2),
    ...         "scale": "log",
    ...     },
    ... ]
    >>> result = optimize_hyperparameters(
    ...     objective, hyperparams, "accuracy", n_trials=50, random_seed=42
    ... )
    """
    hp_configs = []
    for hp in hyperparameters:
        config = HyperparameterConfig(
            name=hp["name"],
            type=hp["type"],
            range=hp["range"],
            scale=hp.get("scale", "linear"),
            default=hp.get("default"),
        )
        hp_configs.append(config)

    exp_config = ExperimentConfig(
        experiment_name=experiment_name,
        optimization_method=OptimizationMethod(optimization_method),
        hyperparameters=hp_configs,
        objective_metric=objective_metric,
        maximize=maximize,
        n_trials=n_trials,
        random_seed=random_seed,
        save_dir=save_dir,
    )

    test_hyperparams = HyperparameterSampler(hp_configs, random_seed).sample()
    test_metrics = objective_function(test_hyperparams)
    metric_names = list(test_metrics.keys())

    objective_wrapper = FunctionObjective(objective_function, metric_names)

    optimizer = HyperparameterOptimizer(exp_config, objective_wrapper)
    return optimizer.optimize(verbose=verbose)

find_learning_rate

find_learning_rate(train_function: Callable[[float], float], reset_function: Callable[[], None], min_lr: float = 1e-07, max_lr: float = 10.0, num_iterations: int = 100, step_mode: str = 'exp', smooth_beta: float = 0.98, verbose: bool = True, plot: bool = True) -> LearningRateFinderResult

Convenience function to find optimal learning rate.

Parameters:

Name	Type	Description	Default
train_function	callable	Function that takes learning_rate (float) and returns loss (float)	required
reset_function	callable	Function to reset model to initial state	required
min_lr	float	Minimum learning rate to test	1e-7
max_lr	float	Maximum learning rate to test	10.0
num_iterations	int	Number of iterations for the test	100
step_mode	str	Learning rate stepping mode ('exp' or 'linear')	'exp'
smooth_beta	float	Smoothing factor for loss curves	0.98
verbose	bool	Whether to print progress	True
plot	bool	Whether to plot results	True

Returns:

Type	Description
LearningRateFinderResult	Results including suggested learning rate

Examples:

>>> # Example with simple quadratic loss
>>> def train_step(lr):
...     # Simulate one training step
...     current_w = getattr(train_step, "w", 1.0)  # Get current weight
...     target_w = 0.5  # Target weight
...     loss = (current_w - target_w) ** 2
...
...     # Gradient descent update
...     gradient = 2 * (current_w - target_w)
...     train_step.w = current_w - lr * gradient
...
...     return loss + np.random.normal(0, 0.01)  # Add noise
>>> def reset_model():
...     train_step.w = 1.0  # Reset to initial weight
>>> result = find_learning_rate(
...     train_step, reset_model, min_lr=1e-4, max_lr=1.0, num_iterations=50
... )
>>> print(f"Suggested learning rate: {result.suggested_lr}")

Source code in src/dlhub/tuning/learning_rate_finder.py

def find_learning_rate(
    train_function: Callable[[float], float],
    reset_function: Callable[[], None],
    min_lr: float = 1e-7,
    max_lr: float = 10.0,
    num_iterations: int = 100,
    step_mode: str = "exp",
    smooth_beta: float = 0.98,
    verbose: bool = True,
    plot: bool = True,
) -> LearningRateFinderResult:
    """
    Convenience function to find optimal learning rate.

    Parameters
    ----------
    train_function : callable
        Function that takes learning_rate (float) and returns loss (float)
    reset_function : callable
        Function to reset model to initial state
    min_lr : float, default=1e-7
        Minimum learning rate to test
    max_lr : float, default=10.0
        Maximum learning rate to test
    num_iterations : int, default=100
        Number of iterations for the test
    step_mode : str, default='exp'
        Learning rate stepping mode ('exp' or 'linear')
    smooth_beta : float, default=0.98
        Smoothing factor for loss curves
    verbose : bool, default=True
        Whether to print progress
    plot : bool, default=True
        Whether to plot results

    Returns
    -------
    LearningRateFinderResult
        Results including suggested learning rate

    Examples
    --------
    >>> # Example with simple quadratic loss
    >>> def train_step(lr):
    ...     # Simulate one training step
    ...     current_w = getattr(train_step, "w", 1.0)  # Get current weight
    ...     target_w = 0.5  # Target weight
    ...     loss = (current_w - target_w) ** 2
    ...
    ...     # Gradient descent update
    ...     gradient = 2 * (current_w - target_w)
    ...     train_step.w = current_w - lr * gradient
    ...
    ...     return loss + np.random.normal(0, 0.01)  # Add noise
    >>> def reset_model():
    ...     train_step.w = 1.0  # Reset to initial weight
    >>> result = find_learning_rate(
    ...     train_step, reset_model, min_lr=1e-4, max_lr=1.0, num_iterations=50
    ... )
    >>> print(f"Suggested learning rate: {result.suggested_lr}")
    """
    trainer = FunctionTrainer(train_function, reset_function)

    finder = LearningRateFinder(
        trainer=trainer,
        min_lr=min_lr,
        max_lr=max_lr,
        num_iterations=num_iterations,
        step_mode=step_mode,
        smooth_beta=smooth_beta,
    )

    result = finder.find(verbose=verbose)

    if plot:
        finder.plot_results(result)

    return result

suggest_learning_rate_schedule

suggest_learning_rate_schedule(result: LearningRateFinderResult, schedule_type: str = 'onecycle') -> dict[str, Any]

Suggest learning rate schedule based on finder results.

Parameters:

Name	Type	Description	Default
result	LearningRateFinderResult	Results from learning rate finder	required
schedule_type	str	Type of schedule to suggest ('onecycle', 'cyclic', 'cosine', 'step')	'onecycle'

Returns:

Type	Description
dict	Suggested schedule parameters

Source code in src/dlhub/tuning/learning_rate_finder.py

def suggest_learning_rate_schedule(
    result: LearningRateFinderResult, schedule_type: str = "onecycle"
) -> dict[str, Any]:
    """
    Suggest learning rate schedule based on finder results.

    Parameters
    ----------
    result : LearningRateFinderResult
        Results from learning rate finder
    schedule_type : str, default='onecycle'
        Type of schedule to suggest ('onecycle', 'cyclic', 'cosine', 'step')

    Returns
    -------
    dict
        Suggested schedule parameters
    """
    max_lr = result.min_gradient_lr
    base_lr = result.suggested_lr

    if schedule_type == "onecycle":
        return {
            "schedule_type": "onecycle",
            "max_lr": max_lr,
            "base_lr": base_lr,
            "pct_start": 0.3,  # 30% warmup
            "final_div_factor": 1e4,  # Final LR = max_lr / final_div_factor
            "description": "One-cycle policy with warmup and annealing",
        }

    elif schedule_type == "cyclic":
        return {
            "schedule_type": "cyclic",
            "base_lr": base_lr,
            "max_lr": max_lr,
            "step_size_up": 2000,  # Steps to increase from base to max
            "mode": "triangular2",  # Decreasing amplitude
            "description": "Cyclical learning rate with triangular policy",
        }

    elif schedule_type == "cosine":
        return {
            "schedule_type": "cosine",
            "initial_lr": max_lr,
            "min_lr": base_lr,
            "T_max": 10,  # Period of cosine annealing
            "description": "Cosine annealing with restarts",
        }

    elif schedule_type == "step":
        return {
            "schedule_type": "step",
            "initial_lr": max_lr / 3,  # Conservative start
            "step_size": 10,  # Epochs between reductions
            "gamma": 0.5,  # Multiplication factor
            "description": "Step decay schedule",
        }

    else:
        raise ValueError(f"Unknown schedule type: {schedule_type}")

analyze_fidelity_correlation

analyze_fidelity_correlation(result: MultiFidelityResult) -> dict[str, float]

Analyze correlation between different fidelity levels.

Parameters:

Name	Type	Description	Default
result	MultiFidelityResult	Results from multi-fidelity optimization	required

Returns:

Type	Description
dict	Correlation analysis between fidelity levels

Source code in src/dlhub/tuning/multifidelity.py

def analyze_fidelity_correlation(result: MultiFidelityResult) -> dict[str, float]:
    """
    Analyze correlation between different fidelity levels.

    Parameters
    ----------
    result : MultiFidelityResult
        Results from multi-fidelity optimization

    Returns
    -------
    dict
        Correlation analysis between fidelity levels
    """
    config_results = defaultdict(dict)
    for res in result.all_results:
        if res.score != -np.inf:
            config_results[res.config_id][res.fidelity] = res.score

    fidelities = sorted(set(res.fidelity for res in result.all_results))
    correlations = {}

    for i, fid1 in enumerate(fidelities[:-1]):
        for fid2 in fidelities[i + 1 :]:
            common_configs = []
            scores1, scores2 = [], []

            for config_id, fid_scores in config_results.items():
                if fid1 in fid_scores and fid2 in fid_scores:
                    scores1.append(fid_scores[fid1])
                    scores2.append(fid_scores[fid2])

            if len(scores1) >= 3:  # Need at least 3 points for correlation
                correlation = np.corrcoef(scores1, scores2)[0, 1]
                if not np.isnan(correlation):
                    correlations[f"fidelity_{fid1}_vs_{fid2}"] = correlation

    return correlations

asha_optimize

asha_optimize(eval_function: Callable[[dict, int], tuple[float, dict]], initial_configurations: list[dict[str, Any]], min_fidelity: int = 1, max_fidelity: int = 81, reduction_factor: int = 3, max_iterations: int = 100, max_concurrent: int = 4, timeout: float | None = None, random_state: int | None = None, verbose: bool = True) -> MultiFidelityResult

Convenience function for ASHA optimization.

Parameters:

Name	Type	Description	Default
eval_function	callable	Function that takes (hyperparams, fidelity) and returns (score, metadata)	required
initial_configurations	list	Initial hyperparameter configurations to evaluate; must be non-empty	required
min_fidelity	int	Minimum fidelity level	1
max_fidelity	int	Maximum fidelity level	81
reduction_factor	int	ASHA reduction factor	3
max_iterations	int	Maximum number of evaluations, performed and recorded alike	100
max_concurrent	int	Maximum concurrent evaluations	4
timeout	float	Timeout in seconds; evaluations already running are still awaited	None
random_state	int	Random seed	None
verbose	bool	Whether to print progress	True

Returns:

Type	Description
MultiFidelityResult	Optimization results. The three best_* fields are None if the run was granted no budget and so recorded nothing.

Raises:

Type	Description
ValueError	If initial_configurations is empty

Examples:

>>> def evaluate_model(hyperparams, fidelity):
...     # Simulate training with given hyperparameters and fidelity
...     lr = hyperparams["learning_rate"]
...     wd = hyperparams["weight_decay"]
...
...     # Simulate performance improving with fidelity
...     base_score = 0.7 - (lr - 0.001) ** 2 - (wd - 0.0001) ** 2
...     fidelity_bonus = 0.2 * (1 - np.exp(-fidelity / 20))
...     noise = np.random.normal(0, 0.01)
...
...     score = base_score + fidelity_bonus + noise
...     metadata = {"fidelity_used": fidelity}
...
...     return score, metadata
>>>
>>> configs = [
...     {"learning_rate": 0.001, "weight_decay": 0.0001},
...     {"learning_rate": 0.01, "weight_decay": 0.001},
...     {"learning_rate": 0.0001, "weight_decay": 0.00001},
... ]
>>>
>>> result = asha_optimize(
...     evaluate_model, configs, min_fidelity=1, max_fidelity=27, max_iterations=20
... )

Source code in src/dlhub/tuning/multifidelity.py

def asha_optimize(
    eval_function: Callable[[dict, int], tuple[float, dict]],
    initial_configurations: list[dict[str, Any]],
    min_fidelity: int = 1,
    max_fidelity: int = 81,
    reduction_factor: int = 3,
    max_iterations: int = 100,
    max_concurrent: int = 4,
    timeout: float | None = None,
    random_state: int | None = None,
    verbose: bool = True,
) -> MultiFidelityResult:
    """
    Convenience function for ASHA optimization.

    Parameters
    ----------
    eval_function : callable
        Function that takes (hyperparams, fidelity) and returns (score, metadata)
    initial_configurations : list
        Initial hyperparameter configurations to evaluate; must be non-empty
    min_fidelity : int, default=1
        Minimum fidelity level
    max_fidelity : int, default=81
        Maximum fidelity level
    reduction_factor : int, default=3
        ASHA reduction factor
    max_iterations : int, default=100
        Maximum number of evaluations, performed and recorded alike
    max_concurrent : int, default=4
        Maximum concurrent evaluations
    timeout : float, optional
        Timeout in seconds; evaluations already running are still awaited
    random_state : int, optional
        Random seed
    verbose : bool, default=True
        Whether to print progress

    Returns
    -------
    MultiFidelityResult
        Optimization results. The three `best_*` fields are None if the run was
        granted no budget and so recorded nothing.

    Raises
    ------
    ValueError
        If `initial_configurations` is empty

    Examples
    --------
    >>> def evaluate_model(hyperparams, fidelity):
    ...     # Simulate training with given hyperparameters and fidelity
    ...     lr = hyperparams["learning_rate"]
    ...     wd = hyperparams["weight_decay"]
    ...
    ...     # Simulate performance improving with fidelity
    ...     base_score = 0.7 - (lr - 0.001) ** 2 - (wd - 0.0001) ** 2
    ...     fidelity_bonus = 0.2 * (1 - np.exp(-fidelity / 20))
    ...     noise = np.random.normal(0, 0.01)
    ...
    ...     score = base_score + fidelity_bonus + noise
    ...     metadata = {"fidelity_used": fidelity}
    ...
    ...     return score, metadata
    >>>
    >>> configs = [
    ...     {"learning_rate": 0.001, "weight_decay": 0.0001},
    ...     {"learning_rate": 0.01, "weight_decay": 0.001},
    ...     {"learning_rate": 0.0001, "weight_decay": 0.00001},
    ... ]
    >>>
    >>> result = asha_optimize(
    ...     evaluate_model, configs, min_fidelity=1, max_fidelity=27, max_iterations=20
    ... )
    """
    evaluator = FunctionEvaluator(eval_function, min_fidelity, max_fidelity)

    optimizer = ASHAOptimizer(
        evaluator=evaluator,
        reduction_factor=reduction_factor,
        min_budget=min_fidelity,
        max_budget=max_fidelity,
        max_concurrent=max_concurrent,
        random_state=random_state,
    )

    return optimizer.optimize(
        initial_configurations=initial_configurations,
        max_iterations=max_iterations,
        timeout=timeout,
        verbose=verbose,
    )

pbt_optimize

pbt_optimize(train_function: Callable[[dict, int], tuple[float, Any]], save_function: Callable[[], Any], load_function: Callable[[Any], None], reset_function: Callable[[], None], initial_hyperparams: list[dict[str, Any]], hyperparam_distributions: dict[str, HyperparameterDistribution], population_size: int = 10, max_steps: int = 10000, eval_interval: int = 100, exploit_fraction: float = 0.2, explore_fraction: float = 0.2, random_state: int | None = None, verbose: bool = True) -> PBTResult

Convenience function for Population-Based Training.

Parameters:

Name	Type	Description	Default
train_function	callable	Function that takes (hyperparams, steps) and returns (score, state)	required
save_function	callable	Function that returns current model state	required
load_function	callable	Function that loads given model state	required
reset_function	callable	Function that resets model to initial state	required
initial_hyperparams	list	Initial hyperparameter configurations	required
hyperparam_distributions	dict	Hyperparameter perturbation distributions	required
population_size	int	Size of population	10
max_steps	int	Maximum training steps	10000
eval_interval	int	Steps between evaluations	100
exploit_fraction	float	Fraction to exploit	0.2
explore_fraction	float	Fraction to explore	0.2
random_state	int	Random seed	None
verbose	bool	Whether to print progress	True

Returns:

Type	Description
PBTResult	Training results

Examples:

>>> # Define training functions
>>> def train_step(hyperparams, steps):
...     # Simulate training
...     lr = hyperparams["learning_rate"]
...     # Performance improves with more steps but depends on lr
...     performance = 0.8 - (lr - 0.001) ** 2 + steps * 0.001
...     return performance, {"step": steps}
>>>
>>> def save_state():
...     return getattr(save_state, "state", {})
>>>
>>> def load_state(state):
...     save_state.state = state
>>>
>>> def reset():
...     save_state.state = {}
>>>
>>> # Define hyperparameters
>>> initial_configs = [
...     {"learning_rate": 0.001},
...     {"learning_rate": 0.01},
...     {"learning_rate": 0.0001},
... ]
>>>
>>> distributions = {
...     "learning_rate": LogUniformPerturbation((0.8, 1.2), (1e-5, 1e-1))
... }
>>>
>>> result = pbt_optimize(
...     train_step,
...     save_state,
...     load_state,
...     reset,
...     initial_configs,
...     distributions,
...     population_size=5,
... )

Source code in src/dlhub/tuning/population_based.py

def pbt_optimize(
    train_function: Callable[[dict, int], tuple[float, Any]],
    save_function: Callable[[], Any],
    load_function: Callable[[Any], None],
    reset_function: Callable[[], None],
    initial_hyperparams: list[dict[str, Any]],
    hyperparam_distributions: dict[str, HyperparameterDistribution],
    population_size: int = 10,
    max_steps: int = 10000,
    eval_interval: int = 100,
    exploit_fraction: float = 0.2,
    explore_fraction: float = 0.2,
    random_state: int | None = None,
    verbose: bool = True,
) -> PBTResult:
    """
    Convenience function for Population-Based Training.

    Parameters
    ----------
    train_function : callable
        Function that takes (hyperparams, steps) and returns (score, state)
    save_function : callable
        Function that returns current model state
    load_function : callable
        Function that loads given model state
    reset_function : callable
        Function that resets model to initial state
    initial_hyperparams : list
        Initial hyperparameter configurations
    hyperparam_distributions : dict
        Hyperparameter perturbation distributions
    population_size : int, default=10
        Size of population
    max_steps : int, default=10000
        Maximum training steps
    eval_interval : int, default=100
        Steps between evaluations
    exploit_fraction : float, default=0.2
        Fraction to exploit
    explore_fraction : float, default=0.2
        Fraction to explore
    random_state : int, optional
        Random seed
    verbose : bool, default=True
        Whether to print progress

    Returns
    -------
    PBTResult
        Training results

    Examples
    --------
    >>> # Define training functions
    >>> def train_step(hyperparams, steps):
    ...     # Simulate training
    ...     lr = hyperparams["learning_rate"]
    ...     # Performance improves with more steps but depends on lr
    ...     performance = 0.8 - (lr - 0.001) ** 2 + steps * 0.001
    ...     return performance, {"step": steps}
    >>>
    >>> def save_state():
    ...     return getattr(save_state, "state", {})
    >>>
    >>> def load_state(state):
    ...     save_state.state = state
    >>>
    >>> def reset():
    ...     save_state.state = {}
    >>>
    >>> # Define hyperparameters
    >>> initial_configs = [
    ...     {"learning_rate": 0.001},
    ...     {"learning_rate": 0.01},
    ...     {"learning_rate": 0.0001},
    ... ]
    >>>
    >>> distributions = {
    ...     "learning_rate": LogUniformPerturbation((0.8, 1.2), (1e-5, 1e-1))
    ... }
    >>>
    >>> result = pbt_optimize(
    ...     train_step,
    ...     save_state,
    ...     load_state,
    ...     reset,
    ...     initial_configs,
    ...     distributions,
    ...     population_size=5,
    ... )
    """

    def worker_factory():
        return FunctionWorker(
            train_function, save_function, load_function, reset_function
        )

    trainer = PopulationBasedTrainer(
        worker_factory=worker_factory,
        initial_hyperparams=initial_hyperparams,
        hyperparam_distributions=hyperparam_distributions,
        population_size=population_size,
        eval_interval=eval_interval,
        exploit_fraction=exploit_fraction,
        explore_fraction=explore_fraction,
        random_state=random_state,
    )

    return trainer.train(max_steps=max_steps, verbose=verbose)

analyze_parameter_importance

analyze_parameter_importance(result: RandomSearchResult, top_n: int = 10) -> dict[str, float]

Analyze parameter importance using correlation with objective values.

Parameters:

Name	Type	Description	Default
result	RandomSearchResult	Results from random search optimization	required
top_n	int	Number of top configurations to analyze	10

Returns:

Type	Description
dict	Parameter importance scores (correlation coefficients)

Source code in src/dlhub/tuning/random_search.py

def analyze_parameter_importance(
    result: RandomSearchResult, top_n: int = 10
) -> dict[str, float]:
    """
    Analyze parameter importance using correlation with objective values.

    Parameters
    ----------
    result : RandomSearchResult
        Results from random search optimization
    top_n : int, default=10
        Number of top configurations to analyze

    Returns
    -------
    dict
        Parameter importance scores (correlation coefficients)
    """
    if len(result.all_params) < 2:
        return {}

    sorted_indices = np.argsort(result.all_scores)[-top_n:]
    top_params = [result.all_params[i] for i in sorted_indices]
    top_scores = [result.all_scores[i] for i in sorted_indices]

    importance_scores = {}
    param_names = list(result.all_params[0].keys())

    for param_name in param_names:
        param_values = []
        for params in top_params:
            val = params[param_name]
            if isinstance(val, (int, float)):
                param_values.append(float(val))
            else:
                # For categorical parameters, skip importance analysis
                continue

        if len(param_values) > 1:
            correlation = np.corrcoef(param_values, top_scores)[0, 1]
            if not np.isnan(correlation):
                importance_scores[param_name] = abs(correlation)

    importance_scores = dict(
        sorted(importance_scores.items(), key=lambda x: x[1], reverse=True)
    )

    return importance_scores

bayesian

Bayesian Optimization for Hyperparameter Tuning

Implements Bayesian optimization using Gaussian Process surrogate models with Expected Improvement acquisition function for efficient hyperparameter search. This approach is particularly effective for expensive black-box optimization problems like neural network hyperparameter tuning.

References

• Snoek, J., Larochelle, H., & Adams, R. P. (2012). "Practical Bayesian Optimization of Machine Learning Algorithms." NIPS.
• Mockus, J. (1994). "Application of Bayesian approach to numerical methods of global and stochastic optimization." Journal of Global Optimization.

Author

Deep Learning Reference Hub

License

MIT License

Notes

This implementation uses a simplified Gaussian Process with RBF kernel. For production use, consider libraries like Optuna, GPyOpt, or scikit-optimize which provide more robust implementations with additional features.

BayesianOptimizationResult dataclass

Container for Bayesian optimization results.

Attributes:

Name	Type	Description
best_params	dict	Best hyperparameter configuration found
best_score	float	Best objective function value achieved
history	list	History of all evaluations
convergence_data	dict	Convergence statistics and diagnostics

Source code in src/dlhub/tuning/bayesian.py

@dataclass
class BayesianOptimizationResult:
    """
    Container for Bayesian optimization results.

    Attributes
    ----------
    best_params : dict
        Best hyperparameter configuration found
    best_score : float
        Best objective function value achieved
    history : list
        History of all evaluations
    convergence_data : dict
        Convergence statistics and diagnostics
    """

    best_params: dict[str, Any]
    best_score: float
    history: list[tuple[dict[str, Any], float]]
    convergence_data: dict[str, Any]

GaussianProcess

Simplified Gaussian Process for Bayesian Optimization.

Implements a GP with RBF kernel for modeling the objective function. This is a educational implementation - production code should use more robust libraries like GPy or scikit-learn.

Parameters:

Name	Type	Description	Default
kernel_lengthscale	float	Length scale parameter for RBF kernel	1.0
kernel_variance	float	Variance parameter for RBF kernel	1.0
noise_variance	float	Noise variance for numerical stability	1e-6

Source code in src/dlhub/tuning/bayesian.py

class GaussianProcess:
    """
    Simplified Gaussian Process for Bayesian Optimization.

    Implements a GP with RBF kernel for modeling the objective function.
    This is a educational implementation - production code should use
    more robust libraries like GPy or scikit-learn.

    Parameters
    ----------
    kernel_lengthscale : float, default=1.0
        Length scale parameter for RBF kernel
    kernel_variance : float, default=1.0
        Variance parameter for RBF kernel
    noise_variance : float, default=1e-6
        Noise variance for numerical stability
    """

    def __init__(
        self,
        kernel_lengthscale: float = 1.0,
        kernel_variance: float = 1.0,
        noise_variance: float = 1e-6,
    ):
        self.kernel_lengthscale = kernel_lengthscale
        self.kernel_variance = kernel_variance
        self.noise_variance = noise_variance
        self.X_train = None
        self.y_train = None
        self.K_inv = None

    def rbf_kernel(self, X1: np.ndarray, X2: np.ndarray) -> np.ndarray:
        """
        Compute RBF (Radial Basis Function) kernel matrix.

        Parameters
        ----------
        X1 : np.ndarray, shape (n1, d)
            First set of input points
        X2 : np.ndarray, shape (n2, d)
            Second set of input points

        Returns
        -------
        np.ndarray, shape (n1, n2)
            Kernel matrix K(X1, X2)
        """
        sq_dists = np.sum((X1[:, np.newaxis, :] - X2[np.newaxis, :, :]) ** 2, axis=2)

        return self.kernel_variance * np.exp(
            -0.5 * sq_dists / (self.kernel_lengthscale**2)
        )

    def fit(self, X: np.ndarray, y: np.ndarray) -> None:
        """
        Fit the Gaussian Process to training data.

        Parameters
        ----------
        X : np.ndarray, shape (n_samples, n_features)
            Training input points
        y : np.ndarray, shape (n_samples,)
            Training target values
        """
        self.X_train = X.copy()
        self.y_train = y.copy()

        # Compute kernel matrix and its inverse
        K = self.rbf_kernel(X, X)
        K += self.noise_variance * np.eye(len(X))

        try:
            self.K_inv = np.linalg.inv(K)
        except np.linalg.LinAlgError:
            warnings.warn("Kernel matrix is singular, using pseudo-inverse")
            self.K_inv = np.linalg.pinv(K)

    def predict(self, X: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
        """
        Make predictions with uncertainty estimates.

        Parameters
        ----------
        X : np.ndarray, shape (n_test, n_features)
            Test input points

        Returns
        -------
        mean : np.ndarray, shape (n_test,)
            Predicted mean values
        std : np.ndarray, shape (n_test,)
            Predicted standard deviations
        """
        if self.X_train is None:
            raise ValueError("GP must be fitted before making predictions")

        # Compute kernel matrices
        K_star = self.rbf_kernel(X, self.X_train)
        K_star_star = self.rbf_kernel(X, X)

        # Compute predictive mean
        mean = K_star @ self.K_inv @ self.y_train  # type: ignore

        var = np.diag(K_star_star) - np.diag(K_star @ self.K_inv @ K_star.T)
        var = np.maximum(var, 1e-10)
        std = np.sqrt(var)

        return mean, std

rbf_kernel

rbf_kernel(X1: ndarray, X2: ndarray) -> np.ndarray

Compute RBF (Radial Basis Function) kernel matrix.

Parameters:

Name	Type	Description	Default
X1	(ndarray, shape(n1, d))	First set of input points	required
X2	(ndarray, shape(n2, d))	Second set of input points	required

Returns:

Type	Description
(ndarray, shape(n1, n2))	Kernel matrix K(X1, X2)

Source code in src/dlhub/tuning/bayesian.py

def rbf_kernel(self, X1: np.ndarray, X2: np.ndarray) -> np.ndarray:
    """
    Compute RBF (Radial Basis Function) kernel matrix.

    Parameters
    ----------
    X1 : np.ndarray, shape (n1, d)
        First set of input points
    X2 : np.ndarray, shape (n2, d)
        Second set of input points

    Returns
    -------
    np.ndarray, shape (n1, n2)
        Kernel matrix K(X1, X2)
    """
    sq_dists = np.sum((X1[:, np.newaxis, :] - X2[np.newaxis, :, :]) ** 2, axis=2)

    return self.kernel_variance * np.exp(
        -0.5 * sq_dists / (self.kernel_lengthscale**2)
    )

fit

fit(X: ndarray, y: ndarray) -> None

Fit the Gaussian Process to training data.

Parameters:

Name	Type	Description	Default
X	(ndarray, shape(n_samples, n_features))	Training input points	required
y	(ndarray, shape(n_samples))	Training target values	required

Source code in src/dlhub/tuning/bayesian.py

def fit(self, X: np.ndarray, y: np.ndarray) -> None:
    """
    Fit the Gaussian Process to training data.

    Parameters
    ----------
    X : np.ndarray, shape (n_samples, n_features)
        Training input points
    y : np.ndarray, shape (n_samples,)
        Training target values
    """
    self.X_train = X.copy()
    self.y_train = y.copy()

    # Compute kernel matrix and its inverse
    K = self.rbf_kernel(X, X)
    K += self.noise_variance * np.eye(len(X))

    try:
        self.K_inv = np.linalg.inv(K)
    except np.linalg.LinAlgError:
        warnings.warn("Kernel matrix is singular, using pseudo-inverse")
        self.K_inv = np.linalg.pinv(K)

predict

predict(X: ndarray) -> tuple[np.ndarray, np.ndarray]

Make predictions with uncertainty estimates.

Parameters:

Name	Type	Description	Default
X	(ndarray, shape(n_test, n_features))	Test input points	required

Returns:

Name	Type	Description
mean	(ndarray, shape(n_test))	Predicted mean values
std	(ndarray, shape(n_test))	Predicted standard deviations

Source code in src/dlhub/tuning/bayesian.py

def predict(self, X: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
    """
    Make predictions with uncertainty estimates.

    Parameters
    ----------
    X : np.ndarray, shape (n_test, n_features)
        Test input points

    Returns
    -------
    mean : np.ndarray, shape (n_test,)
        Predicted mean values
    std : np.ndarray, shape (n_test,)
        Predicted standard deviations
    """
    if self.X_train is None:
        raise ValueError("GP must be fitted before making predictions")

    # Compute kernel matrices
    K_star = self.rbf_kernel(X, self.X_train)
    K_star_star = self.rbf_kernel(X, X)

    # Compute predictive mean
    mean = K_star @ self.K_inv @ self.y_train  # type: ignore

    var = np.diag(K_star_star) - np.diag(K_star @ self.K_inv @ K_star.T)
    var = np.maximum(var, 1e-10)
    std = np.sqrt(var)

    return mean, std

BayesianOptimizer

Bayesian Optimization using Gaussian Process surrogate models.

This implementation uses Expected Improvement as the acquisition function to balance exploration and exploitation in hyperparameter search.

Parameters:

Name	Type	Description	Default
objective_function	callable	Function to optimize. Should take hyperparameter dict and return float	required
search_space	dict	Dictionary defining search space for each hyperparameter. Format: {'param_name': (min_val, max_val)} for continuous parameters	required
acquisition	str	Acquisition function ('ei' for Expected Improvement, 'ucb' for UCB)	'ei'
kappa	float	Exploration parameter for UCB (ignored if acquisition='ei')	2.576
xi	float	Exploration parameter for Expected Improvement	0.01
n_initial	int	Number of random initial evaluations	5
random_state	int	Random seed for reproducibility	None

Source code in src/dlhub/tuning/bayesian.py

class BayesianOptimizer:
    """
    Bayesian Optimization using Gaussian Process surrogate models.

    This implementation uses Expected Improvement as the acquisition function
    to balance exploration and exploitation in hyperparameter search.

    Parameters
    ----------
    objective_function : callable
        Function to optimize. Should take hyperparameter dict and return float
    search_space : dict
        Dictionary defining search space for each hyperparameter.
        Format: {'param_name': (min_val, max_val)} for continuous parameters
    acquisition : str, default='ei'
        Acquisition function ('ei' for Expected Improvement, 'ucb' for UCB)
    kappa : float, default=2.576
        Exploration parameter for UCB (ignored if acquisition='ei')
    xi : float, default=0.01
        Exploration parameter for Expected Improvement
    n_initial : int, default=5
        Number of random initial evaluations
    random_state : int, optional
        Random seed for reproducibility
    """

    def __init__(
        self,
        objective_function: Callable[[dict], float],
        search_space: dict[str, tuple[float, float]],
        acquisition: str = "ei",
        kappa: float = 2.576,
        xi: float = 0.01,
        n_initial: int = 5,
        random_state: int | None = None,
    ):

        self.objective_function = objective_function
        self.search_space = search_space
        self.acquisition = acquisition.lower()
        self.kappa = kappa
        self.xi = xi
        self.n_initial = n_initial

        # Checked here rather than where the acquisition is first evaluated,
        # which is after the initial design has run. Each of those evaluations
        # is a full training run, so a misspelled name must not cost them.
        if self.acquisition not in ("ei", "ucb"):
            raise ValueError(
                f"Unknown acquisition function: {acquisition!r}. Use 'ei' or 'ucb'."
            )

        if random_state is not None:
            np.random.seed(random_state)

        # Initialize internal state
        self.param_names = list(search_space.keys())
        self.bounds = np.array([search_space[name] for name in self.param_names])
        self.gp = GaussianProcess()
        self.X_observed = []
        self.y_observed = []
        self.history = []
        self.best_score = -np.inf
        self.best_params = None

    def _normalize_params(self, X: np.ndarray) -> np.ndarray:
        """Normalize parameters to [0, 1] range."""
        return (X - self.bounds[:, 0]) / (self.bounds[:, 1] - self.bounds[:, 0])

    def _denormalize_params(self, X_norm: np.ndarray) -> np.ndarray:
        """Denormalize parameters from [0, 1] to original range."""
        return X_norm * (self.bounds[:, 1] - self.bounds[:, 0]) + self.bounds[:, 0]

    def _array_to_dict(self, X: np.ndarray) -> dict[str, float]:
        """Convert parameter array to dictionary."""
        return {name: float(val) for name, val in zip(self.param_names, X)}

    def _expected_improvement(self, X: np.ndarray) -> np.ndarray:
        """
        Compute Expected Improvement acquisition function.

        Parameters
        ----------
        X : np.ndarray, shape (n_points, n_params)
            Normalized parameter points to evaluate

        Returns
        -------
        np.ndarray, shape (n_points,)
            Expected improvement values
        """
        if len(self.X_observed) == 0:
            return np.ones(len(X))

        mean, std = self.gp.predict(X)

        f_max = max(self.y_observed)

        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            z = (mean - f_max - self.xi) / std
            ei = (mean - f_max - self.xi) * norm.cdf(z) + std * norm.pdf(z)
            ei[std == 0.0] = 0.0

        return ei

    def _upper_confidence_bound(self, X: np.ndarray) -> np.ndarray:
        """
        Compute Upper Confidence Bound acquisition function.

        Parameters
        ----------
        X : np.ndarray, shape (n_points, n_params)
            Normalized parameter points to evaluate

        Returns
        -------
        np.ndarray, shape (n_points,)
            Upper confidence bound values
        """
        if len(self.X_observed) == 0:
            return np.ones(len(X))

        mean, std = self.gp.predict(X)
        return mean + self.kappa * std

    def _acquisition_function(self, X: np.ndarray) -> np.ndarray:
        """Evaluate the chosen acquisition function."""
        if self.acquisition == "ei":
            return self._expected_improvement(X)
        elif self.acquisition == "ucb":
            return self._upper_confidence_bound(X)
        else:
            raise ValueError(f"Unknown acquisition function: {self.acquisition}")

    def _optimize_acquisition(self) -> np.ndarray:
        """
        Find the point that maximizes the acquisition function.

        Returns
        -------
        np.ndarray, shape (n_params,)
            Normalized parameters that maximize acquisition function
        """

        # Objective to minimize (negative acquisition)
        def objective(x):
            return -self._acquisition_function(x.reshape(1, -1))[0]

        # Try multiple random starting points
        n_restarts = 10
        best_x = None
        best_val = np.inf

        for _ in range(n_restarts):
            x0 = np.random.uniform(0, 1, len(self.param_names))

            try:
                result = minimize(
                    objective,
                    x0,
                    bounds=[(0, 1)] * len(self.param_names),
                    method="L-BFGS-B",
                )

                if result.fun < best_val:
                    best_val = result.fun
                    best_x = result.x
            except Exception:
                continue

        if best_x is None:
            best_x = np.random.uniform(0, 1, len(self.param_names))

        return best_x

    def _evaluate_objective(self, params: dict[str, float]) -> float:
        """
        Evaluate objective function and handle exceptions.

        Parameters
        ----------
        params : dict
            Hyperparameter configuration

        Returns
        -------
        float
            Objective function value (np.nan if evaluation failed)
        """
        try:
            score = self.objective_function(params)
            if np.isnan(score) or np.isinf(score):
                return np.nan
            return float(score)
        except Exception as e:
            warnings.warn(f"Objective evaluation failed: {e}")
            return np.nan

    def optimize(
        self, n_iterations: int = 20, verbose: int = 1
    ) -> BayesianOptimizationResult:
        """
        Run Bayesian optimization.

        Parameters
        ----------
        n_iterations : int, default=20
            Maximum number of optimization iterations
        verbose : int, default=1
            Reporting level: 0 is silent, 1 prints phases and new bests, 2 adds
            every evaluation and the final configuration.

        Returns
        -------
        BayesianOptimizationResult
            Optimization results including best parameters and history
        """
        if verbose >= 1:
            print("Starting Bayesian Optimization...")
            print(f"Search space: {self.search_space}")
            print(f"Acquisition function: {self.acquisition}")

        if verbose >= 1:
            print(f"\nPhase 1: Random initialization ({self.n_initial} points)")

        for i in range(self.n_initial):
            X_raw = np.random.uniform(self.bounds[:, 0], self.bounds[:, 1])
            params = self._array_to_dict(X_raw)

            score = self._evaluate_objective(params)

            if not np.isnan(score):
                X_norm = self._normalize_params(X_raw)
                self.X_observed.append(X_norm)
                self.y_observed.append(score)
                self.history.append((params.copy(), score))

                if score > self.best_score:
                    self.best_score = score
                    self.best_params = params.copy()

                if verbose >= 1:
                    print(f"  {i + 1}/{self.n_initial}: Score = {score:.4f}")

        if len(self.X_observed) == 0:
            raise RuntimeError("All initial evaluations failed")

        if verbose >= 1:
            print(f"\nPhase 2: Bayesian optimization ({n_iterations} iterations)")

        for iteration in range(n_iterations):
            X_train = np.array(self.X_observed)
            y_train = np.array(self.y_observed)
            self.gp.fit(X_train, y_train)

            X_next_norm = self._optimize_acquisition()
            X_next_raw = self._denormalize_params(X_next_norm)
            params_next = self._array_to_dict(X_next_raw)

            score = self._evaluate_objective(params_next)

            if not np.isnan(score):
                self.X_observed.append(X_next_norm)
                self.y_observed.append(score)
                self.history.append((params_next.copy(), score))

                if score > self.best_score:
                    self.best_score = score
                    self.best_params = params_next.copy()

                    if verbose >= 1:
                        print(
                            f"  Iter {iteration + 1}: Score = {score:.4f} (NEW BEST!)"
                        )
                else:
                    if verbose >= 2:
                        print(f"  Iter {iteration + 1}: Score = {score:.4f}")
            else:
                if verbose >= 1:
                    print(f"  Iter {iteration + 1}: Evaluation failed")

        successful_initial = [s for _, s in self.history[: self.n_initial]]
        convergence_data = {
            "n_evaluations": len(self.history),
            "n_failed": n_iterations + self.n_initial - len(self.history),
            # The initial design is random, so its mean is what an equal budget
            # of random search would have averaged. Guarded because every
            # initial point can fail while later ones succeed, and a mean over
            # an empty list is a nan that propagates into the whole summary.
            "improvement_over_random": (
                self.best_score - float(np.mean(successful_initial))
                if successful_initial
                else 0.0
            ),
            "scores": [score for _, score in self.history],
        }

        if verbose >= 1:
            print("\nOptimization completed!")
            print(f"Best score: {self.best_score:.4f}")
        if verbose >= 2:
            print(f"Best parameters: {self.best_params}")
            print(f"Total evaluations: {len(self.history)}")

        return BayesianOptimizationResult(
            best_params=self.best_params,  # type: ignore
            best_score=self.best_score,
            history=self.history,
            convergence_data=convergence_data,
        )

optimize

optimize(n_iterations: int = 20, verbose: int = 1) -> BayesianOptimizationResult

Run Bayesian optimization.

Parameters:

Name	Type	Description	Default
n_iterations	int	Maximum number of optimization iterations	20
verbose	int	Reporting level: 0 is silent, 1 prints phases and new bests, 2 adds every evaluation and the final configuration.	1

Returns:

Type	Description
BayesianOptimizationResult	Optimization results including best parameters and history

Source code in src/dlhub/tuning/bayesian.py

def optimize(
    self, n_iterations: int = 20, verbose: int = 1
) -> BayesianOptimizationResult:
    """
    Run Bayesian optimization.

    Parameters
    ----------
    n_iterations : int, default=20
        Maximum number of optimization iterations
    verbose : int, default=1
        Reporting level: 0 is silent, 1 prints phases and new bests, 2 adds
        every evaluation and the final configuration.

    Returns
    -------
    BayesianOptimizationResult
        Optimization results including best parameters and history
    """
    if verbose >= 1:
        print("Starting Bayesian Optimization...")
        print(f"Search space: {self.search_space}")
        print(f"Acquisition function: {self.acquisition}")

    if verbose >= 1:
        print(f"\nPhase 1: Random initialization ({self.n_initial} points)")

    for i in range(self.n_initial):
        X_raw = np.random.uniform(self.bounds[:, 0], self.bounds[:, 1])
        params = self._array_to_dict(X_raw)

        score = self._evaluate_objective(params)

        if not np.isnan(score):
            X_norm = self._normalize_params(X_raw)
            self.X_observed.append(X_norm)
            self.y_observed.append(score)
            self.history.append((params.copy(), score))

            if score > self.best_score:
                self.best_score = score
                self.best_params = params.copy()

            if verbose >= 1:
                print(f"  {i + 1}/{self.n_initial}: Score = {score:.4f}")

    if len(self.X_observed) == 0:
        raise RuntimeError("All initial evaluations failed")

    if verbose >= 1:
        print(f"\nPhase 2: Bayesian optimization ({n_iterations} iterations)")

    for iteration in range(n_iterations):
        X_train = np.array(self.X_observed)
        y_train = np.array(self.y_observed)
        self.gp.fit(X_train, y_train)

        X_next_norm = self._optimize_acquisition()
        X_next_raw = self._denormalize_params(X_next_norm)
        params_next = self._array_to_dict(X_next_raw)

        score = self._evaluate_objective(params_next)

        if not np.isnan(score):
            self.X_observed.append(X_next_norm)
            self.y_observed.append(score)
            self.history.append((params_next.copy(), score))

            if score > self.best_score:
                self.best_score = score
                self.best_params = params_next.copy()

                if verbose >= 1:
                    print(
                        f"  Iter {iteration + 1}: Score = {score:.4f} (NEW BEST!)"
                    )
            else:
                if verbose >= 2:
                    print(f"  Iter {iteration + 1}: Score = {score:.4f}")
        else:
            if verbose >= 1:
                print(f"  Iter {iteration + 1}: Evaluation failed")

    successful_initial = [s for _, s in self.history[: self.n_initial]]
    convergence_data = {
        "n_evaluations": len(self.history),
        "n_failed": n_iterations + self.n_initial - len(self.history),
        # The initial design is random, so its mean is what an equal budget
        # of random search would have averaged. Guarded because every
        # initial point can fail while later ones succeed, and a mean over
        # an empty list is a nan that propagates into the whole summary.
        "improvement_over_random": (
            self.best_score - float(np.mean(successful_initial))
            if successful_initial
            else 0.0
        ),
        "scores": [score for _, score in self.history],
    }

    if verbose >= 1:
        print("\nOptimization completed!")
        print(f"Best score: {self.best_score:.4f}")
    if verbose >= 2:
        print(f"Best parameters: {self.best_params}")
        print(f"Total evaluations: {len(self.history)}")

    return BayesianOptimizationResult(
        best_params=self.best_params,  # type: ignore
        best_score=self.best_score,
        history=self.history,
        convergence_data=convergence_data,
    )

bayesian_optimize

bayesian_optimize(objective_function: Callable[[dict], float], search_space: dict[str, tuple[float, float]], n_iterations: int = 20, n_initial: int = 5, acquisition: str = 'ei', random_state: int | None = None, verbose: int = 1) -> BayesianOptimizationResult

Run a Bayesian search over continuous bounds, returning its own result type.

The method-specific entry point, alongside :func:~dlhub.tuning.random_search and :func:~dlhub.tuning.asha_optimize. To choose a method by name instead, call :func:~dlhub.tuning.optimize_hyperparameters, which takes the framework's richer search-space format and returns an ExperimentResult.

Takes a scalar objective and a {name: (low, high)} search space; every dimension is continuous, since the Gaussian process interpolates over them.

Parameters:

Name	Type	Description	Default
objective_function	callable	Function to optimize. Should accept hyperparameter dict and return float	required
search_space	dict	Search space definition: {'param_name': (min_val, max_val)}	required
n_iterations	int	Number of optimization iterations after initial random sampling	20
n_initial	int	Number of random initial points	5
acquisition	str	Acquisition function ('ei' or 'ucb')	'ei'
random_state	int	Random seed for reproducibility	None
verbose	int	Reporting level, as in :meth:BayesianOptimizer.optimize	1

Returns:

Type	Description
BayesianOptimizationResult	Optimization results

Examples:

>>> def objective(params):
...     # Simulate training a model and return validation accuracy
...     lr, wd = params["learning_rate"], params["weight_decay"]
...     # Dummy objective (replace with actual model training)
...     return -((lr - 0.001) ** 2) - (wd - 0.0001) ** 2 + np.random.normal(0, 0.01)
>>>
>>> search_space = {"learning_rate": (1e-5, 1e-1), "weight_decay": (1e-6, 1e-2)}
>>>
>>> result = bayesian_optimize(
...     objective, search_space, n_iterations=30, random_state=42
... )
>>> print(f"Best parameters: {result.best_params}")

Source code in src/dlhub/tuning/bayesian.py

def bayesian_optimize(
    objective_function: Callable[[dict], float],
    search_space: dict[str, tuple[float, float]],
    n_iterations: int = 20,
    n_initial: int = 5,
    acquisition: str = "ei",
    random_state: int | None = None,
    verbose: int = 1,
) -> BayesianOptimizationResult:
    """
    Run a Bayesian search over continuous bounds, returning its own result type.

    The method-specific entry point, alongside :func:`~dlhub.tuning.random_search`
    and :func:`~dlhub.tuning.asha_optimize`. To choose a method by name instead,
    call :func:`~dlhub.tuning.optimize_hyperparameters`, which takes the
    framework's richer search-space format and returns an ``ExperimentResult``.

    Takes a scalar objective and a ``{name: (low, high)}`` search space; every
    dimension is continuous, since the Gaussian process interpolates over them.

    Parameters
    ----------
    objective_function : callable
        Function to optimize. Should accept hyperparameter dict and return float
    search_space : dict
        Search space definition: {'param_name': (min_val, max_val)}
    n_iterations : int, default=20
        Number of optimization iterations after initial random sampling
    n_initial : int, default=5
        Number of random initial points
    acquisition : str, default='ei'
        Acquisition function ('ei' or 'ucb')
    random_state : int, optional
        Random seed for reproducibility
    verbose : int, default=1
        Reporting level, as in :meth:`BayesianOptimizer.optimize`

    Returns
    -------
    BayesianOptimizationResult
        Optimization results

    Examples
    --------
    >>> def objective(params):
    ...     # Simulate training a model and return validation accuracy
    ...     lr, wd = params["learning_rate"], params["weight_decay"]
    ...     # Dummy objective (replace with actual model training)
    ...     return -((lr - 0.001) ** 2) - (wd - 0.0001) ** 2 + np.random.normal(0, 0.01)
    >>>
    >>> search_space = {"learning_rate": (1e-5, 1e-1), "weight_decay": (1e-6, 1e-2)}
    >>>
    >>> result = bayesian_optimize(
    ...     objective, search_space, n_iterations=30, random_state=42
    ... )
    >>> print(f"Best parameters: {result.best_params}")
    """
    optimizer = BayesianOptimizer(
        objective_function=objective_function,
        search_space=search_space,
        acquisition=acquisition,
        n_initial=n_initial,
        random_state=random_state,
    )

    return optimizer.optimize(n_iterations=n_iterations, verbose=verbose)

quadratic_objective

quadratic_objective(params)

Example objective function - quadratic with noise.

Source code in src/dlhub/tuning/bayesian.py

def quadratic_objective(params):
    """Example objective function - quadratic with noise."""
    x, y = params["x"], params["y"]
    # Global minimum at (2, -1) with value -5
    return -((x - 2) ** 2) - (y + 1) ** 2 - 5 + np.random.normal(0, 0.1)

nn_objective

nn_objective(params)

Dummy neural network training objective.

Source code in src/dlhub/tuning/bayesian.py

def nn_objective(params):
    """Dummy neural network training objective."""
    lr = params["learning_rate"]
    batch_size = int(params["batch_size"])
    hidden_units = int(params["hidden_units"])

    optimal_lr = 0.003
    optimal_batch = 64
    optimal_hidden = 256

    # Penalty terms for deviations from optimal values
    lr_penalty = -10 * (np.log10(lr) - np.log10(optimal_lr)) ** 2
    batch_penalty = -0.001 * (batch_size - optimal_batch) ** 2
    hidden_penalty = -0.00001 * (hidden_units - optimal_hidden) ** 2

    # Base performance + penalties + noise
    performance = 0.95 + lr_penalty + batch_penalty + hidden_penalty
    performance += np.random.normal(0, 0.02)  # Training noise

    return performance

framework

Modern Hyperparameter Tuning Framework

Production-ready framework integrating multiple optimization strategies with experiment tracking and statistical analysis. This framework provides a unified interface for various hyperparameter optimization methods and includes tools for result comparison, visualization, and reproducibility.

References

• Feurer, M., & Hutter, F. (2019). "Hyperparameter Optimization." Automated Machine Learning: Methods, Systems, Challenges.
• Liaw, R., et al. (2018). "Tune: A Research Platform for Distributed Model Selection and Training." arXiv preprint arXiv:1807.05118.

Author

Deep Learning Reference Hub

License

MIT License

Notes

This framework is designed to be: 1. Flexible - supports multiple optimization strategies 2. Extensible - easy to add new optimizers 3. Reproducible - proper random seeding and logging 4. Production-ready - includes error handling and checkpointing

OptimizationMethod

Bases: Enum

Enumeration of available optimization methods.

Source code in src/dlhub/tuning/framework.py

class OptimizationMethod(Enum):
    """Enumeration of available optimization methods."""

    RANDOM_SEARCH = "random_search"
    BAYESIAN = "bayesian"
    ASHA = "asha"
    PBT = "pbt"
    GRID_SEARCH = "grid_search"

HyperparameterConfig dataclass

Configuration for a single hyperparameter.

Attributes:

Name	Type	Description
name	str	Parameter name
type	str	Parameter type ('continuous', 'integer', 'categorical')
range	tuple or list	Valid range or choices for the parameter
scale	str, default='linear'	Scale for sampling ('linear', 'log')
default	Any(optional)	Default value for said parameter

Source code in src/dlhub/tuning/framework.py

@dataclass
class HyperparameterConfig:
    """
    Configuration for a single hyperparameter.

    Attributes
    ----------
    name : str
        Parameter name
    type : str
        Parameter type ('continuous', 'integer', 'categorical')
    range : tuple or list
        Valid range or choices for the parameter
    scale : str, default='linear'
        Scale for sampling ('linear', 'log')
    default : Any (optional)
        Default value for said parameter
    """

    name: str
    type: str
    range: tuple | list
    scale: str = "linear"
    default: Any = None

ExperimentConfig dataclass

Configuration for hyperparameter optimization experiment.

Attributes:

Name	Type	Description
experiment_name	str	Name of the experiment
optimization_method	OptimizationMethod	Optimization strategy to use
hyperparameters	list	List of HyperparameterConfig objects
objective_metric	str	Name of metric to optimize
maximize	bool, default=True	Whether to maximize the objective metric
n_trials	int, default=100	Number of trials to run
random_seed	int(optional)	Random seed for reproducibility
save_dir	str(optional)	Directory to save results
additional_config	dict(optional)	Additional method-specific configuration

Source code in src/dlhub/tuning/framework.py

@dataclass
class ExperimentConfig:
    """
    Configuration for hyperparameter optimization experiment.

    Attributes
    ----------
    experiment_name : str
        Name of the experiment
    optimization_method : OptimizationMethod
        Optimization strategy to use
    hyperparameters : list
        List of HyperparameterConfig objects
    objective_metric : str
        Name of metric to optimize
    maximize : bool, default=True
        Whether to maximize the objective metric
    n_trials : int, default=100
        Number of trials to run
    random_seed : int (optional)
        Random seed for reproducibility
    save_dir : str (optional)
        Directory to save results
    additional_config : dict (optional)
        Additional method-specific configuration
    """

    experiment_name: str
    optimization_method: OptimizationMethod
    hyperparameters: list[HyperparameterConfig]
    objective_metric: str
    maximize: bool = True
    n_trials: int = 100
    random_seed: int | None = None
    save_dir: str | None = None
    additional_config: dict[str, Any] = field(default_factory=dict)

TrialResult dataclass

Result from a single trial.

Attributes:

Name	Type	Description
trial_id	int	Unique trial identifier
hyperparams	dict	Hyperparameter configuration used
metrics	dict	All metrics recorded
training_time	float	Time taken for training
status	str	Trial status ('success', 'failed', 'cancelled')
metadata	dict	Additional trial information

Source code in src/dlhub/tuning/framework.py

@dataclass
class TrialResult:
    """
    Result from a single trial.

    Attributes
    ----------
    trial_id : int
        Unique trial identifier
    hyperparams : dict
        Hyperparameter configuration used
    metrics : dict
        All metrics recorded
    training_time : float
        Time taken for training
    status : str
        Trial status ('success', 'failed', 'cancelled')
    metadata : dict
        Additional trial information
    """

    trial_id: int
    hyperparams: dict[str, Any]
    metrics: dict[str, float]
    training_time: float
    status: str = "success"
    metadata: dict[str, Any] = field(default_factory=dict)

ExperimentResult dataclass

The outcome of one hyperparameter search: every trial, and the best of them.

Named for the :class:ExperimentConfig it answers. Not to be confused with :class:dlhub.optimizers.OptimizationRun, which traces a single descent.

Attributes:

Name	Type	Description
experiment_config	ExperimentConfig	Configuration used for the experiment
best_trial	TrialResult	Best performing trial
all_trials	list	All trial results
total_time	float	Total optimization time
summary_statistics	dict	Summary statistics and analysis

Source code in src/dlhub/tuning/framework.py

@dataclass
class ExperimentResult:
    """
    The outcome of one hyperparameter search: every trial, and the best of them.

    Named for the :class:`ExperimentConfig` it answers. Not to be confused with
    :class:`dlhub.optimizers.OptimizationRun`, which traces a single descent.

    Attributes
    ----------
    experiment_config : ExperimentConfig
        Configuration used for the experiment
    best_trial : TrialResult
        Best performing trial
    all_trials : list
        All trial results
    total_time : float
        Total optimization time
    summary_statistics : dict
        Summary statistics and analysis
    """

    experiment_config: ExperimentConfig
    best_trial: TrialResult
    all_trials: list[TrialResult]
    total_time: float
    summary_statistics: dict[str, Any] = field(default_factory=dict)

ObjectiveFunction

Bases: ABC

Abstract base class for objective functions.

Defines the interface that objective functions must implement.

Source code in src/dlhub/tuning/framework.py

class ObjectiveFunction(ABC):
    """
    Abstract base class for objective functions.

    Defines the interface that objective functions must implement.
    """

    @abstractmethod
    def evaluate(self, hyperparams: dict[str, Any]) -> dict[str, float]:
        """
        Evaluate hyperparameters and return metrics.

        Parameters
        ----------
        hyperparams : dict
            Hyperparameter configuration

        Returns
        -------
        dict
            Dictionary of metric names to values
        """
        pass

    @abstractmethod
    def get_metric_names(self) -> list[str]:
        """
        Get names of all metrics returned by evaluate.

        Returns
        -------
        list
            List of metric names
        """
        pass

evaluate abstractmethod

evaluate(hyperparams: dict[str, Any]) -> dict[str, float]

Evaluate hyperparameters and return metrics.

Parameters:

Name	Type	Description	Default
hyperparams	dict	Hyperparameter configuration	required

Returns:

Type	Description
dict	Dictionary of metric names to values

Source code in src/dlhub/tuning/framework.py

@abstractmethod
def evaluate(self, hyperparams: dict[str, Any]) -> dict[str, float]:
    """
    Evaluate hyperparameters and return metrics.

    Parameters
    ----------
    hyperparams : dict
        Hyperparameter configuration

    Returns
    -------
    dict
        Dictionary of metric names to values
    """
    pass

get_metric_names abstractmethod

get_metric_names() -> list[str]

Get names of all metrics returned by evaluate.

Returns:

Type	Description
list	List of metric names

Source code in src/dlhub/tuning/framework.py

@abstractmethod
def get_metric_names(self) -> list[str]:
    """
    Get names of all metrics returned by evaluate.

    Returns
    -------
    list
        List of metric names
    """
    pass

FunctionObjective

Bases: ObjectiveFunction

Wrapper for function-based objectives.

Parameters:

Name	Type	Description	Default
eval_function	callable	Function that takes hyperparams and returns metrics dict	required
metric_names	list	Names of metrics returned by eval_function	required

Source code in src/dlhub/tuning/framework.py

class FunctionObjective(ObjectiveFunction):
    """
    Wrapper for function-based objectives.

    Parameters
    ----------
    eval_function : callable
        Function that takes hyperparams and returns metrics dict
    metric_names : list
        Names of metrics returned by eval_function
    """

    def __init__(
        self, eval_function: Callable[[dict], dict[str, float]], metric_names: list[str]
    ):
        self.eval_function = eval_function
        self.metric_names = metric_names

    def evaluate(self, hyperparams: dict[str, Any]) -> dict[str, float]:
        """Evaluate using wrapped function."""
        return self.eval_function(hyperparams)

    def get_metric_names(self) -> list[str]:
        """Get metric names."""
        return self.metric_names

evaluate

evaluate(hyperparams: dict[str, Any]) -> dict[str, float]

Evaluate using wrapped function.

Source code in src/dlhub/tuning/framework.py

def evaluate(self, hyperparams: dict[str, Any]) -> dict[str, float]:
    """Evaluate using wrapped function."""
    return self.eval_function(hyperparams)

get_metric_names

get_metric_names() -> list[str]

Get metric names.

Source code in src/dlhub/tuning/framework.py

def get_metric_names(self) -> list[str]:
    """Get metric names."""
    return self.metric_names

HyperparameterSampler

Utility class for sampling hyperparameters from configurations.

Parameters:

Name	Type	Description	Default
hyperparameter_configs	list	List of HyperparameterConfig objects	required
random_state	int	Random seed for reproducibility	None

Source code in src/dlhub/tuning/framework.py

class HyperparameterSampler:
    """
    Utility class for sampling hyperparameters from configurations.

    Parameters
    ----------
    hyperparameter_configs : list
        List of HyperparameterConfig objects
    random_state : int, optional
        Random seed for reproducibility
    """

    def __init__(
        self,
        hyperparameter_configs: list[HyperparameterConfig],
        random_state: int | None = None,
    ):
        self.configs = hyperparameter_configs
        if random_state is not None:
            np.random.seed(random_state)

    def sample(self) -> dict[str, Any]:
        """
        Sample a hyperparameter configuration.

        Returns
        -------
        dict
            Sampled hyperparameter configuration
        """
        hyperparams = {}

        for config in self.configs:
            if config.type == "continuous":
                hyperparams[config.name] = self._sample_continuous(config)
            elif config.type == "integer":
                hyperparams[config.name] = self._sample_integer(config)
            elif config.type == "categorical":
                hyperparams[config.name] = self._sample_categorical(config)
            else:
                raise ValueError(f"Unknown parameter type: {config.type}")

        return hyperparams

    def _sample_continuous(self, config: HyperparameterConfig) -> float:
        """Sample continuous parameter."""
        low, high = config.range

        if config.scale == "log":
            return np.exp(np.random.uniform(np.log(low), np.log(high)))
        else:
            return np.random.uniform(low, high)

    def _sample_integer(self, config: HyperparameterConfig) -> int:
        """Sample integer parameter."""
        low, high = config.range

        if config.scale == "log":
            log_value = np.random.uniform(np.log(low), np.log(high))
            return int(np.round(np.exp(log_value)))
        else:
            return np.random.randint(low, high + 1)

    def _sample_categorical(self, config: HyperparameterConfig) -> Any:
        """Sample categorical parameter."""
        return np.random.choice(config.range)

    def validate(self, hyperparams: dict[str, Any]) -> bool:
        """
        Validate a hyperparameter configuration.

        Parameters
        ----------
        hyperparams : dict
            Hyperparameter configuration to validate

        Returns
        -------
        bool
            True if configuration is valid
        """
        for config in self.configs:
            if config.name not in hyperparams:
                return False

            value = hyperparams[config.name]

            if config.type in ["continuous", "integer"]:
                low, high = config.range
                if not (low <= value <= high):
                    return False
            elif config.type == "categorical":
                if value not in config.range:
                    return False

        return True

sample

sample() -> dict[str, Any]

Sample a hyperparameter configuration.

Returns:

Type	Description
dict	Sampled hyperparameter configuration

Source code in src/dlhub/tuning/framework.py

def sample(self) -> dict[str, Any]:
    """
    Sample a hyperparameter configuration.

    Returns
    -------
    dict
        Sampled hyperparameter configuration
    """
    hyperparams = {}

    for config in self.configs:
        if config.type == "continuous":
            hyperparams[config.name] = self._sample_continuous(config)
        elif config.type == "integer":
            hyperparams[config.name] = self._sample_integer(config)
        elif config.type == "categorical":
            hyperparams[config.name] = self._sample_categorical(config)
        else:
            raise ValueError(f"Unknown parameter type: {config.type}")

    return hyperparams

validate

validate(hyperparams: dict[str, Any]) -> bool

Validate a hyperparameter configuration.

Parameters:

Name	Type	Description	Default
hyperparams	dict	Hyperparameter configuration to validate	required

Returns:

Type	Description
bool	True if configuration is valid

Source code in src/dlhub/tuning/framework.py

def validate(self, hyperparams: dict[str, Any]) -> bool:
    """
    Validate a hyperparameter configuration.

    Parameters
    ----------
    hyperparams : dict
        Hyperparameter configuration to validate

    Returns
    -------
    bool
        True if configuration is valid
    """
    for config in self.configs:
        if config.name not in hyperparams:
            return False

        value = hyperparams[config.name]

        if config.type in ["continuous", "integer"]:
            low, high = config.range
            if not (low <= value <= high):
                return False
        elif config.type == "categorical":
            if value not in config.range:
                return False

    return True

ExperimentLogger

Logger for experiment results and metadata.

Parameters:

Name	Type	Description	Default
save_dir	str(optional)	Directory to save logs	required
experiment_name	str	Name of the experiment	required

Source code in src/dlhub/tuning/framework.py

class ExperimentLogger:
    """
    Logger for experiment results and metadata.

    Parameters
    ----------
    save_dir : str (optional)
        Directory to save logs
    experiment_name : str
        Name of the experiment
    """

    def __init__(self, save_dir: str | None, experiment_name: str):
        self.experiment_name = experiment_name
        self.save_dir = Path(save_dir) if save_dir else None

        if self.save_dir:
            self.save_dir.mkdir(parents=True, exist_ok=True)
            self.log_file = self.save_dir / f"{experiment_name}_log.jsonl"
        else:
            self.log_file = None

    def log_trial(self, trial_result: TrialResult) -> None:
        """
        Log a trial result.

        Parameters
        ----------
        trial_result : TrialResult
            Trial result to log
        """
        if self.log_file is None:
            return

        # Convert to dictionary
        trial_dict = {
            "trial_id": trial_result.trial_id,
            "hyperparams": trial_result.hyperparams,
            "metrics": trial_result.metrics,
            "training_time": trial_result.training_time,
            "status": trial_result.status,
            "metadata": trial_result.metadata,
            "timestamp": time.time(),
        }

        with open(self.log_file, "a") as f:
            f.write(json.dumps(trial_dict) + "\n")

    def save_results(self, result: ExperimentResult) -> None:
        """
        Save complete optimization results.

        Parameters
        ----------
        result : ExperimentResult
            Optimization results to save
        """
        if self.save_dir is None:
            return

        summary_file = self.save_dir / f"{self.experiment_name}_summary.json"
        summary = {
            "experiment_name": result.experiment_config.experiment_name,
            "optimization_method": result.experiment_config.optimization_method.value,
            "best_trial": {
                "hyperparams": result.best_trial.hyperparams,
                "metrics": result.best_trial.metrics,
                "trial_id": result.best_trial.trial_id,
            },
            "total_time": result.total_time,
            "n_trials": len(result.all_trials),
            "summary_statistics": result.summary_statistics,
        }

        with open(summary_file, "w") as f:
            json.dump(summary, f, indent=2)

    def load_results(self) -> list[TrialResult] | None:
        """
        Load trial results from log file.

        Returns
        -------
        list or None
            List of TrialResult objects, or None if no log exists
        """
        if self.log_file is None or not self.log_file.exists():
            return None

        trials = []
        with open(self.log_file) as f:
            for line in f:
                trial_dict = json.loads(line)
                trial = TrialResult(
                    trial_id=trial_dict["trial_id"],
                    hyperparams=trial_dict["hyperparams"],
                    metrics=trial_dict["metrics"],
                    training_time=trial_dict["training_time"],
                    status=trial_dict["status"],
                    metadata=trial_dict["metadata"],
                )
                trials.append(trial)

        return trials

log_trial

log_trial(trial_result: TrialResult) -> None

Log a trial result.

Parameters:

Name	Type	Description	Default
trial_result	TrialResult	Trial result to log	required

Source code in src/dlhub/tuning/framework.py

def log_trial(self, trial_result: TrialResult) -> None:
    """
    Log a trial result.

    Parameters
    ----------
    trial_result : TrialResult
        Trial result to log
    """
    if self.log_file is None:
        return

    # Convert to dictionary
    trial_dict = {
        "trial_id": trial_result.trial_id,
        "hyperparams": trial_result.hyperparams,
        "metrics": trial_result.metrics,
        "training_time": trial_result.training_time,
        "status": trial_result.status,
        "metadata": trial_result.metadata,
        "timestamp": time.time(),
    }

    with open(self.log_file, "a") as f:
        f.write(json.dumps(trial_dict) + "\n")

save_results

save_results(result: ExperimentResult) -> None

Save complete optimization results.

Parameters:

Name	Type	Description	Default
result	ExperimentResult	Optimization results to save	required

Source code in src/dlhub/tuning/framework.py

def save_results(self, result: ExperimentResult) -> None:
    """
    Save complete optimization results.

    Parameters
    ----------
    result : ExperimentResult
        Optimization results to save
    """
    if self.save_dir is None:
        return

    summary_file = self.save_dir / f"{self.experiment_name}_summary.json"
    summary = {
        "experiment_name": result.experiment_config.experiment_name,
        "optimization_method": result.experiment_config.optimization_method.value,
        "best_trial": {
            "hyperparams": result.best_trial.hyperparams,
            "metrics": result.best_trial.metrics,
            "trial_id": result.best_trial.trial_id,
        },
        "total_time": result.total_time,
        "n_trials": len(result.all_trials),
        "summary_statistics": result.summary_statistics,
    }

    with open(summary_file, "w") as f:
        json.dump(summary, f, indent=2)

load_results

load_results() -> list[TrialResult] | None

Load trial results from log file.

Returns:

Type	Description
list or None	List of TrialResult objects, or None if no log exists

Source code in src/dlhub/tuning/framework.py

def load_results(self) -> list[TrialResult] | None:
    """
    Load trial results from log file.

    Returns
    -------
    list or None
        List of TrialResult objects, or None if no log exists
    """
    if self.log_file is None or not self.log_file.exists():
        return None

    trials = []
    with open(self.log_file) as f:
        for line in f:
            trial_dict = json.loads(line)
            trial = TrialResult(
                trial_id=trial_dict["trial_id"],
                hyperparams=trial_dict["hyperparams"],
                metrics=trial_dict["metrics"],
                training_time=trial_dict["training_time"],
                status=trial_dict["status"],
                metadata=trial_dict["metadata"],
            )
            trials.append(trial)

    return trials

HyperparameterOptimizer

Main hyperparameter optimization framework.

Parameters:

Name	Type	Description	Default
config	ExperimentConfig	Experiment configuration	required
objective	ObjectiveFunction	Objective function to optimize	required

Source code in src/dlhub/tuning/framework.py

class HyperparameterOptimizer:
    """
    Main hyperparameter optimization framework.

    Parameters
    ----------
    config : ExperimentConfig
        Experiment configuration
    objective : ObjectiveFunction
        Objective function to optimize
    """

    def __init__(self, config: ExperimentConfig, objective: ObjectiveFunction):
        self.config = config
        self.objective = objective

        if config.random_seed is not None:
            np.random.seed(config.random_seed)

        self.sampler = HyperparameterSampler(config.hyperparameters, config.random_seed)

        self.logger = ExperimentLogger(config.save_dir, config.experiment_name)

        self.all_trials = []
        self.best_trial = None
        self.trial_counter = 0

    def _evaluate_trial(self, hyperparams: dict[str, Any]) -> TrialResult:
        """
        Evaluate a single trial.

        Parameters
        ----------
        hyperparams : dict
            Hyperparameter configuration

        Returns
        -------
        TrialResult
            Result of the trial
        """
        trial_id = self.trial_counter
        self.trial_counter += 1

        start_time = time.time()

        try:
            if not self.sampler.validate(hyperparams):
                raise ValueError("Invalid hyperparameter configuration")

            metrics = self.objective.evaluate(hyperparams)
            training_time = time.time() - start_time

            if self.config.objective_metric not in metrics:
                raise ValueError(
                    f"Objective metric '{self.config.objective_metric}' "
                    "not found in results"
                )

            result = TrialResult(
                trial_id=trial_id,
                hyperparams=hyperparams.copy(),
                metrics=metrics,
                training_time=training_time,
                status="success",
            )

        except Exception as e:
            training_time = time.time() - start_time
            warnings.warn(f"Trial {trial_id} failed: {e}")

            result = TrialResult(
                trial_id=trial_id,
                hyperparams=hyperparams.copy(),
                metrics={
                    self.config.objective_metric: -np.inf
                    if self.config.maximize
                    else np.inf
                },
                training_time=training_time,
                status="failed",
                metadata={"error": str(e)},
            )

        self.logger.log_trial(result)
        self.all_trials.append(result)

        self._update_best_trial(result)

        return result

    def _update_best_trial(self, trial: TrialResult) -> None:
        """Update best trial if current trial is better."""
        if trial.status != "success":
            return

        current_score = trial.metrics[self.config.objective_metric]

        if self.best_trial is None:
            self.best_trial = trial
        else:
            best_score = self.best_trial.metrics[self.config.objective_metric]

            if self.config.maximize:
                if current_score > best_score:
                    self.best_trial = trial
            else:
                if current_score < best_score:
                    self.best_trial = trial

    def optimize(self, verbose: bool = True) -> ExperimentResult:
        """
        Run hyperparameter optimization.

        Parameters
        ----------
        verbose : bool, default=True
            Whether to print progress

        Returns
        -------
        ExperimentResult
            Optimization results
        """
        if verbose:
            print(
                f"Starting Hyperparameter Optimization: {self.config.experiment_name}"
            )
            print(f"Method: {self.config.optimization_method.value}")
            print(f"Number of trials: {self.config.n_trials}")
            print(
                f"Objective: {'maximize' if self.config.maximize else 'minimize'} "
                f"{self.config.objective_metric}"
            )

        start_time = time.time()

        if self.config.optimization_method == OptimizationMethod.RANDOM_SEARCH:
            self._run_random_search(verbose)
        elif self.config.optimization_method == OptimizationMethod.GRID_SEARCH:
            self._run_grid_search(verbose)
        else:
            raise NotImplementedError(
                f"Method {self.config.optimization_method.value} not implemented "
                "in this simplified framework"
            )

        total_time = time.time() - start_time

        summary_stats = self._compute_summary_statistics()

        result = ExperimentResult(
            experiment_config=self.config,
            best_trial=self.best_trial,  # type: ignore
            all_trials=self.all_trials,
            total_time=total_time,
            summary_statistics=summary_stats,
        )

        self.logger.save_results(result)

        if verbose:
            print(f"\nOptimization completed in {total_time:.2f} seconds")
            print(
                f"Best {self.config.objective_metric}: "
                f"{self.best_trial.metrics[self.config.objective_metric]:.6f}"
            )  # type: ignore
            print(f"Best hyperparameters: {self.best_trial.hyperparams}")  # type: ignore

        return result

    def _run_random_search(self, verbose: bool) -> None:
        """Run random search optimization."""
        for i in range(self.config.n_trials):
            hyperparams = self.sampler.sample()
            trial = self._evaluate_trial(hyperparams)

            if verbose and (i + 1) % max(1, self.config.n_trials // 10) == 0:
                print(
                    f"Trial {i + 1}/{self.config.n_trials}: "
                    f"{self.config.objective_metric} = "
                    f"{trial.metrics.get(self.config.objective_metric, 'N/A')}"
                )

    def _run_grid_search(self, verbose: bool) -> None:
        """Run grid search optimization."""
        grid_configs = self._generate_grid()

        if verbose:
            print(f"Generated grid with {len(grid_configs)} configurations")

        for i, hyperparams in enumerate(grid_configs[: self.config.n_trials]):
            trial = self._evaluate_trial(hyperparams)

            if verbose and (i + 1) % max(1, len(grid_configs) // 10) == 0:
                print(
                    f"Trial {i + 1}/{min(len(grid_configs), self.config.n_trials)}: "
                    f"{self.config.objective_metric} = "
                    f"{trial.metrics.get(self.config.objective_metric, 'N/A')}"
                )

    def _generate_grid(self) -> list[dict[str, Any]]:
        """Generate grid of hyperparameter configurations."""
        from itertools import product

        param_grids = {}
        for config in self.config.hyperparameters:
            if config.type == "categorical":
                param_grids[config.name] = config.range
            elif config.type in ["continuous", "integer"]:
                n_points = 5  # Use 5 points per dimension
                low, high = config.range

                if config.scale == "log":
                    points = np.logspace(np.log10(low), np.log10(high), n_points)
                else:
                    points = np.linspace(low, high, n_points)

                if config.type == "integer":
                    points = np.unique(np.round(points).astype(int))

                param_grids[config.name] = points.tolist()

        keys = list(param_grids.keys())
        values = list(param_grids.values())

        grid_configs = []
        for combo in product(*values):
            config_dict = dict(zip(keys, combo))
            grid_configs.append(config_dict)

        return grid_configs

    def _compute_summary_statistics(self) -> dict[str, Any]:
        """Compute summary statistics from all trials."""
        successful_trials = [t for t in self.all_trials if t.status == "success"]

        if not successful_trials:
            return {}

        objective_scores = [
            t.metrics[self.config.objective_metric] for t in successful_trials
        ]

        training_times = [t.training_time for t in successful_trials]

        statistics = {
            "n_trials": len(self.all_trials),
            "n_successful": len(successful_trials),
            "n_failed": len(self.all_trials) - len(successful_trials),
            "objective_statistics": {
                "mean": float(np.mean(objective_scores)),
                "std": float(np.std(objective_scores)),
                "min": float(np.min(objective_scores)),
                "max": float(np.max(objective_scores)),
                "median": float(np.median(objective_scores)),
                "percentiles": {
                    "25th": float(np.percentile(objective_scores, 25)),
                    "75th": float(np.percentile(objective_scores, 75)),
                    "95th": float(np.percentile(objective_scores, 95)),
                },
            },
            "training_time_statistics": {
                "mean": float(np.mean(training_times)),
                "total": float(np.sum(training_times)),
                "min": float(np.min(training_times)),
                "max": float(np.max(training_times)),
            },
            "improvement_over_random": self._improvement_over_random(objective_scores),
        }

        return statistics

    def _improvement_over_random(self, objective_scores: list[float]) -> float:
        """
        Relative gain of the best trial over the first ten, which for random and
        grid search are as good a stand-in for chance as the run provides.

        Reported as 0.0 below ten trials, and when the baseline mean is zero:
        the relative gain over nothing is not a number, and a metric centred on
        zero is a legitimate thing to optimize.
        """
        if len(objective_scores) < 10:
            return 0.0

        baseline = float(np.mean(objective_scores[:10]))
        if baseline == 0.0:
            return 0.0

        best_score = self.best_trial.metrics[self.config.objective_metric]  # type: ignore
        return float((best_score - baseline) / abs(baseline))

optimize

optimize(verbose: bool = True) -> ExperimentResult

Run hyperparameter optimization.

Parameters:

Name	Type	Description	Default
verbose	bool	Whether to print progress	True

Returns:

Type	Description
ExperimentResult	Optimization results

Source code in src/dlhub/tuning/framework.py

def optimize(self, verbose: bool = True) -> ExperimentResult:
    """
    Run hyperparameter optimization.

    Parameters
    ----------
    verbose : bool, default=True
        Whether to print progress

    Returns
    -------
    ExperimentResult
        Optimization results
    """
    if verbose:
        print(
            f"Starting Hyperparameter Optimization: {self.config.experiment_name}"
        )
        print(f"Method: {self.config.optimization_method.value}")
        print(f"Number of trials: {self.config.n_trials}")
        print(
            f"Objective: {'maximize' if self.config.maximize else 'minimize'} "
            f"{self.config.objective_metric}"
        )

    start_time = time.time()

    if self.config.optimization_method == OptimizationMethod.RANDOM_SEARCH:
        self._run_random_search(verbose)
    elif self.config.optimization_method == OptimizationMethod.GRID_SEARCH:
        self._run_grid_search(verbose)
    else:
        raise NotImplementedError(
            f"Method {self.config.optimization_method.value} not implemented "
            "in this simplified framework"
        )

    total_time = time.time() - start_time

    summary_stats = self._compute_summary_statistics()

    result = ExperimentResult(
        experiment_config=self.config,
        best_trial=self.best_trial,  # type: ignore
        all_trials=self.all_trials,
        total_time=total_time,
        summary_statistics=summary_stats,
    )

    self.logger.save_results(result)

    if verbose:
        print(f"\nOptimization completed in {total_time:.2f} seconds")
        print(
            f"Best {self.config.objective_metric}: "
            f"{self.best_trial.metrics[self.config.objective_metric]:.6f}"
        )  # type: ignore
        print(f"Best hyperparameters: {self.best_trial.hyperparams}")  # type: ignore

    return result

optimize_hyperparameters

optimize_hyperparameters(objective_function: Callable[[dict], dict[str, float]], hyperparameters: list[dict[str, Any]], objective_metric: str, experiment_name: str = 'hyperparameter_optimization', optimization_method: str = 'random_search', n_trials: int = 100, maximize: bool = True, random_seed: int | None = None, save_dir: str | None = None, verbose: bool = True) -> ExperimentResult

Run a hyperparameter search by method name, returning an ExperimentResult.

The general dispatcher: optimization_method selects the strategy. It takes the framework's search-space format, which carries a type per hyperparameter and so covers categorical and integer dimensions as well as continuous ones. The method-specific entry points -- :func:~dlhub.tuning.bayesian_optimize, :func:~dlhub.tuning.random_search -- take their own formats and return their own result types.

Parameters:

Name	Type	Description	Default
objective_function	callable	Function that takes hyperparams dict and returns metrics dict	required
hyperparameters	list	List of hyperparameter definitions (dicts with 'name', 'type', 'range')	required
objective_metric	str	Name of metric to optimize	required
experiment_name	str	Name of the experiment	"hyperparameter_optimization"
optimization_method	str	Optimization method ('random_search' or 'grid_search')	"random_search"
n_trials	int	Number of trials	100
maximize	bool	Whether to maximize objective	True
random_seed	int(optional)	Random seed	None
save_dir	str(optional)	Directory to save results	None
verbose	bool	Whether to print progress	True

Returns:

Type	Description
ExperimentResult	Optimization results

Examples:

>>> def objective(hyperparams):
...     lr = hyperparams["learning_rate"]
...     wd = hyperparams["weight_decay"]
...     accuracy = 0.9 - (lr - 0.001) ** 2 - (wd - 0.0001) ** 2
...     return {"accuracy": accuracy, "loss": 1 - accuracy}
>>> hyperparams = [
...     {
...         "name": "learning_rate",
...         "type": "continuous",
...         "range": (1e-5, 1e-1),
...         "scale": "log",
...     },
...     {
...         "name": "weight_decay",
...         "type": "continuous",
...         "range": (1e-6, 1e-2),
...         "scale": "log",
...     },
... ]
>>> result = optimize_hyperparameters(
...     objective, hyperparams, "accuracy", n_trials=50, random_seed=42
... )

Source code in src/dlhub/tuning/framework.py

def optimize_hyperparameters(
    objective_function: Callable[[dict], dict[str, float]],
    hyperparameters: list[dict[str, Any]],
    objective_metric: str,
    experiment_name: str = "hyperparameter_optimization",
    optimization_method: str = "random_search",
    n_trials: int = 100,
    maximize: bool = True,
    random_seed: int | None = None,
    save_dir: str | None = None,
    verbose: bool = True,
) -> ExperimentResult:
    """
    Run a hyperparameter search by method name, returning an ``ExperimentResult``.

    The general dispatcher: ``optimization_method`` selects the strategy. It takes
    the framework's search-space format, which carries a type per hyperparameter
    and so covers categorical and integer dimensions as well as continuous ones.
    The method-specific entry points -- :func:`~dlhub.tuning.bayesian_optimize`,
    :func:`~dlhub.tuning.random_search` -- take their own formats and return their
    own result types.

    Parameters
    ----------
    objective_function : callable
        Function that takes hyperparams dict and returns metrics dict
    hyperparameters : list
        List of hyperparameter definitions (dicts with 'name', 'type', 'range')
    objective_metric : str
        Name of metric to optimize
    experiment_name : str, default="hyperparameter_optimization"
        Name of the experiment
    optimization_method : str, default="random_search"
        Optimization method ('random_search' or 'grid_search')
    n_trials : int, default=100
        Number of trials
    maximize : bool, default=True
        Whether to maximize objective
    random_seed : int (optional)
        Random seed
    save_dir : str (optional)
        Directory to save results
    verbose : bool, default=True
        Whether to print progress

    Returns
    -------
    ExperimentResult
        Optimization results

    Examples
    --------
    >>> def objective(hyperparams):
    ...     lr = hyperparams["learning_rate"]
    ...     wd = hyperparams["weight_decay"]
    ...     accuracy = 0.9 - (lr - 0.001) ** 2 - (wd - 0.0001) ** 2
    ...     return {"accuracy": accuracy, "loss": 1 - accuracy}
    >>> hyperparams = [
    ...     {
    ...         "name": "learning_rate",
    ...         "type": "continuous",
    ...         "range": (1e-5, 1e-1),
    ...         "scale": "log",
    ...     },
    ...     {
    ...         "name": "weight_decay",
    ...         "type": "continuous",
    ...         "range": (1e-6, 1e-2),
    ...         "scale": "log",
    ...     },
    ... ]
    >>> result = optimize_hyperparameters(
    ...     objective, hyperparams, "accuracy", n_trials=50, random_seed=42
    ... )
    """
    hp_configs = []
    for hp in hyperparameters:
        config = HyperparameterConfig(
            name=hp["name"],
            type=hp["type"],
            range=hp["range"],
            scale=hp.get("scale", "linear"),
            default=hp.get("default"),
        )
        hp_configs.append(config)

    exp_config = ExperimentConfig(
        experiment_name=experiment_name,
        optimization_method=OptimizationMethod(optimization_method),
        hyperparameters=hp_configs,
        objective_metric=objective_metric,
        maximize=maximize,
        n_trials=n_trials,
        random_seed=random_seed,
        save_dir=save_dir,
    )

    test_hyperparams = HyperparameterSampler(hp_configs, random_seed).sample()
    test_metrics = objective_function(test_hyperparams)
    metric_names = list(test_metrics.keys())

    objective_wrapper = FunctionObjective(objective_function, metric_names)

    optimizer = HyperparameterOptimizer(exp_config, objective_wrapper)
    return optimizer.optimize(verbose=verbose)

nn_objective

nn_objective(hyperparams: dict[str, Any]) -> dict[str, float]

Simulate neural network training objective.

Returns multiple metrics for comprehensive evaluation.

Source code in src/dlhub/tuning/framework.py

def nn_objective(hyperparams: dict[str, Any]) -> dict[str, float]:
    """
    Simulate neural network training objective.

    Returns multiple metrics for comprehensive evaluation.
    """
    lr = hyperparams["learning_rate"]
    wd = hyperparams["weight_decay"]
    batch_size = hyperparams["batch_size"]
    dropout = hyperparams["dropout_rate"]
    optimizer_type = hyperparams["optimizer"]

    # Simulate realistic hyperparameter effects
    # Optimal values: lr=1e-3, wd=1e-4, batch_size=64, dropout=0.2
    lr_effect = -5 * (np.log10(lr) + 3) ** 2
    wd_effect = -2 * (np.log10(wd) + 4) ** 2
    batch_effect = -0.001 * (batch_size - 64) ** 2
    dropout_effect = -3 * (dropout - 0.2) ** 2

    optimizer_effects = {"adam": 0.05, "sgd": 0.0, "rmsprop": 0.03}
    opt_effect = optimizer_effects.get(optimizer_type, 0.0)

    base_accuracy = 0.85

    validation_accuracy = (
        base_accuracy
        + lr_effect
        + wd_effect
        + batch_effect
        + dropout_effect
        + opt_effect
    )

    validation_accuracy += np.random.normal(0, 0.02)

    training_accuracy = validation_accuracy + 0.05  # Training usually higher
    training_loss = 1.0 - training_accuracy
    validation_loss = 1.0 - validation_accuracy

    inference_time = 10.0 / batch_size + dropout * 5.0
    model_size = 100.0 * (1 - dropout * 0.3)

    return {
        "validation_accuracy": validation_accuracy,
        "training_accuracy": training_accuracy,
        "validation_loss": validation_loss,
        "training_loss": training_loss,
        "inference_time_ms": inference_time,
        "model_size_mb": model_size,
    }

is_dominated

is_dominated(trial1, trial2)

Check if trial1 is dominated by trial2.

Source code in src/dlhub/tuning/framework.py

def is_dominated(trial1, trial2):
    """Check if trial1 is dominated by trial2."""
    acc1 = trial1.metrics["validation_accuracy"]
    time1 = trial1.metrics["inference_time_ms"]
    acc2 = trial2.metrics["validation_accuracy"]
    time2 = trial2.metrics["inference_time_ms"]

    # trial1 is dominated if trial2 is better in both objectives
    return acc2 >= acc1 and time2 <= time1 and (acc2 > acc1 or time2 < time1)

learning_rate_finder

Learning Rate Finder for Hyperparameter Tuning

Automated learning rate range testing to find optimal learning rate ranges before full training. This technique helps identify good learning rate ranges by monitoring loss behavior during short training runs with exponentially increasing learning rates.

References

• Smith, L. N. (2017). "Cyclical Learning Rates for Training Neural Networks." IEEE Winter Conference on Applications of Computer Vision (WACV).
• Smith, L. N. (2018). "A disciplined approach to neural network hyper-parameters: Part 1 -- learning rate, batch size, momentum, and weight decay." arXiv preprint.

Author

Deep Learning Reference Hub

License

MIT License

Notes

The learning rate finder is particularly useful for: 1. Finding the maximum usable learning rate 2. Identifying learning rate ranges for cyclical learning rate schedules 3. Detecting when the learning rate is too high (loss divergence) 4. Setting appropriate learning rates for different optimizers

LearningRateFinderResult dataclass

Container for learning rate finder results.

Attributes:

Name	Type	Description
learning_rates	ndarray	Array of learning rates tested
losses	ndarray	Corresponding loss values
smoothed_losses	ndarray	Smoothed loss values for trend analysis
suggested_lr	float	Suggested learning rate based on analysis
min_gradient_lr	float	Learning rate with steepest loss decrease
analysis	dict	Additional analysis metrics and diagnostics

Source code in src/dlhub/tuning/learning_rate_finder.py

@dataclass
class LearningRateFinderResult:
    """
    Container for learning rate finder results.

    Attributes
    ----------
    learning_rates : np.ndarray
        Array of learning rates tested
    losses : np.ndarray
        Corresponding loss values
    smoothed_losses : np.ndarray
        Smoothed loss values for trend analysis
    suggested_lr : float
        Suggested learning rate based on analysis
    min_gradient_lr : float
        Learning rate with steepest loss decrease
    analysis : dict
        Additional analysis metrics and diagnostics
    """

    learning_rates: np.ndarray
    losses: np.ndarray
    smoothed_losses: np.ndarray
    suggested_lr: float
    min_gradient_lr: float
    analysis: dict[str, Any]

BaseTrainer

Bases: ABC

Abstract base class for training interface.

Defines the interface that training functions must implement to work with the learning rate finder.

Source code in src/dlhub/tuning/learning_rate_finder.py

class BaseTrainer(ABC):
    """
    Abstract base class for training interface.

    Defines the interface that training functions must implement
    to work with the learning rate finder.
    """

    @abstractmethod
    def train_batch(self, learning_rate: float) -> float:
        """
        Train one batch with given learning rate and return loss.

        Parameters
        ----------
        learning_rate : float
            Learning rate to use for this batch

        Returns
        -------
        float
            Loss value after training step
        """
        pass

    @abstractmethod
    def reset_model(self) -> None:
        """Reset model to initial state."""
        pass

train_batch abstractmethod

train_batch(learning_rate: float) -> float

Train one batch with given learning rate and return loss.

Parameters:

Name	Type	Description	Default
learning_rate	float	Learning rate to use for this batch	required

Returns:

Type	Description
float	Loss value after training step

Source code in src/dlhub/tuning/learning_rate_finder.py

@abstractmethod
def train_batch(self, learning_rate: float) -> float:
    """
    Train one batch with given learning rate and return loss.

    Parameters
    ----------
    learning_rate : float
        Learning rate to use for this batch

    Returns
    -------
    float
        Loss value after training step
    """
    pass

reset_model abstractmethod

reset_model() -> None

Reset model to initial state.

Source code in src/dlhub/tuning/learning_rate_finder.py

@abstractmethod
def reset_model(self) -> None:
    """Reset model to initial state."""
    pass

FunctionTrainer

Bases: BaseTrainer

Trainer wrapper for function-based training.

Wraps user-provided training and reset functions to conform to the BaseTrainer interface.

Parameters:

Name	Type	Description	Default
train_function	callable	Function that takes learning_rate and returns loss	required
reset_function	callable	Function to reset model state	required

Source code in src/dlhub/tuning/learning_rate_finder.py

class FunctionTrainer(BaseTrainer):
    """
    Trainer wrapper for function-based training.

    Wraps user-provided training and reset functions to conform
    to the BaseTrainer interface.

    Parameters
    ----------
    train_function : callable
        Function that takes learning_rate and returns loss
    reset_function : callable
        Function to reset model state
    """

    def __init__(
        self,
        train_function: Callable[[float], float],
        reset_function: Callable[[], None],
    ):
        self.train_function = train_function
        self.reset_function = reset_function

    def train_batch(self, learning_rate: float) -> float:
        """Train one batch with given learning rate."""
        return self.train_function(learning_rate)

    def reset_model(self) -> None:
        """Reset model to initial state."""
        self.reset_function()

train_batch

train_batch(learning_rate: float) -> float

Train one batch with given learning rate.

Source code in src/dlhub/tuning/learning_rate_finder.py

def train_batch(self, learning_rate: float) -> float:
    """Train one batch with given learning rate."""
    return self.train_function(learning_rate)

reset_model

reset_model() -> None

Reset model to initial state.

Source code in src/dlhub/tuning/learning_rate_finder.py

def reset_model(self) -> None:
    """Reset model to initial state."""
    self.reset_function()

LearningRateFinder

Learning Rate Finder for optimal learning rate discovery.

Implements the learning rate range test by training with exponentially increasing learning rates and analyzing the loss curve to suggest optimal learning rate ranges.

Parameters:

Name	Type	Description	Default
trainer	BaseTrainer	Training interface object	required
min_lr	float	Minimum learning rate to test	1e-7
max_lr	float	Maximum learning rate to test	10.0
num_iterations	int	Number of iterations to run the test	100
step_mode	str	How to step learning rate ('exp' for exponential, 'linear' for linear)	'exp'
smooth_beta	float	Smoothing factor for loss smoothing (exponential moving average)	0.98
divergence_threshold	float	Stop if loss > divergence_threshold * min_loss	4.0

Source code in src/dlhub/tuning/learning_rate_finder.py

class LearningRateFinder:
    """
    Learning Rate Finder for optimal learning rate discovery.

    Implements the learning rate range test by training with exponentially
    increasing learning rates and analyzing the loss curve to suggest
    optimal learning rate ranges.

    Parameters
    ----------
    trainer : BaseTrainer
        Training interface object
    min_lr : float, default=1e-7
        Minimum learning rate to test
    max_lr : float, default=10.0
        Maximum learning rate to test
    num_iterations : int, default=100
        Number of iterations to run the test
    step_mode : str, default='exp'
        How to step learning rate ('exp' for exponential, 'linear' for linear)
    smooth_beta : float, default=0.98
        Smoothing factor for loss smoothing (exponential moving average)
    divergence_threshold : float, default=4.0
        Stop if loss > divergence_threshold * min_loss
    """

    def __init__(
        self,
        trainer: BaseTrainer,
        min_lr: float = 1e-7,
        max_lr: float = 10.0,
        num_iterations: int = 100,
        step_mode: str = "exp",
        smooth_beta: float = 0.98,
        divergence_threshold: float = 4.0,
    ):

        self.trainer = trainer
        self.min_lr = min_lr
        self.max_lr = max_lr
        self.num_iterations = num_iterations
        self.step_mode = step_mode.lower()
        self.smooth_beta = smooth_beta
        self.divergence_threshold = divergence_threshold

        # Validated against the normalized value, not the raw argument: lowering
        # the case and then rejecting the un-lowered form makes the normalization
        # unreachable and turns "EXP" into an error.
        if min_lr >= max_lr:
            raise ValueError("min_lr must be less than max_lr")
        if not 0 < smooth_beta < 1:
            raise ValueError("smooth_beta must be between 0 and 1")
        if self.step_mode not in ["exp", "linear"]:
            raise ValueError("step_mode must be 'exp' or 'linear'")

    def _generate_learning_rates(self) -> np.ndarray:
        """
        Generate learning rate schedule.

        Returns
        -------
        np.ndarray
            Array of learning rates to test
        """
        if self.step_mode == "exp":
            return np.logspace(
                np.log10(self.min_lr), np.log10(self.max_lr), self.num_iterations
            )
        else:
            return np.linspace(self.min_lr, self.max_lr, self.num_iterations)

    def _smooth_losses(self, losses: np.ndarray) -> np.ndarray:
        """
        Apply exponential smoothing to losses.

        Parameters
        ----------
        losses : np.ndarray
            Raw loss values

        Returns
        -------
        np.ndarray
            Smoothed loss values
        """
        smoothed = np.zeros_like(losses)
        smoothed[0] = losses[0]

        for i in range(1, len(losses)):
            smoothed[i] = (
                self.smooth_beta * smoothed[i - 1] + (1 - self.smooth_beta) * losses[i]
            )

        return smoothed

    def _detect_divergence(self, losses: np.ndarray, iteration: int) -> bool:
        """
        Detect if training has diverged.

        Parameters
        ----------
        losses : np.ndarray
            Loss values so far
        iteration : int
            Current iteration

        Returns
        -------
        bool
            True if divergence detected
        """
        if iteration < 10:  # Need some history
            return False

        min_loss = np.min(losses[: iteration + 1])
        current_loss = losses[iteration]

        # Cast to a plain bool, as annotated. The comparison yields np.bool_,
        # which is falsy-correct but fails an `is False` identity check.
        return bool(current_loss > self.divergence_threshold * min_loss)

    def _analyze_results(
        self,
        learning_rates: np.ndarray,
        losses: np.ndarray,
        smoothed_losses: np.ndarray,
    ) -> dict[str, Any]:
        """
        Analyze learning rate finder results.

        Parameters
        ----------
        learning_rates : np.ndarray
            Learning rates tested
        losses : np.ndarray
            Raw loss values
        smoothed_losses : np.ndarray
            Smoothed loss values

        Returns
        -------
        dict
            Analysis results and metrics
        """
        analysis = {}

        analysis["min_loss"] = float(np.min(losses))
        analysis["max_loss"] = float(np.max(losses))
        analysis["min_loss_lr"] = float(learning_rates[np.argmin(losses)])

        gradients = np.gradient(smoothed_losses, np.log10(learning_rates))
        min_gradient_idx = np.argmin(gradients)
        analysis["min_gradient_lr"] = float(learning_rates[min_gradient_idx])
        analysis["min_gradient"] = float(gradients[min_gradient_idx])

        suggested_lr = analysis["min_gradient_lr"] / 10
        analysis["suggested_lr"] = float(suggested_lr)

        if len(losses) > 10:
            initial_loss = np.mean(losses[:5])
            min_loss = analysis["min_loss"]
            analysis["loss_reduction_ratio"] = (initial_loss - min_loss) / initial_loss
        else:
            analysis["loss_reduction_ratio"] = 0.0

        loss_variance = np.var(losses)
        analysis["loss_variance"] = float(loss_variance)
        analysis["coefficient_of_variation"] = float(
            np.sqrt(loss_variance) / np.mean(losses)
        )

        if len(losses) > 20:
            first_half_mean = np.mean(losses[: len(losses) // 2])
            second_half_mean = np.mean(losses[len(losses) // 2 :])
            analysis["convergence_ratio"] = second_half_mean / first_half_mean
        else:
            analysis["convergence_ratio"] = 1.0

        return analysis

    def find(self, verbose: bool = True) -> LearningRateFinderResult:
        """
        Run learning rate finder.

        Parameters
        ----------
        verbose : bool, default=True
            Whether to print progress information

        Returns
        -------
        LearningRateFinderResult
            Results of the learning rate finder
        """
        if verbose:
            print("Starting Learning Rate Finder...")
            print(f"Learning rate range: {self.min_lr:.2e} to {self.max_lr:.2e}")
            print(f"Number of iterations: {self.num_iterations}")
            print(f"Step mode: {self.step_mode}")

        learning_rates = self._generate_learning_rates()
        losses = []

        self.trainer.reset_model()

        for i, lr in enumerate(learning_rates):
            try:
                loss = self.trainer.train_batch(lr)

                if np.isnan(loss) or np.isinf(loss):
                    if verbose:
                        print(
                            f"Iteration {i + 1}: Learning rate {lr:.2e} - Invalid loss (nan/inf)"
                        )
                    break

                losses.append(float(loss))

                if self._detect_divergence(np.array(losses), i):
                    if verbose:
                        print(
                            f"Iteration {i + 1}: Learning rate {lr:.2e} - Training diverged"
                        )
                    break

                if verbose and (i + 1) % max(1, self.num_iterations // 10) == 0:
                    print(
                        f"Iteration {i + 1}/{self.num_iterations}: LR = {lr:.2e}, Loss = {loss:.6f}"
                    )

            except Exception as e:
                if verbose:
                    print(
                        f"Iteration {i + 1}: Learning rate {lr:.2e} - Training failed: {e}"
                    )
                break

        if len(losses) < 5:
            raise RuntimeError(
                "Learning rate finder failed - insufficient valid loss values"
            )

        learning_rates = learning_rates[: len(losses)]
        losses = np.array(losses)

        smoothed_losses = self._smooth_losses(losses)

        analysis = self._analyze_results(learning_rates, losses, smoothed_losses)

        if verbose:
            print("\nLearning Rate Finder completed!")
            print(f"Suggested learning rate: {analysis['suggested_lr']:.2e}")
            print(
                f"Learning rate with steepest gradient: {analysis['min_gradient_lr']:.2e}"
            )
            print(f"Minimum loss: {analysis['min_loss']:.6f}")
            print(f"Loss reduction ratio: {analysis['loss_reduction_ratio']:.2%}")

        return LearningRateFinderResult(
            learning_rates=learning_rates,
            losses=losses,
            smoothed_losses=smoothed_losses,
            suggested_lr=analysis["suggested_lr"],
            min_gradient_lr=analysis["min_gradient_lr"],
            analysis=analysis,
        )

    def plot_results(
        self,
        result: LearningRateFinderResult,
        figsize: tuple[int, int] = (12, 8),
        save_path: str | None = None,
    ) -> None:
        """
        Plot learning rate finder results.

        Parameters
        ----------
        result : LearningRateFinderResult
            Results from learning rate finder
        figsize : tuple, default=(12, 8)
            Figure size for the plot
        save_path : str, optional
            Path to save the plot
        """
        fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=figsize)

        # Plot 1: Learning rate vs Loss (log scale)
        ax1.semilogx(
            result.learning_rates, result.losses, "b-", alpha=0.7, label="Raw Loss"
        )
        ax1.semilogx(
            result.learning_rates,
            result.smoothed_losses,
            "r-",
            linewidth=2,
            label="Smoothed Loss",
        )
        ax1.axvline(
            result.suggested_lr,
            color="green",
            linestyle="--",
            alpha=0.8,
            label=f"Suggested LR: {result.suggested_lr:.2e}",
        )
        ax1.axvline(
            result.min_gradient_lr,
            color="orange",
            linestyle="--",
            alpha=0.8,
            label=f"Min Gradient LR: {result.min_gradient_lr:.2e}",
        )
        ax1.set_xlabel("Learning Rate")
        ax1.set_ylabel("Loss")
        ax1.set_title("Learning Rate vs Loss")
        ax1.legend()
        ax1.grid(True, alpha=0.3)

        # Plot 2: Learning rate vs Loss (linear scale, zoomed)
        # Focus on the interesting region around minimum
        min_loss_idx = np.argmin(result.smoothed_losses)
        start_idx = max(0, min_loss_idx - 20)
        end_idx = min(len(result.learning_rates), min_loss_idx + 20)

        ax2.plot(
            result.learning_rates[start_idx:end_idx],
            result.losses[start_idx:end_idx],
            "b-",
            alpha=0.7,
        )
        ax2.plot(
            result.learning_rates[start_idx:end_idx],
            result.smoothed_losses[start_idx:end_idx],
            "r-",
            linewidth=2,
        )
        ax2.axvline(result.suggested_lr, color="green", linestyle="--", alpha=0.8)
        ax2.axvline(result.min_gradient_lr, color="orange", linestyle="--", alpha=0.8)
        ax2.set_xlabel("Learning Rate")
        ax2.set_ylabel("Loss")
        ax2.set_title("Loss (Zoomed Region)")
        ax2.grid(True, alpha=0.3)

        # Plot 3: Loss gradient
        gradients = np.gradient(result.smoothed_losses, np.log10(result.learning_rates))
        ax3.semilogx(result.learning_rates, gradients, "purple", linewidth=2)
        ax3.axvline(result.min_gradient_lr, color="orange", linestyle="--", alpha=0.8)
        ax3.axhline(0, color="black", linestyle="-", alpha=0.3)
        ax3.set_xlabel("Learning Rate")
        ax3.set_ylabel("Loss Gradient")
        ax3.set_title("Loss Gradient vs Learning Rate")
        ax3.grid(True, alpha=0.3)

        # Plot 4: Statistics summary
        ax4.axis("off")
        stats_text = f"""
        Analysis Summary:

        Suggested Learning Rate: {result.suggested_lr:.2e}
        Min Gradient Learning Rate: {result.min_gradient_lr:.2e}

        Minimum Loss: {result.analysis["min_loss"]:.6f}
        Loss Reduction: {result.analysis["loss_reduction_ratio"]:.2%}

        Loss Variance: {result.analysis["loss_variance"]:.6f}
        Coefficient of Variation: {result.analysis["coefficient_of_variation"]:.3f}

        Convergence Ratio: {result.analysis["convergence_ratio"]:.3f}
        """
        ax4.text(
            0.05,
            0.95,
            stats_text,
            transform=ax4.transAxes,
            fontsize=10,
            verticalalignment="top",
            fontfamily="monospace",
            bbox=dict(boxstyle="round", facecolor="lightgray", alpha=0.8),
        )

        plt.tight_layout()

        if save_path:
            plt.savefig(save_path, dpi=300, bbox_inches="tight")
            print(f"Plot saved to {save_path}")

        plt.show()

find

find(verbose: bool = True) -> LearningRateFinderResult

Run learning rate finder.

Parameters:

Name	Type	Description	Default
verbose	bool	Whether to print progress information	True

Returns:

Type	Description
LearningRateFinderResult	Results of the learning rate finder

Source code in src/dlhub/tuning/learning_rate_finder.py

def find(self, verbose: bool = True) -> LearningRateFinderResult:
    """
    Run learning rate finder.

    Parameters
    ----------
    verbose : bool, default=True
        Whether to print progress information

    Returns
    -------
    LearningRateFinderResult
        Results of the learning rate finder
    """
    if verbose:
        print("Starting Learning Rate Finder...")
        print(f"Learning rate range: {self.min_lr:.2e} to {self.max_lr:.2e}")
        print(f"Number of iterations: {self.num_iterations}")
        print(f"Step mode: {self.step_mode}")

    learning_rates = self._generate_learning_rates()
    losses = []

    self.trainer.reset_model()

    for i, lr in enumerate(learning_rates):
        try:
            loss = self.trainer.train_batch(lr)

            if np.isnan(loss) or np.isinf(loss):
                if verbose:
                    print(
                        f"Iteration {i + 1}: Learning rate {lr:.2e} - Invalid loss (nan/inf)"
                    )
                break

            losses.append(float(loss))

            if self._detect_divergence(np.array(losses), i):
                if verbose:
                    print(
                        f"Iteration {i + 1}: Learning rate {lr:.2e} - Training diverged"
                    )
                break

            if verbose and (i + 1) % max(1, self.num_iterations // 10) == 0:
                print(
                    f"Iteration {i + 1}/{self.num_iterations}: LR = {lr:.2e}, Loss = {loss:.6f}"
                )

        except Exception as e:
            if verbose:
                print(
                    f"Iteration {i + 1}: Learning rate {lr:.2e} - Training failed: {e}"
                )
            break

    if len(losses) < 5:
        raise RuntimeError(
            "Learning rate finder failed - insufficient valid loss values"
        )

    learning_rates = learning_rates[: len(losses)]
    losses = np.array(losses)

    smoothed_losses = self._smooth_losses(losses)

    analysis = self._analyze_results(learning_rates, losses, smoothed_losses)

    if verbose:
        print("\nLearning Rate Finder completed!")
        print(f"Suggested learning rate: {analysis['suggested_lr']:.2e}")
        print(
            f"Learning rate with steepest gradient: {analysis['min_gradient_lr']:.2e}"
        )
        print(f"Minimum loss: {analysis['min_loss']:.6f}")
        print(f"Loss reduction ratio: {analysis['loss_reduction_ratio']:.2%}")

    return LearningRateFinderResult(
        learning_rates=learning_rates,
        losses=losses,
        smoothed_losses=smoothed_losses,
        suggested_lr=analysis["suggested_lr"],
        min_gradient_lr=analysis["min_gradient_lr"],
        analysis=analysis,
    )

plot_results

plot_results(result: LearningRateFinderResult, figsize: tuple[int, int] = (12, 8), save_path: str | None = None) -> None

Plot learning rate finder results.

Parameters:

Name	Type	Description	Default
result	LearningRateFinderResult	Results from learning rate finder	required
figsize	tuple	Figure size for the plot	(12, 8)
save_path	str	Path to save the plot	None

Source code in src/dlhub/tuning/learning_rate_finder.py

def plot_results(
    self,
    result: LearningRateFinderResult,
    figsize: tuple[int, int] = (12, 8),
    save_path: str | None = None,
) -> None:
    """
    Plot learning rate finder results.

    Parameters
    ----------
    result : LearningRateFinderResult
        Results from learning rate finder
    figsize : tuple, default=(12, 8)
        Figure size for the plot
    save_path : str, optional
        Path to save the plot
    """
    fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=figsize)

    # Plot 1: Learning rate vs Loss (log scale)
    ax1.semilogx(
        result.learning_rates, result.losses, "b-", alpha=0.7, label="Raw Loss"
    )
    ax1.semilogx(
        result.learning_rates,
        result.smoothed_losses,
        "r-",
        linewidth=2,
        label="Smoothed Loss",
    )
    ax1.axvline(
        result.suggested_lr,
        color="green",
        linestyle="--",
        alpha=0.8,
        label=f"Suggested LR: {result.suggested_lr:.2e}",
    )
    ax1.axvline(
        result.min_gradient_lr,
        color="orange",
        linestyle="--",
        alpha=0.8,
        label=f"Min Gradient LR: {result.min_gradient_lr:.2e}",
    )
    ax1.set_xlabel("Learning Rate")
    ax1.set_ylabel("Loss")
    ax1.set_title("Learning Rate vs Loss")
    ax1.legend()
    ax1.grid(True, alpha=0.3)

    # Plot 2: Learning rate vs Loss (linear scale, zoomed)
    # Focus on the interesting region around minimum
    min_loss_idx = np.argmin(result.smoothed_losses)
    start_idx = max(0, min_loss_idx - 20)
    end_idx = min(len(result.learning_rates), min_loss_idx + 20)

    ax2.plot(
        result.learning_rates[start_idx:end_idx],
        result.losses[start_idx:end_idx],
        "b-",
        alpha=0.7,
    )
    ax2.plot(
        result.learning_rates[start_idx:end_idx],
        result.smoothed_losses[start_idx:end_idx],
        "r-",
        linewidth=2,
    )
    ax2.axvline(result.suggested_lr, color="green", linestyle="--", alpha=0.8)
    ax2.axvline(result.min_gradient_lr, color="orange", linestyle="--", alpha=0.8)
    ax2.set_xlabel("Learning Rate")
    ax2.set_ylabel("Loss")
    ax2.set_title("Loss (Zoomed Region)")
    ax2.grid(True, alpha=0.3)

    # Plot 3: Loss gradient
    gradients = np.gradient(result.smoothed_losses, np.log10(result.learning_rates))
    ax3.semilogx(result.learning_rates, gradients, "purple", linewidth=2)
    ax3.axvline(result.min_gradient_lr, color="orange", linestyle="--", alpha=0.8)
    ax3.axhline(0, color="black", linestyle="-", alpha=0.3)
    ax3.set_xlabel("Learning Rate")
    ax3.set_ylabel("Loss Gradient")
    ax3.set_title("Loss Gradient vs Learning Rate")
    ax3.grid(True, alpha=0.3)

    # Plot 4: Statistics summary
    ax4.axis("off")
    stats_text = f"""
    Analysis Summary:

    Suggested Learning Rate: {result.suggested_lr:.2e}
    Min Gradient Learning Rate: {result.min_gradient_lr:.2e}

    Minimum Loss: {result.analysis["min_loss"]:.6f}
    Loss Reduction: {result.analysis["loss_reduction_ratio"]:.2%}

    Loss Variance: {result.analysis["loss_variance"]:.6f}
    Coefficient of Variation: {result.analysis["coefficient_of_variation"]:.3f}

    Convergence Ratio: {result.analysis["convergence_ratio"]:.3f}
    """
    ax4.text(
        0.05,
        0.95,
        stats_text,
        transform=ax4.transAxes,
        fontsize=10,
        verticalalignment="top",
        fontfamily="monospace",
        bbox=dict(boxstyle="round", facecolor="lightgray", alpha=0.8),
    )

    plt.tight_layout()

    if save_path:
        plt.savefig(save_path, dpi=300, bbox_inches="tight")
        print(f"Plot saved to {save_path}")

    plt.show()

SimulatedNN

Simulated neural network for demonstration.

Source code in src/dlhub/tuning/learning_rate_finder.py

class SimulatedNN:
    """Simulated neural network for demonstration."""

    def __init__(self):
        self.reset()

    def reset(self):
        """Reset network to initial state."""
        self.weights = np.random.normal(0, 1, 10)  # 10 random weights
        self.momentum = np.zeros_like(self.weights)
        self.step_count = 0

    def train_step(self, learning_rate):
        """Simulate one training step."""
        self.step_count += 1

        # Simulate loss function with multiple minima
        base_loss = 2.0 * np.exp(-self.step_count / 20)

        # Learning rate effects
        if learning_rate < 1e-5:
            lr_penalty = 0.5  # Too small - slow convergence
        elif learning_rate > 0.1:
            lr_penalty = (learning_rate - 0.1) * 10  # Too large - instability
        else:
            lr_penalty = 0.0

        gradients = np.random.normal(0, 0.1, len(self.weights))
        self.weights -= learning_rate * gradients

        # Total loss with noise
        loss = base_loss + lr_penalty + np.random.normal(0, 0.01)
        return max(loss, 0.001)

reset

reset()

Reset network to initial state.

Source code in src/dlhub/tuning/learning_rate_finder.py

def reset(self):
    """Reset network to initial state."""
    self.weights = np.random.normal(0, 1, 10)  # 10 random weights
    self.momentum = np.zeros_like(self.weights)
    self.step_count = 0

train_step

train_step(learning_rate)

Simulate one training step.

Source code in src/dlhub/tuning/learning_rate_finder.py

def train_step(self, learning_rate):
    """Simulate one training step."""
    self.step_count += 1

    # Simulate loss function with multiple minima
    base_loss = 2.0 * np.exp(-self.step_count / 20)

    # Learning rate effects
    if learning_rate < 1e-5:
        lr_penalty = 0.5  # Too small - slow convergence
    elif learning_rate > 0.1:
        lr_penalty = (learning_rate - 0.1) * 10  # Too large - instability
    else:
        lr_penalty = 0.0

    gradients = np.random.normal(0, 0.1, len(self.weights))
    self.weights -= learning_rate * gradients

    # Total loss with noise
    loss = base_loss + lr_penalty + np.random.normal(0, 0.01)
    return max(loss, 0.001)

find_learning_rate

find_learning_rate(train_function: Callable[[float], float], reset_function: Callable[[], None], min_lr: float = 1e-07, max_lr: float = 10.0, num_iterations: int = 100, step_mode: str = 'exp', smooth_beta: float = 0.98, verbose: bool = True, plot: bool = True) -> LearningRateFinderResult

Convenience function to find optimal learning rate.

Parameters:

Name	Type	Description	Default
train_function	callable	Function that takes learning_rate (float) and returns loss (float)	required
reset_function	callable	Function to reset model to initial state	required
min_lr	float	Minimum learning rate to test	1e-7
max_lr	float	Maximum learning rate to test	10.0
num_iterations	int	Number of iterations for the test	100
step_mode	str	Learning rate stepping mode ('exp' or 'linear')	'exp'
smooth_beta	float	Smoothing factor for loss curves	0.98
verbose	bool	Whether to print progress	True
plot	bool	Whether to plot results	True

Returns:

Type	Description
LearningRateFinderResult	Results including suggested learning rate

Examples:

>>> # Example with simple quadratic loss
>>> def train_step(lr):
...     # Simulate one training step
...     current_w = getattr(train_step, "w", 1.0)  # Get current weight
...     target_w = 0.5  # Target weight
...     loss = (current_w - target_w) ** 2
...
...     # Gradient descent update
...     gradient = 2 * (current_w - target_w)
...     train_step.w = current_w - lr * gradient
...
...     return loss + np.random.normal(0, 0.01)  # Add noise
>>> def reset_model():
...     train_step.w = 1.0  # Reset to initial weight
>>> result = find_learning_rate(
...     train_step, reset_model, min_lr=1e-4, max_lr=1.0, num_iterations=50
... )
>>> print(f"Suggested learning rate: {result.suggested_lr}")

Source code in src/dlhub/tuning/learning_rate_finder.py

def find_learning_rate(
    train_function: Callable[[float], float],
    reset_function: Callable[[], None],
    min_lr: float = 1e-7,
    max_lr: float = 10.0,
    num_iterations: int = 100,
    step_mode: str = "exp",
    smooth_beta: float = 0.98,
    verbose: bool = True,
    plot: bool = True,
) -> LearningRateFinderResult:
    """
    Convenience function to find optimal learning rate.

    Parameters
    ----------
    train_function : callable
        Function that takes learning_rate (float) and returns loss (float)
    reset_function : callable
        Function to reset model to initial state
    min_lr : float, default=1e-7
        Minimum learning rate to test
    max_lr : float, default=10.0
        Maximum learning rate to test
    num_iterations : int, default=100
        Number of iterations for the test
    step_mode : str, default='exp'
        Learning rate stepping mode ('exp' or 'linear')
    smooth_beta : float, default=0.98
        Smoothing factor for loss curves
    verbose : bool, default=True
        Whether to print progress
    plot : bool, default=True
        Whether to plot results

    Returns
    -------
    LearningRateFinderResult
        Results including suggested learning rate

    Examples
    --------
    >>> # Example with simple quadratic loss
    >>> def train_step(lr):
    ...     # Simulate one training step
    ...     current_w = getattr(train_step, "w", 1.0)  # Get current weight
    ...     target_w = 0.5  # Target weight
    ...     loss = (current_w - target_w) ** 2
    ...
    ...     # Gradient descent update
    ...     gradient = 2 * (current_w - target_w)
    ...     train_step.w = current_w - lr * gradient
    ...
    ...     return loss + np.random.normal(0, 0.01)  # Add noise
    >>> def reset_model():
    ...     train_step.w = 1.0  # Reset to initial weight
    >>> result = find_learning_rate(
    ...     train_step, reset_model, min_lr=1e-4, max_lr=1.0, num_iterations=50
    ... )
    >>> print(f"Suggested learning rate: {result.suggested_lr}")
    """
    trainer = FunctionTrainer(train_function, reset_function)

    finder = LearningRateFinder(
        trainer=trainer,
        min_lr=min_lr,
        max_lr=max_lr,
        num_iterations=num_iterations,
        step_mode=step_mode,
        smooth_beta=smooth_beta,
    )

    result = finder.find(verbose=verbose)

    if plot:
        finder.plot_results(result)

    return result

suggest_learning_rate_schedule

suggest_learning_rate_schedule(result: LearningRateFinderResult, schedule_type: str = 'onecycle') -> dict[str, Any]

Suggest learning rate schedule based on finder results.

Parameters:

Name	Type	Description	Default
result	LearningRateFinderResult	Results from learning rate finder	required
schedule_type	str	Type of schedule to suggest ('onecycle', 'cyclic', 'cosine', 'step')	'onecycle'

Returns:

Type	Description
dict	Suggested schedule parameters

Source code in src/dlhub/tuning/learning_rate_finder.py

def suggest_learning_rate_schedule(
    result: LearningRateFinderResult, schedule_type: str = "onecycle"
) -> dict[str, Any]:
    """
    Suggest learning rate schedule based on finder results.

    Parameters
    ----------
    result : LearningRateFinderResult
        Results from learning rate finder
    schedule_type : str, default='onecycle'
        Type of schedule to suggest ('onecycle', 'cyclic', 'cosine', 'step')

    Returns
    -------
    dict
        Suggested schedule parameters
    """
    max_lr = result.min_gradient_lr
    base_lr = result.suggested_lr

    if schedule_type == "onecycle":
        return {
            "schedule_type": "onecycle",
            "max_lr": max_lr,
            "base_lr": base_lr,
            "pct_start": 0.3,  # 30% warmup
            "final_div_factor": 1e4,  # Final LR = max_lr / final_div_factor
            "description": "One-cycle policy with warmup and annealing",
        }

    elif schedule_type == "cyclic":
        return {
            "schedule_type": "cyclic",
            "base_lr": base_lr,
            "max_lr": max_lr,
            "step_size_up": 2000,  # Steps to increase from base to max
            "mode": "triangular2",  # Decreasing amplitude
            "description": "Cyclical learning rate with triangular policy",
        }

    elif schedule_type == "cosine":
        return {
            "schedule_type": "cosine",
            "initial_lr": max_lr,
            "min_lr": base_lr,
            "T_max": 10,  # Period of cosine annealing
            "description": "Cosine annealing with restarts",
        }

    elif schedule_type == "step":
        return {
            "schedule_type": "step",
            "initial_lr": max_lr / 3,  # Conservative start
            "step_size": 10,  # Epochs between reductions
            "gamma": 0.5,  # Multiplication factor
            "description": "Step decay schedule",
        }

    else:
        raise ValueError(f"Unknown schedule type: {schedule_type}")

quadratic_train_step

quadratic_train_step(lr)

Simulate one training step on quadratic function.

Source code in src/dlhub/tuning/learning_rate_finder.py

def quadratic_train_step(lr):
    """Simulate one training step on quadratic function."""
    current_w = getattr(quadratic_train_step, "w", 1.0)
    target_w = 0.5

    loss = (current_w - target_w) ** 2
    gradient = 2 * (current_w - target_w)

    quadratic_train_step.w = current_w - lr * gradient

    # Add some noise to simulate realistic training
    return loss + np.random.normal(0, 0.001)

reset_quadratic

reset_quadratic()

Reset model to initial state.

Source code in src/dlhub/tuning/learning_rate_finder.py

def reset_quadratic():
    """Reset model to initial state."""
    quadratic_train_step.w = 1.0

multifidelity

Multi-Fidelity Optimization for Hyperparameter Tuning

ASHA (Asynchronous Successive Halving) implementation for efficient resource allocation across hyperparameter candidates. This approach uses cheaper approximations (lower fidelity) to guide the search, then evaluates promising candidates at full fidelity.

References

• Li, L., et al. (2018). "Massively Parallel Hyperparameter Tuning." arXiv preprint arXiv:1810.05934.
• Jamieson, K., & Talwalkar, A. (2016). "Non-stochastic best arm identification and hyperparameter optimization." Artificial Intelligence and Statistics.

Author

Deep Learning Reference Hub

License

MIT License

Notes

Multi-fidelity optimization is particularly effective when: 1. Training time is expensive 2. Early performance correlates with final performance 3. You have many hyperparameter configurations to evaluate 4. Computational resources can be allocated dynamically

FidelityConfig dataclass

Configuration for a fidelity level.

Attributes:

Name	Type	Description
name	str	Name of the fidelity level
budget	int	Budget/resource allocation for this fidelity
min_budget	int	Minimum budget required for this fidelity
max_budget	int	Maximum budget for this fidelity

Source code in src/dlhub/tuning/multifidelity.py

@dataclass
class FidelityConfig:
    """
    Configuration for a fidelity level.

    Attributes
    ----------
    name : str
        Name of the fidelity level
    budget : int
        Budget/resource allocation for this fidelity
    min_budget : int
        Minimum budget required for this fidelity
    max_budget : int
        Maximum budget for this fidelity
    """

    name: str
    budget: int
    min_budget: int = 1
    max_budget: int = 1000

CandidateResult dataclass

Result from evaluating a hyperparameter candidate.

Attributes:

Name	Type	Description
config_id	int	Unique identifier for the configuration
hyperparams	dict	Hyperparameter configuration
fidelity	int	Fidelity level used for evaluation
score	float	Performance score achieved
training_time	float	Time taken for training
metadata	dict	Additional metadata from training

Source code in src/dlhub/tuning/multifidelity.py

@dataclass
class CandidateResult:
    """
    Result from evaluating a hyperparameter candidate.

    Attributes
    ----------
    config_id : int
        Unique identifier for the configuration
    hyperparams : dict
        Hyperparameter configuration
    fidelity : int
        Fidelity level used for evaluation
    score : float
        Performance score achieved
    training_time : float
        Time taken for training
    metadata : dict
        Additional metadata from training
    """

    config_id: int
    hyperparams: dict[str, Any]
    fidelity: int
    score: float
    training_time: float
    metadata: dict[str, Any] = field(default_factory=dict)

MultiFidelityResult dataclass

Results from multi-fidelity optimization.

Attributes:

Name	Type	Description
best_config	dict or None	Best hyperparameter configuration found, or None if the run recorded no results at all (see Notes)
best_score	float or None	Best score achieved, or None for a run with no results
best_fidelity	int or None	Fidelity level of best result, or None for a run with no results
all_results	list	All evaluation results
total_time	float	Total optimization time
total_budget_used	int	Total computational budget consumed
statistics	dict	Optimization statistics and analysis, empty for a run with no results

Notes

A run can legitimately record nothing: max_iterations=0 grants no budget, and a timeout that has already elapsed stops the first submission. Both return this dataclass with the three best_* fields set to None rather than raising, so the three are optional together -- either all are None or none are. all_results is empty and statistics is {} in exactly those runs, so if result.best_config is None and if not result.all_results are equivalent tests.

An empty initial_configurations is caller error rather than an empty run, and ASHAOptimizer.optimize rejects it with a ValueError.

Source code in src/dlhub/tuning/multifidelity.py

@dataclass
class MultiFidelityResult:
    """
    Results from multi-fidelity optimization.

    Attributes
    ----------
    best_config : dict or None
        Best hyperparameter configuration found, or None if the run recorded no
        results at all (see Notes)
    best_score : float or None
        Best score achieved, or None for a run with no results
    best_fidelity : int or None
        Fidelity level of best result, or None for a run with no results
    all_results : list
        All evaluation results
    total_time : float
        Total optimization time
    total_budget_used : int
        Total computational budget consumed
    statistics : dict
        Optimization statistics and analysis, empty for a run with no results

    Notes
    -----
    A run can legitimately record nothing: `max_iterations=0` grants no budget,
    and a `timeout` that has already elapsed stops the first submission. Both
    return this dataclass with the three `best_*` fields set to None rather than
    raising, so the three are optional together -- either all are None or none
    are. `all_results` is empty and `statistics` is `{}` in exactly those runs,
    so `if result.best_config is None` and `if not result.all_results` are
    equivalent tests.

    An empty `initial_configurations` is caller error rather than an empty run,
    and `ASHAOptimizer.optimize` rejects it with a ValueError.
    """

    best_config: dict[str, Any] | None
    best_score: float | None
    best_fidelity: int | None
    all_results: list[CandidateResult]
    total_time: float
    total_budget_used: int
    statistics: dict[str, Any] = field(default_factory=dict)

FidelityEvaluator

Bases: ABC

Abstract base class for fidelity-aware evaluation.

Defines the interface for evaluating hyperparameter configurations at different fidelity levels.

Source code in src/dlhub/tuning/multifidelity.py

class FidelityEvaluator(ABC):
    """
    Abstract base class for fidelity-aware evaluation.

    Defines the interface for evaluating hyperparameter configurations
    at different fidelity levels.
    """

    @abstractmethod
    def evaluate(
        self, hyperparams: dict[str, Any], fidelity: int
    ) -> tuple[float, dict[str, Any]]:
        """
        Evaluate hyperparameters at given fidelity.

        Parameters
        ----------
        hyperparams : dict
            Hyperparameter configuration
        fidelity : int
            Fidelity level (e.g., training epochs, data size)

        Returns
        -------
        tuple
            (score, metadata) where score is performance and metadata contains
            additional information from training
        """
        pass

    @abstractmethod
    def get_fidelity_range(self) -> tuple[int, int]:
        """
        Get the valid fidelity range.

        Returns
        -------
        tuple
            (min_fidelity, max_fidelity)
        """
        pass

evaluate abstractmethod

evaluate(hyperparams: dict[str, Any], fidelity: int) -> tuple[float, dict[str, Any]]

Evaluate hyperparameters at given fidelity.

Parameters:

Name	Type	Description	Default
hyperparams	dict	Hyperparameter configuration	required
fidelity	int	Fidelity level (e.g., training epochs, data size)	required

Returns:

Type	Description
tuple	(score, metadata) where score is performance and metadata contains additional information from training

Source code in src/dlhub/tuning/multifidelity.py

@abstractmethod
def evaluate(
    self, hyperparams: dict[str, Any], fidelity: int
) -> tuple[float, dict[str, Any]]:
    """
    Evaluate hyperparameters at given fidelity.

    Parameters
    ----------
    hyperparams : dict
        Hyperparameter configuration
    fidelity : int
        Fidelity level (e.g., training epochs, data size)

    Returns
    -------
    tuple
        (score, metadata) where score is performance and metadata contains
        additional information from training
    """
    pass

get_fidelity_range abstractmethod

get_fidelity_range() -> tuple[int, int]

Get the valid fidelity range.

Returns:

Type	Description
tuple	(min_fidelity, max_fidelity)

Source code in src/dlhub/tuning/multifidelity.py

@abstractmethod
def get_fidelity_range(self) -> tuple[int, int]:
    """
    Get the valid fidelity range.

    Returns
    -------
    tuple
        (min_fidelity, max_fidelity)
    """
    pass

FunctionEvaluator

Bases: FidelityEvaluator

Function-based evaluator wrapper.

Wraps a user-provided evaluation function to conform to the FidelityEvaluator interface.

Parameters:

Name	Type	Description	Default
eval_function	callable	Function that takes (hyperparams, fidelity) and returns (score, metadata)	required
min_fidelity	int	Minimum fidelity level	1
max_fidelity	int	Maximum fidelity level	100

Source code in src/dlhub/tuning/multifidelity.py

class FunctionEvaluator(FidelityEvaluator):
    """
    Function-based evaluator wrapper.

    Wraps a user-provided evaluation function to conform to the
    FidelityEvaluator interface.

    Parameters
    ----------
    eval_function : callable
        Function that takes (hyperparams, fidelity) and returns (score, metadata)
    min_fidelity : int
        Minimum fidelity level
    max_fidelity : int
        Maximum fidelity level
    """

    def __init__(
        self, eval_function: Callable, min_fidelity: int = 1, max_fidelity: int = 100
    ):
        self.eval_function = eval_function
        self.min_fidelity = min_fidelity
        self.max_fidelity = max_fidelity

    def evaluate(
        self, hyperparams: dict[str, Any], fidelity: int
    ) -> tuple[float, dict[str, Any]]:
        """Evaluate using wrapped function."""
        return self.eval_function(hyperparams, fidelity)

    def get_fidelity_range(self) -> tuple[int, int]:
        """Get fidelity range."""
        return self.min_fidelity, self.max_fidelity

evaluate

evaluate(hyperparams: dict[str, Any], fidelity: int) -> tuple[float, dict[str, Any]]

Evaluate using wrapped function.

Source code in src/dlhub/tuning/multifidelity.py

def evaluate(
    self, hyperparams: dict[str, Any], fidelity: int
) -> tuple[float, dict[str, Any]]:
    """Evaluate using wrapped function."""
    return self.eval_function(hyperparams, fidelity)

get_fidelity_range

get_fidelity_range() -> tuple[int, int]

Get fidelity range.

Source code in src/dlhub/tuning/multifidelity.py

def get_fidelity_range(self) -> tuple[int, int]:
    """Get fidelity range."""
    return self.min_fidelity, self.max_fidelity

ASHAOptimizer

Asynchronous Successive Halving Algorithm (ASHA) for multi-fidelity optimization.

ASHA efficiently allocates computational resources by starting many configurations at low fidelity and promoting the most promising ones to higher fidelities.

Parameters:

Name	Type	Description	Default
evaluator	FidelityEvaluator	Evaluator for hyperparameter configurations	required
reduction_factor	int	Factor by which to reduce number of configurations at each rung	3
min_budget	int	Minimum budget (fidelity) to start configurations	1
max_budget	int	Maximum budget (fidelity) for full evaluation	81
grace_period	int	Minimum budget before first promotion opportunity	1
max_concurrent	int	Maximum number of concurrent evaluations	4
random_state	int	Random seed for reproducibility	None

Source code in src/dlhub/tuning/multifidelity.py

class ASHAOptimizer:
    """
    Asynchronous Successive Halving Algorithm (ASHA) for multi-fidelity optimization.

    ASHA efficiently allocates computational resources by starting many configurations
    at low fidelity and promoting the most promising ones to higher fidelities.

    Parameters
    ----------
    evaluator : FidelityEvaluator
        Evaluator for hyperparameter configurations
    reduction_factor : int, default=3
        Factor by which to reduce number of configurations at each rung
    min_budget : int, default=1
        Minimum budget (fidelity) to start configurations
    max_budget : int, default=81
        Maximum budget (fidelity) for full evaluation
    grace_period : int, default=1
        Minimum budget before first promotion opportunity
    max_concurrent : int, default=4
        Maximum number of concurrent evaluations
    random_state : int, optional
        Random seed for reproducibility
    """

    def __init__(
        self,
        evaluator: FidelityEvaluator,
        reduction_factor: int = 3,
        min_budget: int = 1,
        max_budget: int = 81,
        grace_period: int = 1,
        max_concurrent: int = 4,
        random_state: int | None = None,
    ):

        self.evaluator = evaluator
        self.reduction_factor = reduction_factor
        self.min_budget = min_budget
        self.max_budget = max_budget
        self.grace_period = grace_period
        self.max_concurrent = max_concurrent

        if random_state is not None:
            np.random.seed(random_state)

        # Validate parameters
        eval_min, eval_max = evaluator.get_fidelity_range()
        if min_budget < eval_min or max_budget > eval_max:
            raise ValueError(
                f"Budget range [{min_budget}, {max_budget}] "
                f"outside evaluator range [{eval_min}, {eval_max}]"
            )

        # Initialize internal state
        self.config_counter = 0
        self.rungs = self._create_rungs()
        self.results_history = []
        self.active_evaluations = {}
        self.lock = threading.Lock()

        # Statistics tracking
        self.total_budget_used = 0
        self.best_result = None

    def _create_rungs(self) -> list[dict]:
        """
        Create ASHA rungs (fidelity levels and promotion thresholds).

        Returns
        -------
        list
            List of rung dictionaries with budget and promotion info
        """
        rungs = []
        current_budget = self.min_budget

        while current_budget <= self.max_budget:
            rung = {
                "budget": current_budget,
                "candidates": [],  # (config_id, score) tuples
                "promoted": set(),  # Set of promoted config_ids
                "n_required": 0,  # Number of configs needed for promotion
            }
            rungs.append(rung)
            current_budget *= self.reduction_factor

        # A rung promotes its top 1 / reduction_factor, so it needs at least
        # reduction_factor results before that fraction means anything. The
        # threshold is a property of the reduction factor, not of how many rungs
        # the ladder happens to have: deriving it from the rung count made a
        # taller ladder demand more results at the bottom and stall there.
        for rung in rungs[:-1]:  # The last rung promotes nowhere
            rung["n_required"] = self.reduction_factor

        return rungs

    def _get_rung_for_budget(self, budget: int) -> int | None:
        """Get rung index for given budget."""
        for i, rung in enumerate(self.rungs):
            if rung["budget"] == budget:
                return i
        return None

    def _add_result(self, result: CandidateResult) -> None:
        """Add result and check for promotions."""
        with self.lock:
            self.results_history.append(result)
            self.total_budget_used += result.fidelity

            if self.best_result is None or result.score > self.best_result.score:
                self.best_result = result

            rung_idx = self._get_rung_for_budget(result.fidelity)
            if rung_idx is not None:
                rung = self.rungs[rung_idx]
                rung["candidates"].append((result.config_id, result.score))

    def _get_next_config_to_evaluate(self) -> tuple[int, dict[str, Any], int] | None:
        """
        Claim the next promotion and return the work it implies.

        This is the only place the promotion rule lives, and calling it is not a
        query: the returned configuration is recorded as promoted so that
        concurrent workers cannot claim it twice. Callers must therefore use
        what they get rather than calling it to ask whether work exists.

        Returns
        -------
        tuple or None
            (config_id, hyperparams, fidelity) or None if no work available
        """
        with self.lock:
            for rung_idx in range(len(self.rungs) - 1):
                rung = self.rungs[rung_idx]
                next_rung = self.rungs[rung_idx + 1]

                if len(rung["candidates"]) >= rung["n_required"]:
                    sorted_candidates = sorted(
                        rung["candidates"], key=lambda x: x[1], reverse=True
                    )
                    n_promote = max(1, len(sorted_candidates) // self.reduction_factor)

                    for i in range(min(n_promote, len(sorted_candidates))):
                        config_id, score = sorted_candidates[i]

                        if config_id not in rung["promoted"]:
                            rung["promoted"].add(config_id)

                            hyperparams = None
                            for res in self.results_history:
                                if res.config_id == config_id:
                                    hyperparams = res.hyperparams
                                    break

                            if hyperparams is not None:
                                return config_id, hyperparams, next_rung["budget"]

            return None

    def _evaluate_config(
        self, config_id: int, hyperparams: dict[str, Any], fidelity: int
    ) -> CandidateResult:
        """Evaluate a single configuration."""
        start_time = time.time()

        try:
            score, metadata = self.evaluator.evaluate(hyperparams, fidelity)
            training_time = time.time() - start_time

            if np.isnan(score) or np.isinf(score):
                score = -np.inf

            return CandidateResult(
                config_id=config_id,
                hyperparams=hyperparams.copy(),
                fidelity=fidelity,
                score=float(score),
                training_time=training_time,
                metadata=metadata,
            )

        except Exception as e:
            training_time = time.time() - start_time
            warnings.warn(f"Evaluation failed for config {config_id}: {e}")

            return CandidateResult(
                config_id=config_id,
                hyperparams=hyperparams.copy(),
                fidelity=fidelity,
                score=-np.inf,
                training_time=training_time,
                metadata={"error": str(e)},
            )

    def suggest_initial_configurations(
        self, configurations: list[dict[str, Any]]
    ) -> None:
        """
        Add initial configurations to start evaluation.

        Parameters
        ----------
        configurations : list
            List of hyperparameter configurations to evaluate
        """
        with self.lock:
            for config in configurations:
                config_id = self.config_counter
                self.config_counter += 1
                # Queued at the lowest rung. Every configuration earns its way
                # up from here, which is what makes the search cheap.
                self.active_evaluations[config_id] = (config.copy(), self.min_budget)

    def optimize(
        self,
        initial_configurations: list[dict[str, Any]],
        max_iterations: int = 100,
        timeout: float | None = None,
        verbose: bool = True,
    ) -> MultiFidelityResult:
        """
        Run ASHA optimization.

        Parameters
        ----------
        initial_configurations : list
            Initial hyperparameter configurations to evaluate. Must be non-empty.
        max_iterations : int, default=100
            Maximum number of evaluations to perform. Counted at submission, and
            every submitted evaluation is recorded, so this bounds the results as
            well as the work -- at any `max_concurrent`.
        timeout : float, optional
            Maximum time in seconds (None for no timeout). Evaluations already
            running when the clock runs out are still awaited and recorded; the
            timeout stops new submissions, it does not cancel paid-for work.
        verbose : bool, default=True
            Whether to print progress information

        Returns
        -------
        MultiFidelityResult
            Optimization results. A run granted no budget -- `max_iterations=0`,
            or a `timeout` already elapsed -- records nothing and reports
            `best_config`, `best_score`, and `best_fidelity` as None.

        Raises
        ------
        ValueError
            If `initial_configurations` is empty. A search with no candidates has
            no answer to return, and an empty list is more often a search space
            that filtered down to nothing than a deliberate no-op -- so it is
            reported rather than absorbed into an empty result.
        """
        if not initial_configurations:
            raise ValueError(
                "initial_configurations is empty; ASHA needs at least one "
                "candidate to search"
            )

        if verbose:
            print("Starting ASHA Multi-Fidelity Optimization...")
            print(f"Reduction factor: {self.reduction_factor}")
            print(f"Budget range: [{self.min_budget}, {self.max_budget}]")
            print(f"Initial configurations: {len(initial_configurations)}")
            print(f"Max concurrent evaluations: {self.max_concurrent}")

        start_time = time.time()
        self.suggest_initial_configurations(initial_configurations)

        iteration = 0
        evaluations_completed = 0

        work_queue = []
        for config_id, (config, fidelity) in self.active_evaluations.items():
            work_queue.append((config_id, config, fidelity))
        self.active_evaluations.clear()

        def record(future, config_id: int) -> None:
            """Move one finished evaluation into the results, or warn."""
            nonlocal evaluations_completed

            try:
                result = future.result()
            except Exception as e:
                warnings.warn(f"Future failed for config {config_id}: {e}")
                return

            self._add_result(result)
            evaluations_completed += 1

            # `_add_result` has just run, so `best_result` is set; reading it into
            # a local says that to the type checker, and takes one attribute read
            # rather than two off an object other threads are writing.
            best = self.best_result
            if (
                verbose
                and best is not None
                and evaluations_completed % max(1, max_iterations // 20) == 0
            ):
                print(
                    f"Completed {evaluations_completed} evaluations - "
                    f"Best score: {best.score:.6f} "
                    f"(fidelity {best.fidelity})"
                )

        with ThreadPoolExecutor(max_workers=self.max_concurrent) as executor:
            active_futures = {}

            while iteration < max_iterations and (
                timeout is None or time.time() - start_time < timeout
            ):
                # `_get_next_config_to_evaluate` marks what it returns as
                # promoted, so it is called once per work item and its result is
                # kept. Calling it again as a loop condition would consume a
                # promotion and drop the configuration on the floor.
                while len(active_futures) < self.max_concurrent:
                    if work_queue:
                        config_id, hyperparams, fidelity = work_queue.pop(0)
                    else:
                        next_work = self._get_next_config_to_evaluate()
                        if next_work is None:
                            break
                        config_id, hyperparams, fidelity = next_work

                    # Submit evaluation
                    future = executor.submit(
                        self._evaluate_config, config_id, hyperparams, fidelity
                    )
                    active_futures[future] = (config_id, hyperparams, fidelity)
                    iteration += 1

                    if iteration >= max_iterations:
                        break

                if active_futures:
                    # The one-second bound is a poll interval, not a deadline: it
                    # returns control to the loop so `max_iterations` and
                    # `timeout` get re-checked while evaluations are still
                    # running. `as_completed` reports an elapsed poll by raising,
                    # and a tuner exists to run evaluations that take minutes, so
                    # letting that escape would abort every realistic run the
                    # moment no evaluation happened to finish within a second.
                    # Before 3.11 this is not the builtin `TimeoutError`, so it
                    # is caught under its own name rather than by coincidence.
                    completed_futures = []
                    try:
                        for future in as_completed(active_futures, timeout=1.0):
                            completed_futures.append(future)
                            break  # Process one at a time for responsiveness
                    except FuturesTimeoutError:
                        pass

                    for future in completed_futures:
                        config_id, _, _ = active_futures.pop(future)
                        record(future, config_id)

                # Break if no more work and no active evaluations
                if not active_futures and not work_queue:
                    # Requeued rather than discarded: this call promotes the
                    # configuration it returns, so dropping it would lose the
                    # promotion and end the run one rung short.
                    next_work = self._get_next_config_to_evaluate()
                    if next_work is None:
                        if verbose:
                            print("No more configurations to evaluate - stopping")
                        break
                    work_queue.append(next_work)

            # Whatever is still in flight when the loop exits has already been
            # submitted, so the pool will compute it whether or not anyone waits:
            # `ThreadPoolExecutor.__exit__` joins every worker. Recording it is
            # therefore free, and discarding it would under-report
            # `total_budget_used` by up to `max_concurrent - 1` evaluations --
            # flattering `budget_efficiency` with work the run really paid for.
            # This also keeps `max_iterations` meaning one thing at any worker
            # count: submissions, which now all become results.
            for future, (config_id, _, _) in list(active_futures.items()):
                record(future, config_id)
            active_futures.clear()

        total_time = time.time() - start_time

        statistics = self._compute_statistics()

        if verbose:
            print(f"\nOptimization completed in {total_time:.2f} seconds!")
            print(f"Total evaluations: {evaluations_completed}")
            print(f"Total budget used: {self.total_budget_used}")
            # `best_result` is None exactly when nothing was recorded, which the
            # budget limits make reachable without any caller error: the summary
            # says so rather than dereferencing it.
            if self.best_result is None:
                print("No evaluation completed - no best configuration to report")
            else:
                print(f"Best score: {self.best_result.score:.6f}")
                print(f"Best configuration: {self.best_result.hyperparams}")
                print(f"Best fidelity: {self.best_result.fidelity}")

        best = self.best_result

        return MultiFidelityResult(
            best_config=best.hyperparams if best is not None else None,
            best_score=best.score if best is not None else None,
            best_fidelity=best.fidelity if best is not None else None,
            all_results=self.results_history,
            total_time=total_time,
            total_budget_used=self.total_budget_used,
            statistics=statistics,
        )

    def _compute_statistics(self) -> dict[str, Any]:
        """Compute optimization statistics."""
        if not self.results_history:
            return {}

        fidelity_stats = defaultdict(list)
        for result in self.results_history:
            if result.score != -np.inf:
                fidelity_stats[result.fidelity].append(result.score)

        fidelity_analysis = {}
        for fidelity, scores in fidelity_stats.items():
            fidelity_analysis[fidelity] = {
                "n_evaluations": len(scores),
                "mean_score": np.mean(scores),
                "std_score": np.std(scores),
                "max_score": np.max(scores),
                "min_score": np.min(scores),
            }

        all_scores = [r.score for r in self.results_history if r.score != -np.inf]
        all_times = [r.training_time for r in self.results_history]

        # Reached only past the empty-history guard at the top of the method, so
        # `_add_result` has run and this is set. Testing it anyway is what lets
        # the `type: ignore` come off -- the guard is real, but a checker cannot
        # connect it to this attribute.
        best = self.best_result

        statistics = {
            "total_evaluations": len(self.results_history),
            "successful_evaluations": len(all_scores),
            "failed_evaluations": len(self.results_history) - len(all_scores),
            "mean_score": np.mean(all_scores) if all_scores else 0.0,
            "std_score": np.std(all_scores) if len(all_scores) > 1 else 0.0,
            "mean_training_time": np.mean(all_times),
            "total_training_time": np.sum(all_times),
            "fidelity_analysis": fidelity_analysis,
            "budget_efficiency": (
                best.score / self.total_budget_used
                if best is not None and self.total_budget_used > 0
                else 0.0
            ),
            "rungs_used": len([r for r in self.rungs if r["candidates"]]),
        }

        return statistics

suggest_initial_configurations

suggest_initial_configurations(configurations: list[dict[str, Any]]) -> None

Add initial configurations to start evaluation.

Parameters:

Name	Type	Description	Default
configurations	list	List of hyperparameter configurations to evaluate	required

Source code in src/dlhub/tuning/multifidelity.py

def suggest_initial_configurations(
    self, configurations: list[dict[str, Any]]
) -> None:
    """
    Add initial configurations to start evaluation.

    Parameters
    ----------
    configurations : list
        List of hyperparameter configurations to evaluate
    """
    with self.lock:
        for config in configurations:
            config_id = self.config_counter
            self.config_counter += 1
            # Queued at the lowest rung. Every configuration earns its way
            # up from here, which is what makes the search cheap.
            self.active_evaluations[config_id] = (config.copy(), self.min_budget)

optimize

optimize(initial_configurations: list[dict[str, Any]], max_iterations: int = 100, timeout: float | None = None, verbose: bool = True) -> MultiFidelityResult

Run ASHA optimization.

Parameters:

Name	Type	Description	Default
initial_configurations	list	Initial hyperparameter configurations to evaluate. Must be non-empty.	required
max_iterations	int	Maximum number of evaluations to perform. Counted at submission, and every submitted evaluation is recorded, so this bounds the results as well as the work -- at any max_concurrent.	100
timeout	float	Maximum time in seconds (None for no timeout). Evaluations already running when the clock runs out are still awaited and recorded; the timeout stops new submissions, it does not cancel paid-for work.	None
verbose	bool	Whether to print progress information	True

Returns:

Type	Description
MultiFidelityResult	Optimization results. A run granted no budget -- max_iterations=0, or a timeout already elapsed -- records nothing and reports best_config, best_score, and best_fidelity as None.

Raises:

Type	Description
ValueError	If initial_configurations is empty. A search with no candidates has no answer to return, and an empty list is more often a search space that filtered down to nothing than a deliberate no-op -- so it is reported rather than absorbed into an empty result.

Source code in src/dlhub/tuning/multifidelity.py

def optimize(
    self,
    initial_configurations: list[dict[str, Any]],
    max_iterations: int = 100,
    timeout: float | None = None,
    verbose: bool = True,
) -> MultiFidelityResult:
    """
    Run ASHA optimization.

    Parameters
    ----------
    initial_configurations : list
        Initial hyperparameter configurations to evaluate. Must be non-empty.
    max_iterations : int, default=100
        Maximum number of evaluations to perform. Counted at submission, and
        every submitted evaluation is recorded, so this bounds the results as
        well as the work -- at any `max_concurrent`.
    timeout : float, optional
        Maximum time in seconds (None for no timeout). Evaluations already
        running when the clock runs out are still awaited and recorded; the
        timeout stops new submissions, it does not cancel paid-for work.
    verbose : bool, default=True
        Whether to print progress information

    Returns
    -------
    MultiFidelityResult
        Optimization results. A run granted no budget -- `max_iterations=0`,
        or a `timeout` already elapsed -- records nothing and reports
        `best_config`, `best_score`, and `best_fidelity` as None.

    Raises
    ------
    ValueError
        If `initial_configurations` is empty. A search with no candidates has
        no answer to return, and an empty list is more often a search space
        that filtered down to nothing than a deliberate no-op -- so it is
        reported rather than absorbed into an empty result.
    """
    if not initial_configurations:
        raise ValueError(
            "initial_configurations is empty; ASHA needs at least one "
            "candidate to search"
        )

    if verbose:
        print("Starting ASHA Multi-Fidelity Optimization...")
        print(f"Reduction factor: {self.reduction_factor}")
        print(f"Budget range: [{self.min_budget}, {self.max_budget}]")
        print(f"Initial configurations: {len(initial_configurations)}")
        print(f"Max concurrent evaluations: {self.max_concurrent}")

    start_time = time.time()
    self.suggest_initial_configurations(initial_configurations)

    iteration = 0
    evaluations_completed = 0

    work_queue = []
    for config_id, (config, fidelity) in self.active_evaluations.items():
        work_queue.append((config_id, config, fidelity))
    self.active_evaluations.clear()

    def record(future, config_id: int) -> None:
        """Move one finished evaluation into the results, or warn."""
        nonlocal evaluations_completed

        try:
            result = future.result()
        except Exception as e:
            warnings.warn(f"Future failed for config {config_id}: {e}")
            return

        self._add_result(result)
        evaluations_completed += 1

        # `_add_result` has just run, so `best_result` is set; reading it into
        # a local says that to the type checker, and takes one attribute read
        # rather than two off an object other threads are writing.
        best = self.best_result
        if (
            verbose
            and best is not None
            and evaluations_completed % max(1, max_iterations // 20) == 0
        ):
            print(
                f"Completed {evaluations_completed} evaluations - "
                f"Best score: {best.score:.6f} "
                f"(fidelity {best.fidelity})"
            )

    with ThreadPoolExecutor(max_workers=self.max_concurrent) as executor:
        active_futures = {}

        while iteration < max_iterations and (
            timeout is None or time.time() - start_time < timeout
        ):
            # `_get_next_config_to_evaluate` marks what it returns as
            # promoted, so it is called once per work item and its result is
            # kept. Calling it again as a loop condition would consume a
            # promotion and drop the configuration on the floor.
            while len(active_futures) < self.max_concurrent:
                if work_queue:
                    config_id, hyperparams, fidelity = work_queue.pop(0)
                else:
                    next_work = self._get_next_config_to_evaluate()
                    if next_work is None:
                        break
                    config_id, hyperparams, fidelity = next_work

                # Submit evaluation
                future = executor.submit(
                    self._evaluate_config, config_id, hyperparams, fidelity
                )
                active_futures[future] = (config_id, hyperparams, fidelity)
                iteration += 1

                if iteration >= max_iterations:
                    break

            if active_futures:
                # The one-second bound is a poll interval, not a deadline: it
                # returns control to the loop so `max_iterations` and
                # `timeout` get re-checked while evaluations are still
                # running. `as_completed` reports an elapsed poll by raising,
                # and a tuner exists to run evaluations that take minutes, so
                # letting that escape would abort every realistic run the
                # moment no evaluation happened to finish within a second.
                # Before 3.11 this is not the builtin `TimeoutError`, so it
                # is caught under its own name rather than by coincidence.
                completed_futures = []
                try:
                    for future in as_completed(active_futures, timeout=1.0):
                        completed_futures.append(future)
                        break  # Process one at a time for responsiveness
                except FuturesTimeoutError:
                    pass

                for future in completed_futures:
                    config_id, _, _ = active_futures.pop(future)
                    record(future, config_id)

            # Break if no more work and no active evaluations
            if not active_futures and not work_queue:
                # Requeued rather than discarded: this call promotes the
                # configuration it returns, so dropping it would lose the
                # promotion and end the run one rung short.
                next_work = self._get_next_config_to_evaluate()
                if next_work is None:
                    if verbose:
                        print("No more configurations to evaluate - stopping")
                    break
                work_queue.append(next_work)

        # Whatever is still in flight when the loop exits has already been
        # submitted, so the pool will compute it whether or not anyone waits:
        # `ThreadPoolExecutor.__exit__` joins every worker. Recording it is
        # therefore free, and discarding it would under-report
        # `total_budget_used` by up to `max_concurrent - 1` evaluations --
        # flattering `budget_efficiency` with work the run really paid for.
        # This also keeps `max_iterations` meaning one thing at any worker
        # count: submissions, which now all become results.
        for future, (config_id, _, _) in list(active_futures.items()):
            record(future, config_id)
        active_futures.clear()

    total_time = time.time() - start_time

    statistics = self._compute_statistics()

    if verbose:
        print(f"\nOptimization completed in {total_time:.2f} seconds!")
        print(f"Total evaluations: {evaluations_completed}")
        print(f"Total budget used: {self.total_budget_used}")
        # `best_result` is None exactly when nothing was recorded, which the
        # budget limits make reachable without any caller error: the summary
        # says so rather than dereferencing it.
        if self.best_result is None:
            print("No evaluation completed - no best configuration to report")
        else:
            print(f"Best score: {self.best_result.score:.6f}")
            print(f"Best configuration: {self.best_result.hyperparams}")
            print(f"Best fidelity: {self.best_result.fidelity}")

    best = self.best_result

    return MultiFidelityResult(
        best_config=best.hyperparams if best is not None else None,
        best_score=best.score if best is not None else None,
        best_fidelity=best.fidelity if best is not None else None,
        all_results=self.results_history,
        total_time=total_time,
        total_budget_used=self.total_budget_used,
        statistics=statistics,
    )

asha_optimize

asha_optimize(eval_function: Callable[[dict, int], tuple[float, dict]], initial_configurations: list[dict[str, Any]], min_fidelity: int = 1, max_fidelity: int = 81, reduction_factor: int = 3, max_iterations: int = 100, max_concurrent: int = 4, timeout: float | None = None, random_state: int | None = None, verbose: bool = True) -> MultiFidelityResult

Convenience function for ASHA optimization.

Parameters:

Name	Type	Description	Default
eval_function	callable	Function that takes (hyperparams, fidelity) and returns (score, metadata)	required
initial_configurations	list	Initial hyperparameter configurations to evaluate; must be non-empty	required
min_fidelity	int	Minimum fidelity level	1
max_fidelity	int	Maximum fidelity level	81
reduction_factor	int	ASHA reduction factor	3
max_iterations	int	Maximum number of evaluations, performed and recorded alike	100
max_concurrent	int	Maximum concurrent evaluations	4
timeout	float	Timeout in seconds; evaluations already running are still awaited	None
random_state	int	Random seed	None
verbose	bool	Whether to print progress	True

Returns:

Type	Description
MultiFidelityResult	Optimization results. The three best_* fields are None if the run was granted no budget and so recorded nothing.

Raises:

Type	Description
ValueError	If initial_configurations is empty

Examples:

>>> def evaluate_model(hyperparams, fidelity):
...     # Simulate training with given hyperparameters and fidelity
...     lr = hyperparams["learning_rate"]
...     wd = hyperparams["weight_decay"]
...
...     # Simulate performance improving with fidelity
...     base_score = 0.7 - (lr - 0.001) ** 2 - (wd - 0.0001) ** 2
...     fidelity_bonus = 0.2 * (1 - np.exp(-fidelity / 20))
...     noise = np.random.normal(0, 0.01)
...
...     score = base_score + fidelity_bonus + noise
...     metadata = {"fidelity_used": fidelity}
...
...     return score, metadata
>>>
>>> configs = [
...     {"learning_rate": 0.001, "weight_decay": 0.0001},
...     {"learning_rate": 0.01, "weight_decay": 0.001},
...     {"learning_rate": 0.0001, "weight_decay": 0.00001},
... ]
>>>
>>> result = asha_optimize(
...     evaluate_model, configs, min_fidelity=1, max_fidelity=27, max_iterations=20
... )

Source code in src/dlhub/tuning/multifidelity.py

def asha_optimize(
    eval_function: Callable[[dict, int], tuple[float, dict]],
    initial_configurations: list[dict[str, Any]],
    min_fidelity: int = 1,
    max_fidelity: int = 81,
    reduction_factor: int = 3,
    max_iterations: int = 100,
    max_concurrent: int = 4,
    timeout: float | None = None,
    random_state: int | None = None,
    verbose: bool = True,
) -> MultiFidelityResult:
    """
    Convenience function for ASHA optimization.

    Parameters
    ----------
    eval_function : callable
        Function that takes (hyperparams, fidelity) and returns (score, metadata)
    initial_configurations : list
        Initial hyperparameter configurations to evaluate; must be non-empty
    min_fidelity : int, default=1
        Minimum fidelity level
    max_fidelity : int, default=81
        Maximum fidelity level
    reduction_factor : int, default=3
        ASHA reduction factor
    max_iterations : int, default=100
        Maximum number of evaluations, performed and recorded alike
    max_concurrent : int, default=4
        Maximum concurrent evaluations
    timeout : float, optional
        Timeout in seconds; evaluations already running are still awaited
    random_state : int, optional
        Random seed
    verbose : bool, default=True
        Whether to print progress

    Returns
    -------
    MultiFidelityResult
        Optimization results. The three `best_*` fields are None if the run was
        granted no budget and so recorded nothing.

    Raises
    ------
    ValueError
        If `initial_configurations` is empty

    Examples
    --------
    >>> def evaluate_model(hyperparams, fidelity):
    ...     # Simulate training with given hyperparameters and fidelity
    ...     lr = hyperparams["learning_rate"]
    ...     wd = hyperparams["weight_decay"]
    ...
    ...     # Simulate performance improving with fidelity
    ...     base_score = 0.7 - (lr - 0.001) ** 2 - (wd - 0.0001) ** 2
    ...     fidelity_bonus = 0.2 * (1 - np.exp(-fidelity / 20))
    ...     noise = np.random.normal(0, 0.01)
    ...
    ...     score = base_score + fidelity_bonus + noise
    ...     metadata = {"fidelity_used": fidelity}
    ...
    ...     return score, metadata
    >>>
    >>> configs = [
    ...     {"learning_rate": 0.001, "weight_decay": 0.0001},
    ...     {"learning_rate": 0.01, "weight_decay": 0.001},
    ...     {"learning_rate": 0.0001, "weight_decay": 0.00001},
    ... ]
    >>>
    >>> result = asha_optimize(
    ...     evaluate_model, configs, min_fidelity=1, max_fidelity=27, max_iterations=20
    ... )
    """
    evaluator = FunctionEvaluator(eval_function, min_fidelity, max_fidelity)

    optimizer = ASHAOptimizer(
        evaluator=evaluator,
        reduction_factor=reduction_factor,
        min_budget=min_fidelity,
        max_budget=max_fidelity,
        max_concurrent=max_concurrent,
        random_state=random_state,
    )

    return optimizer.optimize(
        initial_configurations=initial_configurations,
        max_iterations=max_iterations,
        timeout=timeout,
        verbose=verbose,
    )

analyze_fidelity_correlation

analyze_fidelity_correlation(result: MultiFidelityResult) -> dict[str, float]

Analyze correlation between different fidelity levels.

Parameters:

Name	Type	Description	Default
result	MultiFidelityResult	Results from multi-fidelity optimization	required

Returns:

Type	Description
dict	Correlation analysis between fidelity levels

Source code in src/dlhub/tuning/multifidelity.py

def analyze_fidelity_correlation(result: MultiFidelityResult) -> dict[str, float]:
    """
    Analyze correlation between different fidelity levels.

    Parameters
    ----------
    result : MultiFidelityResult
        Results from multi-fidelity optimization

    Returns
    -------
    dict
        Correlation analysis between fidelity levels
    """
    config_results = defaultdict(dict)
    for res in result.all_results:
        if res.score != -np.inf:
            config_results[res.config_id][res.fidelity] = res.score

    fidelities = sorted(set(res.fidelity for res in result.all_results))
    correlations = {}

    for i, fid1 in enumerate(fidelities[:-1]):
        for fid2 in fidelities[i + 1 :]:
            common_configs = []
            scores1, scores2 = [], []

            for config_id, fid_scores in config_results.items():
                if fid1 in fid_scores and fid2 in fid_scores:
                    scores1.append(fid_scores[fid1])
                    scores2.append(fid_scores[fid2])

            if len(scores1) >= 3:  # Need at least 3 points for correlation
                correlation = np.corrcoef(scores1, scores2)[0, 1]
                if not np.isnan(correlation):
                    correlations[f"fidelity_{fid1}_vs_{fid2}"] = correlation

    return correlations

quadratic_eval

quadratic_eval(hyperparams: dict[str, Any], fidelity: int) -> tuple[float, dict[str, Any]]

Simulate model evaluation with fidelity-dependent performance.

Source code in src/dlhub/tuning/multifidelity.py

def quadratic_eval(
    hyperparams: dict[str, Any], fidelity: int
) -> tuple[float, dict[str, Any]]:
    """Simulate model evaluation with fidelity-dependent performance."""
    x, y = hyperparams["x"], hyperparams["y"]

    base_score = -((x - 2) ** 2 + (y + 1) ** 2)

    fidelity_bonus = 2 * (1 - np.exp(-fidelity / 10))

    # Add noise (decreases with fidelity)
    noise_std = 0.1 / np.sqrt(fidelity)
    noise = np.random.normal(0, noise_std)

    final_score = base_score + fidelity_bonus + noise
    metadata = {
        "base_score": base_score,
        "fidelity_bonus": fidelity_bonus,
        "noise": noise,
    }

    return final_score, metadata

nn_eval

nn_eval(hyperparams: dict[str, Any], fidelity: int) -> tuple[float, dict[str, Any]]

Simulate neural network training with different fidelities.

Source code in src/dlhub/tuning/multifidelity.py

def nn_eval(
    hyperparams: dict[str, Any], fidelity: int
) -> tuple[float, dict[str, Any]]:
    """Simulate neural network training with different fidelities."""
    lr = hyperparams["learning_rate"]
    wd = hyperparams["weight_decay"]
    batch_size = hyperparams["batch_size"]

    optimal_lr = 0.001
    optimal_wd = 0.0001

    lr_penalty = -5 * (np.log10(lr) - np.log10(optimal_lr)) ** 2
    wd_penalty = -2 * (np.log10(wd) - np.log10(optimal_wd)) ** 2
    batch_penalty = -0.0001 * (batch_size - 64) ** 2

    base_score = 0.8 + lr_penalty + wd_penalty + batch_penalty

    # Performance improves with training time (fidelity = epochs)
    if fidelity <= 20:
        fidelity_bonus = 0.15 * (1 - np.exp(-fidelity / 5))
    else:
        # Potential overfitting for very long training
        fidelity_bonus = 0.15 * (1 - np.exp(-20 / 5)) - 0.01 * (fidelity - 20)

    noise_std = 0.02 / np.sqrt(fidelity)
    noise = np.random.normal(0, noise_std)

    final_score = base_score + fidelity_bonus + noise

    training_time = fidelity * (1 + batch_size / 1000)

    metadata = {
        "base_score": base_score,
        "fidelity_bonus": fidelity_bonus,
        "training_time": training_time,
        "epochs_trained": fidelity,
    }

    return final_score, metadata

random_search_baseline

random_search_baseline(eval_func, configs, max_budget, n_evals)

Simple random search baseline at maximum fidelity.

Source code in src/dlhub/tuning/multifidelity.py

def random_search_baseline(eval_func, configs, max_budget, n_evals):
    """Simple random search baseline at maximum fidelity."""
    best_score = -np.inf
    best_config = None
    total_budget = 0

    sampled_configs = np.random.choice(
        len(configs), min(n_evals, len(configs)), replace=False
    )

    for i in sampled_configs:
        config = configs[i]
        score, metadata = eval_func(config, max_budget)
        total_budget += max_budget

        if score > best_score:
            best_score = score
            best_config = config

    return best_score, best_config, total_budget

population_based

Population-Based Training (PBT) for Hyperparameter Optimization

Complete PBT implementation with online hyperparameter adaptation during training. PBT simultaneously trains multiple models with different hyperparameters and periodically updates hyperparameters based on population performance, enabling discovery of time-varying optimal hyperparameters.

References

• Jaderberg, M., et al. (2017). "Population Based Training of Neural Networks." arXiv preprint arXiv:1711.09846.
• Parker-Holder, J., et al. (2020). "Effective Diversity in Population Based Reinforcement Learning." NeurIPS.

Author

Deep Learning Reference Hub

License

MIT License

Notes

PBT is particularly effective for: 1. Long training runs where optimal hyperparameters may change over time 2. Scenarios where early performance may not predict final performance 3. Reinforcement learning where environment complexity increases 4. Large-scale distributed training with multiple workers

WorkerState dataclass

State of a single worker in the population.

Attributes:

Name	Type	Description
worker_id	int	Unique identifier for the worker
hyperparams	dict	Current hyperparameter configuration
performance_history	list	History of performance scores
training_step	int	Current training step
model_state	any	Current model state (implementation dependent)
metadata	dict	Additional worker metadata

Source code in src/dlhub/tuning/population_based.py

@dataclass
class WorkerState:
    """
    State of a single worker in the population.

    Attributes
    ----------
    worker_id : int
        Unique identifier for the worker
    hyperparams : dict
        Current hyperparameter configuration
    performance_history : list
        History of performance scores
    training_step : int
        Current training step
    model_state : any
        Current model state (implementation dependent)
    metadata : dict
        Additional worker metadata
    """

    worker_id: int
    hyperparams: dict[str, Any]
    performance_history: list[float] = field(default_factory=list)
    training_step: int = 0
    model_state: Any = None
    metadata: dict[str, Any] = field(default_factory=dict)

PBTResult dataclass

Results from Population-Based Training.

Attributes:

Name	Type	Description
best_worker	WorkerState	Best performing worker at the end
final_population	list	Final state of all workers
population_history	list	History of population states over time
total_training_time	float	Total time spent training
total_steps	int	Total training steps across all workers
statistics	dict	Training statistics and analysis

Source code in src/dlhub/tuning/population_based.py

@dataclass
class PBTResult:
    """
    Results from Population-Based Training.

    Attributes
    ----------
    best_worker : WorkerState
        Best performing worker at the end
    final_population : list
        Final state of all workers
    population_history : list
        History of population states over time
    total_training_time : float
        Total time spent training
    total_steps : int
        Total training steps across all workers
    statistics : dict
        Training statistics and analysis
    """

    best_worker: WorkerState
    final_population: list[WorkerState]
    population_history: list[list[WorkerState]]
    total_training_time: float
    total_steps: int
    statistics: dict[str, Any] = field(default_factory=dict)

HyperparameterDistribution

Bases: ABC

Abstract class for hyperparameter distributions used in exploration.

Source code in src/dlhub/tuning/population_based.py

class HyperparameterDistribution(ABC):
    """Abstract class for hyperparameter distributions used in exploration."""

    @abstractmethod
    def perturb(self, value: Any) -> Any:
        """
        Perturb a hyperparameter value.

        Parameters
        ----------
        value : any
            Current hyperparameter value

        Returns
        -------
        any
            Perturbed hyperparameter value
        """
        pass

    @abstractmethod
    def resample(self) -> Any:
        """
        Resample a hyperparameter value from the distribution.

        Returns
        -------
        any
            New hyperparameter value
        """
        pass

perturb abstractmethod

perturb(value: Any) -> Any

Perturb a hyperparameter value.

Parameters:

Name	Type	Description	Default
value	any	Current hyperparameter value	required

Returns:

Type	Description
any	Perturbed hyperparameter value

Source code in src/dlhub/tuning/population_based.py

@abstractmethod
def perturb(self, value: Any) -> Any:
    """
    Perturb a hyperparameter value.

    Parameters
    ----------
    value : any
        Current hyperparameter value

    Returns
    -------
    any
        Perturbed hyperparameter value
    """
    pass

resample abstractmethod

resample() -> Any

Resample a hyperparameter value from the distribution.

Returns:

Type	Description
any	New hyperparameter value

Source code in src/dlhub/tuning/population_based.py

@abstractmethod
def resample(self) -> Any:
    """
    Resample a hyperparameter value from the distribution.

    Returns
    -------
    any
        New hyperparameter value
    """
    pass

LogUniformPerturbation

Bases: HyperparameterDistribution

Log-uniform perturbation for hyperparameters that vary over orders of magnitude.

Parameters:

Name	Type	Description	Default
factor_range	tuple	Range of multiplicative factors for perturbation	(0.8, 1.2)
bounds	tuple	(min, max) bounds for the hyperparameter	None

Source code in src/dlhub/tuning/population_based.py

class LogUniformPerturbation(HyperparameterDistribution):
    """
    Log-uniform perturbation for hyperparameters that vary over orders of magnitude.

    Parameters
    ----------
    factor_range : tuple, default=(0.8, 1.2)
        Range of multiplicative factors for perturbation
    bounds : tuple, optional
        (min, max) bounds for the hyperparameter
    """

    def __init__(
        self,
        factor_range: tuple[float, float] = (0.8, 1.2),
        bounds: tuple[float, float] | None = None,
    ):
        self.factor_range = factor_range
        self.bounds = bounds

    def perturb(self, value: float) -> float:
        """Perturb value by random multiplicative factor."""
        factor = np.random.uniform(*self.factor_range)
        new_value = value * factor

        if self.bounds is not None:
            new_value = np.clip(new_value, *self.bounds)

        return new_value

    def resample(self) -> float:
        """Resample from log-uniform distribution."""
        if self.bounds is None:
            raise ValueError("Bounds required for resampling")
        return np.exp(np.random.uniform(np.log(self.bounds[0]), np.log(self.bounds[1])))

perturb

perturb(value: float) -> float

Perturb value by random multiplicative factor.

Source code in src/dlhub/tuning/population_based.py

def perturb(self, value: float) -> float:
    """Perturb value by random multiplicative factor."""
    factor = np.random.uniform(*self.factor_range)
    new_value = value * factor

    if self.bounds is not None:
        new_value = np.clip(new_value, *self.bounds)

    return new_value

resample

resample() -> float

Resample from log-uniform distribution.

Source code in src/dlhub/tuning/population_based.py

def resample(self) -> float:
    """Resample from log-uniform distribution."""
    if self.bounds is None:
        raise ValueError("Bounds required for resampling")
    return np.exp(np.random.uniform(np.log(self.bounds[0]), np.log(self.bounds[1])))

UniformPerturbation

Bases: HyperparameterDistribution

Uniform perturbation for continuous hyperparameters.

Parameters:

Name	Type	Description	Default
noise_std	float	Standard deviation of Gaussian noise to add	0.1
bounds	tuple	(min, max) bounds for the hyperparameter	None

Source code in src/dlhub/tuning/population_based.py

class UniformPerturbation(HyperparameterDistribution):
    """
    Uniform perturbation for continuous hyperparameters.

    Parameters
    ----------
    noise_std : float, default=0.1
        Standard deviation of Gaussian noise to add
    bounds : tuple, optional
        (min, max) bounds for the hyperparameter
    """

    def __init__(
        self, noise_std: float = 0.1, bounds: tuple[float, float] | None = None
    ):
        self.noise_std = noise_std
        self.bounds = bounds

    def perturb(self, value: float) -> float:
        """Perturb value by adding Gaussian noise."""
        new_value = value + np.random.normal(0, self.noise_std)

        if self.bounds is not None:
            new_value = np.clip(new_value, *self.bounds)

        return new_value

    def resample(self) -> float:
        """Resample from uniform distribution."""
        if self.bounds is None:
            raise ValueError("Bounds required for resampling")
        return np.random.uniform(*self.bounds)

perturb

perturb(value: float) -> float

Perturb value by adding Gaussian noise.

Source code in src/dlhub/tuning/population_based.py

def perturb(self, value: float) -> float:
    """Perturb value by adding Gaussian noise."""
    new_value = value + np.random.normal(0, self.noise_std)

    if self.bounds is not None:
        new_value = np.clip(new_value, *self.bounds)

    return new_value

resample

resample() -> float

Resample from uniform distribution.

Source code in src/dlhub/tuning/population_based.py

def resample(self) -> float:
    """Resample from uniform distribution."""
    if self.bounds is None:
        raise ValueError("Bounds required for resampling")
    return np.random.uniform(*self.bounds)

ChoicePerturbation

Bases: HyperparameterDistribution

Perturbation for categorical hyperparameters.

Parameters:

Name	Type	Description	Default
choices	list	List of possible values	required
change_probability	float	Probability of changing to a different value	0.3

Source code in src/dlhub/tuning/population_based.py

class ChoicePerturbation(HyperparameterDistribution):
    """
    Perturbation for categorical hyperparameters.

    Parameters
    ----------
    choices : list
        List of possible values
    change_probability : float, default=0.3
        Probability of changing to a different value
    """

    def __init__(self, choices: list[Any], change_probability: float = 0.3):
        self.choices = choices
        self.change_probability = change_probability

    def perturb(self, value: Any) -> Any:
        """Perturb categorical value."""
        if np.random.random() < self.change_probability:
            # Choose different value
            other_choices = [c for c in self.choices if c != value]
            if other_choices:
                return np.random.choice(other_choices)
        return value

    def resample(self) -> Any:
        """Resample from choices."""
        return np.random.choice(self.choices)

perturb

perturb(value: Any) -> Any

Perturb categorical value.

Source code in src/dlhub/tuning/population_based.py

def perturb(self, value: Any) -> Any:
    """Perturb categorical value."""
    if np.random.random() < self.change_probability:
        # Choose different value
        other_choices = [c for c in self.choices if c != value]
        if other_choices:
            return np.random.choice(other_choices)
    return value

resample

resample() -> Any

Resample from choices.

Source code in src/dlhub/tuning/population_based.py

def resample(self) -> Any:
    """Resample from choices."""
    return np.random.choice(self.choices)

WorkerInterface

Bases: ABC

Abstract interface for training workers in PBT.

Defines the methods that workers must implement to participate in population-based training.

Source code in src/dlhub/tuning/population_based.py

class WorkerInterface(ABC):
    """
    Abstract interface for training workers in PBT.

    Defines the methods that workers must implement to participate
    in population-based training.
    """

    @abstractmethod
    def train_step(
        self, hyperparams: dict[str, Any], steps: int = 1
    ) -> tuple[float, Any]:
        """
        Train for specified number of steps.

        Parameters
        ----------
        hyperparams : dict
            Current hyperparameter configuration
        steps : int, default=1
            Number of training steps to perform

        Returns
        -------
        tuple
            (performance_score, model_state)
        """
        pass

    @abstractmethod
    def save_state(self) -> Any:
        """
        Save current model state.

        Returns
        -------
        any
            Serializable model state
        """
        pass

    @abstractmethod
    def load_state(self, state: Any) -> None:
        """
        Load model state.

        Parameters
        ----------
        state : any
            Model state to load
        """
        pass

    @abstractmethod
    def reset(self) -> None:
        """Reset worker to initial state."""
        pass

train_step abstractmethod

train_step(hyperparams: dict[str, Any], steps: int = 1) -> tuple[float, Any]

Train for specified number of steps.

Parameters:

Name	Type	Description	Default
hyperparams	dict	Current hyperparameter configuration	required
steps	int	Number of training steps to perform	1

Returns:

Type	Description
tuple	(performance_score, model_state)

Source code in src/dlhub/tuning/population_based.py

@abstractmethod
def train_step(
    self, hyperparams: dict[str, Any], steps: int = 1
) -> tuple[float, Any]:
    """
    Train for specified number of steps.

    Parameters
    ----------
    hyperparams : dict
        Current hyperparameter configuration
    steps : int, default=1
        Number of training steps to perform

    Returns
    -------
    tuple
        (performance_score, model_state)
    """
    pass

save_state abstractmethod

save_state() -> Any

Save current model state.

Returns:

Type	Description
any	Serializable model state

Source code in src/dlhub/tuning/population_based.py

@abstractmethod
def save_state(self) -> Any:
    """
    Save current model state.

    Returns
    -------
    any
        Serializable model state
    """
    pass

load_state abstractmethod

load_state(state: Any) -> None

Load model state.

Parameters:

Name	Type	Description	Default
state	any	Model state to load	required

Source code in src/dlhub/tuning/population_based.py

@abstractmethod
def load_state(self, state: Any) -> None:
    """
    Load model state.

    Parameters
    ----------
    state : any
        Model state to load
    """
    pass

reset abstractmethod

reset() -> None

Reset worker to initial state.

Source code in src/dlhub/tuning/population_based.py

@abstractmethod
def reset(self) -> None:
    """Reset worker to initial state."""
    pass

FunctionWorker

Bases: WorkerInterface

Function-based worker implementation.

Wraps user-provided training functions to conform to WorkerInterface.

Parameters:

Name	Type	Description	Default
train_function	callable	Function that takes (hyperparams, steps) and returns (score, state)	required
save_function	callable	Function that returns current state	required
load_function	callable	Function that loads given state	required
reset_function	callable	Function that resets to initial state	required

Source code in src/dlhub/tuning/population_based.py

class FunctionWorker(WorkerInterface):
    """
    Function-based worker implementation.

    Wraps user-provided training functions to conform to WorkerInterface.

    Parameters
    ----------
    train_function : callable
        Function that takes (hyperparams, steps) and returns (score, state)
    save_function : callable
        Function that returns current state
    load_function : callable
        Function that loads given state
    reset_function : callable
        Function that resets to initial state
    """

    def __init__(
        self,
        train_function: Callable,
        save_function: Callable,
        load_function: Callable,
        reset_function: Callable,
    ):
        self.train_function = train_function
        self.save_function = save_function
        self.load_function = load_function
        self.reset_function = reset_function

    def train_step(
        self, hyperparams: dict[str, Any], steps: int = 1
    ) -> tuple[float, Any]:
        """Train using wrapped function."""
        return self.train_function(hyperparams, steps)

    def save_state(self) -> Any:
        """Save state using wrapped function."""
        return self.save_function()

    def load_state(self, state: Any) -> None:
        """Load state using wrapped function."""
        self.load_function(state)

    def reset(self) -> None:
        """Reset using wrapped function."""
        self.reset_function()

train_step

train_step(hyperparams: dict[str, Any], steps: int = 1) -> tuple[float, Any]

Train using wrapped function.

Source code in src/dlhub/tuning/population_based.py

def train_step(
    self, hyperparams: dict[str, Any], steps: int = 1
) -> tuple[float, Any]:
    """Train using wrapped function."""
    return self.train_function(hyperparams, steps)

save_state

save_state() -> Any

Save state using wrapped function.

Source code in src/dlhub/tuning/population_based.py

def save_state(self) -> Any:
    """Save state using wrapped function."""
    return self.save_function()

load_state

load_state(state: Any) -> None

Load state using wrapped function.

Source code in src/dlhub/tuning/population_based.py

def load_state(self, state: Any) -> None:
    """Load state using wrapped function."""
    self.load_function(state)

reset

reset() -> None

Reset using wrapped function.

Source code in src/dlhub/tuning/population_based.py

def reset(self) -> None:
    """Reset using wrapped function."""
    self.reset_function()

PopulationBasedTrainer

Population-Based Training optimizer.

Manages a population of workers, periodically evaluating performance and updating hyperparameters through exploitation and exploration.

Parameters:

Name	Type	Description	Default
worker_factory	callable	Factory function that creates new WorkerInterface instances	required
initial_hyperparams	list	Initial hyperparameter configurations for population	required
hyperparam_distributions	dict	Mapping from hyperparameter names to HyperparameterDistribution objects	required
population_size	int	Size of the population	10
eval_interval	int	Training steps between population evaluations	100
exploit_fraction	float	Fraction of worst performers to replace	0.2
explore_fraction	float	Fraction of hyperparameters to perturb during exploration	0.2
truncation_selection	bool	Whether to use truncation selection (replace worst with best)	True
random_state	int	Random seed for reproducibility	None

Source code in src/dlhub/tuning/population_based.py

class PopulationBasedTrainer:
    """
    Population-Based Training optimizer.

    Manages a population of workers, periodically evaluating performance
    and updating hyperparameters through exploitation and exploration.

    Parameters
    ----------
    worker_factory : callable
        Factory function that creates new WorkerInterface instances
    initial_hyperparams : list
        Initial hyperparameter configurations for population
    hyperparam_distributions : dict
        Mapping from hyperparameter names to HyperparameterDistribution objects
    population_size : int, default=10
        Size of the population
    eval_interval : int, default=100
        Training steps between population evaluations
    exploit_fraction : float, default=0.2
        Fraction of worst performers to replace
    explore_fraction : float, default=0.2
        Fraction of hyperparameters to perturb during exploration
    truncation_selection : bool, default=True
        Whether to use truncation selection (replace worst with best)
    random_state : int, optional
        Random seed for reproducibility
    """

    def __init__(
        self,
        worker_factory: Callable[[], WorkerInterface],
        initial_hyperparams: list[dict[str, Any]],
        hyperparam_distributions: dict[str, HyperparameterDistribution],
        population_size: int = 10,
        eval_interval: int = 100,
        exploit_fraction: float = 0.2,
        explore_fraction: float = 0.2,
        truncation_selection: bool = True,
        random_state: int | None = None,
    ):

        self.worker_factory = worker_factory
        self.initial_hyperparams = initial_hyperparams
        self.hyperparam_distributions = hyperparam_distributions
        self.population_size = population_size
        self.eval_interval = eval_interval
        self.exploit_fraction = exploit_fraction
        self.explore_fraction = explore_fraction
        self.truncation_selection = truncation_selection

        if random_state is not None:
            np.random.seed(random_state)

        # Initialize population
        self.population = []
        self.population_history = []
        self.total_steps = 0
        self.generation = 0

    def _initialize_population(self) -> None:
        """Initialize the population of workers."""
        self.population = []

        configs = self.initial_hyperparams.copy()
        while len(configs) < self.population_size:
            config = {}
            for param, dist in self.hyperparam_distributions.items():
                config[param] = dist.resample()
            configs.append(config)

        for i in range(self.population_size):
            config = configs[i % len(configs)]
            worker_state = WorkerState(
                worker_id=i,
                hyperparams=config.copy(),
                performance_history=[],
                training_step=0,
                model_state=None,
                metadata={"generation_created": 0},
            )
            self.population.append(worker_state)

    def _evaluate_population(self, workers: list[WorkerInterface]) -> list[float]:
        """
        Evaluate current performance of all workers.

        Parameters
        ----------
        workers : list
            List of worker instances

        Returns
        -------
        list
            Performance scores for each worker
        """
        scores = []
        for i, worker in enumerate(workers):
            try:
                # Train for evaluation interval
                score, model_state = worker.train_step(
                    self.population[i].hyperparams, self.eval_interval
                )

                # A diverged run reports nan, and nan sorts to the end of
                # np.argsort -- so the worker that just blew up would be read as
                # the population's best and have its hyperparameters copied into
                # everyone else. Treated as the worst possible score instead.
                if not np.isfinite(score):
                    score = -np.inf

                # Update worker state
                self.population[i].performance_history.append(score)
                self.population[i].training_step += self.eval_interval
                self.population[i].model_state = model_state

                scores.append(score)

            except Exception as e:
                warnings.warn(f"Worker {i} evaluation failed: {e}")
                scores.append(-np.inf)

        return scores

    def _exploit_and_explore(
        self, workers: list[WorkerInterface], scores: list[float]
    ) -> None:
        """
        Perform exploitation and exploration step.

        Parameters
        ----------
        workers : list
            List of worker instances
        scores : list
            Current performance scores
        """
        if len(scores) < 2:
            return

        sorted_indices = np.argsort(scores)

        # Truncation selection needs the two ends to be disjoint. Above half the
        # population they overlap, which puts a top performer in the list of
        # workers to overwrite: its state is replaced and then perturbed, so the
        # generation's best result is destroyed by the step meant to spread it.
        n_exploit = max(1, int(self.exploit_fraction * len(scores)))
        n_exploit = min(n_exploit, len(scores) // 2)

        worst_indices = sorted_indices[:n_exploit]
        best_indices = sorted_indices[-n_exploit:]

        for worst_idx in worst_indices:
            if self.truncation_selection:
                best_idx = np.random.choice(best_indices)

                self.population[worst_idx].hyperparams = self.population[
                    best_idx
                ].hyperparams.copy()

                if self.population[best_idx].model_state is not None:
                    workers[worst_idx].load_state(self.population[best_idx].model_state)
                    self.population[worst_idx].model_state = self.population[
                        best_idx
                    ].model_state

                self.population[worst_idx].performance_history = []
                self.population[worst_idx].metadata["generation_created"] = (
                    self.generation
                )

            self._perturb_hyperparams(worst_idx)

    def _perturb_hyperparams(self, worker_idx: int) -> None:
        """
        Perturb hyperparameters for exploration.

        Parameters
        ----------
        worker_idx : int
            Index of worker to perturb
        """
        hyperparams = self.population[worker_idx].hyperparams

        param_names = list(hyperparams.keys())
        n_perturb = max(1, int(self.explore_fraction * len(param_names)))
        params_to_perturb = np.random.choice(param_names, n_perturb, replace=False)

        for param in params_to_perturb:
            if param in self.hyperparam_distributions:
                dist = self.hyperparam_distributions[param]
                hyperparams[param] = dist.perturb(hyperparams[param])

    def train(
        self,
        max_steps: int = 10000,
        max_generations: int = 100,
        timeout: float | None = None,
        verbose: bool = True,
    ) -> PBTResult:
        """
        Run Population-Based Training.

        Parameters
        ----------
        max_steps : int, default=10000
            Maximum total training steps
        max_generations : int, default=100
            Maximum number of generations
        timeout : float, optional
            Maximum training time in seconds
        verbose : bool, default=True
            Whether to print progress information

        Returns
        -------
        PBTResult
            Training results including best worker and population history
        """
        if verbose:
            print("Starting Population-Based Training...")
            print(f"Population size: {self.population_size}")
            print(f"Evaluation interval: {self.eval_interval}")
            print(f"Exploit fraction: {self.exploit_fraction}")
            print(f"Explore fraction: {self.explore_fraction}")

        start_time = time.time()

        self._initialize_population()

        workers = [self.worker_factory() for _ in range(self.population_size)]

        for i, worker in enumerate(workers):
            worker.reset()

        best_score = -np.inf
        best_worker = None

        while (
            self.total_steps < max_steps
            and self.generation < max_generations
            and (timeout is None or time.time() - start_time < timeout)
        ):
            scores = self._evaluate_population(workers)
            self.total_steps += self.population_size * self.eval_interval

            valid_scores = [s for s in scores if s != -np.inf]
            if valid_scores:
                max_score_idx = np.argmax(scores)
                if scores[max_score_idx] > best_score:
                    best_score = scores[max_score_idx]
                    best_worker = copy.deepcopy(self.population[max_score_idx])

            population_snapshot = copy.deepcopy(self.population)
            self.population_history.append(population_snapshot)

            if verbose:
                if valid_scores:
                    mean_score = np.mean(valid_scores)
                    std_score = np.std(valid_scores) if len(valid_scores) > 1 else 0.0
                else:
                    mean_score = float("nan")
                    std_score = float("nan")

                print(
                    f"Generation {self.generation}: "
                    f"Best={best_score:.6f}, "
                    f"Mean={mean_score:.6f}±{std_score:.6f}, "
                    f"Steps={self.total_steps}"
                )

            self._exploit_and_explore(workers, scores)

            self.generation += 1

        total_time = time.time() - start_time

        if best_worker is None and self.population:
            best_overall_score = -np.inf
            for worker in self.population:
                if worker.performance_history:
                    worker_best = max(worker.performance_history)
                    if worker_best > best_overall_score:
                        best_overall_score = worker_best
                        best_worker = copy.deepcopy(worker)

        if best_worker is None and self.population:
            best_worker = copy.deepcopy(self.population[0])
            best_score = -np.inf

        final_scores = [
            (
                np.max(worker.performance_history)
                if worker.performance_history
                else -np.inf
            )
            for worker in self.population
        ]

        valid_final_scores = [s for s in final_scores if s != -np.inf]

        statistics = {
            "generations_completed": self.generation,
            "total_training_time": total_time,
            "final_population_mean": (
                np.mean(valid_final_scores) if valid_final_scores else float("nan")
            ),
            "final_population_std": (
                np.std(valid_final_scores) if len(valid_final_scores) > 1 else 0.0
            ),
            "best_score_progression": [
                max(
                    [
                        max(w.performance_history) if w.performance_history else -np.inf
                        for w in gen
                    ]
                )
                for gen in self.population_history
            ],
            "population_diversity": self._compute_diversity_metrics(),
            "convergence_generation": self._find_convergence_generation(),
        }

        if verbose:
            print(f"\nTraining completed in {total_time:.2f} seconds!")
            print(f"Generations: {self.generation}")
            print(f"Total steps: {self.total_steps}")
            print(f"Best score: {best_score:.6f}")
            if best_worker is not None:
                print(f"Best hyperparameters: {best_worker.hyperparams}")
            else:
                print("No valid workers found during training")

        return PBTResult(
            best_worker=best_worker,  # type: ignore
            final_population=self.population,
            population_history=self.population_history,
            total_training_time=total_time,
            total_steps=self.total_steps,
            statistics=statistics,
        )

    def _compute_diversity_metrics(self) -> dict[str, float]:
        """Compute population diversity metrics."""
        if not self.population_history:
            return {}

        diversity_over_time = []

        for generation in self.population_history:
            param_diversities = []

            for param_name in self.hyperparam_distributions.keys():
                values = []
                for worker in generation:
                    if param_name in worker.hyperparams:
                        val = worker.hyperparams[param_name]
                        if isinstance(val, (int, float)):
                            values.append(val)

                if len(values) > 1:
                    diversity = np.std(values) / (np.mean(values) + 1e-8)
                    param_diversities.append(diversity)

            if param_diversities:
                diversity_over_time.append(np.mean(param_diversities))

        return {
            "initial_diversity": diversity_over_time[0] if diversity_over_time else 0.0,
            "final_diversity": diversity_over_time[-1] if diversity_over_time else 0.0,
            "mean_diversity": (  # type: ignore
                np.mean(diversity_over_time) if diversity_over_time else 0.0
            ),
            "diversity_trend": (
                diversity_over_time[-1] - diversity_over_time[0]
                if len(diversity_over_time) > 1
                else 0.0
            ),
        }

    def _find_convergence_generation(self) -> int | None:
        """Find the generation where population converged."""
        if len(self.population_history) < 5:
            return None

        best_scores = []
        for generation in self.population_history:
            scores = [
                max(w.performance_history) if w.performance_history else -np.inf
                for w in generation
            ]
            best_scores.append(max(scores))

        improvement_threshold = 0.001
        window_size = 5

        for i in range(window_size, len(best_scores)):
            recent_improvement = (
                best_scores[i] - best_scores[i - window_size]
            ) / window_size
            if recent_improvement < improvement_threshold:
                return i

        return None

train

train(max_steps: int = 10000, max_generations: int = 100, timeout: float | None = None, verbose: bool = True) -> PBTResult

Run Population-Based Training.

Parameters:

Name	Type	Description	Default
max_steps	int	Maximum total training steps	10000
max_generations	int	Maximum number of generations	100
timeout	float	Maximum training time in seconds	None
verbose	bool	Whether to print progress information	True

Returns:

Type	Description
PBTResult	Training results including best worker and population history

Source code in src/dlhub/tuning/population_based.py

def train(
    self,
    max_steps: int = 10000,
    max_generations: int = 100,
    timeout: float | None = None,
    verbose: bool = True,
) -> PBTResult:
    """
    Run Population-Based Training.

    Parameters
    ----------
    max_steps : int, default=10000
        Maximum total training steps
    max_generations : int, default=100
        Maximum number of generations
    timeout : float, optional
        Maximum training time in seconds
    verbose : bool, default=True
        Whether to print progress information

    Returns
    -------
    PBTResult
        Training results including best worker and population history
    """
    if verbose:
        print("Starting Population-Based Training...")
        print(f"Population size: {self.population_size}")
        print(f"Evaluation interval: {self.eval_interval}")
        print(f"Exploit fraction: {self.exploit_fraction}")
        print(f"Explore fraction: {self.explore_fraction}")

    start_time = time.time()

    self._initialize_population()

    workers = [self.worker_factory() for _ in range(self.population_size)]

    for i, worker in enumerate(workers):
        worker.reset()

    best_score = -np.inf
    best_worker = None

    while (
        self.total_steps < max_steps
        and self.generation < max_generations
        and (timeout is None or time.time() - start_time < timeout)
    ):
        scores = self._evaluate_population(workers)
        self.total_steps += self.population_size * self.eval_interval

        valid_scores = [s for s in scores if s != -np.inf]
        if valid_scores:
            max_score_idx = np.argmax(scores)
            if scores[max_score_idx] > best_score:
                best_score = scores[max_score_idx]
                best_worker = copy.deepcopy(self.population[max_score_idx])

        population_snapshot = copy.deepcopy(self.population)
        self.population_history.append(population_snapshot)

        if verbose:
            if valid_scores:
                mean_score = np.mean(valid_scores)
                std_score = np.std(valid_scores) if len(valid_scores) > 1 else 0.0
            else:
                mean_score = float("nan")
                std_score = float("nan")

            print(
                f"Generation {self.generation}: "
                f"Best={best_score:.6f}, "
                f"Mean={mean_score:.6f}±{std_score:.6f}, "
                f"Steps={self.total_steps}"
            )

        self._exploit_and_explore(workers, scores)

        self.generation += 1

    total_time = time.time() - start_time

    if best_worker is None and self.population:
        best_overall_score = -np.inf
        for worker in self.population:
            if worker.performance_history:
                worker_best = max(worker.performance_history)
                if worker_best > best_overall_score:
                    best_overall_score = worker_best
                    best_worker = copy.deepcopy(worker)

    if best_worker is None and self.population:
        best_worker = copy.deepcopy(self.population[0])
        best_score = -np.inf

    final_scores = [
        (
            np.max(worker.performance_history)
            if worker.performance_history
            else -np.inf
        )
        for worker in self.population
    ]

    valid_final_scores = [s for s in final_scores if s != -np.inf]

    statistics = {
        "generations_completed": self.generation,
        "total_training_time": total_time,
        "final_population_mean": (
            np.mean(valid_final_scores) if valid_final_scores else float("nan")
        ),
        "final_population_std": (
            np.std(valid_final_scores) if len(valid_final_scores) > 1 else 0.0
        ),
        "best_score_progression": [
            max(
                [
                    max(w.performance_history) if w.performance_history else -np.inf
                    for w in gen
                ]
            )
            for gen in self.population_history
        ],
        "population_diversity": self._compute_diversity_metrics(),
        "convergence_generation": self._find_convergence_generation(),
    }

    if verbose:
        print(f"\nTraining completed in {total_time:.2f} seconds!")
        print(f"Generations: {self.generation}")
        print(f"Total steps: {self.total_steps}")
        print(f"Best score: {best_score:.6f}")
        if best_worker is not None:
            print(f"Best hyperparameters: {best_worker.hyperparams}")
        else:
            print("No valid workers found during training")

    return PBTResult(
        best_worker=best_worker,  # type: ignore
        final_population=self.population,
        population_history=self.population_history,
        total_training_time=total_time,
        total_steps=self.total_steps,
        statistics=statistics,
    )

pbt_optimize

pbt_optimize(train_function: Callable[[dict, int], tuple[float, Any]], save_function: Callable[[], Any], load_function: Callable[[Any], None], reset_function: Callable[[], None], initial_hyperparams: list[dict[str, Any]], hyperparam_distributions: dict[str, HyperparameterDistribution], population_size: int = 10, max_steps: int = 10000, eval_interval: int = 100, exploit_fraction: float = 0.2, explore_fraction: float = 0.2, random_state: int | None = None, verbose: bool = True) -> PBTResult

Convenience function for Population-Based Training.

Parameters:

Name	Type	Description	Default
train_function	callable	Function that takes (hyperparams, steps) and returns (score, state)	required
save_function	callable	Function that returns current model state	required
load_function	callable	Function that loads given model state	required
reset_function	callable	Function that resets model to initial state	required
initial_hyperparams	list	Initial hyperparameter configurations	required
hyperparam_distributions	dict	Hyperparameter perturbation distributions	required
population_size	int	Size of population	10
max_steps	int	Maximum training steps	10000
eval_interval	int	Steps between evaluations	100
exploit_fraction	float	Fraction to exploit	0.2
explore_fraction	float	Fraction to explore	0.2
random_state	int	Random seed	None
verbose	bool	Whether to print progress	True

Returns:

Type	Description
PBTResult	Training results

Examples:

>>> # Define training functions
>>> def train_step(hyperparams, steps):
...     # Simulate training
...     lr = hyperparams["learning_rate"]
...     # Performance improves with more steps but depends on lr
...     performance = 0.8 - (lr - 0.001) ** 2 + steps * 0.001
...     return performance, {"step": steps}
>>>
>>> def save_state():
...     return getattr(save_state, "state", {})
>>>
>>> def load_state(state):
...     save_state.state = state
>>>
>>> def reset():
...     save_state.state = {}
>>>
>>> # Define hyperparameters
>>> initial_configs = [
...     {"learning_rate": 0.001},
...     {"learning_rate": 0.01},
...     {"learning_rate": 0.0001},
... ]
>>>
>>> distributions = {
...     "learning_rate": LogUniformPerturbation((0.8, 1.2), (1e-5, 1e-1))
... }
>>>
>>> result = pbt_optimize(
...     train_step,
...     save_state,
...     load_state,
...     reset,
...     initial_configs,
...     distributions,
...     population_size=5,
... )

Source code in src/dlhub/tuning/population_based.py

def pbt_optimize(
    train_function: Callable[[dict, int], tuple[float, Any]],
    save_function: Callable[[], Any],
    load_function: Callable[[Any], None],
    reset_function: Callable[[], None],
    initial_hyperparams: list[dict[str, Any]],
    hyperparam_distributions: dict[str, HyperparameterDistribution],
    population_size: int = 10,
    max_steps: int = 10000,
    eval_interval: int = 100,
    exploit_fraction: float = 0.2,
    explore_fraction: float = 0.2,
    random_state: int | None = None,
    verbose: bool = True,
) -> PBTResult:
    """
    Convenience function for Population-Based Training.

    Parameters
    ----------
    train_function : callable
        Function that takes (hyperparams, steps) and returns (score, state)
    save_function : callable
        Function that returns current model state
    load_function : callable
        Function that loads given model state
    reset_function : callable
        Function that resets model to initial state
    initial_hyperparams : list
        Initial hyperparameter configurations
    hyperparam_distributions : dict
        Hyperparameter perturbation distributions
    population_size : int, default=10
        Size of population
    max_steps : int, default=10000
        Maximum training steps
    eval_interval : int, default=100
        Steps between evaluations
    exploit_fraction : float, default=0.2
        Fraction to exploit
    explore_fraction : float, default=0.2
        Fraction to explore
    random_state : int, optional
        Random seed
    verbose : bool, default=True
        Whether to print progress

    Returns
    -------
    PBTResult
        Training results

    Examples
    --------
    >>> # Define training functions
    >>> def train_step(hyperparams, steps):
    ...     # Simulate training
    ...     lr = hyperparams["learning_rate"]
    ...     # Performance improves with more steps but depends on lr
    ...     performance = 0.8 - (lr - 0.001) ** 2 + steps * 0.001
    ...     return performance, {"step": steps}
    >>>
    >>> def save_state():
    ...     return getattr(save_state, "state", {})
    >>>
    >>> def load_state(state):
    ...     save_state.state = state
    >>>
    >>> def reset():
    ...     save_state.state = {}
    >>>
    >>> # Define hyperparameters
    >>> initial_configs = [
    ...     {"learning_rate": 0.001},
    ...     {"learning_rate": 0.01},
    ...     {"learning_rate": 0.0001},
    ... ]
    >>>
    >>> distributions = {
    ...     "learning_rate": LogUniformPerturbation((0.8, 1.2), (1e-5, 1e-1))
    ... }
    >>>
    >>> result = pbt_optimize(
    ...     train_step,
    ...     save_state,
    ...     load_state,
    ...     reset,
    ...     initial_configs,
    ...     distributions,
    ...     population_size=5,
    ... )
    """

    def worker_factory():
        return FunctionWorker(
            train_function, save_function, load_function, reset_function
        )

    trainer = PopulationBasedTrainer(
        worker_factory=worker_factory,
        initial_hyperparams=initial_hyperparams,
        hyperparam_distributions=hyperparam_distributions,
        population_size=population_size,
        eval_interval=eval_interval,
        exploit_fraction=exploit_fraction,
        explore_fraction=explore_fraction,
        random_state=random_state,
    )

    return trainer.train(max_steps=max_steps, verbose=verbose)

simple_train_step

simple_train_step(hyperparams: dict[str, Any], steps: int) -> tuple[float, Any]

Simulate training step with time-varying optimal hyperparameters.

Source code in src/dlhub/tuning/population_based.py

def simple_train_step(hyperparams: dict[str, Any], steps: int) -> tuple[float, Any]:
    """Simulate training step with time-varying optimal hyperparameters."""
    worker_id = threading.current_thread().ident

    if worker_id not in worker_states_simple:
        worker_states_simple[worker_id] = {"total_steps": 0, "momentum": 0.0}

    state = worker_states_simple[worker_id]
    lr = hyperparams["learning_rate"]

    total_steps = state["total_steps"]
    optimal_lr = 0.01 * np.exp(-total_steps / 1000)  # Decreasing over time

    base_performance = 0.9 - (lr - optimal_lr) ** 2 * 100

    momentum_effect = state["momentum"] * 0.1
    performance = base_performance + momentum_effect

    state["total_steps"] += steps
    state["momentum"] = 0.9 * state["momentum"] + 0.1 * performance

    performance += np.random.normal(0, 0.01)
    return performance, state.copy()

simple_save_state

simple_save_state()

Save current worker state.

Source code in src/dlhub/tuning/population_based.py

def simple_save_state():
    """Save current worker state."""
    worker_id = threading.current_thread().ident
    return worker_states_simple.get(worker_id, {}).copy()

simple_load_state

simple_load_state(state)

Load worker state.

Source code in src/dlhub/tuning/population_based.py

def simple_load_state(state):
    """Load worker state."""
    worker_id = threading.current_thread().ident
    worker_states_simple[worker_id] = state.copy()

simple_reset

simple_reset()

Reset worker state.

Source code in src/dlhub/tuning/population_based.py

def simple_reset():
    """Reset worker state."""
    worker_id = threading.current_thread().ident
    worker_states_simple[worker_id] = {"total_steps": 0, "momentum": 0.0}

nn_train_step

nn_train_step(hyperparams: dict[str, Any], steps: int) -> tuple[float, Any]

Simulate neural network training with multiple hyperparameters.

Source code in src/dlhub/tuning/population_based.py

def nn_train_step(hyperparams: dict[str, Any], steps: int) -> tuple[float, Any]:
    """Simulate neural network training with multiple hyperparameters."""
    worker_id = threading.current_thread().ident

    if worker_id not in worker_states_nn:
        worker_states_nn[worker_id] = {
            "total_steps": 0,
            "validation_history": [],
            "overfitting_penalty": 0.0,
        }

    state = worker_states_nn[worker_id]

    lr = hyperparams["learning_rate"]
    wd = hyperparams["weight_decay"]
    batch_size = hyperparams["batch_size"]
    dropout = hyperparams["dropout_rate"]

    total_steps = state["total_steps"]
    base_performance = 0.85 * (1 - np.exp(-total_steps / 500))

    lr_effect = -2 * (np.log10(lr) + 3) ** 2  # Optimal around 1e-3
    wd_effect = -0.5 * (np.log10(wd) + 4) ** 2  # Optimal around 1e-4
    batch_effect = -0.001 * (batch_size - 64) ** 2  # Optimal around 64
    dropout_effect = -2 * (dropout - 0.2) ** 2  # Optimal around 0.2

    if total_steps > 800:
        overfitting = 0.1 * (total_steps - 800) / 1000
        overfitting_protection = dropout * 0.2
        state["overfitting_penalty"] = overfitting - overfitting_protection

    performance = (
        base_performance
        + lr_effect
        + wd_effect
        + batch_effect
        + dropout_effect
        - state.get("overfitting_penalty", 0.0)
    )

    state["total_steps"] += steps
    state["validation_history"].append(performance)

    if len(state["validation_history"]) > 20:
        state["validation_history"] = state["validation_history"][-20:]

    performance += np.random.normal(0, 0.01)
    return performance, state.copy()

nn_save_state

nn_save_state()

Save current worker state.

Source code in src/dlhub/tuning/population_based.py

def nn_save_state():
    """Save current worker state."""
    worker_id = threading.current_thread().ident
    return worker_states_nn.get(
        worker_id,
        {
            "total_steps": 0,
            "validation_history": [],
            "overfitting_penalty": 0.0,
        },
    ).copy()

nn_load_state

nn_load_state(state)

Load worker state.

Source code in src/dlhub/tuning/population_based.py

def nn_load_state(state):
    """Load worker state."""
    worker_id = threading.current_thread().ident
    worker_states_nn[worker_id] = state.copy()

nn_reset

nn_reset()

Reset worker state.

Source code in src/dlhub/tuning/population_based.py

def nn_reset():
    """Reset worker state."""
    worker_id = threading.current_thread().ident
    worker_states_nn[worker_id] = {
        "total_steps": 0,
        "validation_history": [],
        "overfitting_penalty": 0.0,
    }

fixed_nn_train_step

fixed_nn_train_step(hyperparams: dict[str, Any], steps: int) -> tuple[float, Any]

Simulate neural network training with fixed hyperparameters.

Source code in src/dlhub/tuning/population_based.py

def fixed_nn_train_step(
    hyperparams: dict[str, Any], steps: int
) -> tuple[float, Any]:
    """Simulate neural network training with fixed hyperparameters."""
    worker_id = threading.current_thread().ident

    if worker_id not in worker_states_fixed:
        worker_states_fixed[worker_id] = {
            "total_steps": 0,
            "validation_history": [],
            "overfitting_penalty": 0.0,
        }

    state = worker_states_fixed[worker_id]

    lr = hyperparams["learning_rate"]
    wd = hyperparams["weight_decay"]
    batch_size = hyperparams["batch_size"]
    dropout = hyperparams["dropout_rate"]

    total_steps = state["total_steps"]
    base_performance = 0.85 * (1 - np.exp(-total_steps / 500))

    lr_effect = -2 * (np.log10(lr) + 3) ** 2
    wd_effect = -0.5 * (np.log10(wd) + 4) ** 2
    batch_effect = -0.001 * (batch_size - 64) ** 2
    dropout_effect = -2 * (dropout - 0.2) ** 2

    if total_steps > 800:
        overfitting = 0.1 * (total_steps - 800) / 1000
        overfitting_protection = dropout * 0.2
        state["overfitting_penalty"] = overfitting - overfitting_protection

    performance = (
        base_performance
        + lr_effect
        + wd_effect
        + batch_effect
        + dropout_effect
        - state.get("overfitting_penalty", 0.0)
    )

    state["total_steps"] += steps
    state["validation_history"].append(performance)

    if len(state["validation_history"]) > 20:
        state["validation_history"] = state["validation_history"][-20:]

    performance += np.random.normal(0, 0.01)
    return performance, state.copy()

random_search

Random Search for Hyperparameter Tuning

Comprehensive random search implementation with proper probability distributions and parallel evaluation support. Random search has been shown to be more effective than grid search for high-dimensional hyperparameter optimization problems.

References

• Bergstra, J., & Bengio, Y. (2012). "Random search for hyper-parameter optimization." Journal of Machine Learning Research, 13, 281-305.
• Li, L., et al. (2017). "Hyperband: A novel bandit-based approach to hyperparameter optimization." Journal of Machine Learning Research, 18, 1-52.

Author

Deep Learning Reference Hub

License

MIT License

Notes

Random search is particularly effective when only a few hyperparameters matter for the final performance. It explores the hyperparameter space more efficiently than grid search by sampling more unique values per dimension.

RandomSearchResult dataclass

Container for random search optimization results.

Attributes:

Name	Type	Description
best_params	dict	Best hyperparameter configuration found
best_score	float	Best objective function value achieved
all_params	list	All parameter configurations evaluated
all_scores	list	All scores corresponding to parameter configurations
search_time	float	Total search time in seconds
statistics	dict	Search statistics and analysis

Source code in src/dlhub/tuning/random_search.py

@dataclass
class RandomSearchResult:
    """
    Container for random search optimization results.

    Attributes
    ----------
    best_params : dict
        Best hyperparameter configuration found
    best_score : float
        Best objective function value achieved
    all_params : list
        All parameter configurations evaluated
    all_scores : list
        All scores corresponding to parameter configurations
    search_time : float
        Total search time in seconds
    statistics : dict
        Search statistics and analysis
    """

    best_params: dict[str, Any]
    best_score: float
    all_params: list[dict[str, Any]]
    all_scores: list[float]
    search_time: float
    statistics: dict[str, Any] = field(default_factory=dict)

ParameterDistribution

Base class for hyperparameter distributions.

Defines the interface for sampling hyperparameters from different probability distributions.

Source code in src/dlhub/tuning/random_search.py

class ParameterDistribution:
    """
    Base class for hyperparameter distributions.

    Defines the interface for sampling hyperparameters from different
    probability distributions.
    """

    def sample(self) -> Any:
        """Sample a value from the distribution."""
        raise NotImplementedError

    def __repr__(self) -> str:
        return f"{self.__class__.__name__}()"

sample

sample() -> Any

Sample a value from the distribution.

Source code in src/dlhub/tuning/random_search.py

def sample(self) -> Any:
    """Sample a value from the distribution."""
    raise NotImplementedError

UniformDistribution

Bases: ParameterDistribution

Uniform distribution for continuous parameters.

Parameters:

Name	Type	Description	Default
low	float	Lower bound of the distribution	required
high	float	Upper bound of the distribution	required

Source code in src/dlhub/tuning/random_search.py

class UniformDistribution(ParameterDistribution):
    """
    Uniform distribution for continuous parameters.

    Parameters
    ----------
    low : float
        Lower bound of the distribution
    high : float
        Upper bound of the distribution
    """

    def __init__(self, low: float, high: float):
        self.low = low
        self.high = high
        self._dist = uniform(loc=low, scale=high - low)

    def sample(self) -> float:
        """Sample from uniform distribution."""
        return self._dist.rvs()

    def __repr__(self) -> str:
        return f"UniformDistribution(low={self.low}, high={self.high})"

sample

sample() -> float

Sample from uniform distribution.

Source code in src/dlhub/tuning/random_search.py

def sample(self) -> float:
    """Sample from uniform distribution."""
    return self._dist.rvs()

LogUniformDistribution

Bases: ParameterDistribution

Log-uniform distribution for parameters that vary over orders of magnitude.

Particularly useful for learning rates, regularization parameters, etc.

Parameters:

Name	Type	Description	Default
low	float	Lower bound (must be positive)	required
high	float	Upper bound (must be positive)	required

Source code in src/dlhub/tuning/random_search.py

class LogUniformDistribution(ParameterDistribution):
    """
    Log-uniform distribution for parameters that vary over orders of magnitude.

    Particularly useful for learning rates, regularization parameters, etc.

    Parameters
    ----------
    low : float
        Lower bound (must be positive)
    high : float
        Upper bound (must be positive)
    """

    def __init__(self, low: float, high: float):
        if low <= 0 or high <= 0:
            raise ValueError("Log-uniform distribution requires positive bounds")
        self.low = low
        self.high = high
        self._dist = loguniform(a=low, b=high)

    def sample(self) -> float:
        """Sample from log-uniform distribution."""
        return self._dist.rvs()

    def __repr__(self) -> str:
        return f"LogUniformDistribution(low={self.low}, high={self.high})"

sample

sample() -> float

Sample from log-uniform distribution.

Source code in src/dlhub/tuning/random_search.py

def sample(self) -> float:
    """Sample from log-uniform distribution."""
    return self._dist.rvs()

IntegerDistribution

Bases: ParameterDistribution

Discrete uniform distribution for integer parameters.

Parameters:

Name	Type	Description	Default
low	int	Lower bound (inclusive)	required
high	int	Upper bound (exclusive)	required

Source code in src/dlhub/tuning/random_search.py

class IntegerDistribution(ParameterDistribution):
    """
    Discrete uniform distribution for integer parameters.

    Parameters
    ----------
    low : int
        Lower bound (inclusive)
    high : int
        Upper bound (exclusive)
    """

    def __init__(self, low: int, high: int):
        self.low = low
        self.high = high
        self._dist = randint(low=low, high=high)

    def sample(self) -> int:
        """Sample from discrete uniform distribution."""
        return int(self._dist.rvs())

    def __repr__(self) -> str:
        return f"IntegerDistribution(low={self.low}, high={self.high})"

sample

sample() -> int

Sample from discrete uniform distribution.

Source code in src/dlhub/tuning/random_search.py

def sample(self) -> int:
    """Sample from discrete uniform distribution."""
    return int(self._dist.rvs())

ChoiceDistribution

Bases: ParameterDistribution

Categorical distribution for discrete choices.

Parameters:

Name	Type	Description	Default
choices	list	List of possible values to choose from	required
probabilities	list	Probability weights for each choice (uniform if None)	None

Source code in src/dlhub/tuning/random_search.py

class ChoiceDistribution(ParameterDistribution):
    """
    Categorical distribution for discrete choices.

    Parameters
    ----------
    choices : list
        List of possible values to choose from
    probabilities : list, optional
        Probability weights for each choice (uniform if None)
    """

    def __init__(self, choices: list[Any], probabilities: list[float] | None = None):
        self.choices = choices
        if probabilities is None:
            self.probabilities = [1.0 / len(choices)] * len(choices)
        else:
            if len(probabilities) != len(choices):
                raise ValueError("Probabilities must match number of choices")
            # Normalize probabilities
            total = sum(probabilities)
            self.probabilities = [p / total for p in probabilities]

    def sample(self) -> Any:
        """Sample from categorical distribution."""
        return np.random.choice(self.choices, p=self.probabilities)

    def __repr__(self) -> str:
        return f"ChoiceDistribution(choices={self.choices})"

sample

sample() -> Any

Sample from categorical distribution.

Source code in src/dlhub/tuning/random_search.py

def sample(self) -> Any:
    """Sample from categorical distribution."""
    return np.random.choice(self.choices, p=self.probabilities)

PowerDistribution

Bases: ParameterDistribution

Power law distribution for parameters with non-uniform preferences.

Useful when smaller values are preferred (common in regularization).

Parameters:

Name	Type	Description	Default
low	float	Lower bound	required
high	float	Upper bound	required
power	float	Power parameter. Above 1 the mass concentrates near low, at 1 the distribution is uniform, and below 1 it concentrates near high.	2.0

Source code in src/dlhub/tuning/random_search.py

class PowerDistribution(ParameterDistribution):
    """
    Power law distribution for parameters with non-uniform preferences.

    Useful when smaller values are preferred (common in regularization).

    Parameters
    ----------
    low : float
        Lower bound
    high : float
        Upper bound
    power : float, default=2.0
        Power parameter. Above 1 the mass concentrates near `low`, at 1 the
        distribution is uniform, and below 1 it concentrates near `high`.
    """

    def __init__(self, low: float, high: float, power: float = 2.0):
        self.low = low
        self.high = high
        self.power = power

    def sample(self) -> float:
        """Sample from power distribution."""
        # u ** power, not u ** (1 / power). The latter is the standard
        # power-function distribution, whose mass moves toward `high` as the
        # exponent grows: the default power=2 would draw a mean of 2/3 of the
        # range, which is the opposite of what this class is documented to do
        # and useless for the weight-decay sweeps it exists for.
        u = np.random.random()
        return self.low + (self.high - self.low) * (u**self.power)

    def __repr__(self) -> str:
        return (
            f"PowerDistribution(low={self.low}, high={self.high}, power={self.power})"
        )

sample

sample() -> float

Sample from power distribution.

Source code in src/dlhub/tuning/random_search.py

def sample(self) -> float:
    """Sample from power distribution."""
    # u ** power, not u ** (1 / power). The latter is the standard
    # power-function distribution, whose mass moves toward `high` as the
    # exponent grows: the default power=2 would draw a mean of 2/3 of the
    # range, which is the opposite of what this class is documented to do
    # and useless for the weight-decay sweeps it exists for.
    u = np.random.random()
    return self.low + (self.high - self.low) * (u**self.power)

RandomSearchOptimizer

Random Search optimizer for hyperparameter tuning.

Implements efficient random sampling of hyperparameters with support for different probability distributions, parallel evaluation, and early stopping.

Parameters:

Name	Type	Description	Default
objective_function	callable	Function to optimize. Should take hyperparameter dict and return float	required
search_space	dict	Dictionary mapping parameter names to ParameterDistribution objects	required
n_iter	int	Number of parameter configurations to sample and evaluate	100
random_state	int(optional)	Random seed for reproducibility	None
n_jobs	int	Number of parallel jobs (-1 for all available cores)	1
early_stopping	bool	Whether to use early stopping based on improvement	False
patience	int	Number of iterations without improvement before stopping	10

Source code in src/dlhub/tuning/random_search.py

class RandomSearchOptimizer:
    """
    Random Search optimizer for hyperparameter tuning.

    Implements efficient random sampling of hyperparameters with support for
    different probability distributions, parallel evaluation, and early stopping.

    Parameters
    ----------
    objective_function : callable
        Function to optimize. Should take hyperparameter dict and return float
    search_space : dict
        Dictionary mapping parameter names to ParameterDistribution objects
    n_iter : int, default=100
        Number of parameter configurations to sample and evaluate
    random_state : int (optional)
        Random seed for reproducibility
    n_jobs : int, default=1
        Number of parallel jobs (-1 for all available cores)
    early_stopping : bool, default=False
        Whether to use early stopping based on improvement
    patience : int, default=10
        Number of iterations without improvement before stopping
    """

    def __init__(
        self,
        objective_function: Callable[[dict], float],
        search_space: dict[str, ParameterDistribution],
        n_iter: int = 100,
        random_state: int | None = None,
        n_jobs: int = 1,
        early_stopping: bool = False,
        patience: int = 10,
    ) -> None:

        self.objective_function = objective_function
        self.search_space = search_space
        self.n_iter = n_iter
        self.random_state = random_state
        self.n_jobs = n_jobs if n_jobs != -1 else mp.cpu_count()
        self.early_stopping = early_stopping
        self.patience = patience

        if random_state is not None:
            np.random.seed(random_state)

        self.results_history = []
        self.best_score = -np.inf
        self.best_params = None
        self.iterations_without_improvement = 0

    def sample_parameters(self) -> dict[str, Any]:
        """
        Sample a single parameter configuration from the search space.

        Returns
        -------
        dict
            Sampled hyperparameter configuration
        """
        params = {}
        for param_name, distribution in self.search_space.items():
            params[param_name] = distribution.sample()
        return params

    def sample_multiple_parameters(self, n_samples: int) -> list[dict[str, Any]]:
        """
        Sample multiple parameter configurations.

        Parameters
        ----------
        n_samples : int
            Number of configurations to sample

        Returns
        -------
        list
            List of parameter configurations
        """
        return [self.sample_parameters() for _ in range(n_samples)]

    def _evaluate_single(self, params: dict[str, Any]) -> tuple[dict[str, Any], float]:
        """
        Evaluate objective function for a single parameter configuration.

        Parameters
        ----------
        params : dict
            Parameter configuration to evaluate

        Returns
        -------
        tuple
            (parameters, score) tuple
        """
        try:
            score = self.objective_function(params)
            if np.isnan(score) or np.isinf(score):
                return params, -np.inf
            return params, float(score)
        except Exception as e:
            warnings.warn(f"Evaluation failed for {params}: {e}")
            return params, -np.inf

    def _evaluate_batch_sequential(
        self, param_list: list[dict[str, Any]]
    ) -> list[tuple[dict[str, Any], float]]:
        """Evaluate parameters sequentially."""
        results = []
        for params in param_list:
            result = self._evaluate_single(params)
            results.append(result)
        return results

    def _evaluate_batch_parallel(
        self, param_list: list[dict[str, Any]]
    ) -> list[tuple[dict[str, Any], float]]:
        """Evaluate parameters in parallel."""
        results = []
        with ProcessPoolExecutor(max_workers=self.n_jobs) as executor:
            # Submit all jobs
            future_to_params = {
                executor.submit(self._evaluate_single, params): params
                for params in param_list
            }

            # Collect results as they complete
            for future in as_completed(future_to_params):
                try:
                    result = future.result()
                    results.append(result)
                except Exception as e:
                    params = future_to_params[future]
                    warnings.warn(f"Parallel evaluation failed for {params}: {e}")
                    results.append((params, -np.inf))

        return results

    def _should_stop_early(self) -> bool:
        """Check if early stopping criteria are met."""
        if not self.early_stopping:
            return False
        return self.iterations_without_improvement >= self.patience

    def optimize(self, verbose: bool = True) -> RandomSearchResult:
        """
        Run random search optimization.

        Parameters
        ----------
        verbose : bool, default=True
            Whether to print progress information

        Returns
        -------
        RandomSearchResult
            Optimization results
        """
        if verbose:
            print("Starting Random Search Optimization...")
            print(f"Search space: {len(self.search_space)} parameters")
            print(f"Number of iterations: {self.n_iter}")
            print(f"Parallel jobs: {self.n_jobs}")
            if self.early_stopping:
                print(f"Early stopping: patience={self.patience}")

        start_time = time.time()
        all_params = []
        all_scores = []

        if self.n_jobs > 1:
            batch_size = min(self.n_jobs * 2, self.n_iter)
        else:
            batch_size = 10  # Process in small batches even for sequential

        iterations_completed = 0

        while iterations_completed < self.n_iter:
            remaining_iterations = self.n_iter - iterations_completed
            current_batch_size = min(batch_size, remaining_iterations)

            param_batch = self.sample_multiple_parameters(current_batch_size)

            if self.n_jobs > 1:
                batch_results = self._evaluate_batch_parallel(param_batch)
            else:
                batch_results = self._evaluate_batch_sequential(param_batch)

            improvement_found = False
            for params, score in batch_results:
                all_params.append(params)
                all_scores.append(score)

                if score > self.best_score:
                    self.best_score = score
                    self.best_params = params.copy()
                    improvement_found = True
                    self.iterations_without_improvement = 0

                    if verbose:
                        print(
                            f"Iteration {iterations_completed + 1}: "
                            f"New best score = {score:.6f}"
                        )
                else:
                    self.iterations_without_improvement += 1

                iterations_completed += 1

                if self._should_stop_early():
                    if verbose:
                        print(f"Early stopping at iteration {iterations_completed}")
                    break

            if self._should_stop_early():
                break

            # Progress update
            if verbose and iterations_completed % max(1, self.n_iter // 10) == 0:
                print(
                    f"Progress: {iterations_completed}/{self.n_iter} "
                    f"({100 * iterations_completed / self.n_iter:.1f}%) "
                    f"- Best score: {self.best_score:.6f}"
                )

        search_time = time.time() - start_time

        valid_scores = [s for s in all_scores if s != -np.inf]
        statistics = {
            "total_evaluations": len(all_scores),
            "successful_evaluations": len(valid_scores),
            "failed_evaluations": len(all_scores) - len(valid_scores),
            "mean_score": np.mean(valid_scores) if valid_scores else -np.inf,
            "std_score": np.std(valid_scores) if len(valid_scores) > 1 else 0.0,
            "score_percentiles": {
                "25th": np.percentile(valid_scores, 25) if valid_scores else -np.inf,
                "50th": np.percentile(valid_scores, 50) if valid_scores else -np.inf,
                "75th": np.percentile(valid_scores, 75) if valid_scores else -np.inf,
                "95th": np.percentile(valid_scores, 95) if valid_scores else -np.inf,
            },
            "improvement_over_random": (self.best_score - np.mean(valid_scores[:10]))
            if len(valid_scores) >= 10
            else 0.0,
            "early_stopped": self._should_stop_early(),
            "iterations_completed": iterations_completed,
        }

        if verbose:
            print(f"\nOptimization completed in {search_time:.2f} seconds!")
            print(f"Best score: {self.best_score:.6f}")
            print(f"Best parameters: {self.best_params}")
            print(f"Total evaluations: {statistics['total_evaluations']}")
            print(
                f"Success rate: {statistics['successful_evaluations'] / statistics['total_evaluations']:.2%}"
            )

        return RandomSearchResult(
            best_params=self.best_params,  # type: ignore
            best_score=self.best_score,
            all_params=all_params,
            all_scores=all_scores,
            search_time=search_time,
            statistics=statistics,
        )

sample_parameters

sample_parameters() -> dict[str, Any]

Sample a single parameter configuration from the search space.

Returns:

Type	Description
dict	Sampled hyperparameter configuration

Source code in src/dlhub/tuning/random_search.py

def sample_parameters(self) -> dict[str, Any]:
    """
    Sample a single parameter configuration from the search space.

    Returns
    -------
    dict
        Sampled hyperparameter configuration
    """
    params = {}
    for param_name, distribution in self.search_space.items():
        params[param_name] = distribution.sample()
    return params

sample_multiple_parameters

sample_multiple_parameters(n_samples: int) -> list[dict[str, Any]]

Sample multiple parameter configurations.

Parameters:

Name	Type	Description	Default
n_samples	int	Number of configurations to sample	required

Returns:

Type	Description
list	List of parameter configurations

Source code in src/dlhub/tuning/random_search.py

def sample_multiple_parameters(self, n_samples: int) -> list[dict[str, Any]]:
    """
    Sample multiple parameter configurations.

    Parameters
    ----------
    n_samples : int
        Number of configurations to sample

    Returns
    -------
    list
        List of parameter configurations
    """
    return [self.sample_parameters() for _ in range(n_samples)]

optimize

optimize(verbose: bool = True) -> RandomSearchResult

Run random search optimization.

Parameters:

Name	Type	Description	Default
verbose	bool	Whether to print progress information	True

Returns:

Type	Description
RandomSearchResult	Optimization results

Source code in src/dlhub/tuning/random_search.py

def optimize(self, verbose: bool = True) -> RandomSearchResult:
    """
    Run random search optimization.

    Parameters
    ----------
    verbose : bool, default=True
        Whether to print progress information

    Returns
    -------
    RandomSearchResult
        Optimization results
    """
    if verbose:
        print("Starting Random Search Optimization...")
        print(f"Search space: {len(self.search_space)} parameters")
        print(f"Number of iterations: {self.n_iter}")
        print(f"Parallel jobs: {self.n_jobs}")
        if self.early_stopping:
            print(f"Early stopping: patience={self.patience}")

    start_time = time.time()
    all_params = []
    all_scores = []

    if self.n_jobs > 1:
        batch_size = min(self.n_jobs * 2, self.n_iter)
    else:
        batch_size = 10  # Process in small batches even for sequential

    iterations_completed = 0

    while iterations_completed < self.n_iter:
        remaining_iterations = self.n_iter - iterations_completed
        current_batch_size = min(batch_size, remaining_iterations)

        param_batch = self.sample_multiple_parameters(current_batch_size)

        if self.n_jobs > 1:
            batch_results = self._evaluate_batch_parallel(param_batch)
        else:
            batch_results = self._evaluate_batch_sequential(param_batch)

        improvement_found = False
        for params, score in batch_results:
            all_params.append(params)
            all_scores.append(score)

            if score > self.best_score:
                self.best_score = score
                self.best_params = params.copy()
                improvement_found = True
                self.iterations_without_improvement = 0

                if verbose:
                    print(
                        f"Iteration {iterations_completed + 1}: "
                        f"New best score = {score:.6f}"
                    )
            else:
                self.iterations_without_improvement += 1

            iterations_completed += 1

            if self._should_stop_early():
                if verbose:
                    print(f"Early stopping at iteration {iterations_completed}")
                break

        if self._should_stop_early():
            break

        # Progress update
        if verbose and iterations_completed % max(1, self.n_iter // 10) == 0:
            print(
                f"Progress: {iterations_completed}/{self.n_iter} "
                f"({100 * iterations_completed / self.n_iter:.1f}%) "
                f"- Best score: {self.best_score:.6f}"
            )

    search_time = time.time() - start_time

    valid_scores = [s for s in all_scores if s != -np.inf]
    statistics = {
        "total_evaluations": len(all_scores),
        "successful_evaluations": len(valid_scores),
        "failed_evaluations": len(all_scores) - len(valid_scores),
        "mean_score": np.mean(valid_scores) if valid_scores else -np.inf,
        "std_score": np.std(valid_scores) if len(valid_scores) > 1 else 0.0,
        "score_percentiles": {
            "25th": np.percentile(valid_scores, 25) if valid_scores else -np.inf,
            "50th": np.percentile(valid_scores, 50) if valid_scores else -np.inf,
            "75th": np.percentile(valid_scores, 75) if valid_scores else -np.inf,
            "95th": np.percentile(valid_scores, 95) if valid_scores else -np.inf,
        },
        "improvement_over_random": (self.best_score - np.mean(valid_scores[:10]))
        if len(valid_scores) >= 10
        else 0.0,
        "early_stopped": self._should_stop_early(),
        "iterations_completed": iterations_completed,
    }

    if verbose:
        print(f"\nOptimization completed in {search_time:.2f} seconds!")
        print(f"Best score: {self.best_score:.6f}")
        print(f"Best parameters: {self.best_params}")
        print(f"Total evaluations: {statistics['total_evaluations']}")
        print(
            f"Success rate: {statistics['successful_evaluations'] / statistics['total_evaluations']:.2%}"
        )

    return RandomSearchResult(
        best_params=self.best_params,  # type: ignore
        best_score=self.best_score,
        all_params=all_params,
        all_scores=all_scores,
        search_time=search_time,
        statistics=statistics,
    )

random_search

random_search(objective_function: Callable[[dict], float], search_space: dict[str, ParameterDistribution | tuple | list], n_iter: int = 100, random_state: int | None = None, n_jobs: int = 1, early_stopping: bool = False, patience: int = 10, verbose: bool = True) -> RandomSearchResult

Convenience function for random search hyperparameter optimization.

Parameters:

Name	Type	Description	Default
objective_function	callable	Function to optimize. Should accept hyperparameter dict and return float	required
search_space	dict	Search space definition. Can contain: - ParameterDistribution objects - Tuples (low, high) for numeric ranges. Two integer bounds give an inclusive integer range, and a ratio above 100 switches to log-uniform so that wide ranges are sampled by order of magnitude. - Lists for categorical choices, whatever their length	required
n_iter	int	Number of parameter configurations to evaluate	100
random_state	int(optional)	Random seed for reproducibility	None
n_jobs	int	Number of parallel jobs (-1 for all cores)	1
early_stopping	bool	Whether to use early stopping	False
patience	int	Early stopping patience	10
verbose	bool	Whether to print progress	True

Returns:

Type	Description
RandomSearchResult	Optimization results

Examples:

>>> def objective(params):
...     lr, wd = params["learning_rate"], params["weight_decay"]
...     return -((lr - 0.001) ** 2) - (wd - 0.0001) ** 2 + np.random.normal(0, 0.01)
>>> # Simple tuple/list format
>>> search_space = {
...     "learning_rate": (1e-5, 1e-1),  # Will use log-uniform
...     "batch_size": [16, 32, 64, 128],  # Will use choice
...     "hidden_units": (64, 512),  # Will use uniform
... }
>>> result = random_search(objective, search_space, n_iter=50, random_state=42)
>>> # Advanced distribution format
>>> from scipy.stats import truncnorm
>>> search_space_advanced = {
...     "learning_rate": LogUniformDistribution(1e-5, 1e-1),
...     "weight_decay": PowerDistribution(1e-6, 1e-2, power=2.0),
...     "batch_size": ChoiceDistribution([16, 32, 64, 128], [0.1, 0.3, 0.4, 0.2]),
...     "dropout_rate": UniformDistribution(0.0, 0.5),
... }

Source code in src/dlhub/tuning/random_search.py

def random_search(
    objective_function: Callable[[dict], float],
    search_space: dict[str, ParameterDistribution | tuple | list],
    n_iter: int = 100,
    random_state: int | None = None,
    n_jobs: int = 1,
    early_stopping: bool = False,
    patience: int = 10,
    verbose: bool = True,
) -> RandomSearchResult:
    """
    Convenience function for random search hyperparameter optimization.

    Parameters
    ----------
    objective_function : callable
        Function to optimize. Should accept hyperparameter dict and return float
    search_space : dict
        Search space definition. Can contain:
        - ParameterDistribution objects
        - Tuples ``(low, high)`` for numeric ranges. Two integer bounds give an
          inclusive integer range, and a ratio above 100 switches to
          log-uniform so that wide ranges are sampled by order of magnitude.
        - Lists for categorical choices, whatever their length
    n_iter : int, default=100
        Number of parameter configurations to evaluate
    random_state : int (optional)
        Random seed for reproducibility
    n_jobs : int, default=1
        Number of parallel jobs (-1 for all cores)
    early_stopping : bool, default=False
        Whether to use early stopping
    patience : int, default=10
        Early stopping patience
    verbose : bool, default=True
        Whether to print progress

    Returns
    -------
    RandomSearchResult
        Optimization results

    Examples
    --------
    >>> def objective(params):
    ...     lr, wd = params["learning_rate"], params["weight_decay"]
    ...     return -((lr - 0.001) ** 2) - (wd - 0.0001) ** 2 + np.random.normal(0, 0.01)
    >>> # Simple tuple/list format
    >>> search_space = {
    ...     "learning_rate": (1e-5, 1e-1),  # Will use log-uniform
    ...     "batch_size": [16, 32, 64, 128],  # Will use choice
    ...     "hidden_units": (64, 512),  # Will use uniform
    ... }
    >>> result = random_search(objective, search_space, n_iter=50, random_state=42)
    >>> # Advanced distribution format
    >>> from scipy.stats import truncnorm
    >>> search_space_advanced = {
    ...     "learning_rate": LogUniformDistribution(1e-5, 1e-1),
    ...     "weight_decay": PowerDistribution(1e-6, 1e-2, power=2.0),
    ...     "batch_size": ChoiceDistribution([16, 32, 64, 128], [0.1, 0.3, 0.4, 0.2]),
    ...     "dropout_rate": UniformDistribution(0.0, 0.5),
    ... }
    """
    # Convert simple formats to distribution objects
    processed_space = {}
    for param_name, param_spec in search_space.items():
        if isinstance(param_spec, ParameterDistribution):
            processed_space[param_name] = param_spec
        elif isinstance(param_spec, list):
            # A list is always categorical. Checking the length first would
            # capture any two-option list — ["relu", "tanh"] — and try to read
            # it as a numeric range.
            processed_space[param_name] = ChoiceDistribution(param_spec)
        elif isinstance(param_spec, tuple) and len(param_spec) == 2:
            low, high = param_spec
            if isinstance(low, (int, float)) and isinstance(high, (int, float)):
                if low > 0 and high / low > 100:  # Use log-uniform for wide ranges
                    processed_space[param_name] = LogUniformDistribution(low, high)
                elif isinstance(low, int) and isinstance(high, int):
                    processed_space[param_name] = IntegerDistribution(low, high + 1)
                else:
                    processed_space[param_name] = UniformDistribution(low, high)
            else:
                raise ValueError(f"Invalid tuple format for {param_name}: {param_spec}")
        else:
            raise ValueError(
                f"Unsupported search space format for {param_name}: {param_spec}"
            )

    optimizer = RandomSearchOptimizer(
        objective_function=objective_function,
        search_space=processed_space,
        n_iter=n_iter,
        random_state=random_state,
        n_jobs=n_jobs,
        early_stopping=early_stopping,
        patience=patience,
    )

    return optimizer.optimize(verbose=verbose)

analyze_parameter_importance

analyze_parameter_importance(result: RandomSearchResult, top_n: int = 10) -> dict[str, float]

Analyze parameter importance using correlation with objective values.

Parameters:

Name	Type	Description	Default
result	RandomSearchResult	Results from random search optimization	required
top_n	int	Number of top configurations to analyze	10

Returns:

Type	Description
dict	Parameter importance scores (correlation coefficients)

Source code in src/dlhub/tuning/random_search.py

def analyze_parameter_importance(
    result: RandomSearchResult, top_n: int = 10
) -> dict[str, float]:
    """
    Analyze parameter importance using correlation with objective values.

    Parameters
    ----------
    result : RandomSearchResult
        Results from random search optimization
    top_n : int, default=10
        Number of top configurations to analyze

    Returns
    -------
    dict
        Parameter importance scores (correlation coefficients)
    """
    if len(result.all_params) < 2:
        return {}

    sorted_indices = np.argsort(result.all_scores)[-top_n:]
    top_params = [result.all_params[i] for i in sorted_indices]
    top_scores = [result.all_scores[i] for i in sorted_indices]

    importance_scores = {}
    param_names = list(result.all_params[0].keys())

    for param_name in param_names:
        param_values = []
        for params in top_params:
            val = params[param_name]
            if isinstance(val, (int, float)):
                param_values.append(float(val))
            else:
                # For categorical parameters, skip importance analysis
                continue

        if len(param_values) > 1:
            correlation = np.corrcoef(param_values, top_scores)[0, 1]
            if not np.isnan(correlation):
                importance_scores[param_name] = abs(correlation)

    importance_scores = dict(
        sorted(importance_scores.items(), key=lambda x: x[1], reverse=True)
    )

    return importance_scores

quadratic_objective

quadratic_objective(params)

Example objective - quadratic function with noise.

Source code in src/dlhub/tuning/random_search.py

def quadratic_objective(params):
    """Example objective - quadratic function with noise."""
    x, y = params["x"], params["y"]
    return -((x - 2) ** 2) - (y + 1) ** 2 - 5 + np.random.normal(0, 0.1)

nn_objective

nn_objective(params)

Realistic neural network objective function simulation.

Source code in src/dlhub/tuning/random_search.py

def nn_objective(params):
    """Realistic neural network objective function simulation."""
    lr = params["learning_rate"]
    wd = params["weight_decay"]
    batch_size = params["batch_size"]
    dropout = params["dropout_rate"]
    optimizer = params["optimizer"]

    # Simulate realistic hyperparameter interactions
    base_score = 0.85

    lr_effect = -2 * (np.log10(lr) + 3) ** 2  # Optimum at 1e-3

    wd_effect = -0.5 * (np.log10(wd) + 4) ** 2  # Optimum around 1e-4

    batch_effect = -0.01 * (batch_size - 64) ** 2 / 100

    dropout_effect = -2 * (dropout - 0.2) ** 2

    optimizer_effects = {"adam": 0.05, "sgd": 0.0, "rmsprop": 0.02}
    opt_effect = optimizer_effects.get(optimizer, 0.0)

    score = (
        base_score + lr_effect + wd_effect + batch_effect + dropout_effect + opt_effect
    )
    score += np.random.normal(0, 0.02)  # Add realistic noise

    return score

simple_nn_objective

simple_nn_objective(params)

Simple neural network objective for testing.

Source code in src/dlhub/tuning/random_search.py

def simple_nn_objective(params):
    """Simple neural network objective for testing."""
    lr = params["learning_rate"]
    wd = params["weight_decay"]
    return (
        -((np.log10(lr) + 3) ** 2) - (np.log10(wd) + 4) ** 2 + np.random.normal(0, 0.1)
    )