Functional

Reference information for the trainers Functional API.

eva.core.trainers.functional.run_evaluation_session

Runs a downstream evaluation session out-of-place.

It performs an evaluation run (with configurable stages) on the model multiple times. Note that as the input base_trainer and base_model would be cloned, the input object would not be modified.

Parameters:

Name	Type	Description	Default
base_trainer	Trainer	The base trainer module to use.	required
base_model	ModelModule	The base model module to use.	required
datamodule	DataModule	The data module.	required
n_runs	int	The number of runs to perform.	1
stages	List[Literal['fit', 'validate', 'test']] | None	List of stages to execute. Options: "fit", "validate", "test".	None
record_datasets_as_runs	bool	If True, when multiple validation and/or test datasets are configured, each dataset output is recorded as a separate run in the session summary. Otherwise, dataset outputs are logged separately within the same run.	False
verbose	bool	Whether to verbose the session metrics instead of those of each individual run and vice-versa.	True

Source code in src/eva/core/trainers/functional.py

def run_evaluation_session(
    base_trainer: eva_trainer.Trainer,
    base_model: modules.ModelModule,
    datamodule: datamodules.DataModule,
    *,
    n_runs: int = 1,
    stages: List[Literal["fit", "validate", "test"]] | None = None,
    record_datasets_as_runs: bool = False,
    verbose: bool = True,
) -> None:
    """Runs a downstream evaluation session out-of-place.

    It performs an evaluation run (with configurable stages) on the model
    multiple times. Note that as the input `base_trainer` and
    `base_model` would be cloned, the input object would not
    be modified.

    Args:
        base_trainer: The base trainer module to use.
        base_model: The base model module to use.
        datamodule: The data module.
        n_runs: The number of runs to perform.
        stages: List of stages to execute. Options: "fit", "validate", "test".
        record_datasets_as_runs: If True, when multiple validation and/or test datasets are
            configured, each dataset output is recorded as a separate run in the session
            summary. Otherwise, dataset outputs are logged separately within the same run.
        verbose: Whether to verbose the session metrics instead of
            those of each individual run and vice-versa.
    """
    if not stages:
        stages = ["fit", "validate", "test"]
    recorder = _recorder.SessionRecorder(output_dir=base_trainer.default_log_dir, verbose=verbose)
    for run_index in range(n_runs):
        validation_scores, test_scores = run_evaluation(
            base_trainer,
            base_model,
            datamodule,
            run_id=run_index,
            stages=stages,
            record_datasets_as_runs=record_datasets_as_runs,
            verbose=not verbose,
        )
        if validation_scores or test_scores:
            if record_datasets_as_runs:
                for val_result, test_result in zip_longest(
                    validation_scores or [], test_scores or []
                ):
                    recorder.update(
                        [val_result] if val_result is not None else None,
                        [test_result] if test_result is not None else None,
                    )
            else:
                recorder.update(validation_scores, test_scores)
    recorder.save()

eva.core.trainers.functional.run_evaluation

Runs the specified evaluation stages out-of-place.

Parameters:

Name	Type	Description	Default
base_trainer	Trainer	The base trainer to use but not modify.	required
base_model	ModelModule	The model module to use but not modify.	required
datamodule	DataModule	The data module.	required
run_id	int | None	The run id to be appended to the output log directory. If None, it will use the log directory of the trainer as is.	None
stages	List[Literal['fit', 'validate', 'test']] | None	List of stages to execute. Options: "fit", "validate", "test".	None
record_datasets_as_runs	bool	If True, evaluate each dataloader separately so that metrics are computed independently per dataset.	False
verbose	bool	Whether to print the validation and test metrics in the end of the training.	True

Returns:

Type	Description
_EVALUATE_OUTPUT | None	A tuple with the validation and the test metrics (if executed).
_EVALUATE_OUTPUT | None	If a stage is not executed, its value will be None.

Source code in src/eva/core/trainers/functional.py

def run_evaluation(
    base_trainer: eva_trainer.Trainer,
    base_model: modules.ModelModule,
    datamodule: datamodules.DataModule,
    *,
    run_id: int | None = None,
    stages: List[Literal["fit", "validate", "test"]] | None = None,
    record_datasets_as_runs: bool = False,
    verbose: bool = True,
) -> Tuple[_EVALUATE_OUTPUT | None, _EVALUATE_OUTPUT | None]:
    """Runs the specified evaluation stages out-of-place.

    Args:
        base_trainer: The base trainer to use but not modify.
        base_model: The model module to use but not modify.
        datamodule: The data module.
        run_id: The run id to be appended to the output log directory.
            If `None`, it will use the log directory of the trainer as is.
        stages: List of stages to execute. Options: "fit", "validate", "test".
        record_datasets_as_runs: If True, evaluate each dataloader separately
            so that metrics are computed independently per dataset.
        verbose: Whether to print the validation and test metrics
            in the end of the training.

    Returns:
        A tuple with the validation and the test metrics (if executed).
        If a stage is not executed, its value will be None.
    """
    if not stages:
        stages = ["fit", "validate", "test"]
    trainer = _utils.clone(base_trainer)
    model = _utils.clone(base_model) if "fit" in stages else base_model

    model.configure_model()

    trainer.init_logger_run(run_id)

    validation_scores = None
    test_scores = None

    if "fit" in stages:
        trainer.fit(model, datamodule=datamodule)
    if "validate" in stages and getattr(datamodule.datasets, "val", None) is not None:
        validation_scores = _evaluate_stage(
            trainer,
            model,
            datamodule,
            stage="validate",
            record_datasets_as_runs=record_datasets_as_runs,
            verbose=verbose,
        )
    if "test" in stages and getattr(datamodule.datasets, "test", None) is not None:
        test_scores = _evaluate_stage(
            trainer,
            model,
            datamodule,
            stage="test",
            record_datasets_as_runs=record_datasets_as_runs,
            verbose=verbose,
        )
    trainer.finish_logger_run(run_id)
    return validation_scores, test_scores

eva.core.trainers.functional.infer_model

Performs model inference.

Parameters:

Name	Type	Description	Default
trainer	Trainer	The trainer to use.	required
model	ModelModule	The model module to use.	required
datamodule	DataModule	The data module.	required
return_predictions	bool	Whether to return the model predictions.	False
enable_clone	bool	Whether to clone the trainer before inference.	False

Source code in src/eva/core/trainers/functional.py

def infer_model(
    trainer: eva_trainer.Trainer,
    model: modules.ModelModule,
    datamodule: datamodules.DataModule,
    *,
    return_predictions: bool = False,
    enable_clone: bool = False,
) -> _PREDICT_OUTPUT | None:
    """Performs model inference.

    Args:
        trainer: The trainer to use.
        model: The model module to use.
        datamodule: The data module.
        return_predictions: Whether to return the model predictions.
        enable_clone: Whether to clone the trainer before inference.
    """
    trainer = _utils.clone(trainer) if enable_clone else trainer
    return trainer.predict(
        model=model,
        datamodule=datamodule,
        return_predictions=return_predictions,
    )