config

optimus_dl.recipe.eval.config

Configuration for evaluation recipe.

EvalCommonConfig dataclass

Common evaluation configuration.

Parameters:

Name	Type	Description	Default
checkpoint_path	str | None	Path to checkpoint directory or metadata file	'???'
model	Any	Model to build (if you want to load model not from checkpoint)	None
use_gpu	bool	Use gpu if available	True
seed	int	Random seed for evaluation	42
tokenizer	Any		'???'

Source code in optimus_dl/recipe/eval/config.py

@dataclass
class EvalCommonConfig:
    """Common evaluation configuration."""

    checkpoint_path: str | None = field(
        default=MISSING,
        metadata={"description": "Path to checkpoint directory or metadata file"},
    )
    model: Any = field(
        default=None,
        metadata={
            "description": "Model to build (if you want to load model not from checkpoint)"
        },
    )
    use_gpu: bool = field(
        default=True,
        metadata={"description": "Use gpu if available"},
    )
    seed: int = field(
        default=42, metadata={"description": "Random seed for evaluation"}
    )
    tokenizer: Any = MISSING

EvalConfig dataclass

Main evaluation configuration.

Parameters:

Name	Type	Description	Default
common	EvalCommonConfig	Common evaluation configuration.	<dynamic>
lm_eval	LMEvalConfig	Configuration for lm_eval harness evaluation.	<dynamic>

Source code in optimus_dl/recipe/eval/config.py

@dataclass
class EvalConfig:
    """Main evaluation configuration."""

    common: EvalCommonConfig = field(default_factory=EvalCommonConfig)
    lm_eval: LMEvalConfig = field(default_factory=LMEvalConfig)

    def __post_init__(self):
        """Validate configuration."""
        if self.common.checkpoint_path == MISSING:
            raise ValueError("checkpoint_path is required")

        # Convert checkpoint_path to Path for validation
        checkpoint_path = Path(self.common.checkpoint_path)
        if not (checkpoint_path.exists() or checkpoint_path.parent.exists()):
            raise ValueError(f"Checkpoint path does not exist: {checkpoint_path}")

__post_init__()

Validate configuration.

Source code in optimus_dl/recipe/eval/config.py

def __post_init__(self):
    """Validate configuration."""
    if self.common.checkpoint_path == MISSING:
        raise ValueError("checkpoint_path is required")

    # Convert checkpoint_path to Path for validation
    checkpoint_path = Path(self.common.checkpoint_path)
    if not (checkpoint_path.exists() or checkpoint_path.parent.exists()):
        raise ValueError(f"Checkpoint path does not exist: {checkpoint_path}")

LMEvalConfig dataclass

Configuration for lm_eval harness evaluation.

Parameters:

Name	Type	Description	Default
tasks	list[str]	List of lm_eval tasks to evaluate on	['hellaswag']
num_fewshot	int	Number of few-shot examples	0
batch_size	int	Batch size for evaluation	1
limit	int | None	Limit number of examples per task	None
output_path	str | None	Path to save evaluation results	None

Source code in optimus_dl/recipe/eval/config.py

@dataclass
class LMEvalConfig:
    """Configuration for lm_eval harness evaluation."""

    tasks: list[str] = field(
        default_factory=lambda: ["hellaswag"],
        metadata={"description": "List of lm_eval tasks to evaluate on"},
    )
    num_fewshot: int = field(
        default=0, metadata={"description": "Number of few-shot examples"}
    )
    batch_size: int = field(
        default=1, metadata={"description": "Batch size for evaluation"}
    )
    limit: int | None = field(
        default=None, metadata={"description": "Limit number of examples per task"}
    )
    output_path: str | None = field(
        default=None, metadata={"description": "Path to save evaluation results"}
    )