API Reference¶

async def evaluate(
    test_cases: list[EvalTestCase],
    metrics: list[MetricPattern],
    verbose: bool = True,
    show_dashboard: bool = False,
    session_name: str = None
) -> list[tuple[None, list[TestCaseResult]]]:

async def evaluate_conversations(
    conv_cases: list[ConversationalEvalTestCase],
    metrics: list[ConversationalMetricPattern],
    verbose: bool = True
) -> list[tuple[None, list[ConversationalTestCaseResult]]]:

async def chat_complete(
    llm: str | tuple[str, str] | LLMDescriptor | CustomLLMClient,
    messages: list[dict[str, str]],
    temperature: float = 0.0,
    *,
    api_key: str | None = None,
    api_base: str | None = None,
    extra_kwargs: dict | None = None,
) -> tuple[str, float | None]:

async def get_embeddings(
    model: str | tuple[str, str] | LLMDescriptor | CustomLLMClient,
    texts: list[str]
) -> tuple[list[list[float]], float | None]:

def score_agg(
    scores: list[float],
    temperature: float = 0.5,
    penalty: float = 0.1,
    eps_for_neg_p: float = 1e-9
) -> float:

def extract_json_block(text: str) -> str:

class EvalTestCase(BaseModel):
    input: str                                      # User query (required)
    actual_output: str                              # AI response (required)
    expected_output: str | None = None              # Reference answer
    retrieval_context: list[str] | None = None      # Retrieved documents
    tools_called: list[str] | None = None           # Tools actually called
    expected_tools: list[str] | None = None         # Expected tool calls
    reasoning: str | None = None                    # Chain of thought
    name: str | None = None                         # Test case label

class ConversationalEvalTestCase(BaseModel):
    turns: list[EvalTestCase]                       # Conversation turns
    chatbot_role: str | None = None                 # System prompt / role
    name: str | None = None                         # Test case label

class ToolCall(BaseModel):
    name: str                                       # Tool name
    description: str | None = None                  # Tool description
    reasoning: str | None = None                    # Why tool was called

@dataclass
class MetricResult:
    name: str                               # Metric name
    score: float                            # Score (0.0-1.0)
    threshold: float                        # Pass threshold
    success: bool                           # score >= threshold
    evaluation_cost: float | None           # Cost in USD
    reason: str                             # Human-readable explanation
    evaluation_model: str                   # Model used for evaluation
    evaluation_log: dict | None = None      # Detailed evaluation data

@dataclass
class TestCaseResult:
    input: str
    actual_output: str
    expected_output: str | None
    retrieval_context: list[str] | None
    success: bool                           # All metrics passed
    metrics_data: list[MetricResult]
    tools_called: list[str] | None = None
    expected_tools: list[str] | None = None

@dataclass
class ConversationalTestCaseResult:
    dialogue: list[dict[str, str]]          # role/content pairs
    success: bool                           # All metrics passed
    metrics_data: list[MetricResult]

class Provider(str, Enum):
    OLLAMA = "ollama"
    MLX = "mlx"
    ZHIPU = "zhipu"
    CUSTOM = "custom"

class LLMDescriptor:
    provider: Provider | str
    model: str
    api_key: str | None = None
    api_base: str | None = None

class CustomLLMClient(ABC):
    @abstractmethod
    async def chat_complete(self, messages: list, temperature: float) -> tuple[str, float | None]: ...

    @abstractmethod
    def get_model_name(self) -> str: ...

    async def get_embeddings(
        self, texts: list, model: str = "text-embedding-3-small"
    ) -> tuple[list[list[float]], float | None]: ...

class MetricPattern:
    name: str

    def __init__(self, model: str, threshold: float, verbose: bool = False): ...
    async def evaluate(self, test_case: EvalTestCase) -> dict[str, Any]: ...

class ConversationalMetricPattern:
    name: str

    def __init__(self, model: str, threshold: float, verbose: bool = False): ...
    async def evaluate(self, test_case: ConversationalEvalTestCase) -> dict[str, Any]: ...