Core API Reference¶

def agent_test(
    criteria: Optional[Union[str, List[str]]] = None,
    tags: Optional[Union[str, List[str]]] = None,
    timeout: Optional[int] = None,
    retry_count: int = 0,
    **metadata
) -> Callable

from agent_test import agent_test

@agent_test(
    criteria=['similarity', 'llm_judge'],
    tags=['summarization', 'quality'],
    timeout=60,
    retry_count=2
)
def test_summarization():
    return {
        "input": "Long article...",
        "actual": agent_summary,
        "expected": "Expected summary..."
    }

def run_test(
    test_func: Callable,
    config: Optional[Config] = None
) -> TestResult

from agent_test import run_test, agent_test

@agent_test(criteria=['similarity'])
def my_test():
    return {"input": "test", "actual": "result", "expected": "result"}

result = run_test(my_test)
print(f"Test passed: {result.passed}")

class Config:
    def __init__(
        self,
        project_name: str = "AgentTest Project",
        evaluators: List[EvaluatorConfig] = None,
        llm: Optional[LLMConfig] = None,
        git: Optional[GitConfig] = None,
        logging: Optional[LoggingConfig] = None
    )

# Load from default location (.agenttest/config.yaml)
config = Config.load()

# Load from specific path
config = Config.load("custom/config.yaml")

config.save()  # Save to default location
config.save("backup/config.yaml")  # Save to specific path

@dataclass
class EvaluatorConfig:
    name: str
    type: str
    config: Dict[str, Any] = field(default_factory=dict)
    weight: float = 1.0
    enabled: bool = True

@dataclass
class LLMConfig:
    provider: str  # 'openai', 'anthropic', 'gemini'
    model: str
    api_key: Optional[str] = None
    temperature: float = 0.0
    max_tokens: Optional[int] = None
    timeout: float = 30.0

class EvaluatorRegistry:
    def __init__(self, config: Config)

registry = EvaluatorRegistry(config)
similarity_evaluator = registry.get_evaluator('similarity')

custom_evaluator = MyCustomEvaluator(config)
registry.register_evaluator('custom', custom_evaluator)

evaluators = registry.list_evaluators()
# Returns: {'similarity': 'StringSimilarityEvaluator', ...}

results = registry.evaluate_with_multiple(
    test_output=test_data,
    criteria=['similarity', 'llm_judge']
)

@dataclass
class TestResult:
    test_name: str
    passed: bool
    score: Optional[float] = None
    duration: float = 0.0
    error: Optional[str] = None
    details: Dict[str, Any] = field(default_factory=dict)
    evaluations: Dict[str, Any] = field(default_factory=dict)

@dataclass
class TestResults:
    test_results: List[TestResult] = field(default_factory=list)
    summary: Dict[str, Any] = field(default_factory=dict)
    metadata: Dict[str, Any] = field(default_factory=dict)

results = TestResults()
results.add_result(test_result)

if results.has_failures():
    print("Some tests failed!")

pass_rate = results.get_pass_rate()
print(f"Pass rate: {pass_rate:.1f}%")

summary = results.get_summary()
# Returns:
# {
#     "total_tests": 10,
#     "passed": 8,
#     "failed": 2,
#     "pass_rate": 80.0,
#     "average_score": 0.85,
#     "total_duration": 45.2
# }

results.save_to_file("test_results.json")

class TestRunner:
    def __init__(self, config: Config)

def run_tests(
    self,
    path: Optional[Path] = None,
    pattern: str = "test_*.py",
    tags: Optional[List[str]] = None,
    verbose: bool = False
) -> TestResults

runner = TestRunner(config)
results = runner.run_tests(
    path=Path("tests/"),
    pattern="test_*.py",
    tags=["integration"],
    verbose=True
)

def discover_tests(
    self,
    path: Optional[Path] = None,
    pattern: str = "test_*.py"
) -> List[TestCase]

@dataclass
class TestCase:
    name: str
    function: Callable
    criteria: List[str]
    tags: List[str] = field(default_factory=list)
    timeout: Optional[int] = None
    retry_count: int = 0
    metadata: Dict[str, Any] = field(default_factory=dict)
    module: Optional[str] = None
    file_path: Optional[str] = None

from agent_test.core.decorators import get_registered_tests

tests = get_registered_tests()
for test_id, test_case in tests.items():
    print(f"Test: {test_case.name}")

from agent_test.core.decorators import clear_test_registry

clear_test_registry()

from agent_test.core.decorators import is_agent_test

if is_agent_test(my_function):
    print("This is an agent test")

from agent_test.core.decorators import get_test_case

test_case = get_test_case(my_test_function)
if test_case:
    print(f"Test criteria: {test_case.criteria}")

class AgentTestError(Exception):
    """Base exception for AgentTest."""
    pass

class EvaluationError(AgentTestError):
    """Exception for evaluation errors."""
    pass

class ConfigurationError(AgentTestError):
    """Exception for configuration errors."""
    pass

class GenerationError(AgentTestError):
    """Exception for test generation errors."""
    pass

from agent_test import agent_test, Config, TestRunner

# Configure AgentTest
config = Config.load()

# Define a test
@agent_test(criteria=['similarity'])
def test_my_agent():
    return {
        "input": "Hello",
        "actual": "Hello world",
        "expected": "Hello world"
    }

# Run tests
runner = TestRunner(config)
results = runner.run_tests()

# Check results
if results.has_failures():
    print("Tests failed!")
else:
    print("All tests passed!")

from agent_test import Config, EvaluatorRegistry
from agent_test.evaluators.base import BaseEvaluator, EvaluationResult

class CustomEvaluator(BaseEvaluator):
    @property
    def name(self) -> str:
        return "custom"

    def evaluate(self, test_output) -> EvaluationResult:
        # Custom evaluation logic
        return EvaluationResult(passed=True, score=1.0)

# Register custom evaluator
config = Config.load()
registry = EvaluatorRegistry(config)
registry.register_evaluator('custom', CustomEvaluator())

from agent_test import Config, EvaluatorConfig, LLMConfig

# Create configuration programmatically
config = Config(
    project_name="My AI Project",
    evaluators=[
        EvaluatorConfig(
            name="similarity",
            type="string_similarity",
            config={"threshold": 0.8},
            weight=1.0
        ),
        EvaluatorConfig(
            name="llm_judge",
            type="llm_as_judge",
            config={"criteria": ["accuracy", "clarity"]},
            weight=2.0
        )
    ],
    llm=LLMConfig(
        provider="openai",
        model="gpt-4",
        temperature=0.0
    )
)

# Save configuration
config.save(".agenttest/config.yaml")