Collaborative testing for LLM & agentic apps

import os

from rhesis.sdk.synthesizers.multi_turn import MultiTurnSynthesizer, GenerationConfig
from rhesis.penelope import PenelopeAgent, EndpointTarget

# Set the environment variables
os.environ["RHESIS_BASE_URL"] = "https://api.rhesis.ai"
os.environ["RHESIS_API_KEY"] = "rh-XXXXXXXXXXXXXXXXXXXX"

# Step 1: Generate multi-turn test case
config = GenerationConfig(
    generation_prompt="Generate a test for an AI travel guide planning complex multi-city trips"
)

synthesizer = MultiTurnSynthesizer(config=config)
test_set = synthesizer.generate(num_tests=1)
test = test_set.tests[0]

# Step 2: Execute with Penelope
agent = PenelopeAgent()
target = EndpointTarget(endpoint_id="travel-guide-prod")

result = agent.execute_test(
    target=target,
    goal=test['test_configuration']['goal'],
    instructions=test['test_configuration']['instructions'],
    restrictions=test['test_configuration']['restrictions'],
    scenario=test['test_configuration']['scenario']
)

# Step 3: Access results
print(f"Goal Achieved: {result.goal_achieved}")
print(f"Turns: {result.turns_used}")
print(f"Duration: {result.duration_seconds:.2f}s")

for metric_name, metric_data in result.metrics.items():
    print(f"{metric_name}: {metric_data['score']}")