from fair_forge.storage import create_local_storage
from pathlib import Path

storage = create_local_storage(
    tests_dir=Path("./test_datasets"),
    results_dir=Path("./test_results"),
    enabled_suites=None,  # Load all suites
)

test_datasets/
├── regression/
│   ├── api_tests.json
│   └── ui_tests.json
├── smoke/
│   └── basic_tests.json
└── standalone_tests.json

test_results/
├── test_run_20240115_143022_abc123.json
├── test_run_20240115_150000_def456.json
└── test_run_20240116_091530_ghi789.json

# Load all datasets
datasets = storage.load_datasets()

print(f"Loaded {len(datasets)} dataset(s)")
for ds in datasets:
    print(f"  - {ds.session_id}: {len(ds.conversation)} batches")

from datetime import datetime
import uuid

result_path = storage.save_results(
    datasets=executed_datasets,
    run_id=str(uuid.uuid4()),
    timestamp=datetime.now(),
)

print(f"Saved to: {result_path}")
# Output: test_results/test_run_20240115_143022_abc123.json

{
  "session_id": "test-001",
  "assistant_id": "my-assistant",
  "language": "english",
  "context": "You are a helpful assistant.",
  "conversation": [
    {
      "qa_id": "q1",
      "query": "What is the capital of France?",
      "assistant": "",
      "ground_truth_assistant": "Paris"
    }
  ]
}

{
  "run_id": "abc123",
  "timestamp": "2024-01-15T14:30:22",
  "datasets": [
    {
      "session_id": "test-001",
      "assistant_id": "my-assistant",
      "conversation": [
        {
          "qa_id": "q1",
          "query": "What is the capital of France?",
          "assistant": "The capital of France is Paris.",
          "ground_truth_assistant": "Paris"
        }
      ]
    }
  ]
}

# Only load specific test suites
storage = create_local_storage(
    tests_dir="./test_datasets",
    results_dir="./results",
    enabled_suites=["regression", "smoke"],
)

# Will only load from:
# - test_datasets/regression/
# - test_datasets/smoke/

import asyncio
from datetime import datetime
import uuid
from fair_forge.storage import create_local_storage
from fair_forge.runners import AlquimiaRunner

async def run_tests():
    # Setup storage
    storage = create_local_storage(
        tests_dir="./test_datasets",
        results_dir="./test_results",
    )

    # Setup runner
    runner = AlquimiaRunner(
        base_url="...",
        api_key="...",
        agent_id="...",
        channel_id="...",
    )

    # Load datasets
    datasets = storage.load_datasets()
    print(f"Loaded {len(datasets)} dataset(s)")

    # Execute
    executed = []
    for dataset in datasets:
        print(f"Running: {dataset.session_id}")
        updated, summary = await runner.run_dataset(dataset)
        executed.append(updated)
        print(f"  Result: {summary['successes']}/{summary['total_batches']}")

    # Save results
    result_path = storage.save_results(
        datasets=executed,
        run_id=str(uuid.uuid4()),
        timestamp=datetime.now(),
    )
    print(f"\nResults saved: {result_path}")

asyncio.run(run_tests())