from abc import ABC, abstractmethod
from typing import Type
from fair_forge.core.retriever import Retriever

class FairForge(ABC):
    def __init__(self, retriever: Type[Retriever], verbose: bool = False, **kwargs):
        self.retriever = retriever(**kwargs)
        self.metrics = []
        self.verbose = verbose

    @abstractmethod
    def batch(self, session_id: str, context: str, assistant_id: str,
              batch: list[Batch], language: str | None) -> None:
        """Process a batch of conversations. Implemented by each metric."""
        pass

    @classmethod
    def run(cls, retriever: Type[Retriever], **kwargs) -> list:
        """One-shot execution: instantiate and process."""
        instance = cls(retriever, **kwargs)
        instance._process()
        return instance.metrics

from abc import ABC, abstractmethod
from fair_forge.schemas.common import Dataset

class Retriever(ABC):
    def __init__(self, **kwargs):
        pass

    @abstractmethod
    def load_dataset(self) -> list[Dataset]:
        """Load and return datasets for evaluation."""
        pass

class Dataset(BaseModel):
    session_id: str          # Unique session identifier
    assistant_id: str        # ID of the assistant being evaluated
    language: str | None     # Language code (e.g., "english")
    context: str             # System context/instructions
    conversation: list[Batch] # List of Q&A interactions

class Batch(BaseModel):
    qa_id: str                          # Unique interaction ID
    query: str                          # User question
    assistant: str                      # Assistant response
    ground_truth_assistant: str | None  # Expected response
    observation: str | None             # Additional notes
    agentic: dict | None                # Metadata
    ground_truth_agentic: dict | None   # Expected metadata
    logprobs: dict | None               # Log probabilities

from fair_forge.core.base import FairForge

class MyMetric(FairForge):
    def __init__(self, retriever, verbose=False, **kwargs):
        super().__init__(retriever, verbose, **kwargs)
        # Initialize metric-specific components

    def batch(self, session_id, context, assistant_id, batch, language):
        # Process the batch and compute metrics
        result = self._compute(batch)
        self.metrics.append(result)

fair_forge/
├── core/
│   ├── base.py           # FairForge base class
│   ├── retriever.py      # Retriever abstract class
│   ├── guardian.py       # Guardian interface (bias detection)
│   ├── sentiment.py      # Sentiment analyzer interface
│   ├── loader.py         # Toxicity loader interface
│   └── extractor.py      # Group extractor interface
├── metrics/
│   ├── toxicity.py       # Toxicity metric
│   ├── bias.py           # Bias metric
│   ├── context.py        # Context metric
│   ├── conversational.py # Conversational metric
│   ├── humanity.py       # Humanity metric
│   └── best_of.py        # BestOf metric
├── schemas/
│   ├── common.py         # Dataset, Batch schemas
│   ├── toxicity.py       # Toxicity result schemas
│   ├── bias.py           # Bias result schemas
│   └── ...               # Other metric schemas
├── statistical/
│   ├── base.py           # StatisticalMode interface
│   ├── frequentist.py    # Frequentist implementation
│   └── bayesian.py       # Bayesian implementation
├── generators/           # Test dataset generation
├── runners/              # Test execution
├── storage/              # Storage backends
├── llm/                  # LLM integration (Judge)
├── guardians/            # Guardian implementations
├── extractors/           # Group extractor implementations
└── loaders/              # Toxicity lexicon loaders