1. Load regulatory corpus (markdown files)
2. Chunk documents with configurable size/overlap
3. For each interaction:
   a. Retrieve relevant chunks via embeddings (user query + agent response)
   b. Rerank chunks → classify as SUPPORTS or CONTRADICTS
   c. compliance_score = supporting / (supporting + contradicting)
   d. verdict = COMPLIANT / NON_COMPLIANT / IRRELEVANT
4. Session aggregate: weighted mean of per-interaction scores
5. Session verdict derived from the aggregated score

uv add "alquimia-fair-forge[regulatory]"

# Weight high-risk interactions more heavily
Batch(qa_id="billing_dispute",  ..., weight=0.5),   # Critical compliance area
Batch(qa_id="general_inquiry",  ..., weight=0.3),
Batch(qa_id="small_talk",       ..., weight=0.2),

class RegulatoryMetric(BaseMetric):
    session_id: str
    assistant_id: str
    n_interactions: int                    # Number of interactions evaluated
    compliance_score: float                # Weighted mean compliance score (0.0-1.0)
    compliance_score_ci_low: float | None  # Lower credible bound — Bayesian only
    compliance_score_ci_high: float | None # Upper credible bound — Bayesian only
    verdict: Literal["COMPLIANT", "NON_COMPLIANT", "IRRELEVANT"]
    total_supporting_chunks: int           # Sum across all interactions
    total_contradicting_chunks: int        # Sum across all interactions
    interactions: list[RegulatoryInteraction]  # Per-QA detail

class RegulatoryInteraction(BaseModel):
    qa_id: str
    query: str
    assistant: str
    compliance_score: float                # supporting / (supporting + contradicting)
    verdict: Literal["COMPLIANT", "NON_COMPLIANT", "IRRELEVANT"]
    supporting_chunks: int
    contradicting_chunks: int
    retrieved_chunks: list[RegulatoryChunk]  # Chunk-level evidence
    insight: str                             # Human-readable explanation

class RegulatoryChunk(BaseModel):
    text: str              # Chunk content
    source: str            # Source document filename
    chunk_index: int       # Position in source document
    similarity: float      # Cosine similarity from retrieval (0-1)
    reranker_score: float  # Reranker score (higher = supports)
    verdict: Literal["SUPPORTS", "CONTRADICTS"]

from fair_forge.connectors import LocalCorpusConnector

connector = LocalCorpusConnector("path/to/corpus/")
documents = connector.load_documents()
print(f"Loaded {len(documents)} documents")

from fair_forge.connectors.lakefs import LakeFSCorpusConnector

connector = LakeFSCorpusConnector(
    host="https://lakefs.example.com",
    username="your-username",
    password="your-password",
    repo_id="regulations",
    corpus_prefix="compliance/",
    branch_name="main",
)

from fair_forge.connectors import LocalCorpusConnector
from fair_forge.core.retriever import Retriever
from fair_forge.embedders import QwenEmbedder
from fair_forge.metrics.regulatory import Regulatory
from fair_forge.rerankers import QwenReranker
from fair_forge.statistical import BayesianMode
from fair_forge.schemas.common import Dataset, Batch

class ComplianceRetriever(Retriever):
    def load_dataset(self) -> list[Dataset]:
        return [
            Dataset(
                session_id="compliance_test_001",
                assistant_id="callcenter_bot",
                language="english",
                context="Call center regulatory compliance",
                conversation=[
                    Batch(
                        qa_id="qa_001",
                        query="I don't want any more calls from you!",
                        assistant="I've immediately added your number to our do-not-call list.",
                        ground_truth_assistant="Add to do-not-call list",
                        weight=0.6,  # High-stakes interaction
                    ),
                    Batch(
                        qa_id="qa_002",
                        query="Can I get a refund? I bought this 45 days ago.",
                        assistant="Our standard refund policy is 30 days. I can offer store credit instead.",
                        ground_truth_assistant="Explain 30-day limit, offer alternatives",
                        weight=0.4,
                    ),
                ],
            ),
        ]

corpus_connector = LocalCorpusConnector("./regulations/")

metrics = Regulatory.run(
    ComplianceRetriever,
    corpus_connector=corpus_connector,
    embedder=QwenEmbedder(),
    reranker=QwenReranker(),
    statistical_mode=BayesianMode(mc_samples=5000, ci_level=0.95),
    compliance_threshold=0.5,
    verbose=True,
)

for metric in metrics:
    print(f"Session: {metric.session_id}")
    ci = f"  [{metric.compliance_score_ci_low:.1%}, {metric.compliance_score_ci_high:.1%}]" \
         if metric.compliance_score_ci_low is not None else ""
    print(f"  Compliance: {metric.compliance_score:.1%}{ci}")
    print(f"  Verdict:    {metric.verdict}")
    print(f"  Evidence:   {metric.total_supporting_chunks} supporting, {metric.total_contradicting_chunks} contradicting")
    print()

    icon = {"COMPLIANT": "✅", "NON_COMPLIANT": "❌", "IRRELEVANT": "⚠️"}
    for interaction in metric.interactions:
        print(f"  {icon[interaction.verdict]} [{interaction.qa_id}] {interaction.verdict}  score={interaction.compliance_score:.1%}")
        print(f"     {interaction.insight}")
        for chunk in interaction.retrieved_chunks[:2]:
            print(f"     [{chunk.verdict}] {chunk.source}  sim={chunk.similarity:.2f}  rerank={chunk.reranker_score:.2f}")

# Call Center Policy

## Do-Not-Call Regulations

### Customer Rights
- All customers have the right to request removal from call lists at any time
- Requests must be honored within 24 hours of receipt
- No calls between 9:00 PM and 8:00 AM local time

## Refund Policy
- Full refunds available within 30 days of purchase with original receipt
- Refunds processed within 5-7 business days