uv pip install "alquimia-fair-forge[context]"
uv pip install langchain-groq  # Or your preferred LLM provider

from fair_forge.metrics.context import Context
from langchain_groq import ChatGroq
from your_retriever import MyRetriever

# Initialize the judge model
judge_model = ChatGroq(
    model="llama-3.3-70b-versatile",
    api_key="your-api-key",
    temperature=0.0,
)

# Run the metric
metrics = Context.run(
    MyRetriever,
    model=judge_model,
    use_structured_output=True,
    verbose=True,
)

# Analyze results
for metric in metrics:
    print(f"QA ID: {metric.qa_id}")
    print(f"Context Awareness: {metric.context_awareness}")
    print(f"Insight: {metric.context_insight}")

class ContextMetric(BaseMetric):
    session_id: str
    assistant_id: str
    qa_id: str                    # ID of the evaluated interaction
    context_awareness: float      # Alignment score (0-1)
    context_insight: str          # Explanation of the assessment
    context_thinkings: str        # Judge's reasoning (if available)

Score: 1.0
Insight: "The assistant's response directly answers the user's query about
the CEO of Alquimia, matching the ground truth provided in the context.
No extraneous information is included."

Score: 0.2
Insight: "The assistant's response repeats a generic statement that
doesn't reflect the detailed information in the context. It fails to
mention the key attributes."

Score: 0.0
Insight: "The assistant provided no response to the user's queries,
failing to address the context."

import os
from fair_forge.metrics.context import Context
from fair_forge.core.retriever import Retriever
from fair_forge.schemas.common import Dataset, Batch
from langchain_groq import ChatGroq

# Define retriever with context
class ContextTestRetriever(Retriever):
    def load_dataset(self) -> list[Dataset]:
        context = """You are a helpful customer service assistant for TechStore.
        Key policies:
        - Returns accepted within 30 days with receipt
        - Free shipping on orders over $50
        - Support hours: Mon-Fri 9am-5pm EST
        Always be polite and offer to help further."""

        return [
            Dataset(
                session_id="context-eval-001",
                assistant_id="techstore-bot",
                language="english",
                context=context,
                conversation=[
                    Batch(
                        qa_id="q1",
                        query="What's your return policy?",
                        assistant="Our return policy allows returns within 30 days with a receipt. Is there anything else I can help with?",
                        ground_truth_assistant="Returns within 30 days with receipt.",
                    ),
                    Batch(
                        qa_id="q2",
                        query="Do you offer free shipping?",
                        assistant="Yes, we offer free shipping on orders over $50. Would you like help with anything else?",
                        ground_truth_assistant="Free shipping on orders over $50.",
                    ),
                    Batch(
                        qa_id="q3",
                        query="What are your hours?",
                        assistant="We're open 24/7!",  # Wrong - context says Mon-Fri 9-5
                        ground_truth_assistant="Mon-Fri 9am-5pm EST.",
                    ),
                ]
            )
        ]

# Initialize judge
judge = ChatGroq(
    model="llama-3.3-70b-versatile",
    api_key=os.getenv("GROQ_API_KEY"),
    temperature=0.0,
)

# Run context evaluation
metrics = Context.run(
    ContextTestRetriever,
    model=judge,
    use_structured_output=True,
    verbose=True,
)

# Analyze results
print("Context Evaluation Results")
print("=" * 60)

for metric in metrics:
    status = "PASS" if metric.context_awareness >= 0.8 else "FAIL"
    print(f"\n[{status}] QA ID: {metric.qa_id}")
    print(f"Score: {metric.context_awareness:.2f}")
    print(f"Insight: {metric.context_insight}")

# Calculate average
avg_score = sum(m.context_awareness for m in metrics) / len(metrics)
print(f"\nAverage Context Awareness: {avg_score:.2f}")

from langchain_groq import ChatGroq

judge = ChatGroq(
    model="llama-3.3-70b-versatile",
    api_key="your-api-key",
    temperature=0.0,
)

from langchain_openai import ChatOpenAI

judge = ChatOpenAI(
    model="gpt-4o",
    api_key="your-api-key",
    temperature=0.0,
)

from langchain_anthropic import ChatAnthropic

judge = ChatAnthropic(
    model="claude-3-sonnet-20240229",
    api_key="your-api-key",
    temperature=0.0,
)

from langchain_ollama import ChatOllama

judge = ChatOllama(
    model="llama3.1:8b",
    temperature=0.0,
)

metrics = Context.run(
    MyRetriever,
    model=judge,
    use_structured_output=True,  # Uses .with_structured_output()
)

metrics = Context.run(
    MyRetriever,
    model=judge,
    use_structured_output=False,
    bos_json_clause="```json",
    eos_json_clause="```",
)