Utiliser les évaluateurs intégrés

from weave.scorers import HallucinationFreeScorer

scorer = HallucinationFreeScorer()

import asyncio
import weave
from weave.scorers import HallucinationFreeScorer

# Initialiser le scorer avec un mappage de colonnes si nécessaire.
hallucination_scorer = HallucinationFreeScorer(
    model_id="openai/gpt-4o", # ou tout autre modèle pris en charge par litellm
    column_map={"context": "input", "output": "other_col"}
)

# Créer le jeu de données
dataset = [
    {"input": "John likes various types of cheese."},
    {"input": "Pepe likes various types of cheese."},
]

@weave.op
def model(input: str) -> str:
    return "The person's favorite cheese is cheddar."

# Lancer l'évaluation
evaluation = weave.Evaluation(
    dataset=dataset,
    scorers=[hallucination_scorer],
)
result = asyncio.run(evaluation.evaluate(model))
print(result)
# Exemple de sortie :
# {'HallucinationFreeScorer': {'has_hallucination': {'true_count': 2, 'true_fraction': 1.0}}, 'model_latency': {'mean': ...}}

from weave.scorers import SummarizationScorer

scorer = SummarizationScorer(
    model_id="openai/gpt-4o"  # ou tout autre modèle pris en charge par litellm
)

import asyncio
import weave
from weave.scorers import SummarizationScorer

class SummarizationModel(weave.Model):
    @weave.op()
    async def predict(self, input: str) -> str:
        return "This is a summary of the input text."

# Initialiser le scorer
summarization_scorer = SummarizationScorer(
    model_id="openai/gpt-4o"  # ou tout autre modèle pris en charge par litellm
)
# Créer le jeu de données
dataset = [
    {"input": "The quick brown fox jumps over the lazy dog."},
    {"input": "Artificial Intelligence is revolutionizing various industries."}
]
# Lancer l'évaluation
evaluation = weave.Evaluation(dataset=dataset, scorers=[summarization_scorer])
results = asyncio.run(evaluation.evaluate(SummarizationModel()))
print(results)
# Exemple de sortie :
# {'SummarizationScorer': {'is_entity_dense': {'true_count': 0, 'true_fraction': 0.0}, 'summarization_eval_score': {'mean': 0.0}, 'entity_density': {'mean': 0.0}}, 'model_latency': {'mean': ...}}

from weave.scorers import OpenAIModerationScorer

scorer = OpenAIModerationScorer()

import asyncio
import weave
from weave.scorers import OpenAIModerationScorer

class MyModel(weave.Model):
    @weave.op
    async def predict(self, input: str) -> str:
        return input

# Initialiser le scorer
moderation_scorer = OpenAIModerationScorer()

# Créer le jeu de données
dataset = [
    {"input": "I love puppies and kittens!"},
    {"input": "I hate everyone and want to hurt them."}
]

# Lancer l'évaluation
evaluation = weave.Evaluation(dataset=dataset, scorers=[moderation_scorer])
results = asyncio.run(evaluation.evaluate(MyModel()))
print(results)
# Exemple de sortie :
# {'OpenAIModerationScorer': {'flagged': {'true_count': 1, 'true_fraction': 0.5}, 'categories': {'violence': {'true_count': 1, 'true_fraction': 1.0}}}, 'model_latency': {'mean': ...}}

from weave.scorers import EmbeddingSimilarityScorer

similarity_scorer = EmbeddingSimilarityScorer(
    model_id="openai/text-embedding-3-small",  # ou tout autre modèle pris en charge par litellm
    threshold=0.4  # le seuil de similarité cosinus
)

import asyncio
import weave
from weave.scorers import EmbeddingSimilarityScorer

# Initialiser le scorer
similarity_scorer = EmbeddingSimilarityScorer(
    model_id="openai/text-embedding-3-small",  # ou tout autre modèle pris en charge par litellm
    threshold=0.7
)
# Créer le jeu de données
dataset = [
    {
        "input": "He's name is John",
        "target": "John likes various types of cheese.",
    },
    {
        "input": "He's name is Pepe.",
        "target": "Pepe likes various types of cheese.",
    },
]
# Définir le modèle
@weave.op
def model(input: str) -> str:
    return "John likes various types of cheese."

# Lancer l'évaluation
evaluation = weave.Evaluation(
    dataset=dataset,
    scorers=[similarity_scorer],
)
result = asyncio.run(evaluation.evaluate(model))
print(result)
# Exemple de sortie :
# {'EmbeddingSimilarityScorer': {'is_similar': {'true_count': 1, 'true_fraction': 0.5}, 'similarity_score': {'mean': 0.844851403}}, 'model_latency': {'mean': ...}}

from weave.scorers import ValidJSONScorer

json_scorer = ValidJSONScorer()

import asyncio
import weave
from weave.scorers import ValidJSONScorer

class JSONModel(weave.Model):
    @weave.op()
    async def predict(self, input: str) -> str:
        # Ceci est un espace réservé.
        # Dans un scénario réel, cela générerait du JSON.
        return '{"key": "value"}'

model = JSONModel()
json_scorer = ValidJSONScorer()

dataset = [
    {"input": "Generate a JSON object with a key and value"},
    {"input": "Create an invalid JSON"}
]

evaluation = weave.Evaluation(dataset=dataset, scorers=[json_scorer])
results = asyncio.run(evaluation.evaluate(model))
print(results)
# Exemple de sortie :
# {'ValidJSONScorer': {'json_valid': {'true_count': 2, 'true_fraction': 1.0}}, 'model_latency': {'mean': ...}}

from weave.scorers import ValidXMLScorer

xml_scorer = ValidXMLScorer()

import asyncio
import weave
from weave.scorers import ValidXMLScorer

class XMLModel(weave.Model):
    @weave.op()
    async def predict(self, input: str) -> str:
        # Ceci est un espace réservé. Dans un scénario réel, cela générerait du XML.
        return '<root><element>value</element></root>'

model = XMLModel()
xml_scorer = ValidXMLScorer()

dataset = [
    {"input": "Generate a valid XML with a root element"},
    {"input": "Create an invalid XML"}
]

evaluation = weave.Evaluation(dataset=dataset, scorers=[xml_scorer])
results = asyncio.run(evaluation.evaluate(model))
print(results)
# Exemple de sortie :
# {'ValidXMLScorer': {'xml_valid': {'true_count': 2, 'true_fraction': 1.0}}, 'model_latency': {'mean': ...}}

from weave.scorers import PydanticScorer
from pydantic import BaseModel

class FinancialReport(BaseModel):
    revenue: int
    year: str

pydantic_scorer = PydanticScorer(model=FinancialReport)

from weave.scorers import ContextEntityRecallScorer

entity_recall_scorer = ContextEntityRecallScorer(
    model_id="openai/gpt-4o"
)

from weave.scorers import ContextRelevancyScorer

relevancy_scorer = ContextRelevancyScorer(
    model_id="openai/gpt-4o",  # ou tout autre modèle pris en charge par litellm
    relevancy_prompt="""
Given the following question and context, rate the relevancy of the context to the question on a scale from 0 to 1.

Question: {question}
Context: {context}
Relevancy Score (0-1):
"""
)

import asyncio
from textwrap import dedent
import weave
from weave.scorers import ContextEntityRecallScorer, ContextRelevancyScorer

class RAGModel(weave.Model):
    @weave.op()
    async def predict(self, question: str) -> str:
        "Récupérer le contexte pertinent"
        return "Paris is the capital of France."

# Définir les prompts
relevancy_prompt: str = dedent("""
    Étant donné la question et le contexte suivants, évaluez la pertinence du contexte par rapport à la question sur une échelle de 0 à 1.

    Question : {question}
    Contexte : {context}
    Score de pertinence (0-1) :
    """)
# Initialiser les évaluateurs
entity_recall_scorer = ContextEntityRecallScorer()
relevancy_scorer = ContextRelevancyScorer(relevancy_prompt=relevancy_prompt)
# Créer le jeu de données
dataset = [
    {
        "question": "Quelle est la capitale de la France ?",
        "context": "Paris est la capitale de la France."
    },
    {
        "question": "Qui a écrit Roméo et Juliette ?",
        "context": "William Shakespeare a écrit de nombreuses pièces célèbres."
    }
]
# Exécuter l'évaluation
evaluation = weave.Evaluation(
    dataset=dataset,
    scorers=[entity_recall_scorer, relevancy_scorer]
)
results = asyncio.run(evaluation.evaluate(RAGModel()))
print(results)
# Exemple de sortie :
# {'ContextEntityRecallScorer': {'recall': {'mean': ...}}, 
# 'ContextRelevancyScorer': {'relevancy_score': {'mean': ...}}, 
# 'model_latency': {'mean': ...}}

from weave.scorers import SummarizationScorer

# Passer au modèle Claude d'Anthropic
summarization_scorer = SummarizationScorer(
    model_id="anthropic/claude-3-5-sonnet-20240620"
)