用RAGAS框架系统评估RAG系统质量

from ragas import evaluatefrom ragas.metrics import faithfulness, answer_relevancyfrom datasets import Dataset # 构建评估数据集data = {    "question": ["RAGAS的忠实度指标如何计算？"],    "answer": ["RAGAS通过将答案分解为原子声明，逐条验证每个声明是否有上下文支撑来计算忠实度分数。"],    "contexts": [["RAGAS忠实度衡量答案中所有声明是否能从检索上下文中找到支撑。计算过程包括声明分解和逐条验证。"]],    "ground_truth": ["通过将答案分解为原子声明并验证上下文支撑来计算"]}dataset = Dataset.from_dict(data) # 执行评估result = evaluate(    dataset=dataset,    metrics=[faithfulness, answer_relevancy])print(f"Faithfulness: {result['faithfulness']:.4f}")print(f"Answer Relevancy: {result['answer_relevancy']:.4f}")

from ragas import evaluatefrom ragas.metrics import (    faithfulness, answer_relevancy,    context_precision, context_recall)from datasets import Dataset # 模拟评估数据集questions = ["什么是RAG？", "RAGAS有哪些核心指标？"]answers = [    "RAG是检索增强生成，通过检索外部知识来增强LLM的生成能力。",    "RAGAS有四个核心指标：忠实度、答案相关性、上下文精确率和上下文召回率。"]contexts_list = [    ["RAG（Retrieval Augmented Generation）是一种结合检索和生成的AI范式。"],    ["RAGAS框架包含忠实度、答案相关性、上下文精确率、上下文召回率四个核心指标。"]]ground_truths = [    "RAG是检索增强生成，通过检索外部知识库来增强大语言模型的生成能力",    "四个核心指标是忠实度、答案相关性、上下文精确率和上下文召回率"] eval_dataset = Dataset.from_dict({    "question": questions,    "answer": answers,    "contexts": contexts_list,    "ground_truth": ground_truths}) result = evaluate(    dataset=eval_dataset,    metrics=[faithfulness, answer_relevancy, context_precision, context_recall]) for metric_name, score in result.items():    print(f"{metric_name}: {score:.4f}")

import json # RAGAS RAG评估核心指标模拟计算# 在真实场景中需安装ragas库，此处演示核心计算逻辑 def compute_faithfulness(answer, contexts):    """    忠实度计算：将答案分解为原子声明，逐条验证是否有上下文支撑    真实场景由LLM完成声明分解和验证，此处模拟计算过程    """    # 模拟原子声明分解    statements = answer.split("。")    statements = [s.strip() for s in statements if s.strip()]        supported_count = 0    for stmt in statements:        # 模拟LLM验证：检查声明关键词是否出现在上下文中        context_text = " ".join(contexts)        keywords = stmt[:6] if len(stmt) >= 6 else stmt        if keywords in context_text:            supported_count += 1        faithfulness_score = supported_count / len(statements) if statements else 0    return faithfulness_score, len(statements), supported_count  def compute_answer_relevancy(question, answer):    """    答案相关性计算：通过反向生成问题与原问题的余弦相似度衡量    此处用字符重叠度模拟余弦相似度    """    # 模拟反向生成的问题    reverse_questions = [        answer[:20] + "是什么",        "关于" + answer[:15] + "的问题",        question  # 理想情况下反向生成的问题应接近原问题    ]        # 模拟余弦相似度计算（字符级Jaccard近似）    q_chars = set(question)    similarities = []    for rq in reverse_questions:        rq_chars = set(rq)        intersection = len(q_chars & rq_chars)        union = len(q_chars | rq_chars)        sim = intersection / union if union > 0 else 0        similarities.append(sim)        relevancy_score = sum(similarities) / len(similarities)    return relevancy_score, similarities  def compute_context_precision(contexts, answer):    """    上下文精确率：衡量检索结果中相关chunk的占比    """    relevant_count = 0    for ctx in contexts:        # 模拟判断chunk是否与答案相关        overlap = len(set(ctx) & set(answer))        total = len(set(ctx) | set(answer))        if total > 0 and overlap / total > 0.1:            relevant_count += 1        precision = relevant_count / len(contexts) if contexts else 0    return precision, relevant_count  def compute_context_recall(contexts, ground_truth):    """    上下文召回率：衡量关键信息是否被检索到    需要参考答案作为基准    """    # 模拟将ground_truth分解为关键信息点    info_points = [p.strip() for p in ground_truth.split("，") if p.strip()]    if not info_points:        info_points = [ground_truth]        context_text = " ".join(contexts)    recalled_count = 0    for point in info_points:        # 检查信息点是否在上下文中存在        key_part = point[:4] if len(point) >= 4 else point        if key_part in context_text:            recalled_count += 1        recall = recalled_count / len(info_points) if info_points else 0    return recall, recalled_count, len(info_points)  # === 执行评估 ===question = "RAGAS框架的四个核心指标是什么？"answer = "RAGAS框架包含忠实度、答案相关性、上下文精确率和上下文召回率四个核心指标。忠实度衡量答案与上下文的一致性。"contexts = [    "RAGAS是专门用于RAG系统评估的开源框架，包含四个核心指标。",    "忠实度衡量生成答案中的所有声明是否都能从检索上下文中找到支撑。",    "上下文精确率衡量检索结果中相关文档片段的占比。",    "天气今天晴朗适合出行"  # 噪声chunk]ground_truth = "RAGAS四个核心指标是忠实度、答案相关性、上下文精确率、上下文召回率" # 计算四维指标faith_score, total_stmt, supported = compute_faithfulness(answer, contexts)relevancy_score, sims = compute_answer_relevancy(question, answer)precision_score, relevant_ctx = compute_context_precision(contexts, answer)recall_score, recalled, total_points = compute_context_recall(contexts, ground_truth) # 输出评估报告print("=" * 50)print("RAGAS RAG评估报告")print("=" * 50)print(f"忠实度 (Faithfulness):        {faith_score:.4f}  ({supported}/{total_stmt} 声明有支撑)")print(f"答案相关性 (Answer Relevancy): {relevancy_score:.4f}")print(f"上下文精确率 (Ctx Precision):  {precision_score:.4f}  ({relevant_ctx}/{len(contexts)} chunks相关)")print(f"上下文召回率 (Ctx Recall):     {recall_score:.4f}  ({recalled}/{total_points} 信息点被检索)")print("=" * 50) # 诊断建议issues = []if faith_score < 0.8:    issues.append("忠实度偏低：建议优化生成Prompt，增加引用约束")if relevancy_score < 0.7:    issues.append("答案相关性偏低：建议改进查询理解和意图识别")if precision_score < 0.7:    issues.append("上下文精确率偏低：建议优化检索策略，减少噪声chunk")if recall_score < 0.7:    issues.append("上下文召回率偏低：建议增大chunk尺寸或引入混合检索") if issues:    print("诊断建议：")    for issue in issues:        print(f"  - {issue}")else:    print("所有指标达标，RAG系统质量良好")