DeepEval：LLM评估框架

from deepeval import evaluatefrom deepeval.test_case import LLMTestCasefrom deepeval.metrics import FaithfulnessMetric test_case = LLMTestCase(    input="What is the statute of limitations for contract disputes?",    actual_output="The statute of limitations is 6 years.",    expected_output="The statute of limitations for contract disputes is 6 years under UCC.",    retrieval_context=["UCC Section 2-725: An action for breach must be commenced within 6 years."])metric = FaithfulnessMetric(threshold=0.7)evaluate([test_case], [metric])

from deepeval.metrics import BaseMetricfrom deepeval.test_case import LLMTestCase class CitationAccuracyMetric(BaseMetric):    def __init__(self, threshold=0.8):        self.threshold = threshold        self.score = 0.0        def measure(self, test_case):        citations = test_case.expected_output.split('; ')        actual = test_case.actual_output        correct = sum(1 for c in citations if c.strip() in actual)        self.score = correct / len(citations) if citations else 0.0        self.success = self.score >= self.threshold        return self.score        async def a_measure(self, test_case):        return self.measure(test_case)        def is_successful(self):        return self.success        @property    def __name__(self):        return "Citation Accuracy" tc = LLMTestCase(input="q", actual_output="See Art.1; Art.3", expected_output="Art.1; Art.3; Art.5", retrieval_context=[])m = CitationAccuracyMetric(threshold=0.7)score = m.measure(tc)print(f"Citation Accuracy: {score:.2f}, Pass: {m.is_successful()}")

from deepeval.test_case import LLMTestCasefrom deepeval.dataset import EvaluationDataset dataset = EvaluationDataset()cases = [    LLMTestCase(input="合同纠纷诉讼时效?", actual_output="6年", expected_output="根据UCC第2-725条，合同纠纷诉讼时效为6年", retrieval_context=["UCC 2-725: 6年时效"]),    LLMTestCase(input="侵权诉讼时效?", actual_output="2年", expected_output="根据民法典第188条，侵权诉讼时效为3年", retrieval_context=["民法典188条: 3年时效"])]for tc in cases:    dataset.add_test_case(tc)print(f"Dataset size: {len(dataset.test_cases)}")for i, tc in enumerate(dataset.test_cases):    print(f"  Case {i+1}: {tc.input} -> {tc.actual_output}")

import jsonfrom dataclasses import dataclass, fieldfrom typing import List, Optional @dataclassclass LLMTestCase:    input: str    actual_output: str    expected_output: str = ""    retrieval_context: List[str] = field(default_factory=list) class BaseMetric:    def __init__(self, threshold=0.7):        self.threshold = threshold        self.score = 0.0        self.success = False    def measure(self, test_case):        raise NotImplementedError    def is_successful(self):        return self.success    @property    def __name__(self):        return self.__class__.__name__ class FaithfulnessMetric(BaseMetric):    def measure(self, test_case):        if not test_case.retrieval_context:            self.score = 0.0            self.success = False            return self.score        context = ' '.join(test_case.retrieval_context).lower()        output_words = set(test_case.actual_output.lower().split())        context_words = set(context.split())        supported = len(output_words & context_words)        total = len(output_words)        self.score = supported / total if total > 0 else 0.0        self.success = self.score >= self.threshold        return self.score class AnswerRelevancyMetric(BaseMetric):    def measure(self, test_case):        input_words = set(test_case.input.lower().replace('?', '').split())        output_words = set(test_case.actual_output.lower().split())        overlap = len(input_words & output_words)        self.score = overlap / len(input_words) if input_words else 0.0        self.success = self.score >= self.threshold        return self.score class EvaluationDataset:    def __init__(self):        self.test_cases = []    def add_test_case(self, test_case):        self.test_cases.append(test_case)    def evaluate(self, metrics):        results = []        for tc in self.test_cases:            case_results = {}            for metric in metrics:                score = metric.measure(tc)                case_results[metric.__name__] = {                    'score': round(score, 4),                    'passed': metric.is_successful()                }            results.append(case_results)        return results class CIGate:    def __init__(self, thresholds):        self.thresholds = thresholds    def check(self, dataset_results):        all_passed = True        for i, result in enumerate(dataset_results):            for metric_name, data in result.items():                if not data['passed']:                    print(f"  FAIL: Case {i+1} {metric_name} score={data['score']} < threshold")                    all_passed = False        return all_passed tc1 = LLMTestCase(input="合同纠纷诉讼时效", actual_output="合同纠纷诉讼时效为6年根据UCC", expected_output="6年", retrieval_context=["UCC第2-725条规定合同纠纷诉讼时效为6年"])tc2 = LLMTestCase(input="侵权诉讼时效", actual_output="侵权诉讼时效为3年", expected_output="3年", retrieval_context=["民法典第188条规定侵权诉讼时效为3年"])tc3 = LLMTestCase(input="商标注册有效期", actual_output="商标注册有效期为10年可续展", expected_output="10年", retrieval_context=["商标法规定注册有效期为10年"])dataset = EvaluationDataset()for tc in [tc1, tc2, tc3]:    dataset.add_test_case(tc)metrics = [FaithfulnessMetric(threshold=0.5), AnswerRelevancyMetric(threshold=0.3)]results = dataset.evaluate(metrics)for i, result in enumerate(results):    print(f"Case {i+1}: {result}")gate = CIGate(thresholds={'FaithfulnessMetric': 0.5, 'AnswerRelevancyMetric': 0.3})passed = gate.check(results)print(f"\nCI Gate: {'PASSED' if passed else 'FAILED - blocking deployment'}")