AI输出审查过滤方案：给大模型当“内容安全编导”

import re  class OutputReviewer:    """AI输出审查器：规则过滤 + 模型打分 + 误报白名单"""     def __init__(self, threshold=0.6):        # 敏感词表（示例数据，生产环境按业务维护）        self.sensitive_words = ["赌博", "暴力", "违法", "违禁品"]        # 正则变体规则：处理谐音、拆字、拼音等变形        self.regex_rules = [            re.compile(r"赌[\s\-_.]*博"),            re.compile(r"d[bp]o", re.IGNORECASE),            re.compile(r"\d{11}"),  # 疑似手机号        ]        self.threshold = threshold        self.whitelist_cache = set()        self.review_count = 0        self.block_count = 0     def rule_filter(self, text):        """规则过滤：敏感词/正则命中立即拦截"""        for word in self.sensitive_words:            if word in text:                return True, "敏感词命中"        for rule in self.regex_rules:            if rule.search(text):                return True, "正则命中"        return False, ""     def model_scorer(self, text):        """模拟语义模型打分：分数越高越可能违规"""        score = 0.0        if len(text) > 50:            score += 0.1        if "攻击" in text or "报复" in text:            score += 0.6        if "私下交易" in text:            score += 0.5        if text.startswith("我不确定"):            score -= 0.3        return max(0.0, min(1.0, score))     def review(self, text):        """审查总入口：白名单免审，规则先行，模型兜底"""        self.review_count += 1        if text in self.whitelist_cache:            return True, "白名单免审", 0.0        hit, reason = self.rule_filter(text)        if hit:            self.block_count += 1            return False, reason, 1.0        score = self.model_scorer(text)        if score >= self.threshold:            self.block_count += 1            return False, "语义违规", score        return True, "通过", score     def add_whitelist(self, text):        """误报处理：确认安全的文本加入白名单"""        self.whitelist_cache.add(text)  def main():    reviewer = OutputReviewer(threshold=0.6)    samples = [        "你好，请问今天天气怎么样？",        "我来教你一招快速赚钱的赌博技巧",        "这个产品我不确定是否合规，建议咨询法务",        "他想报复前公司，准备找人暗中攻击",        "我的电话号码是13812345678",    ]    for text in samples:        ok, reason, score = reviewer.review(text)        tag = "PASS" if ok else "BLOCK"        print("{} | {} | 分数={:.2f} | 原因={}".format(tag, text[:18], score, reason))    # 误报处理演示：人工确认后加入白名单    reviewer.add_whitelist("我的电话号码是13812345678")    ok, reason, score = reviewer.review("我的电话号码是13812345678")    print("白名单复核后: PASS" if ok else "白名单复核后: BLOCK")    print("总审核 {} 条，拦截 {} 条".format(reviewer.review_count, reviewer.block_count))  if __name__ == "__main__":    main()