多模态工具调用：API、搜索与代码的系统化实战

# -*- coding: utf-8 -*-"""多模态工具调用Agent简易演示：管家感知多模态请求，按需调用API/搜索/代码工具"""import json  TOOL_SCHEMAS = [    {"name": "weather_api", "params": {"city": "string"}, "desc": "查询城市天气"},    {"name": "hotel_api", "params": {"city": "string"}, "desc": "预订酒店房间"},    {"name": "translate_api", "params": {"text": "string"}, "desc": "翻译文本"},    {"name": "search_docs", "params": {"query": "string"}, "desc": "检索知识库"},    {"name": "run_python", "params": {"code": "string"}, "desc": "沙箱执行代码"},]  class ToolBox:    """三层工具箱：API / 搜索 / 代码"""     def __init__(self):        self.api_map = {            "weather_api": lambda p: {"weather": "晴", "temp": 24, "city": p["city"]},            "hotel_api": lambda p: {"order_id": "BK-2024-001", "status": "confirmed"},            "translate_api": lambda p: {"translated": "hello " + p["text"]},        }        self.search_index = ["杭州今日天气", "西湖附近酒店", "英文翻译服务"]        self.code_runner = lambda c: eval(c, {"__builtins__": {}}, {})     def api_call(self, name, params):        """模拟调用外部API，返回JSON结果"""        if name not in self.api_map:            return {"error": "unknown api"}        return self.api_map[name](params)     def search_docs(self, query):        """模拟检索：把关键词命中的文档作为结果返回"""        hits = [doc for doc in self.search_index if query in doc]        return {"hits": hits if hits else []}     def run_python(self, code):        """模拟沙箱执行：捕获异常并返回结果或报错"""        try:            return {"result": self.code_runner(code)}        except Exception as exc:            return {"error": str(exc)}  class MultimodalAgent:    """管家Agent：感知 -> 规划 -> 执行 -> 观察 的决策循环"""     def __init__(self, tools):        self.tools = tools        self.memory = []      # 短期记忆：本轮对话与行动历史        self.gc_count = 0     # 计数器，不与任何方法同名     def perceive(self, text_note, image_note, audio_note):        """多模态感知：把图像与音频的文本描述汇总成一句话"""        merged = " ".join(x for x in [text_note, image_note, audio_note] if x)        self.memory.append({"role": "perceive", "content": merged})        return merged     def plan(self, query):        """规则式规划：根据关键词从五件工具中选一件"""        self.gc_count += 1        if "天气" in query:            return "weather_api"        if "酒店" in query:            return "hotel_api"        if "翻译" in query:            return "translate_api"        if query.startswith("计算"):            return "run_python"        return "search_docs"     def execute(self, action, query):        """执行：调用工具并把结果写回记忆"""        if action == "weather_api":            result = self.tools.api_call("weather_api", {"city": query[2:4]})        elif action == "hotel_api":            result = self.tools.api_call("hotel_api", {"city": "杭州"})        elif action == "translate_api":            result = self.tools.api_call("translate_api", {"text": query[3:]})        elif action == "run_python":            result = self.tools.run_python(query[3:])        else:            result = self.tools.search_docs(query)        self.memory.append({"role": "action", "tool": action, "result": result})        return result     def run(self, query, text_note="", image_note="", audio_note=""):        """完整Agent循环：感知 -> 规划 -> 执行 -> 返回"""        self.perceive(text_note, image_note, audio_note)        action = self.plan(query)        result = self.execute(action, query)        return {"query": query, "tool": action, "result": result}  def main():    """演示三个场景：天气API、代码计算、知识库检索"""    tools = ToolBox()    agent = MultimodalAgent(tools)    cases = [        {"query": "查询杭州天气", "image_note": "[图片] 阴雨中的城市街景"},        {"query": "计算 3*8+2", "text_note": "[文本] 用户需要精确结果"},        {"query": "找一下西湖附近酒店", "audio_note": "[语音转文字] 用户口述"},    ]    for case in cases:        print(json.dumps(agent.run(**case), ensure_ascii=False, indent=2))    print("Agent记忆轮次:", len(agent.memory))  if __name__ == "__main__":    main()