Grafana可视化：监控告警面板

# 使用 Grafana HTTP API 创建 Dashboard 并定义变量模板import jsonimport requests GRAFANA_URL = "http://localhost:3000"API_KEY = "your-api-key" headers = {    "Authorization": f"Bearer {API_KEY}",    "Content-Type": "application/json"} # 定义变量template_variables = [    {        "name": "instance",        "type": "query",        "datasource": "Prometheus",        "query": "label_values(up, instance)",        "refresh": 1,        "multi": True,        "includeAll": True    },    {        "name": "model_version",        "type": "custom",        "current": {"text": "v2.1", "value": "v2.1"},        "options": [            {"text": "v1.0", "value": "v1.0", "selected": False},            {"text": "v2.0", "value": "v2.0", "selected": False},            {"text": "v2.1", "value": "v2.1", "selected": True}        ]    }] # 定义面板dashboard_json = {    "dashboard": {        "title": "AI Inference Service Monitor",        "templating": {"list": template_variables},        "panels": [            {                "id": 1,                "title": "GPU Utilization - $instance",                "type": "timeseries",                "datasource": "Prometheus",                "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},                "targets": [                    {                        "expr": "nvidia_gpu_utilization{instance=~\"$instance\"}",                        "legendFormat": "{{gpu}} - {{instance}}"                    }                ]            },            {                "id": 2,                "title": "Token Throughput",                "type": "timeseries",                "datasource": "Prometheus",                "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},                "targets": [                    {                        "expr": "rate(inference_tokens_total{model_version=\"$model_version\"}[5m])",                        "legendFormat": "{{model_version}}"                    }                ]            }        ]    },    "overwrite": True} # 发送创建请求resp = requests.post(    f"{GRAFANA_URL}/api/dashboards/db",    headers=headers,    json=dashboard_json)print(f"Status: {resp.status_code}")print(f"Response: {resp.json()}")

# 使用 Grafana HTTP API 创建告警规则import jsonimport requests GRAFANA_URL = "http://localhost:3000"API_KEY = "your-api-key" headers = {    "Authorization": f"Bearer {API_KEY}",    "Content-Type": "application/json"} # 定义告警规则alert_rule = {    "name": "GPU Temperature Critical",    "ruleGroup": "ai_inference_alerts",    "evalOffset": "0s",    "condition": "C",    "data": [        {            "refId": "A",            "relativeTimeRange": {"from": 300, "to": 0},            "datasourceUid": "prometheus-ds-uid",            "model": {                "expr": "nvidia_gpu_temperature_celsius",                "intervalMs": 1000,                "maxDataPoints": 43200,                "refId": "A"            }        },        {            "refId": "B",            "relativeTimeRange": {"from": 0, "to": 0},            "datasourceUid": "__expr__",            "model": {                "conditions": [{"evaluator": {"params": [85], "type": "gt"}}],                "datasource": {"type": "__expr__", "uid": "__expr__"},                "expression": "A",                "reducer": "last",                "refId": "B",                "type": "threshold"            }        },        {            "refId": "C",            "relativeTimeRange": {"from": 0, "to": 0},            "datasourceUid": "__expr__",            "model": {                "conditions": [],                "datasource": {"type": "__expr__", "uid": "__expr__"},                "expression": "B",                "refId": "C",                "reducer": "last",                "type": "threshold"            }        }    ],    "noDataState": "OK",    "execErrState": "Error",    "for": "5m",    "annotations": {        "summary": "GPU {{ $labels.gpu }} temperature is {{ $values.A }} C",        "description": "GPU temperature exceeded 85C threshold for 5 minutes"    },    "labels": {"severity": "critical", "team": "ai-infra"}} # 发送创建请求resp = requests.post(    f"{GRAFANA_URL}/api/v1/provisioning/alert-rules",    headers=headers,    json=alert_rule)print(f"Status: {resp.status_code}")print(f"Response: {json.dumps(resp.json(), indent=2)}")

# 使用 Grafana API 创建 Loki 数据源并配置指标日志关联import jsonimport requests GRAFANA_URL = "http://localhost:3000"API_KEY = "your-api-key" headers = {    "Authorization": f"Bearer {API_KEY}",    "Content-Type": "application/json"} # 1. 创建 Loki 数据源loki_datasource = {    "name": "Loki",    "type": "loki",    "access": "proxy",    "url": "http://loki:3100",    "jsonData": {        "maxLines": 1000,        "derivedFields": [            {                "name": "TraceID",                "matcherRegex": "trace_id=(\\w+)",                "datasourceUid": "tempo-ds-uid",                "url": ""            }        ]    }} resp = requests.post(    f"{GRAFANA_URL}/api/datasources",    headers=headers,    json=loki_datasource)print(f"Loki datasource: {resp.status_code}") # 2. 配置 Prometheus 数据源的指标日志关联# 在 Prometheus 数据源的 jsonData 中配置 derivedFieldsprom_update = {    "id": 1,    "jsonData": {        "httpMethod": "POST",        "manageAlerts": True,        "prometheusType": "Prometheus",        "exemplarTraceIdDestinations": [            {                "name": "traceID",                "datasourceUid": "tempo-ds-uid"            }        ]    }} resp = requests.put(    f"{GRAFANA_URL}/api/datasources/1",    headers=headers,    json=prom_update)print(f"Prometheus update: {resp.status_code}") # 3. 定义 LogQL 查询示例logql_examples = {    "error_rate": 'rate({app="inference-service"} |= "error" [5m])',    "latency_log": '{app="inference-service"} | json | latency > 5000',    "model_load_log": '{app="inference-service"} |= "model" |= "loaded"',    "gpu_oom": '{app="inference-service"} |= "CUDA out of memory"'} print("\nLogQL Query Examples:")for name, query in logql_examples.items():    print(f"  {name}: {query}")

# Dashboard 版本管理与 Provisioning 配置示例import jsonimport os # 1. 生成 Provisioning 配置文件provisioning_config = {    "apiVersion": 1,    "providers": [        {            "name": "AI Dashboards",            "orgId": 1,            "folder": "AI Monitoring",            "folderUid": "ai-monitoring",            "type": "file",            "disableDeletion": True,            "updateIntervalSeconds": 30,            "allowUiUpdates": False,            "options": {                "path": "/var/lib/grafana/dashboards/ai",                "foldersFromFilesStructure": True            }        }    ]} # 保存 provisioning 配置print("=== Provisioning Configuration ===")print(json.dumps(provisioning_config, indent=2)) # 2. Dashboard JSON 模板（参数化）dashboard_template = {    "uid": "ai-inference-overview",    "title": "AI Inference Service - $environment",    "tags": ["ai", "inference", "monitoring"],    "timezone": "browser",    "refresh": "30s",    "time": {"from": "now-6h", "to": "now"},    "templating": {        "list": [            {                "name": "environment",                "type": "custom",                "current": {"text": "production", "value": "production"},                "options": [                    {"text": "staging", "value": "staging", "selected": False},                    {"text": "production", "value": "production", "selected": True}                ]            }        ]    },    "panels": [        {            "id": 1,            "title": "Request Rate",            "type": "timeseries",            "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},            "targets": [                {                    "expr": "sum(rate(http_requests_total{environment=\"$environment\"}[5m]))",                    "legendFormat": "requests/sec"                }            ]        }    ],    "schemaVersion": 38,    "version": 1} # 3. 模拟版本管理流程def save_dashboard_version(dashboard, version, output_dir="/tmp/dashboards"):    """保存 Dashboard 版本到文件"""    os.makedirs(output_dir, exist_ok=True)    filename = f"{dashboard['uid']}_v{version}.json"    filepath = os.path.join(output_dir, filename)    with open(filepath, "w") as f:        json.dump(dashboard, f, indent=2)    print(f"Saved: {filepath}")    return filepath # 保存初始版本path = save_dashboard_version(dashboard_template, version=1)print(f"\nDashboard saved to: {path}")print(f"UID: {dashboard_template['uid']}")print(f"Version: {dashboard_template['version']}")

# Grafana Dashboard 面板配置生成器# 使用 Python 动态生成 AI 推理服务监控 Dashboard JSONimport json def create_panel(panel_id, title, panel_type, expr, grid_pos, legend=None, unit=None):    """创建一个 Grafana 面板配置"""    panel = {        "id": panel_id,        "title": title,        "type": panel_type,        "datasource": {"type": "prometheus", "uid": "prometheus"},        "gridPos": grid_pos,        "targets": [{"expr": expr, "refId": "A"}]    }    if legend:        panel["targets"][0]["legendFormat"] = legend    if unit:        panel["fieldConfig"] = {            "defaults": {"unit": unit}        }    return panel # 定义面板列表panels = []row_y = 0 # Row 1: GPU 监控panels.append({"id": 100, "title": "GPU Monitoring", "type": "row", "gridPos": {"h": 1, "w": 24, "x": 0, "y": row_y}, "collapsed": False})row_y += 1panels.append(create_panel(1, "GPU Utilization", "timeseries", "avg(nvidia_gpu_utilization) by (gpu)", {"h": 8, "w": 12, "x": 0, "y": row_y}, legend="GPU {{gpu}}", unit="percent"))panels.append(create_panel(2, "GPU Memory", "timeseries", "nvidia_gpu_memory_used_bytes / nvidia_gpu_memory_total_bytes * 100", {"h": 8, "w": 12, "x": 12, "y": row_y}, legend="GPU {{gpu}}", unit="percent"))row_y += 8 # Row 2: 推理性能panels.append({"id": 101, "title": "Inference Performance", "type": "row", "gridPos": {"h": 1, "w": 24, "x": 0, "y": row_y}, "collapsed": False})row_y += 1panels.append(create_panel(3, "Token Throughput", "timeseries", "rate(inference_tokens_total[5m])", {"h": 8, "w": 8, "x": 0, "y": row_y}, legend="{{model_version}}", unit="short"))panels.append(create_panel(4, "P99 Latency", "timeseries", "histogram_quantile(0.99, rate(inference_latency_bucket[5m]))", {"h": 8, "w": 8, "x": 8, "y": row_y}, legend="P99", unit="ms"))panels.append(create_panel(5, "Error Rate", "stat", "sum(rate(inference_errors_total[5m])) / sum(rate(inference_requests_total[5m])) * 100", {"h": 8, "w": 8, "x": 16, "y": row_y}, unit="percent")) # 组装 Dashboard JSONdashboard = {    "title": "AI Inference Service Monitor",    "uid": "ai-inference-monitor",    "tags": ["ai", "inference", "gpu"],    "timezone": "browser",    "refresh": "30s",    "time": {"from": "now-6h", "to": "now"},    "panels": panels,    "schemaVersion": 38,    "version": 1} # 输出 JSONresult = json.dumps(dashboard, indent=2, ensure_ascii=False)print(f"Dashboard: {dashboard['title']}")print(f"Total panels: {len([p for p in panels if p['type'] != 'row'])}")print(f"Total rows: {len([p for p in panels if p['type'] == 'row'])}")print(f"JSON size: {len(result)} bytes")