# 更新软件源并安装基础工具(含 tree,验证目录用)
 apt update && apt install -y curl wget vim tree net-tools

 # 安装 docker 依赖与 GPG 密钥(阿里云源)
 apt install -y apt-transport-https ca-certificates curl gnupg lsb-release
 curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
 ​
 # 添加 docker 软件源(阿里云镜像)
 echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://mirrors.aliyun.com/docker-ce/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
 ​
 # 安装 docker 与 compose 插件
 apt update && apt install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
 ​
 # 开机自启
 systemctl enable docker && systemctl start docker
 ​
 # 验证安装
 docker --version && docker compose version

 # 备份原配置
 cp /etc/docker/daemon.json /etc/docker/daemon.json.bak 2>/dev/null
 ​
 # 写入加速配置(JSON 不能带注释;dns 用国内,8.8.8.8 容器解析会失败)
 cat > /etc/docker/daemon.json << 'EOF'
 {
   "registry-mirrors": [
     "https://ry6uhvsydk0j90kjv4.xuanyuan.dev",
     "https://hub.rat.dev",
     "https://docker.1panel.live",
     "https://dockerpull.org",
     "https://docker.m.daocloud.io"
   ],
   "dns": ["223.5.5.5", "119.29.29.29"],
   "log-driver": "json-file",
   "log-opts": { "max-size": "10m", "max-file": "3" }
 }
 EOF
 ​
 # 加载新配置并重启 docker
 systemctl daemon-reload && systemctl restart docker

 # 创建配置目录(数据目录不用建,compose 命名卷自动管理权限)
 mkdir -p /root/prom/yml /root/prom/rules
 cd /root/prom

 cat > /root/prom/docker-compose.yml << 'EOF'
 services:
   node-exporter:                 # 节点监控(CPU/内存/磁盘),告警主要靠它
     image: prom/node-exporter:v1.8.2
     container_name: node-exporter
     restart: always
     ports: ["9100:9100"]
     # 关键:挂载宿主 /,否则只看到容器自身,磁盘/文件系统指标全失效
     volumes:
       - /proc:/host/proc:ro
       - /sys:/host/sys:ro
       - /:/rootfs:ro
     command:
       - '--path.rootfs=/host'
       - '--path.procfs=/host/proc'
       - '--path.sysfs=/host/sys'
 ​
   cadvisor:                      # 容器监控(每容器的 CPU/内存/网络)
     image: ry6uhvsydk0j90kjv4-gcr.xuanyuan.dev/cadvisor/cadvisor:v0.47.0   # 轩辕 gcr 专属(官方镜像)
     container_name: cadvisor
     restart: always
     ports: ["8080:8080"]
     privileged: true
     devices: [/dev/kmsg]
     volumes:
       - /:/rootfs:ro
       - /var/run:/var/run:ro
       - /sys:/sys:ro
       - /var/lib/docker/:/var/lib/docker:ro
       - /dev/disk/:/dev/disk:ro
 ​
   prometheus:                    # 抓取指标+算告警规则
     image: prom/prometheus:v2.54.1
     container_name: prometheus
     restart: always
     ports: ["9090:9090"]
     volumes:
       - "./yml/prometheus.yml:/etc/prometheus/prometheus.yml"
       - "prometheus_data:/prometheus"          # 数据卷,持久化
       - "./rules:/etc/prometheus/rules"
 ​
   grafana:                       # 可视化
     image: grafana/grafana:11.1.0
     container_name: grafana
     restart: always
     ports: ["3000:3000"]
     environment:
       - GF_SECURITY_ADMIN_PASSWORD=Admin@2026  # 默认密码,首登后改
     volumes:
       - "grafana_data:/var/lib/grafana"
 ​
   alertmanager:                  # 告警收敛/分发
     image: prom/alertmanager:v0.27.0
     container_name: alertmanager
     restart: always
     ports: ["9093:9093"]
     volumes:
       - "./yml/alertmanager.yml:/etc/alertmanager/alertmanager.yml"
 ​
   webhook:                       # 把告警转成钉钉消息
     image: timonwong/prometheus-webhook-dingtalk:v2.0.0
     container_name: webhook
     restart: always
     ports: ["8060:8060"]
     volumes:
       - "./yml/config.yml:/etc/prometheus-webhook-dingtalk/config.yml"
 ​
 volumes:
   prometheus_data:
   grafana_data:
 EOF

 cat > /root/prom/yml/prometheus.yml << 'EOF'
 global:
   scrape_interval: 15s        # 抓取频率;测试用 15s 秒级反馈,上线可调 1m
   evaluation_interval: 15s    # 规则评估频率
 ​
 alerting:
   alertmanagers:
     - static_configs:
         - targets: ["alertmanager:9093"]   # 用 compose 服务名,勿写宿主 IP
 ​
 rule_files:
   - "/etc/prometheus/rules/*.yml"
 ​
 scrape_configs:
   - job_name: "prometheus"
     static_configs:
       - targets: ["localhost:9090"]
   - job_name: "node"                        # 节点指标(node-exporter)
     static_configs:
       - targets: ["node-exporter:9100"]
   - job_name: "cadvisor"                    # 容器指标(cadvisor)
     static_configs:
       - targets: ["cadvisor:8080"]
 EOF

 cat > /root/prom/yml/alertmanager.yml << 'EOF'
 global:
   resolve_timeout: 5m
 ​
 route:
   receiver: webhook
   group_wait: 10s            # 首个告警等 10s 再发(测试秒级看到;生产 30s)
   group_interval: 5m
   repeat_interval: 2h        # 未恢复时重发间隔,生产 2-4h 防轰炸;测试可 1m
   group_by: [alertname]
 ​
 receivers:
   - name: webhook
     webhook_configs:
       - url: http://webhook:8060/dingtalk/webhook1/send   # compose 服务名
         send_resolved: true                                # 恢复后也发通知
 EOF

 cat > /root/prom/yml/config.yml << 'EOF'
 targets:
   webhook1:                                       # 与 alertmanager url 中的 webhook1 对应
     url: https://oapi.dingtalk.com/robot/send?access_token=请替换为你的access_token
     secret: 请替换为你的加签密钥
 EOF

cat > /root/prom/rules/rules.yml << 'EOF'
groups:
  - name: 系统告警
    rules:
      # CPU: (1-idle比例)*100 = 使用率
      - alert: CPU使用率告警
        expr: (1 - avg by (instance)(rate(node_cpu_seconds_total{mode="idle"}[2m]))) * 100 > 50
        for: 1m            # 持续 1m 才触发,防抖动
        labels: { severity: warning }
        annotations:
          summary: "告警 CPU使用率正在飙升"
          description: "CPU使用率超过50%(当前 {{ $value }}%)"

      # 磁盘:只盯根分区 /,避免 /boot 等小分区误报
      - alert: 磁盘使用率告警
        expr: (1 - node_filesystem_free_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 > 80
        for: 1m
        labels: { severity: warning }
        annotations:
          summary: "告警 硬盘分区使用率过高"
          description: "根分区使用率超过80%(当前 {{ $value }}%)"

      - alert: 内存使用率告警
        expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 50
        for: 1m
        labels: { severity: warning }
        annotations:
          summary: "告警 服务器可用内存不足"
          description: "内存使用率超过50%(当前 {{ $value }}%)"

      # up==0 = 抓取目标宕机
      - alert: 实例存活告警
        expr: up == 0
        for: 1m
        labels: { severity: warning }
        annotations:
          summary: "告警 主机宕机"
          description: "实例 {{ $labels.instance }} 已宕机"
EOF

# 进入项目目录
cd /root/prom

# 拉镜像并启动所有服务(轩辕通道,约 1 分钟内完成)
docker compose up -d

# 查看状态:6 个容器全 Up 即正常
docker compose ps

# 重启 webhook/alertmanager,使新 token 生效
cd /root/prom && docker compose restart webhook alertmanager

# 验证 webhook 日志(告警触发后应看到 resp_status=200)
docker logs webhook --tail=20

# 1) 停 node-exporter → 触发"实例存活告警"
docker stop node-exporter

# 2) 看状态:http://192.168.7.7:9090/alerts   INACTIVE→PENDING→FIRING(约1-2分钟)
# 3) 钉钉群应收到告警(15s 评估+group_wait 10s,很快)

# 4) 恢复,验证"恢复通知"
docker start node-exporter

# 调试(收不到时看这两个日志)
docker logs alertmanager --tail=30
docker logs webhook --tail=30

# 进入项目目录
cd /root/prom

# 启动/更新所有服务
docker compose up -d

# 停止所有服务(数据卷保留)
docker compose down

# 重启单个服务
docker compose restart prometheus

# 实时查看日志
docker compose logs -f

# 查看服务状态
docker compose ps

┌────────────────────────────────────────────────────────────┐
│                    虚拟机 192.168.7.7                       │
│  ┌───────────┐ ┌──────────┐ ┌───────────┐ ┌───────────┐   │
│  │node-exporter││ cAdvisor │ │  Grafana  │ │  webhook  │   │
│  │   9100     │ │   8080   │ │   3000    │ │   8060    │   │
│  └─────┬──────┘ └────┬─────┘ └─────┬─────┘ └─────┬─────┘   │
│        └─────────────┼─────────────┘             │         │
│                  ┌───▼───┐                       │         │
│                  │Prometheus│ (服务名互通)        │         │
│                  │  9090   │                     │         │
│                  └───┬───┘                       │         │
│                  ┌───▼────┐                      │         │
│                  │Alertmanager│ 9093 ────────────┘         │
│                  └─────────┘                               │
└─────────────────────────────┬──────────────────────────────┘
                              │ HTTPS
                       ┌──────▼──────┐
                       │  钉钉机器人  │ (加签)
                       └─────────────┘

cat > /root/prom/dingtalk_webhook.py << 'EOF'
#!/usr/bin/env python3
# 自研钉钉告警 Webhook:Alertmanager -> 自定义消息 -> 钉钉
import os, json, time, hmac, hashlib, base64, urllib.parse, urllib.request
from http.server import BaseHTTPRequestHandler, HTTPServer

TOKEN  = os.environ.get("DING_TOKEN", "")
SECRET = os.environ.get("DING_SECRET", "")
DING_URL = "https://oapi.dingtalk.com/robot/send"

def send_ding(title, text):
    ts = str(round(time.time()*1000))              # 钉钉加签(timestamp+secret)
    s = urllib.parse.quote_plus(base64.b64encode(
        hmac.new(SECRET.encode(), f"{ts}\n{SECRET}".encode(), hashlib.sha256).digest()))
    url = f"{DING_URL}?access_token={TOKEN}&timestamp={ts}&sign={s}"
    body = {"msgtype": "markdown", "markdown": {"title": title, "text": text}}
    req = urllib.request.Request(url, data=json.dumps(body).encode(),
                                 headers={"Content-Type": "application/json"})
    urllib.request.urlopen(req, timeout=10)

def build(msg):
    title, parts = "监控通知", []
    for a in msg.get("alerts", []):
        st = a.get("status", "firing")
        L, A = a.get("labels", {}), a.get("annotations", {})
        name, inst, job, sev = L.get("alertname","?"), L.get("instance",""), L.get("job",""), L.get("severity","")
        if st == "firing":                         # 🔴 告警中
            title = f"🚨【告警中】{name}"
            parts.append(f"🚨 **{name}**\n- 级别:{sev} | 主机:{inst} ({job})\n- 详情:{A.get('description','')}")
        else:                                      # 🟢 已恢复
            title = f"✅【已恢复】{name}"
            parts.append(f"✅ **{name} 已恢复**\n- 主机:{inst} ({job}) 已恢复正常")
    body = "\n\n".join(parts) + f"\n\n📋 告警面板:[点此查看]({msg.get('externalURL','')})"
    return title, body

class H(BaseHTTPRequestHandler):
    def do_POST(self):
        try:
            msg = json.loads(self.rfile.read(int(self.headers.get("Content-Length",0))) or b"{}")
            t, b = build(msg); send_ding(t, b)
            self.send_response(200); self.end_headers(); self.wfile.write(b"ok")
        except Exception as e:
            self.send_response(500); self.end_headers(); self.wfile.write(str(e).encode())
    def log_message(self, *a): pass

HTTPServer(("0.0.0.0", 8060), H).serve_forever()
EOF

# 在 /root/prom 下执行:自动把 webhook 服务换成自研容器,并从 yml/config.yml 读 token/secret
cd /root/prom && python3 << 'PY'
import re
c  = open('docker-compose.yml').read()
cfg = open('yml/config.yml').read()
tok = re.search(r'access_token=([A-Za-z0-9]+)', cfg).group(1)
sec = re.search(r'secret:\s*(\S+)', cfg).group(1)
new = f'''  webhook:                        # 告警 -> 自定义消息 -> 钉钉
    image: python:3.11-slim
    container_name: webhook
    restart: always
    ports: ["8060:8060"]
    environment:
      - DING_TOKEN={tok}
      - DING_SECRET={sec}
      - TZ=Asia/Shanghai
    volumes:
      - "./dingtalk_webhook.py:/app/dingtalk_webhook.py:ro"
    command: ["python", "/app/dingtalk_webhook.py"]
'''
c2 = re.sub(r'  webhook:.*?(?=^volumes:)', new, c, flags=re.S|re.M)
assert c2 != c, '替换未生效,请检查 compose 里 webhook 段'
open('docker-compose.yml','w').write(c2)
print('OK: webhook 段已替换')
PY

# 把 webhook 接收地址改成自研服务(任何 POST 都收,路径随意)
sed -i 's#http://webhook:8060/dingtalk/webhook1/send#http://webhook:8060/#' /root/prom/yml/alertmanager.yml
grep 'url:' /root/prom/yml/alertmanager.yml    # 应显示 http://webhook:8060/

# 1) 重建 webhook(拉取 python:3.11-slim 镜像)+ alertmanager
cd /root/prom && docker compose up -d webhook alertmanager

# 2) 手动发测试,验证自定义格式(返回 ok = 钉钉已收到)
# 已恢复:
curl -s -X POST http://127.0.0.1:8060/ -H 'Content-Type: application/json' -d '{"status":"resolved","externalURL":"http://192.168.7.7:9093","alerts":[{"status":"resolved","labels":{"alertname":"实例存活告警","severity":"warning","instance":"node-exporter:9100","job":"node"},"annotations":{"description":"实例 node-exporter:9100 已宕机"}}]}'
# 告警中:
curl -s -X POST http://127.0.0.1:8060/ -H 'Content-Type: application/json' -d '{"status":"firing","externalURL":"http://192.168.7.7:9093","alerts":[{"status":"firing","labels":{"alertname":"CPU使用率告警","severity":"warning","instance":"node-exporter:9100","job":"node"},"annotations":{"description":"CPU使用率超过50%"}}]}'

# ① 一键回退 compose 的 webhook 段为 timonwong v2(A 版,恢复 config.yml 挂载)
cd /root/prom && python3 << 'PY'
import re
c = open('docker-compose.yml').read()
new = '''  webhook:                       # 把告警转成钉钉消息(A 方案)
    image: timonwong/prometheus-webhook-dingtalk:v2.0.0
    container_name: webhook
    restart: always
    ports: ["8060:8060"]
    environment:
      - TZ=Asia/Shanghai
    volumes:
      - "./yml/config.yml:/etc/prometheus-webhook-dingtalk/config.yml"
'''
c2 = re.sub(r'  webhook:.*?(?=^volumes:)', new, c, flags=re.S|re.M)
assert c2 != c, '替换未生效,请检查 compose 里 webhook 段'
open('docker-compose.yml','w').write(c2)
print('OK: 已回退 webhook 段为 A 版')
PY

# ② 还原 alertmanager 接收地址(v2 专用路径)
sed -i 's#http://webhook:8060/#http://webhook:8060/dingtalk/webhook1/send#' /root/prom/yml/alertmanager.yml

# ③ 重建生效
cd /root/prom && docker compose up -d webhook alertmanager

curl -s -X POST http://127.0.0.1:8060/dingtalk/webhook1/send -H 'Content-Type: application/json' -d '{"version":"4","status":"firing","receiver":"webhook","groupLabels":{"alertname":"测试"},"externalURL":"http://192.168.7.7:9093","alerts":[{"status":"firing","labels":{"alertname":"测试","severity":"warning","instance":"node-exporter:9100"},"annotations":{"summary":"告警 回退测试","description":"告警 已回到A方案"}}]}'
# 预期:返回 OK;钉钉消息头部为 [FIRING:1],正文是默认 Description/Graph/Details 格式