prometheus

架构图

prometheus-server

setup.sh

config/prometheus.yml

#!/bin/bash
echo "Asia/Shanghai" > /etc/timezone

mkdir -p $(pwd)/data
chown -R 65534:65534 $(pwd)/data

port=9090

docker stop prometheus
docker rm prometheus
docker run -d --net host \
--name prometheus \
-v /etc/timezone:/etc/timezone:ro \
-v /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime:ro \
-v $(pwd)/config/:/etc/prometheus/ \
-v $(pwd)/data/:/prometheus/ \
-v $(pwd)/groups/:/usr/local/prometheus/groups/ \
-v $(pwd)/rules/:/usr/local/prometheus/rules/ \
prom/prometheus:v2.46.0 \
--config.file=/etc/prometheus/prometheus.yml \
--web.console.libraries=/etc/prometheus/console_libraries \
--web.console.templates=/etc/prometheus/consoles \
--storage.tsdb.path=/prometheus \
--storage.tsdb.retention.time=60d \
--web.enable-admin-api

global:
  scrape_interval:     60s
  evaluation_interval: 60s

rule_files:
  - "/usr/local/prometheus/rules/*.rules"

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ["127.0.0.1:9090"]

Tip

如果在调试的时候需要查看请求的日志, 在启动脚本中新增--log.level=debug配置

exporter

集合不同节点数据

主配置

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ["127.0.0.1:9090"]

  - job_name: 'federate'
    scrape_interval: 5s
    scrape_timeout: 3s
    metrics_path: '/federate'
    honor_labels: true
    params:
      'match[]':
        - '{job!=""}'
    file_sd_configs:
      - files: ['/usr/local/prometheus/groups/federate/*.json','/usr/local/prometheus/groups/federate/*.yml']

组配置

- targets: [ "127.0.0.1:9090"]
  labels:
    job_name: test
    instance: test
    comment: "test"

pushgetway

启动

#!/bin/bash

docker kill pushgateway
docker rm pushgateway
docker run -d --net host \
--name pushgateway \
--restart=always \
-v /etc/localtime:/etc/localtime:ro \
-v /etc/tiemzone:/etc/timezone:ro \
prom/pushgateway:v1.7.0 \
--web.listen-address=0.0.0.0:9091 \
--web.enable-admin-api \
--web.telemetry-path=/metrics

prometheus

在prometheus.yaml中增加：

- job_name: 'aliyun-server-info'
  scrape_interval: 60s
  scrape_timeout: 35s
  static_configs:
    - targets: ['192.168.1.1:9091']
      labels:
        instance: aliyun-server-info

推送数据

一般uri格式为/metrics/job/<job_name>/instance/<instance_name>

cat <<EOF | curl --data-binary @- http://192.168.1.1:9091/metrics/job/balanceinfo/instance/${account}
${balance}
${balancetime}
EOF

grafana

数据源

添加Prometheus，修改HTTP中的url为http://10.0.18.2:9090

仪表盘

在https://grafana.com/grafana/dashboards/中寻找模板

推荐模板：

在mysql的仪表盘中可以添加一个监控项

mysql_version_info{instance="$host"}

options中选择instant，legend填写{{version}}

右侧选择stat,Text mode选择Name

推送

prometheus-webhook-dingtalk

config/config.yml

templates/alertmanager-dingtalk.tmpl

setup.sh

# Request timeout
timeout: 5s

# Customizable templates path
templates:
  - templates/alertmanager-dingtalk.tmpl

targets:
  webhook:
    # 运维内部群机器人
    url: https://oapi.dingtalk.com/robot/send?access_token=1adaa314f6d04b7
    # secret for signature
    secret: SEC9e23
    message:
      text: '{{ template "dingtalk.to.message" . }}'

{{ define "dingtalk.to.message" }}

{{- if gt (len .Alerts.Firing) 0 -}}
{{- range $index, $alert := .Alerts -}}

=========  **监控告警** =========

**告警程序:**     Alertmanager
**告警类型:**    {{ $alert.Labels.alertname }}
**告警级别:**    {{ $alert.Labels.severity }}
**告警状态:**    {{ .Status }}
**故障主机:**    {{ $alert.Labels.instance }} {{ $alert.Labels.device }}
**告警主题:**    {{ .Annotations.summary }}
**告警详情:**    {{ $alert.Annotations.message }}{{ $alert.Annotations.description}}
**告警图形:**    [📈]({{ .GeneratorURL }})
**主机标签:**    {{ range .Labels.SortedPairs  }}  </br> [{{ .Name }}: {{ .Value | markdown | html }} ]
{{- end }} </br>

**故障时间:**    {{ $alert.StartsAt.Local.Format "2006-01-02 15:04:05" }}

========= = end =  =========
{{- end }}
{{- end }}

{{- if gt (len .Alerts.Resolved) 0 -}}
{{- range $index, $alert := .Alerts -}}

========= 告警恢复 =========
**告警程序:**     Alertmanager
**告警主题:**    {{ $alert.Annotations.summary }}
**告警主机:**    {{ .Labels.instance }}
**告警类型:**    {{ .Labels.alertname }}
**告警级别:**    {{ $alert.Labels.severity }}
**告警状态:**    {{   .Status }}
**告警详情:**    {{ $alert.Annotations.message }}{{ $alert.Annotations.description}}
**告警图形:**    [📈]({{ .GeneratorURL }})
**故障时间:**    {{ $alert.StartsAt.Local.Format "2006-01-02 15:04:05" }}
**恢复时间:**    {{ $alert.EndsAt.Local.Format "2006-01-02 15:04:05" }}

========= = **end** =  =========
{{- end }}
{{- end }}
{{- end }}

#!/bin/bash

version="v2.1.0"

docker stop prometheus-webhook-dingtalk
docker rm prometheus-webhook-dingtalk
docker run -d \
--net host \
--name prometheus-webhook-dingtalk \
-v /etc/timezone:/etc/timezone:ro \
-v /etc/localtime:/etc/localtime:ro \
-v $(pwd)/config/:/etc/prometheus-webhook-dingtalk/ \
-v $(pwd)/templates/:/prometheus-webhook-dingtalk/templates/ \
timonwong/prometheus-webhook-dingtalk:${version} \
--web.listen-address=:8060 \
--web.enable-ui \
--web.enable-lifecycle \
--config.file=/etc/prometheus-webhook-dingtalk/config.yml \
--log.level=info \
--log.format=logfmt

alertmanager

config/alertmanager.yml

templates/alertmanager-email.tmpl

setup.sh

global:
  # 每2分钟检查一次是否恢复
  resolve_timeout: 2m
# route用来设置报警的分发策略

route:
  receiver: 'ops-dingtalk'
  group_by: ['...']
  group_wait: 3s
  group_interval: 1m
  repeat_interval: 5m
  routes:
    - receiver: 'ops-dingtalk'
      group_by: ['...']
      # 等待时间，如果同一个组有新的告警会被合并到同一个消息内
      group_wait: 3s
      # 异常持续报警间隔时间为 group_interval + repeat_interval 总和
      # group_interval 相同的Group之间发送告警通知的时间间隔
      group_interval: 1m
      # 一条成功发送的告警，在最终发送通知之前的等待时间
      repeat_interval: 5m
      matchers:
        - severity=~"^信息$|^警告$|^一般严重$|^严重$|^灾难$|^测试模板$"
receivers:
  - name: 'ops-dingtalk'
    webhook_configs:
      - send_resolved: true
        url: 'http://10.0.18.2:8060/dingtalk/webhook/send'
        max_alerts: 0

#########基于alertmanager官方模板修改，内容可删减#########

{{ define "__alertmanager" }}Alertmanager{{ end }}

{{ define "__subject" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}{{ end }}
{{ define "__description" }}{{ end }}

{{ define "email.to.html" }}
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<!--
Style and HTML derived from https://github.com/mailgun/transactional-email-templates

The MIT License (MIT)

Copyright (c) 2014 Mailgun

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
-->
<html xmlns="http://www.w3.org/1999/xhtml" xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
<head style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
<meta name="viewport" content="width=device-width" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" />
<title style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">{{ template "__subject" . }}</title>

</head>

<body itemscope="" itemtype="http://schema.org/EmailMessage" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; height: 100%; line-height: 1.6em; width: 100% !important; background-color: #f6f6f6; margin: 0; padding: 0;" bgcolor="#f6f6f6">

<table style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;" bgcolor="#f6f6f6">
  <div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 0;">
    <table width="100%" cellpadding="0" cellspacing="0" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px solid #e9e9e9;" bgcolor="#fff">
      <tr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
        <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: top; color: #fff; font-weight: 500; text-align: center; border-radius: 3px 3px 0 0; background-color: #E6522C; margin: 0; padding: 20px;" align="center" bgcolor="#E6522C" valign="top">
          发生 {{ .Alerts | len }}  个 {{ range .GroupLabels.SortedPairs }}
                {{ .Value }}
          {{ end }} 告警 ！！请尽快处理
        </td>
      </tr>
      <tr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
        <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 10px;" valign="top">
          <table border="1" cellpadding="2" cellspacing="0" width="100%" cellpadding="0" cellspacing="0" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
            <tr border="1" cellpadding="2" cellspacing="0" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
            </tr>
          <table border="1" cellpadding="2" cellspacing="0" width="100%" cellpadding="0" cellspacing="0" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
            <tr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
              <td width="50px" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: middle; margin: 0; padding: 3 3 3px;" valign="top" >
                <strong>告警名称</strong>
              </td>
              <td width="50px" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: middle; margin: 0; padding: 3 3 3px;" valign="top" >
                <strong>告警级别</strong>
              <td width="50px" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: middle; margin: 0; padding: 3 3 3px;" valign="top" >
                <strong>实例</strong>
              <td width="50px" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: middle; margin: 0; padding: 3 3 3px;" valign="top" >
                <strong>所属系统</strong>
              <td width="50px" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: middle; margin: 0; padding: 3 3 3px;" valign="top" >
                <strong>厂商</strong>
              <td width="50px" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: middle; margin: 0; padding: 3 3 3px;" valign="top" >
                <strong>触发时间</strong>
              </td>
              <td width="50px" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 16px; vertical-align: middle; margin: 0; padding: 3 3 3px;" valign="top" >
                <strong>说明</strong>
              </td>
            </tr>
            {{ range .Alerts.Firing }}
            <tr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
              <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 3 3 3px;" valign="top">
               <!-- {{ .Labels.alertname }} -->
                {{ .Labels.severity }}
              </td>
              <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 3 3 3px;" valign="top">
                {{ .Status }}
              </td>
              <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 3 3 3px;" valign="top">
                {{ .Labels.instance }}
              </td>
              <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 3 3 3px;" valign="top">
                {{ .Labels.ownningsystem }}
              </td>
              <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 3 3 3px;" valign="top">
                {{ .Labels.company }}
              </td>
              <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 3 3 3px;" valign="top">
                {{  .StartsAt.Format "2006-01-02 15:04:05"  }}
              </td>
              <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 3 3 3px;" valign="top">
                {{ .Annotations.description  }}
              </td>
            </tr>
            {{ end }}
          </table>
        </td>
      </tr>
      </table>
    </table>

    <div style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; clear: both; color: #999; margin: 0; padding: 20px;">
      <table width="100%" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
        <tr style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
          <td style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 12px; vertical-align: top; text-align: center; color: #999; margin: 0; padding: 0 0 20px;" align="center" valign="top"><a href="{{ .ExternalURL }}" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 12px; color: #999; text-decoration: underline; margin: 0;">Sent by {{ template "__alertmanager" . }}</a></td>
        </tr>
      </table>
    </div></div>
</table>

</body>
</html>
{{ end }}

#!/bin/bash
echo "Asia/Shanghai" > /etc/timezone

port=9093

docker stop alertmanager
docker rm alertmanager
docker run --name alertmanager -d \
--net host \
-v /etc/timezone:/etc/timezone:ro \
-v /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime:ro \
-v $(pwd)/config/alertmanager.yml:/etc/alertmanager/alertmanager.yml \
-v $(pwd)/templates/:/etc/alertmanager/templates/ \
prom/alertmanager:v0.26.0

配置增加

alerting:
  alertmanagers:
  - static_configs:
    - targets:
      - '10.0.18.2:9093'

#prometheus

#架构图

#prometheus-server

#exporter

#集合不同节点数据

#主配置

#组配置

#pushgetway

#启动

#prometheus

#推送数据

#grafana

#数据源

#仪表盘

#推送

#prometheus-webhook-dingtalk

#alertmanager

#配置增加

#prometheus

#架构图

#prometheus-server

#exporter

#集合不同节点数据

#主配置

#组配置

#pushgetway

#启动

#prometheus

#推送数据

#grafana

#数据源

#仪表盘

#推送

#prometheus-webhook-dingtalk

#alertmanager

#配置增加

prometheus

架构图

prometheus-server

exporter

集合不同节点数据

主配置

组配置

pushgetway

启动

prometheus

推送数据

grafana

数据源

仪表盘

推送

prometheus-webhook-dingtalk

alertmanager

配置增加

prometheus

架构图

prometheus-server

exporter

集合不同节点数据

主配置

组配置

pushgetway

启动

prometheus

推送数据

grafana

数据源

仪表盘

推送

prometheus-webhook-dingtalk

alertmanager

配置增加