prometheus + grafana指标监控系统安装与配置

整体架构

1. promethueus: 一个开源的服务监控系统和时间序列数据库。附带丰富的数据采集器，支持告警处理。
2. grafana： 一款开源的数据可视化工具，可以非常轻松的将数据转成图表的展现形式来做到数据监控以及数据统计。支持promethueus、influxDB等多种数据源。

prometheus-architecture.png

基础安装

本文采用的主要安装方式为docker-compose

prometheus

version: '3.8'
services:
  prometheus:
    image: prom/prometheus:v2.37.6
    ports:
      - 9090:9090
    volumes:
      - ./config:/etc/prometheus
      - ./prom-data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus/data'
      - '--web.console.libraries=/prometheus/console_libraries'
      - '--web.console.templates=/prometheus/consoles'
    user: '0'

grafana

version: '3.8'
services:
  grafana:
    image: grafana/grafana-oss:9.4.3
    ports:
      - 3000:3000
    volumes:
      - ./gfdata:/var/lib/grafana
      - TZ=Asia/Shanghai
    environment:
      - GF_SERVER_ROOT_URL=http://100.127.9.84:3000/
    user: '0'

image-20230310120537493.png

监控配置

服务器

wget https://github.com/prometheus/node_exporter/releases/download/v1.5.0/node_exporter-1.5.0.linux-amd64.tar.gz
tar -xvf node_exporter-1.5.0.linux-amd64.tar.gz & mv node_exporter-1.5.0.linux-amd64 node_exporter
cd node_exporter
nohup ./node_exporter >/dev/null 2>&1 &

scrape_configs:
  - job_name: "node"
    static_configs:
      - targets: ['192.168.0.184:9100','192.168.0.59:9100','192.168.0.182:9100','192.168.1.10:9100','192.168.3.220:9100']

image-20230310132748412.png

mysql

CREATE USER 'exporter'@'%' IDENTIFIED BY 'xxxxxx' WITH MAX_USER_CONNECTIONS 3;
GRANT PROCESS, REPLICATION CLIENT, SELECT ON *.* TO 'exporter'@'%';

version: '3.8'
services:
  mysqld-exporter:
    image: prom/mysqld-exporter:v0.14.0
    ports:
      - "9104:9104"
    environment:
      - DATA_SOURCE_NAME=exporter:xxxxxx@(host:port)/

scrape_configs:
  - job_name: "mysql"
    static_configs:
      - targets:
        # All mysql hostnames to monitor.
        - 192.168.0.184:2206
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        # The mysqld_exporter host:port
        replacement: 192.168.3.39:9104

image-20230310133226180.png

mongodb

db.getSiblingDB("admin").createUser({
    user: "exporter",
    pwd: "xxxxxx",
    roles: [
        { role: "clusterMonitor", db: "admin" },
        { role: "read", db: "local" }
    ]
})

wget https://github.com/percona/mongodb_exporter/releases/download/v0.11.0/mongodb_exporter-0.11.0.linux-amd64.tar.gz
tar -xvf mongodb_exporter-0.11.0.linux-amd64.tar.gz

FROM alpine:3.4
EXPOSE 9216
RUN apk add --update ca-certificates
COPY mongodb_exporter /bin/mongodb_exporter
RUN chmod 755 /bin/mongodb_exporter
CMD [ "mongodb_exporter" ]

version: '3.8'
services:
  mongodb-exporter:
    build:
      context: ./
      dockerfile: Dockerfile
    image: percona/mongodb_exporter:0.11.0
    ports:
      - 9216:9216
    environment:
      - MONGODB_URI=mongodb://exporter:xxxxxx@host:port/?authSource=admin

scrape_configs:
  - job_name: "mongodb"
    static_configs:
      - targets: ["192.168.3.39:9216"]

image-20230310134222718.png

redis

version: '3.8'
services:
  redis-exporter:
    image: oliver006/redis_exporter:v1.48.0
    ports:
      - "9121:9121"
    environment:
      - REDIS_ADDR=redis://host:port # 连接集群中一个节点
      - REDIS_PASSWORD=xxxxxxx
      - REDIS_EXPORTER_INCL_SYSTEM_METRICS=true

scrape_configs:
  - job_name: 'redis_exporter_targets'
    static_configs:
      - targets:
        - redis://192.168.0.182:8001
        - redis://192.168.0.182:8002
        - redis://192.168.0.59:8001
        - redis://192.168.0.59:8002
        - redis://192.168.0.184:8001
        - redis://192.168.0.184:8002
    metrics_path: /scrape
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 192.168.3.39:9121
## config for scraping the exporter itself
  - job_name: 'redis_exporter'
    static_configs:
      - targets:
        - 192.168.3.39:9121

image-20230310134847117.png

rocketmq

curl -O https://github.com/apache/rocketmq-exporter/archive/refs/heads/master.zip
unzip master.zip
cd rocketmq-exporter-master
mvn package
cd target

FROM openjdk:11
EXPOSE 5557
COPY rocketmq-exporter-0.0.2-SNAPSHOT-exec.jar /usr/local/rocketmq-exporter.jar
CMD [ "java", "-jar", "/usr/local/rocketmq-exporter.jar" ]

version: '3.8'
services:
  rockermq-exporter:
    build:
      context: ./
      dockerfile: Dockerfile
    image: apache/rocketmq_exporter:0.0.2
    ports:
      - 5557:5557
    environment:
      - ROCKETMQ_CONFIG_NAMESRVADDR=host:port
      - ROCKETMQ_CONFIG_ROCKETMQ_VERSION=V4_4_0

scrape_configs:
  - job_name: 'rockermq_exporter'
    static_configs:
      - targets:
        - 192.168.3.39:5557

c530acf5ba82fa9196afdab371abc36.png

minIO

docker容器

elasticsearch

version: '3.8'
services:
  elasticsearch_exporter:
    image: prometheuscommunity/elasticsearch-exporter:v1.2.1
    command:
      - '--es.uri=http://user1:password1@192.168.0.59:9200' # 连接集群中的一个节点
      - '--es.all'
      - '--es.indices'
      - '--es.clusterinfo.interval=5m'
    ports:
      - "9114:9114"

scrape_configs:
  - job_name: 'elasticsearch_exporter'
    static_configs:
      - targets:
        - 192.168.3.39:9114

image-20230320105112512.png

告警配置