監(jiān)控新星 | 實戰(zhàn)Prometheus搭建監(jiān)控系統(tǒng)

$ wget https://github.com/prometheus/prometheus/releases/download/v2.4.3/prometheus-2.4.3.linux-amd64.tar.gz$ tar xvfz prometheus-2.4.3.linux-amd64.tar.gz

$ cd prometheus-2.4.3.linux-amd64$ ./prometheus --versionprometheus, version 2.4.3 (branch: HEAD, revision: 167a4b4e73a8eca8df648d2d2043e21bdb9a7449)  build user:       root@1e42b46043e9  build date:       20181004-08:42:02  go version:       go1.11.1

$ ./prometheus --config.file=prometheus.yml

$ sudo docker run -d -p 9090:9090 prom/prometheus

$ sudo docker run -d -p 9090:9090 \    -v ~/docker/prometheus/:/etc/prometheus/ \    prom/prometheus

$ sudo docker inspect 0c[...]"Id": "0c4c2d0eed938395bcecf1e8bb4b6b87091fc4e6385ce5b404b6bb7419010f46","Created": "2018-10-15T22:27:34.56050369Z","Path": "/bin/prometheus","Args": ["--config.file=/etc/prometheus/prometheus.yml","--storage.tsdb.path=/prometheus","--web.console.libraries=/usr/share/prometheus/console_libraries","--web.console.templates=/usr/share/prometheus/consoles"        ],[...]

/etc/prometheus $ cat prometheus.yml # my global configglobal:scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.  # scrape_timeout is set to the global default (10s).
# Alertmanager configurationalerting:alertmanagers:- static_configs:- targets:      # - alertmanager:9093
# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.rule_files:  # - "first_rules.yml"  # - "second_rules.yml"
# A scrape configuration containing exactly one endpoint to scrape:# Here it's Prometheus itself.scrape_configs:  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.- job_name: 'prometheus'
    # metrics_path defaults to '/metrics'    # scheme defaults to 'http'.
static_configs:- targets: ['localhost:9090']

promhttp_metric_handler_requests_total{code="200",instance="192.168.0.107:9090",job="prometheus"} 106

# 表示 Prometheus 能否抓取 target 的指標(biāo)，用于 target 的健康檢查up

up{instance="192.168.0.107:9090",job="prometheus"}    1up{instance="192.168.0.108:9090",job="prometheus"}    1up{instance="192.168.0.107:9100",job="server"}    1up{instance="192.168.0.108:9104",job="mysql"}    0

up{job="prometheus"}

up{job!="prometheus"}up{job=~"server|mysql"}up{job=~"192\.168\.0\.107.+"}

http_requests_total[5m]

# 計算的是每秒的平均值，適用于變化很慢的 counter# per-second average rate of increase, for slow-moving countersrate(http_requests_total[5m])
# 計算的是每秒瞬時增加速率，適用于變化很快的 counter# per-second instant rate of increase, for volatile and fast-moving countersirate(http_requests_total[5m])

$ docker run -d -p 3000:3000 grafana/grafana

$ wget https://github.com/prometheus/node_exporter/releases/download/v0.16.0/node_exporter-0.16.0.linux-amd64.tar.gz$ tar xvfz node_exporter-0.16.0.linux-amd64.tar.gz$ cd node_exporter-0.16.0.linux-amd64$ ./node_exporter

$ curl http://localhost:9100/metrics

scrape_configs:  - job_name: 'prometheus'    static_configs:      - targets: ['192.168.0.107:9090']  - job_name: 'server'    static_configs:      - targets: ['192.168.0.107:9100']

$ killall -HUP prometheus

docker run -d \  --net="host" \  --pid="host" \  -v "/:/host:ro,rslave" \  quay.io/prometheus/node-exporter \  --path.rootfs /host

$ wget https://github.com/prometheus/mysqld_exporter/releases/download/v0.11.0/mysqld_exporter-0.11.0.linux-amd64.tar.gz$ tar xvfz mysqld_exporter-0.11.0.linux-amd64.tar.gz$ cd mysqld_exporter-0.11.0.linux-amd64/

$ export DATA_SOURCE_NAME='root:123456@(192.168.0.107:3306)/'$ ./mysqld_exporter另一種方式是通過配置文

$ ./mysqld_exporter --config.my-cnf=".my.cnf"

$ cat .my.cnf[client]host=localhostport=3306user=rootpassword=123456

CREATE USER 'exporter'@'localhost' IDENTIFIED BY 'password' WITH MAX_USER_CONNECTIONS 3;GRANT PROCESS, REPLICATION CLIENT, SELECT ON *.* TO 'exporter'@'localhost';

$ wget https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.3.1/jmx_prometheus_javaagent-0.3.1.jar

$ java -javaagent:jmx_prometheus_javaagent-0.3.1.jar=9404:config.yml -jar spring-boot-sample-1.0-SNAPSHOT.jar

$ curl http://localhost:9404/metrics

rule_files:  - "alert.rules"

groups:- name: examplerules:  # Alert for any instance that is unreachable for >5 minutes.- alert: InstanceDownexpr: up == 0for: 5mlabels:severity: pageannotations:summary: "Instance {{ $labels.instance }} down"description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."  # Alert for any instance that has a median request latency >1s.- alert: APIHighRequestLatencyexpr: api_http_request_latencies_second{quantile="0.5"} > 1for: 10mannotations:summary: "High request latency on {{ $labels.instance }}"description: "{{ $labels.instance }} has a median request latency above 1s (current value: {{ $value }}s)"

$ wget https://github.com/prometheus/alertmanager/releases/download/v0.15.2/alertmanager-0.15.2.linux-amd64.tar.gz$ tar xvfz alertmanager-0.15.2.linux-amd64.tar.gz$ cd alertmanager-0.15.2.linux-amd64$ ./alertmanager

alerting:alertmanagers:- scheme: httpstatic_configs:- targets:- "192.168.0.107:9093"

$ ./prometheus -alertmanager.url=http://192.168.0.107:9093

global:resolve_timeout: 5m
route:group_by: ['alertname']group_wait: 10sgroup_interval: 10srepeat_interval: 1hreceiver: 'web.hook'receivers:- name: 'web.hook'webhook_configs:- url: 'http://127.0.0.1:5001/'inhibit_rules:- source_match:severity: 'critical'target_match:severity: 'warning'equal: ['alertname', 'dev', 'instance']

routes:- receiver: 'database-pager'group_wait: 10smatch_re:service: mysql|cassandra
- receiver: 'frontend-pager'group_by: [product, environment]match:team: frontend