Prometheus

sudo useradd --no-create-home --shell /bin/false prometheus
sudo mkdir /etc/prometheus /var/lib/prometheus
sudo chown prometheus:prometheus /var/lib/prometheus
cd /tmp/
wget https://github.com/prometheus/prometheus/releases/download/v2.7.1/prometheus-2.7.1.linux-amd64.tar.gz
tar -xvf prometheus-2.7.1.linux-amd64.tar.gz
cd prometheus-2.7.1.linux-amd64/
sudo mv {console*,prometheus.yml} /etc/prometheus
sudo chown -R prometheus:prometheus /etc/prometheus
sudo mv {prometheus,promtool} /usr/local/bin/
sudo chown prometheus:prometheus /usr/local/bin/prometheus /usr/local/bin/promtool
cat <<EOF | sudo tee -a /etc/systemd/system/prometheus.service
[Unit]
Description=Prometheus
Wants=network-online.target
After=network-online.target

[Service]
User=prometheus
Group=prometheus
Type=simple
ExecStart=/usr/local/bin/prometheus \
    --config.file /etc/prometheus/prometheus.yml \
    --storage.tsdb.path /var/lib/prometheus/ \
    --web.console.templates=/etc/prometheus/consoles \
    --web.console.libraries=/etc/prometheus/console_libraries

[Install]
WantedBy=multi-user.target
sudo systemctl start prometheus
sudo systemctl enable prometheus
sudo systemctl status prometheus
curl localhost:9090
curl localhost:9090/metrics

http_request_duration_seconds_bucket{l1="0.3"}
http_request_duration_seconds_bucket{l1="0.36}
http_request_duration_seconds_bucket{l1="1.0"}

http_request_duration_seconds{quantile="0.95"}

+ addition  
- substraction  
* multiplication  
/ division  
% modulo  
^ exponentiation  

== equal
!= not equal
> greater than
< less than
>= greater than or equal
<= less than or equal

sum - add all values together
min - select the samallest value
max - select the largest value
avg - calculate the average of all values
stddev - calculate population standard deviation over all values
stdvar - calculate population standard variance over all values
count - count number of values
count_values - cound the number of values with the same value
bottomk - smallest number (k) of elements
topk - largest number (k) of elements
quantile - calculate the quantile for particular dimension

<aggr-op> [without|by (<label list>)] ([parameter,] <vector expression>)

curl PROMETHEUS_URL/api/v1/query?query=node_cpu_seconds_total

/api/v1/query
/api/v1/labels
/api/v1/rules
/api/v1/alerts
/api/v1/targets
/api/v1/status/config
/api/v1/status/flags

sudo useradd -M -r -s /bin/false apache_exporter
wget https://github.com/Lusitaniae/apache_exporter/releases/download/v0.9.0/apache_exporter-0.9.0.linux-amd64.tar.gz
tar -zxvf apache_exporter-0.9.0.linux-amd64.tar.gz
sudo cp apache_exporter-0.9.0.linux-amd64/apache_exporter /usr/local/bin/
sudo chown apache_exporter:apache_exporter /usr/local/bin//apache_exporter

cat <<EOF | sudo tee -a /etc/systemd/system/apache_exporter.service
[Unit]
Description=Prometheus Apache Exporter
Wants=network-online.target
After=network-online.target

[Service]
User=apache_exporter
Group=apache_exporter
Type=simple
ExecStart=/usr/local/bin/apache_exporter

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl daemon-reload
suod systemctl start apache_exporter
sudo systemctl status apache_exporter
sudo systemctl enable apache_exporter

sudo useradd -M -r -s /bin/false node_exporter
wget https://github.com/prometheus/node_exporter/releases/download/v1.1.2/node_exporter-1.1.2.linux-amd64.tar.gz
tar -zxvf node_exporter-1.1.2.linux-amd64.tar.gz
sudo cp node_exporter-1.1.2.linux-amd64/node_exporter /usr/local/bin/
sudo chown node_exporter:node_exporter /usr/local/bin/node_exporter

cat <<EOF | tee -a /etc/systemd/system/node_exporter.service
[Unit]
Description=Prometheus Node Exporter
Wants=network-online.target
After=network-online.target

[Service]
User=node_exporter
Group=node_exporter
Type=simple
ExecStart=/usr/local/bin/node_exporter

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl deaemon-reaload
sudo systemctl start node_exporter
sudo systemctl enable node_exporter

# test metrics on target
curl localhost:9100/metrics

+- job: api-server
  +- instance 1: 1.2.4.4:5670
  +- instance 2: 1.2.4.4:5671
  +- instance 3: 5.6.7.8:5670

sudo useradd -M -r -s /bin/false pushgateway
wget https://github.com/prometheus/pushgateway/releases/download/v1.4.1/pushgateway-1.4.1.linux-amd64.tar.gz
tar -zxvf pushgateway-1.4.1.linux-amd64.tar.gz
sudo cp pushgateway-1.4.1.linux-amd64/pushgateway /usr/local/bin/
sudo chown pushgateway:pushgateway /usr/local/bin/pushgateway

cat <<EOF | sudo tee -a /etc/systemd/system/pushgateway.service
[Unit]
Description=Prometheus Pushgateway
Wants=network-online.target
After=network-online.target

[Service]
User=pushgateway
Group=pushgateway
Type=simple
ExecStart=/usr/local/bin/pushgateway

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl daemon-reload
sudo systemctl start pushgatway
sudo systemctl status pushgatway
sudo systemctl enable pushgatway
# check pushgateway metrics
curl localhost:9091/metrics

# TYPE some_metric counter
# HELP Example metric
some_metric{label="val1"} 42
# TYPE another_metric gauge
anohter_metric 12

cat <<EOF| curl --data-binary @- http://localhost:9091/metrics/job/my_job/instance/my_instance
> # TYPE temperature gauge
> temperature{location="room1"} 31
> temperature{location="room2"} 33
> # TYPE my_metric gauge
> # HELP my_metric An example
> my_metric 5
> EOF

groups:
- name: my_rule_group
  rules:
  - record: my_custom_metric
    exor: up{job="My Job"}

groups:
# the name of the group.must be unique within a file
- name: <string>
# how owfet rules in the group are evaluated
[intarval: <duration> | default = global.evaluation_interval]
rules:
# the name of the metric where results will be stored
- record: <string>
 # the expression or query used to calculate the value
 expr: <string>
 # Labels to add or overwrite before storing the result.
 labels:
   [ <labelname>: <labelvalue> ]

sudo useradd -M -r -s /bin/false alertmanager
wget https://github.com/prometheus/alertmanager/releases/download/v0.22.2/alertmanager-0.22.2.linux-amd64.tar.gz
tar -zxvf alertmanager-0.22.2.linux-amd64.tar.gz
sudo cp alertmanager-0.22.2.linux-amd64/{alertmanager,amtool} /usr/local/bin/
sudo chown alertmanager:alertmanager /usr/local/bin/{alertmanager,amtool}
sudo mkdir -p /etc/alertmanager
sudo cp alertmanager-0.22.2.linux-amd64/alertmanager.yml /etc/alertmanager/
sudo chown -R alertmanager:alertmanager /etc/alertmanager
sudo mkdir -p /var/lib/alertmanager
sudo chown alertmanager:alertmanager /var/lib/alertmanager
# optional
sudo mkdir -p /etc/amtool

cat <<EOF | sudo tee -a /etc/amtool/config.yml
alertmanager: http://localhost:9093
EOF

cat <<EOF | sudo tee -a /etc/systemd/system/alertmanager.service
[Unit]
Description=Prometheus Alertmanager
Wants=network-online.target
After=network-online.target

[Service]
User=alertmanager
Group=alertmanager
Type=simple
ExecStart=/usr/local/bin/alertmanager --config.file /etc/alertmanager/alertmanager.yml --storage.path /var/lib/alertmanager/

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl daemon-reload
sudo systemctl start alertmanager
sudo systemctl status alertmanager
sudo systemctl enable alertmanager
amtool config show 

groups:
- name: example
  rules:
  # Alert for any instance that is unreachable for >5 minutes.
  - alert: InstanceDown
    expr: up == 0
    for: 5m
    labels:
      severity: page
    annotations:
      summary: "Instance {{ $labels.instance }} down"
      description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."

# from alertmanager
curl -s localhost:9093/api/v2/alerts | python -m json.tool
# from prometheus
curl -s localhost:9090/api/v1/alerts | python -mjson.tool

route:
  group_by: ['alertname']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 1h
  receiver: 'web.hook'
  routes:
  - receiver: 'web.hook'
    # one can group the alerts which has same labels
    group_by: ['service']
    
    # regex which alertnames will be evaluated
    # DEPRECATED, use matchers
    # match_re:
    #  alertname: 'Server.*Down'
    matchers:
    - alertname =~ "Server.*Down"
receivers:
- name: 'web.hook'
  webhook_configs:
  - url: 'http://127.0.0.1:5001/'
inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'dev', 'instance']

    # a list of matchers for which one or more alerts have to exist for the inhibition to take effect
  - source_matchers: [ alertname =~ "Server.*Down" ]
    # a list of matchers that have to be fulfilled by the target alerts to be muted
    target_matchers: [ alertname = "DownstreamServerDown" ]

scrape_configs:
  - job_name: 'federate'
    scrape_interval: 15s

    honor_labels: true
    metrics_path: '/federate' ## use instead /metrics

    # narrow the time-series data you are retrieving with the `match[]` paramter
    # match on metric names and/or lables to retrieve a subset of the data
    # in this case, all records which has expression `job="prometheus"` or which name has name like `job:.*`
    params:
      'match[]':
        - '{job="prometheus"}'
        - '{__name__=~"job:.*"}'

    static_configs:
      - targets:
        - 'source-prometheus-1:9090'
        - 'source-prometheus-2:9090'
        - 'source-prometheus-3:9090'