探索 vmalert

1. 查看 grafana deployment

$ kubectl get deployment vmalert-vm-stack-victoria-metrics-k8s-stack -n kube-vm -o yaml
...
  template:
 ...
    spec:
      containers:
      - args:
        - -datasource.url=http://vmselect-vm-stack-victoria-metrics-k8s-stack.kube-vm.svc:8481/select/0/prometheus
        - -evaluationInterval=15s
        - -httpListenAddr=:8080
        - -notifier.url=http://vmalertmanager-vm-stack-victoria-metrics-k8s-stack.kube-vm.svc:9093
        - -remoteRead.url=http://vmselect-vm-stack-victoria-metrics-k8s-stack.kube-vm.svc:8481/select/0/prometheus
        - -remoteWrite.disablePathAppend=true
        - -remoteWrite.url=http://vminsert-vm-stack-victoria-metrics-k8s-stack.kube-vm.svc:8480/insert/0/prometheus/api/v1/write
        - -rule="/etc/vmalert/config/vm-vm-stack-victoria-metrics-k8s-stack-rulefiles-0/*.yaml"
        image: victoriametrics/vmalert:v1.89.1
...
        volumeMounts:
        - mountPath: /etc/vmalert/remote_secrets
          name: remote-secrets
          readOnly: true
        - mountPath: /etc/vmagent-tls/certs
          name: tls-assets
          readOnly: true
        - mountPath: /etc/vmalert/config/vm-vm-stack-victoria-metrics-k8s-stack-rulefiles-0
          name: vm-vm-stack-victoria-metrics-k8s-stack-rulefiles-0
      - args:
        - -volume-dir=/etc/vmalert/config/vm-vm-stack-victoria-metrics-k8s-stack-rulefiles-0
        - -webhook-url=http://localhost:8080/-/reload
        image: jimmidyson/configmap-reload:v0.3.0
...
        volumeMounts:
        - mountPath: /etc/vmalert/config/vm-vm-stack-victoria-metrics-k8s-stack-rulefiles-0
          name: vm-vm-stack-victoria-metrics-k8s-stack-rulefiles-0
...
      volumes:
      - name: remote-secrets
        secret:
          defaultMode: 420
          secretName: vmalert-vm-stack-victoria-metrics-k8s-stack
      - name: tls-assets
        secret:
          defaultMode: 420
          secretName: tls-assets-vmalert-vm-stack-victoria-metrics-k8s-stack
      - configMap:
          defaultMode: 420
          name: vm-vm-stack-victoria-metrics-k8s-stack-rulefiles-0
        name: vm-vm-stack-victoria-metrics-k8s-stack-rulefiles-0

2. 查看 vmrules

// 查看定义的 rule

$ kubectl get vmrules -n kube-vm
$ kubectl get vmrules vm-stack-victoria-metrics-k8s-stack-etcd -n kube-vm -o yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  name: vm-stack-victoria-metrics-k8s-stack-etcd
...
spec:
  groups:
  - name: etcd
    rules:
    - alert: etcdInsufficientMembers
      annotations:
        message: 'etcd cluster "{{ $labels.job }}": insufficient members ({{ $value
          }}).'
      expr: sum(up{job=~".*etcd.*"} == bool 1) by (job) < ((count(up{job=~".*etcd.*"})
        by (job) + 1) / 2)
      for: 3m
      labels:
        severity: critical
    - alert: etcdNoLeader
      annotations:
        message: 'etcd cluster "{{ $labels.job }}": member {{ $labels.instance }}
          has no leader.'
      expr: etcd_server_has_leader{job=~".*etcd.*"} == 0
      for: 1m
      labels:
        severity: critical
...

3. 查看 vmalert operator

$ kubectl get vmalert -n kube-vm

对 vmalert 的修改需要在这里进行