探索 vmagent

1. 查看 vmagent pod

$ kubectl get pod -n kube-vm | grep vmagent
vmagent-victoria-metrics-k8s-stack-0-1682237547-cfd64b8ff-kxlfv   2/2     Running   0          41h

从以上可以看出，vmagent 有两个容器，让我们具体查看两个容器都用来做什么？

$ kubectl get pod vmagent-victoria-metrics-k8s-stack-0-1682237547-cfd64b8ff-kxlfv -n kube-vm -o yaml
$ kubectl get deploy -n kube-vm
$ kubectl describe deploy vmagent-vm-stack-victoria-metrics-k8s-stack -n kube-vm
...
Pod Template:
...
  Service Account:  vmagent-vm-stack-victoria-metrics-k8s-stack
  Containers:
   config-reloader:
    # 辅助镜像，用于监控配置，并生成配置文件
    Image:      quay.io/prometheus-operator/prometheus-config-reloader:v0.58.0
...
    Command:
      /bin/prometheus-config-reloader
    Args:
      --reload-url=http://localhost:8429/-/reload      # vmagent 热加载配置
      --config-envsubst-file=/etc/vmagent/config_out/vmagent.env.yaml
      --watched-dir=/etc/vm/relabeling       # 监控目录
      --watched-dir=/etc/vm/stream-aggr
      --config-file=/etc/vmagent/config/vmagent.yaml.gz    # vmagent 要使用的配置文件
...
    Mounts:
      /etc/vm/relabeling from relabeling-assets (ro)
      /etc/vm/stream-aggr from stream-aggr-conf (ro)
      /etc/vmagent/config from config (rw)
      /etc/vmagent/config_out from config-out (rw)
   vmagent:
    Image:      victoriametrics/vmagent:v1.89.1      # vmagent 主镜像
    Port:       8429/TCP
    Host Port:  0/TCP
    Args:
      -httpListenAddr=:8429
      -promscrape.config=/etc/vmagent/config_out/vmagent.env.yaml
      -promscrape.streamParse=true                  # 使用参数解析
      -remoteWrite.maxDiskUsagePerURL=1073741824
      -remoteWrite.tmpDataPath=/tmp/vmagent-remotewrite-data
      -remoteWrite.url=http://vminsert-vm-stack-victoria-metrics-k8s-stack.kube-vm.svc:8480/insert/0/prometheus/api/v1/write
...
    Mounts:
      /etc/vm/relabeling from relabeling-assets (ro)
      /etc/vm/stream-aggr from stream-aggr-conf (ro)
      /etc/vmagent-tls/certs from tls-assets (ro)
      /etc/vmagent/config from config (ro)
      /etc/vmagent/config_out from config-out (ro)
      /tmp/vmagent-remotewrite-data from persistent-queue-data (rw)
  Volumes:
   persistent-queue-data:        # 写入远端的临时数据（使用空白目录）
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:
    SizeLimit:  <unset>
   config:                     # 从 secret 中挂载 vmagent 主配置文件 vmagent.yaml.gz
    Type:        Secret (a volume populated by a Secret)
    SecretName:  vmagent-vm-stack-victoria-metrics-k8s-stack
    Optional:    false
   tls-assets:              # 连接集群使用的证书
    Type:        Secret (a volume populated by a Secret)
    SecretName:  tls-assets-vmagent-vm-stack-victoria-metrics-k8s-stack
    Optional:    false
   config-out:             # vmagent的配置（使用空白目录）
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:
    SizeLimit:  <unset>
   relabeling-assets:       # 从 configmap 中挂载 重新打标的配置，默认是空
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      relabelings-assets-vmagent-vm-stack-victoria-metrics-k8s-stack
    Optional:  false
   stream-aggr-conf:        # 从 configmap 中挂载 配置参数，默认是空
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      stream-aggr-vmagent-vm-stack-victoria-metrics-k8s-stack
    Optional:  false

2. 查看 vmagent metrics job

// 查看 crd

$ kubectl api-resources | grep victoriametrics.com
vmagents                                       operator.victoriametrics.com/v1beta1   true         VMAgent
vmalertmanagerconfigs                          operator.victoriametrics.com/v1beta1   true         VMAlertmanagerConfig
vmalertmanagers                   vma          operator.victoriametrics.com/v1beta1   true         VMAlertmanager
vmalerts                                       operator.victoriametrics.com/v1beta1   true         VMAlert
vmauths                                        operator.victoriametrics.com/v1beta1   true         VMAuth
vmclusters                                     operator.victoriametrics.com/v1beta1   true         VMCluster
vmnodescrapes                                  operator.victoriametrics.com/v1beta1   true         VMNodeScrape
vmpodscrapes                                   operator.victoriametrics.com/v1beta1   true         VMPodScrape
vmprobes                                       operator.victoriametrics.com/v1beta1   true         VMProbe
vmrules                                        operator.victoriametrics.com/v1beta1   true         VMRule
vmservicescrapes                               operator.victoriametrics.com/v1beta1   true         VMServiceScrape
vmsingles                                      operator.victoriametrics.com/v1beta1   true         VMSingle
vmstaticscrapes                                operator.victoriametrics.com/v1beta1   true         VMStaticScrape
vmusers                                        operator.victoriametrics.com/v1beta1   true         VMUser

$ kubectl get vmnodescrapes -n kube-vm
NAME                                           AGE
vm-stack-victoria-metrics-k8s-stack-cadvisor   73m
vm-stack-victoria-metrics-k8s-stack-kubelet    73m
vm-stack-victoria-metrics-k8s-stack-probes     73m

$ kubectl get vmpodscrapes -n kube-vm
No resources found in kube-vm namespace.

$ kubectl get vmservicescrapes -n kube-vm
NAME                                                          AGE
vm-stack-victoria-metrics-k8s-stack-apiserver                 73m
vm-stack-victoria-metrics-k8s-stack-coredns                   73m
vm-stack-victoria-metrics-k8s-stack-grafana                   73m
vm-stack-victoria-metrics-k8s-stack-kube-controller-manager   73m
vm-stack-victoria-metrics-k8s-stack-kube-etcd                 73m
vm-stack-victoria-metrics-k8s-stack-kube-scheduler            73m
vm-stack-victoria-metrics-k8s-stack-kube-state-metrics        73m
vm-stack-victoria-metrics-k8s-stack-node-exporter             73m
vm-stack-victoria-metrics-k8s-stack-operator                  73m
vmagent-vm-stack-victoria-metrics-k8s-stack                   73m
vmalert-vm-stack-victoria-metrics-k8s-stack                   73m
vmalertmanager-vm-stack-victoria-metrics-k8s-stack            73m
vminsert-vm-stack-victoria-metrics-k8s-stack                  73m
vmselect-vm-stack-victoria-metrics-k8s-stack                  73m
vmstorage-vm-stack-victoria-metrics-k8s-stack                 73m

（1）vmnodescrapes

$ kubectl describe vmnodescrapes -n kube-vm

cadvisor 用于监控容器指标，运行在 kubelet 程序上，路径为：https://NodeIP:10250/metrics/cadvisor , 常用指标如下：

// 获取 token

$ kubectl exec -it vmagent-vm-stack-victoria-metrics-k8s-stack-5b46f86b5b-lt9b8 -n kube-vm -- cat /var/run/secrets/kubernetes.io/serviceaccount/token

// 使用 token 访问

$ TOKEN="上面获的 token"
$ curl -X GET -H "Authorization: Bearer $TOKEN" \
  https://192.168.10.209:10250/metrics/cadvisor --insecure

// 常见的几个指标

# cpu使用时间
container_cpu_usage_seconds_total
# 分配cpu limit数
container_spec_cpu_quota
# 内存使用量
container_memory_usage_bytes
# 分配内存 limit数
container_spec_memory_limit_bytes
# 网络接收流量
container_network_receive_bytes_total
# 网络发送流量
container_network_transmit_bytes_total
# 磁盘使用量
container_fs_usage_bytes
# 磁盘写
container_fs_writes_bytes_total
# 磁盘读
container_fs_reads_bytes_total

kubelet https://NodeIP:10250/metrics
probes https://NodeIP:10250/metrics/probes

（2）vmservicescrapes

$ kubectl describe vmservicescrapes -n kube-vm

apiserver https://MasterIP:6443/metrics
kube-controller-manager https://MasterIP:10257/metrics
kube-etcd https://MasterIP:2379/metrics
kube-scheduler https://MasterIP:10251/metrics
coredns http://ContainerIP:9153/metrics

kube-state-metrics http://ContainerIP:8080/metrics
node-exporter http://NodeIP:9100/metrics
k8s-stack-operator http://ContainerIP:8080/metrics

vmagent http://ContainerIP:8429/metrics
vmalert http://ContainerIP:8080/metrics
vmalertmanager http://ContainerIP:9093/metrics
vminsert http://ContainerIP:8480/metrics
vmselect http://ContainerIP:8481/metrics
vmstorage http://ContainerIP:8482/metrics

grafana http://ContainerIP:3000/metrics

3. 查看 vmagent oprator

$ kubectl get vmagent -n kube-vm
NAME                                      SHARDS COUNT   REPLICA COUNT
victoria-metrics-k8s-stack-0-1682237547   0              1

$ kubectl get vmagent victoria-metrics-k8s-stack-0-1682237547 -n kube-vm -o yaml
$ kubectl explain vmagent