K8S结合Prometheus+Grafana
OS: Ubuntu Server 22.04 LTS
一: 安装Kube-Prometheus
1) Clone Kube-Prometheus
root@srv1:~# apt install git -y
root@srv1:~# git clone https://github.com/coreos/kube-prometheus
Cloning into 'kube-prometheus'...
remote: Enumerating objects: 16368, done.
remote: Counting objects: 100% (289/289), done.
remote: Compressing objects: 100% (118/118), done.
remote: Total 16368 (delta 202), reused 217 (delta 160), pack-reused 16079
Receiving objects: 100% (16368/16368), 8.15 MiB | 799.00 KiB/s, done.
Resolving deltas: 100% (10542/10542), done2) 确认kube-prometheus文件
root@srv1:~# cd kube-prometheus/manifests
root@srv1:~/kube-prometheus/manifests# ls -l
total 1976
-rw-r--r-- 1 root root 928 Jun 25 17:07 alertmanager-alertmanager.yaml
-rw-r--r-- 1 root root 977 Jun 25 17:07 alertmanager-networkPolicy.yaml
-rw-r--r-- 1 root root 561 Jun 25 17:07 alertmanager-podDisruptionBudget.yaml
-rw-r--r-- 1 root root 7072 Jun 25 17:07 alertmanager-prometheusRule.yaml
-rw-r--r-- 1 root root 1443 Jun 25 17:07 alertmanager-secret.yaml
-rw-r--r-- 1 root root 650 Jun 25 17:07 alertmanager-service.yaml
-rw-r--r-- 1 root root 351 Jun 25 17:07 alertmanager-serviceAccount.yaml
-rw-r--r-- 1 root root 637 Jun 25 17:07 alertmanager-serviceMonitor.yaml
-rw-r--r-- 1 root root 287 Jun 25 17:07 blackboxExporter-clusterRole.yaml
-rw-r--r-- 1 root root 485 Jun 25 17:07 blackboxExporter-clusterRoleBinding.yaml
-rw-r--r-- 1 root root 1392 Jun 25 17:07 blackboxExporter-configuration.yaml
-rw-r--r-- 1 root root 3545 Jun 25 17:07 blackboxExporter-deployment.yaml
-rw-r--r-- 1 root root 722 Jun 25 17:07 blackboxExporter-networkPolicy.yaml
-rw-r--r-- 1 root root 540 Jun 25 17:07 blackboxExporter-service.yaml
-rw-r--r-- 1 root root 315 Jun 25 17:07 blackboxExporter-serviceAccount.yaml
-rw-r--r-- 1 root root 680 Jun 25 17:07 blackboxExporter-serviceMonitor.yaml
-rw-r--r-- 1 root root 344 Jun 25 17:07 grafana-config.yaml
-rw-r--r-- 1 root root 680 Jun 25 17:07 grafana-dashboardDatasources.yaml
-rw-r--r-- 1 root root 1548477 Jun 25 17:07 grafana-dashboardDefinitions.yaml
-rw-r--r-- 1 root root 658 Jun 25 17:07 grafana-dashboardSources.yaml
-rw-r--r-- 1 root root 9290 Jun 25 17:07 grafana-deployment.yaml
-rw-r--r-- 1 root root 651 Jun 25 17:07 grafana-networkPolicy.yaml
-rw-r--r-- 1 root root 1427 Jun 25 17:07 grafana-prometheusRule.yaml
-rw-r--r-- 1 root root 452 Jun 25 17:07 grafana-service.yaml
-rw-r--r-- 1 root root 293 Jun 25 17:07 grafana-serviceAccount.yaml
-rw-r--r-- 1 root root 398 Jun 25 17:07 grafana-serviceMonitor.yaml
-rw-r--r-- 1 root root 4361 Jun 25 17:07 kubePrometheus-prometheusRule.yaml
-rw-r--r-- 1 root root 1712 Jun 25 17:07 kubeStateMetrics-clusterRole.yaml
-rw-r--r-- 1 root root 464 Jun 25 17:07 kubeStateMetrics-clusterRoleBinding.yaml
-rw-r--r-- 1 root root 3423 Jun 25 17:07 kubeStateMetrics-deployment.yaml
-rw-r--r-- 1 root root 723 Jun 25 17:07 kubeStateMetrics-networkPolicy.yaml
-rw-r--r-- 1 root root 3152 Jun 25 17:07 kubeStateMetrics-prometheusRule.yaml
-rw-r--r-- 1 root root 580 Jun 25 17:07 kubeStateMetrics-service.yaml
-rw-r--r-- 1 root root 316 Jun 25 17:07 kubeStateMetrics-serviceAccount.yaml
-rw-r--r-- 1 root root 1011 Jun 25 17:07 kubeStateMetrics-serviceMonitor.yaml
-rw-r--r-- 1 root root 71499 Jun 25 17:07 kubernetesControlPlane-prometheusRule.yaml
-rw-r--r-- 1 root root 6997 Jun 25 17:07 kubernetesControlPlane-serviceMonitorApiserver.yaml
-rw-r--r-- 1 root root 591 Jun 25 17:07 kubernetesControlPlane-serviceMonitorCoreDNS.yaml
-rw-r--r-- 1 root root 6516 Jun 25 17:07 kubernetesControlPlane-serviceMonitorKubeControllerManager.yaml
-rw-r--r-- 1 root root 577 Jun 25 17:07 kubernetesControlPlane-serviceMonitorKubeScheduler.yaml
-rw-r--r-- 1 root root 7714 Jun 25 17:07 kubernetesControlPlane-serviceMonitorKubelet.yaml
-rw-r--r-- 1 root root 485 Jun 25 17:07 nodeExporter-clusterRole.yaml
-rw-r--r-- 1 root root 468 Jun 25 17:07 nodeExporter-clusterRoleBinding.yaml
-rw-r--r-- 1 root root 3588 Jun 25 17:07 nodeExporter-daemonset.yaml
-rw-r--r-- 1 root root 671 Jun 25 17:07 nodeExporter-networkPolicy.yaml
-rw-r--r-- 1 root root 14553 Jun 25 17:07 nodeExporter-prometheusRule.yaml
-rw-r--r-- 1 root root 492 Jun 25 17:07 nodeExporter-service.yaml
-rw-r--r-- 1 root root 306 Jun 25 17:07 nodeExporter-serviceAccount.yaml
-rw-r--r-- 1 root root 850 Jun 25 17:07 nodeExporter-serviceMonitor.yaml
-rw-r--r-- 1 root root 430 Jun 25 17:07 prometheus-clusterRole.yaml
-rw-r--r-- 1 root root 483 Jun 25 17:07 prometheus-clusterRoleBinding.yaml
-rw-r--r-- 1 root root 922 Jun 25 17:07 prometheus-networkPolicy.yaml
-rw-r--r-- 1 root root 546 Jun 25 17:07 prometheus-podDisruptionBudget.yaml
-rw-r--r-- 1 root root 1238 Jun 25 17:07 prometheus-prometheus.yaml
-rw-r--r-- 1 root root 15726 Jun 25 17:07 prometheus-prometheusRule.yaml
-rw-r--r-- 1 root root 507 Jun 25 17:07 prometheus-roleBindingConfig.yaml
-rw-r--r-- 1 root root 1661 Jun 25 17:07 prometheus-roleBindingSpecificNamespaces.yaml
-rw-r--r-- 1 root root 402 Jun 25 17:07 prometheus-roleConfig.yaml
-rw-r--r-- 1 root root 2161 Jun 25 17:07 prometheus-roleSpecificNamespaces.yaml
-rw-r--r-- 1 root root 637 Jun 25 17:07 prometheus-service.yaml
-rw-r--r-- 1 root root 343 Jun 25 17:07 prometheus-serviceAccount.yaml
-rw-r--r-- 1 root root 624 Jun 25 17:07 prometheus-serviceMonitor.yaml
-rw-r--r-- 1 root root 482 Jun 25 17:07 prometheusAdapter-apiService.yaml
-rw-r--r-- 1 root root 433 Jun 25 17:07 prometheusAdapter-clusterRole.yaml
-rw-r--r-- 1 root root 600 Jun 25 17:07 prometheusAdapter-clusterRoleAggregatedMetricsReader.yaml
-rw-r--r-- 1 root root 495 Jun 25 17:07 prometheusAdapter-clusterRoleBinding.yaml
-rw-r--r-- 1 root root 518 Jun 25 17:07 prometheusAdapter-clusterRoleBindingDelegator.yaml
-rw-r--r-- 1 root root 402 Jun 25 17:07 prometheusAdapter-clusterRoleServerResources.yaml
-rw-r--r-- 1 root root 2204 Jun 25 17:07 prometheusAdapter-configMap.yaml
-rw-r--r-- 1 root root 3171 Jun 25 17:07 prometheusAdapter-deployment.yaml
-rw-r--r-- 1 root root 564 Jun 25 17:07 prometheusAdapter-networkPolicy.yaml
-rw-r--r-- 1 root root 501 Jun 25 17:07 prometheusAdapter-podDisruptionBudget.yaml
-rw-r--r-- 1 root root 515 Jun 25 17:07 prometheusAdapter-roleBindingAuthReader.yaml
-rw-r--r-- 1 root root 501 Jun 25 17:07 prometheusAdapter-service.yaml
-rw-r--r-- 1 root root 323 Jun 25 17:07 prometheusAdapter-serviceAccount.yaml
-rw-r--r-- 1 root root 906 Jun 25 17:07 prometheusAdapter-serviceMonitor.yaml
-rw-r--r-- 1 root root 1401 Jun 25 17:07 prometheusOperator-clusterRole.yaml
-rw-r--r-- 1 root root 471 Jun 25 17:07 prometheusOperator-clusterRoleBinding.yaml
-rw-r--r-- 1 root root 2631 Jun 25 17:07 prometheusOperator-deployment.yaml
-rw-r--r-- 1 root root 694 Jun 25 17:07 prometheusOperator-networkPolicy.yaml
-rw-r--r-- 1 root root 5819 Jun 25 17:07 prometheusOperator-prometheusRule.yaml
-rw-r--r-- 1 root root 515 Jun 25 17:07 prometheusOperator-service.yaml
-rw-r--r-- 1 root root 321 Jun 25 17:07 prometheusOperator-serviceAccount.yaml
-rw-r--r-- 1 root root 715 Jun 25 17:07 prometheusOperator-serviceMonitor.yaml
drwxr-xr-x 2 root root 4096 Jun 25 17:07 setup3) 修改源
root@srv1:~/kube-prometheus/manifests# sed -i 's/quay.io/quay.mirrors.ustc.edu.cn/g' ./prometheusOperator-deployment.yaml
root@srv1:~/kube-prometheus/manifests# sed -i 's/quay.io/quay.mirrors.ustc.edu.cn/g' ./prometheus-prometheus.yaml
root@srv1:~/kube-prometheus/manifests# sed -i 's/quay.io/quay.mirrors.ustc.edu.cn/g' ./alertmanager-alertmanager.yaml
root@srv1:~/kube-prometheus/manifests# sed -i 's/quay.io/quay.mirrors.ustc.edu.cn/g' ./kubeStateMetrics-deployment.yaml
root@srv1:~/kube-prometheus/manifests# sed -i 's/quay.io/quay.mirrors.ustc.edu.cn/g' ./nodeExporter-daemonset.yaml
root@srv1:~/kube-prometheus/manifests# sed -i 's/quay.io/quay.mirrors.ustc.edu.cn/g' ./blackboxExporter-deployment.yaml
root@srv1:~/kube-prometheus/manifests# sed -i 's/k8s.gcr.io\/kube-state-metrics\/kube-state-metrics:v2.5.0/bitnami\/kube-state-metrics:2.5.0/g' ./kubeStateMetrics-deployment.yaml
root@srv1:~/kube-prometheus/manifests# sed -i 's/k8s.gcr.io\/prometheus-adapter\/prometheus-adapter:v0.9.1/selina5288\/prometheus-adapter:v0.9.1/g' ./prometheusAdapter-deployment.yaml4) 设定NodePort
root@srv1:~/kube-prometheus/manifests# vim prometheus-service.yaml
apiVersion: v1
kind: Service
metadata:
labels:
prometheus: k8s
name: prometheus-k8s
namespace: monitoring
spec:
# 新增
type: NodePort
ports:
- name: web
port: 9090
targetPort: web
# 新增
nodePort: 30090
selector:
app: prometheus
prometheus: k8s
sessionAffinity: ClientIP
root@srv1:~/kube-prometheus/manifests# vim alertmanager-service.yaml
apiVersion: v1
kind: Service
metadata:
labels:
alertmanager: main
name: alertmanager-main
namespace: monitoring
spec:
# 新增
type: NodePort
ports:
- name: web
port: 9093
targetPort: web
# 新增
nodePort: 30093
selector:
alertmanager: main
app: alertmanager
sessionAffinity: ClientIP
root@srv1:~/kube-prometheus/manifests# vim grafana-service.yaml
apiVersion: v1
kind: Service
metadata:
labels:
app: grafana
name: grafana
namespace: monitoring
spec:
# 新增
type: NodePort
ports:
- name: http
port: 3000
targetPort: http
# 新增
nodePort: 32000
selector:
app: grafana二: 安装Kube-Prometheus
1) 生成DashBoard
root@srv1:~/kube-prometheus/manifests# kubectl create -f setup/
namespace/monitoring created
customresourcedefinition.apiextensions.k8s.io/alertmanagerconfigs.monitoring.coreos.com created
customresourcedefinition.apiextensions.k8s.io/alertmanagers.monitoring.coreos.com created
customresourcedefinition.apiextensions.k8s.io/podmonitors.monitoring.coreos.com created
customresourcedefinition.apiextensions.k8s.io/probes.monitoring.coreos.com created
customresourcedefinition.apiextensions.k8s.io/prometheuses.monitoring.coreos.com created
customresourcedefinition.apiextensions.k8s.io/prometheusrules.monitoring.coreos.com created
customresourcedefinition.apiextensions.k8s.io/servicemonitors.monitoring.coreos.com created
customresourcedefinition.apiextensions.k8s.io/thanosrulers.monitoring.coreos.com created
namespace/monitoring created2) 安装Prometheus、Grafana等服务
root@srv1:~/kube-prometheus/manifests# kubectl create -f .
alertmanager.monitoring.coreos.com/main created
networkpolicy.networking.k8s.io/alertmanager-main created
poddisruptionbudget.policy/alertmanager-main created
prometheusrule.monitoring.coreos.com/alertmanager-main-rules created
secret/alertmanager-main created
service/alertmanager-main created
serviceaccount/alertmanager-main created
servicemonitor.monitoring.coreos.com/alertmanager-main created
clusterrole.rbac.authorization.k8s.io/blackbox-exporter created
clusterrolebinding.rbac.authorization.k8s.io/blackbox-exporter created
configmap/blackbox-exporter-configuration created
deployment.apps/blackbox-exporter created
networkpolicy.networking.k8s.io/blackbox-exporter created
service/blackbox-exporter created
serviceaccount/blackbox-exporter created
servicemonitor.monitoring.coreos.com/blackbox-exporter created
secret/grafana-config created
secret/grafana-datasources created
configmap/grafana-dashboard-alertmanager-overview created
configmap/grafana-dashboard-apiserver created
configmap/grafana-dashboard-cluster-total created
configmap/grafana-dashboard-controller-manager created
configmap/grafana-dashboard-grafana-overview created
configmap/grafana-dashboard-k8s-resources-cluster created
configmap/grafana-dashboard-k8s-resources-namespace created
configmap/grafana-dashboard-k8s-resources-node created
configmap/grafana-dashboard-k8s-resources-pod created
configmap/grafana-dashboard-k8s-resources-workload created
configmap/grafana-dashboard-k8s-resources-workloads-namespace created
configmap/grafana-dashboard-kubelet created
configmap/grafana-dashboard-namespace-by-pod created
configmap/grafana-dashboard-namespace-by-workload created
configmap/grafana-dashboard-node-cluster-rsrc-use created
configmap/grafana-dashboard-node-rsrc-use created
configmap/grafana-dashboard-nodes-darwin created
configmap/grafana-dashboard-nodes created
configmap/grafana-dashboard-persistentvolumesusage created
configmap/grafana-dashboard-pod-total created
configmap/grafana-dashboard-prometheus-remote-write created
configmap/grafana-dashboard-prometheus created
configmap/grafana-dashboard-proxy created
configmap/grafana-dashboard-scheduler created
configmap/grafana-dashboard-workload-total created
configmap/grafana-dashboards created
deployment.apps/grafana created
networkpolicy.networking.k8s.io/grafana created
prometheusrule.monitoring.coreos.com/grafana-rules created
service/grafana created
serviceaccount/grafana created
servicemonitor.monitoring.coreos.com/grafana created
prometheusrule.monitoring.coreos.com/kube-prometheus-rules created
clusterrole.rbac.authorization.k8s.io/kube-state-metrics created
clusterrolebinding.rbac.authorization.k8s.io/kube-state-metrics created
deployment.apps/kube-state-metrics created
networkpolicy.networking.k8s.io/kube-state-metrics created
prometheusrule.monitoring.coreos.com/kube-state-metrics-rules created
service/kube-state-metrics created
serviceaccount/kube-state-metrics created
servicemonitor.monitoring.coreos.com/kube-state-metrics created
prometheusrule.monitoring.coreos.com/kubernetes-monitoring-rules created
servicemonitor.monitoring.coreos.com/kube-apiserver created
servicemonitor.monitoring.coreos.com/coredns created
servicemonitor.monitoring.coreos.com/kube-controller-manager created
servicemonitor.monitoring.coreos.com/kube-scheduler created
servicemonitor.monitoring.coreos.com/kubelet created
clusterrole.rbac.authorization.k8s.io/node-exporter created
clusterrolebinding.rbac.authorization.k8s.io/node-exporter created
daemonset.apps/node-exporter created
networkpolicy.networking.k8s.io/node-exporter created
prometheusrule.monitoring.coreos.com/node-exporter-rules created
service/node-exporter created
serviceaccount/node-exporter created
servicemonitor.monitoring.coreos.com/node-exporter created
clusterrole.rbac.authorization.k8s.io/prometheus-k8s created
clusterrolebinding.rbac.authorization.k8s.io/prometheus-k8s created
networkpolicy.networking.k8s.io/prometheus-k8s created
poddisruptionbudget.policy/prometheus-k8s created
prometheus.monitoring.coreos.com/k8s created
prometheusrule.monitoring.coreos.com/prometheus-k8s-prometheus-rules created
rolebinding.rbac.authorization.k8s.io/prometheus-k8s-config created
rolebinding.rbac.authorization.k8s.io/prometheus-k8s created
rolebinding.rbac.authorization.k8s.io/prometheus-k8s created
rolebinding.rbac.authorization.k8s.io/prometheus-k8s created
role.rbac.authorization.k8s.io/prometheus-k8s-config created
role.rbac.authorization.k8s.io/prometheus-k8s created
role.rbac.authorization.k8s.io/prometheus-k8s created
role.rbac.authorization.k8s.io/prometheus-k8s created
service/prometheus-k8s created
serviceaccount/prometheus-k8s created
servicemonitor.monitoring.coreos.com/prometheus-k8s created
apiservice.apiregistration.k8s.io/v1beta1.metrics.k8s.io configured
clusterrole.rbac.authorization.k8s.io/prometheus-adapter created
clusterrole.rbac.authorization.k8s.io/system:aggregated-metrics-reader configured
clusterrolebinding.rbac.authorization.k8s.io/prometheus-adapter created
clusterrolebinding.rbac.authorization.k8s.io/resource-metrics:system:auth-delegator created
clusterrole.rbac.authorization.k8s.io/resource-metrics-server-resources created
configmap/adapter-config created
deployment.apps/prometheus-adapter created
networkpolicy.networking.k8s.io/prometheus-adapter created
poddisruptionbudget.policy/prometheus-adapter created
rolebinding.rbac.authorization.k8s.io/resource-metrics-auth-reader created
service/prometheus-adapter created
serviceaccount/prometheus-adapter created
servicemonitor.monitoring.coreos.com/prometheus-adapter created
clusterrole.rbac.authorization.k8s.io/prometheus-operator created
clusterrolebinding.rbac.authorization.k8s.io/prometheus-operator created
deployment.apps/prometheus-operator created
networkpolicy.networking.k8s.io/prometheus-operator created
prometheusrule.monitoring.coreos.com/prometheus-operator-rules created
service/prometheus-operator created
serviceaccount/prometheus-operator created
servicemonitor.monitoring.coreos.com/prometheus-operator created3) 确认运行状态
root@srv1:~/kube-prometheus/manifests# kubectl get pod -n monitoring
NAME READY STATUS RESTARTS AGE
alertmanager-main-0 2/2 Running 0 25m
alertmanager-main-1 2/2 Running 0 25m
alertmanager-main-2 2/2 Running 0 25m
blackbox-exporter-656d46c758-x7mj8 3/3 Running 0 25m
grafana-7bf795c5b-27mxz 1/1 Running 0 25m
kube-state-metrics-6f6f9898ff-2wwnp 3/3 Running 0 25m
node-exporter-nvfng 2/2 Running 0 25m
node-exporter-v76q4 2/2 Running 0 25m
node-exporter-xjljn 2/2 Running 0 25m
prometheus-adapter-6957bd4fcc-7qw2k 1/1 Running 0 25m
prometheus-adapter-6957bd4fcc-fccmq 1/1 Running 0 25m
prometheus-k8s-0 2/2 Running 0 25m
prometheus-k8s-1 2/2 Running 0 25m
prometheus-operator-5756996bfb-c7k4f 2/2 Running 0 25m三: 访问测试
1) 访问Prometheus
[浏览器]==>[http://srv1.1000y.cloud:30090]
2) 访问Alert-Manager
[浏览器]==>[http://srv1.1000y.cloud:30093]
3) 访问Grafana
[浏览器]==>[http://srv1.1000y.cloud:32000]
四. 配置Granfa
1) 更改admin账户的密码
2) 测试Prometheus数据源是否正常
3) 添加监控模板
更多k8s监控模板请查询: https://grafana.com/dashboards?dataSource=prometheus&search=kubewu
五: 显示监控信息
本文暂时没有评论，来添加一个吧(●'◡'●)