新集群安装kube-prometheus套件
背景
检查兼容性
下载main版本，对应k8s的1.36
git clone https://g.1ab.asia/https://github.com/prometheus-operator/kube-prometheus.git
检索配置文件，并修改成内网
root@qat-k8s-master:~/kube-pro/kube-prometheus/manifests# grep "image:" ./*
./alertmanager-alertmanager.yaml: image: quay.io/prometheus/alertmanager:v0.32.1
./blackboxExporter-deployment.yaml: image: quay.io/prometheus/blackbox-exporter:v0.28.0
./blackboxExporter-deployment.yaml: image: ghcr.io/jimmidyson/configmap-reload:v0.15.0
./blackboxExporter-deployment.yaml: image: quay.io/brancz/kube-rbac-proxy:v0.22.0
./grafana-deployment.yaml: image: grafana/grafana:13.0.1
./kubeStateMetrics-deployment.yaml: image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.19.0
./kubeStateMetrics-deployment.yaml: image: quay.io/brancz/kube-rbac-proxy:v0.22.0
./kubeStateMetrics-deployment.yaml: image: quay.io/brancz/kube-rbac-proxy:v0.22.0
./nodeExporter-daemonset.yaml: image: quay.io/prometheus/node-exporter:v1.11.1
./nodeExporter-daemonset.yaml: image: quay.io/brancz/kube-rbac-proxy:v0.22.0
./prometheusAdapter-deployment.yaml: image: registry.k8s.io/prometheus-adapter/prometheus-adapter:v0.12.0
./prometheusOperator-deployment.yaml: image: quay.io/prometheus-operator/prometheus-operator:v0.91.0
./prometheusOperator-deployment.yaml: image: quay.io/brancz/kube-rbac-proxy:v0.22.0
./prometheus-prometheus.yaml: image: quay.io/prometheus/prometheus:v3.12.0
grep: ./setup: Is a directory
本地下载好这些镜像，推到内网仓库。
register.a.cn/quay.io/prometheus/alertmanager:v0.32.1
register.a.cn/quay.io/prometheus/blackbox-exporter:v0.28.0
register.a.cn/ghcr.io/jimmidyson/configmap-reload:v0.15.0
register.a.cn/quay.io/brancz/kube-rbac-proxy:v0.22.0
register.a.cn/grafana/grafana:13.0.1
register.a.cn/registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.19.0
register.a.cn/quay.io/prometheus/node-exporter:v1.11.1
register.a.cn/registry.k8s.io/prometheus-adapter/prometheus-adapter:v0.12.0
register.a.cn/quay.io/prometheus-operator/prometheus-operator:v0.91.0
register.a.cn/quay.io/prometheus/prometheus:v3.12.0
修改manifests/grafana-service.yaml为nodeport方式
apiVersion: v1
kind: Service
metadata:
labels:
app.kubernetes.io/component: grafana
app.kubernetes.io/name: grafana
app.kubernetes.io/part-of: kube-prometheus
app.kubernetes.io/version: 13.0.1
name: grafana
namespace: monitoring
spec:
ports:
- name: http
nodePort: 30002
port: 3000
targetPort: http
selector:
app.kubernetes.io/component: grafana
app.kubernetes.io/name: grafana
app.kubernetes.io/part-of: kube-prometheus
type: NodePort
调整grafana的limit
默认limit是200m和200Mi 查询会崩溃
resources:
limits:
cpu: 200m
memory: 200Mi
requests:
cpu: 100m
memory: 100Mi
调整后如下
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app.kubernetes.io/component: grafana
app.kubernetes.io/name: grafana
app.kubernetes.io/part-of: kube-prometheus
app.kubernetes.io/version: 13.0.1
name: grafana
namespace: monitoring
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/component: grafana
app.kubernetes.io/name: grafana
app.kubernetes.io/part-of: kube-prometheus
template:
metadata:
annotations:
checksum/grafana-config: 602d284d964ca8083712d2a9ada8c855
checksum/grafana-dashboardproviders: b72ac93e1a01807cd416f74afcbe3a9f
checksum/grafana-datasources: 1f4d8b201b67eeadc38a68792b6b7536
labels:
app.kubernetes.io/component: grafana
app.kubernetes.io/name: grafana
app.kubernetes.io/part-of: kube-prometheus
app.kubernetes.io/version: 13.0.1
spec:
automountServiceAccountToken: false
containers:
- env: []
image: register.hrttest.cn/grafana/grafana:13.0.1
name: grafana
ports:
- containerPort: 3000
name: http
readinessProbe:
httpGet:
path: /api/health
port: http
resources:
limits:
cpu: 2000m
memory: 2000Mi
requests:
cpu: 100m
memory: 100Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
seccompProfile:
type: RuntimeDefault
volumeMounts:
- mountPath: /var/lib/grafana
name: grafana-storage
readOnly: false
- mountPath: /etc/grafana/provisioning/datasources
name: grafana-datasources
readOnly: false
- mountPath: /etc/grafana/provisioning/dashboards
name: grafana-dashboards
readOnly: false
- mountPath: /tmp
name: tmp-plugins
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/alertmanager-overview
name: grafana-dashboard-alertmanager-overview
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/apiserver
name: grafana-dashboard-apiserver
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/cluster-total
name: grafana-dashboard-cluster-total
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/controller-manager
name: grafana-dashboard-controller-manager
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/grafana-overview
name: grafana-dashboard-grafana-overview
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/k8s-resources-cluster
name: grafana-dashboard-k8s-resources-cluster
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/k8s-resources-multicluster
name: grafana-dashboard-k8s-resources-multicluster
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/k8s-resources-namespace
name: grafana-dashboard-k8s-resources-namespace
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/k8s-resources-node
name: grafana-dashboard-k8s-resources-node
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/k8s-resources-nodes-overview
name: grafana-dashboard-k8s-resources-nodes-overview
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/k8s-resources-pod
name: grafana-dashboard-k8s-resources-pod
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/k8s-resources-windows-cluster
name: grafana-dashboard-k8s-resources-windows-cluster
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/k8s-resources-windows-namespace
name: grafana-dashboard-k8s-resources-windows-namespace
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/k8s-resources-windows-pod
name: grafana-dashboard-k8s-resources-windows-pod
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/k8s-resources-workload
name: grafana-dashboard-k8s-resources-workload
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/k8s-resources-workloads-namespace
name: grafana-dashboard-k8s-resources-workloads-namespace
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/k8s-windows-cluster-rsrc-use
name: grafana-dashboard-k8s-windows-cluster-rsrc-use
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/k8s-windows-node-rsrc-use
name: grafana-dashboard-k8s-windows-node-rsrc-use
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/kubelet
name: grafana-dashboard-kubelet
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/namespace-by-pod
name: grafana-dashboard-namespace-by-pod
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/namespace-by-workload
name: grafana-dashboard-namespace-by-workload
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/node-cluster-rsrc-use
name: grafana-dashboard-node-cluster-rsrc-use
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/node-rsrc-use
name: grafana-dashboard-node-rsrc-use
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/nodes-aix
name: grafana-dashboard-nodes-aix
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/nodes-darwin
name: grafana-dashboard-nodes-darwin
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/nodes
name: grafana-dashboard-nodes
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/persistentvolumesusage
name: grafana-dashboard-persistentvolumesusage
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/pod-total
name: grafana-dashboard-pod-total
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/prometheus-remote-write
name: grafana-dashboard-prometheus-remote-write
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/prometheus
name: grafana-dashboard-prometheus
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/proxy
name: grafana-dashboard-proxy
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/scheduler
name: grafana-dashboard-scheduler
readOnly: false
- mountPath: /grafana-dashboard-definitions/0/workload-total
name: grafana-dashboard-workload-total
readOnly: false
- mountPath: /etc/grafana
name: grafana-config
readOnly: false
nodeSelector:
kubernetes.io/os: linux
securityContext:
fsGroup: 65534
runAsGroup: 65534
runAsNonRoot: true
runAsUser: 65534
serviceAccountName: grafana
volumes:
- emptyDir: {}
name: grafana-storage
- name: grafana-datasources
secret:
secretName: grafana-datasources
- configMap:
name: grafana-dashboards
name: grafana-dashboards
- emptyDir:
medium: Memory
name: tmp-plugins
- configMap:
name: grafana-dashboard-alertmanager-overview
name: grafana-dashboard-alertmanager-overview
- configMap:
name: grafana-dashboard-apiserver
name: grafana-dashboard-apiserver
- configMap:
name: grafana-dashboard-cluster-total
name: grafana-dashboard-cluster-total
- configMap:
name: grafana-dashboard-controller-manager
name: grafana-dashboard-controller-manager
- configMap:
name: grafana-dashboard-grafana-overview
name: grafana-dashboard-grafana-overview
- configMap:
name: grafana-dashboard-k8s-resources-cluster
name: grafana-dashboard-k8s-resources-cluster
- configMap:
name: grafana-dashboard-k8s-resources-multicluster
name: grafana-dashboard-k8s-resources-multicluster
- configMap:
name: grafana-dashboard-k8s-resources-namespace
name: grafana-dashboard-k8s-resources-namespace
- configMap:
name: grafana-dashboard-k8s-resources-node
name: grafana-dashboard-k8s-resources-node
- configMap:
name: grafana-dashboard-k8s-resources-nodes-overview
name: grafana-dashboard-k8s-resources-nodes-overview
- configMap:
name: grafana-dashboard-k8s-resources-pod
name: grafana-dashboard-k8s-resources-pod
- configMap:
name: grafana-dashboard-k8s-resources-windows-cluster
name: grafana-dashboard-k8s-resources-windows-cluster
- configMap:
name: grafana-dashboard-k8s-resources-windows-namespace
name: grafana-dashboard-k8s-resources-windows-namespace
- configMap:
name: grafana-dashboard-k8s-resources-windows-pod
name: grafana-dashboard-k8s-resources-windows-pod
- configMap:
name: grafana-dashboard-k8s-resources-workload
name: grafana-dashboard-k8s-resources-workload
- configMap:
name: grafana-dashboard-k8s-resources-workloads-namespace
name: grafana-dashboard-k8s-resources-workloads-namespace
- configMap:
name: grafana-dashboard-k8s-windows-cluster-rsrc-use
name: grafana-dashboard-k8s-windows-cluster-rsrc-use
- configMap:
name: grafana-dashboard-k8s-windows-node-rsrc-use
name: grafana-dashboard-k8s-windows-node-rsrc-use
- configMap:
name: grafana-dashboard-kubelet
name: grafana-dashboard-kubelet
- configMap:
name: grafana-dashboard-namespace-by-pod
name: grafana-dashboard-namespace-by-pod
- configMap:
name: grafana-dashboard-namespace-by-workload
name: grafana-dashboard-namespace-by-workload
- configMap:
name: grafana-dashboard-node-cluster-rsrc-use
name: grafana-dashboard-node-cluster-rsrc-use
- configMap:
name: grafana-dashboard-node-rsrc-use
name: grafana-dashboard-node-rsrc-use
- configMap:
name: grafana-dashboard-nodes-aix
name: grafana-dashboard-nodes-aix
- configMap:
name: grafana-dashboard-nodes-darwin
name: grafana-dashboard-nodes-darwin
- configMap:
name: grafana-dashboard-nodes
name: grafana-dashboard-nodes
- configMap:
name: grafana-dashboard-persistentvolumesusage
name: grafana-dashboard-persistentvolumesusage
- configMap:
name: grafana-dashboard-pod-total
name: grafana-dashboard-pod-total
- configMap:
name: grafana-dashboard-prometheus-remote-write
name: grafana-dashboard-prometheus-remote-write
- configMap:
name: grafana-dashboard-prometheus
name: grafana-dashboard-prometheus
- configMap:
name: grafana-dashboard-proxy
name: grafana-dashboard-proxy
- configMap:
name: grafana-dashboard-scheduler
name: grafana-dashboard-scheduler
- configMap:
name: grafana-dashboard-workload-total
name: grafana-dashboard-workload-total
- name: grafana-config
secret:
secretName: grafana-config
创建
#部署kube-prometheeus
kubectl apply --server-side -f ./setup
kubectl create -f ./
#Prometheus Operator 默认设置了 NetworkPolicy，需要手动删除后才能访问
kubectl delete -f manifests/prometheus-networkPolicy.yaml
kubectl delete -f manifests/grafana-networkPolicy.yaml
kubectl delete -f manifests/alertmanager-networkPolicy.yaml
配置nginx转发
#nginx部分
#grafana
server {
listen 80;
server_name grafana2.a.cn;
rewrite ^ https://server_namerequest_uri? permanent;
}
server {
listen 443 ssl;
server_name grafana2.a.cn;
ssl_certificate /opt/ssl/all.a.cn/a.cn.pem;
ssl_certificate_key /opt/ssl/all.a.cn/a.cn.key;
ssl_session_cache shared:SSL:1m;
ssl_session_timeout 5m;
#ssl_ciphers HIGH:!aNULL:!MD5;
ssl_ciphers 'HIGH:!aNULL:!MD5:!SHA:!DSS';
ssl_prefer_server_ciphers on;
location / {
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $host;
proxy_read_timeout 86400;
proxy_send_timeout 86400;
proxy_pass http://192.168.0.18:30002;
}
# 在 server 块内添加
location /api/live/ws {
proxy_pass http://192.168.0.18:30002;
proxy_http_version 1.1;
proxy_set_header Upgrade http_upgrade;
proxy_set_header Connection connection_upgrade;
proxy_set_header Host $host;
proxy_read_timeout 86400;
}
}