kubernetes 性能與監(jiān)控 部署Dashboard 部署Prometheus grafana安裝
部署metrics-server
# spec.containers.command 最下面手動添加如下一行 --enable-aggregator-routing=true root@master ~ # systemctl restart kubelet root@master ~ # kubectl -n kube-system get pod kube-apiserver-master -o yaml |grep enable-aggregator-routing --enable-aggregator-routing=true
root@master ~ # vim /etc/kubernetes/manifests/kube-apiserver.yaml要在所有節(jié)點執(zhí)行(master,node-0001,node-0002,node-0003)
申請的多余證書可以使用 (kubectl delete certificatesigningrequests 證書名稱) 刪除
[root@node-0001 ~]# vim /var/lib/kubelet/config.yaml
# 在文件的最后一行添加
serverTLSBootstrap: true
[root@node-0001 ~]# systemctl restart kubelet
#-------------------------簽發(fā)證書必須在 master 上執(zhí)行 ------------------------------------
[root@master ~]# kubectl get certificatesigningrequests
NAME? ? ? ? AGE? ?REQUESTOR? ? ? ? ? ? CONDITION
csr-wsfz7? ?8s? ? system:node:master? ?Pending
[root@master ~]# kubectl certificate approve csr-wsfz7
[root@master ~]# kubectl get certificatesigningrequests
NAME? ? ? ? AGE? ?REQUESTOR? ? ? ? ? ? CONDITION
csr-wsfz7? ?86s? ?system:node:master? ?Approved,Issued
安裝mertics插件
[root@master metrics]# docker load -i metrisc-server.tar.gz
[root@master metrics]# docker tag gcr.io/k8s-staging-metrics-server/metrics-server:master 192.168.1.100:5000/metrics-server:master
[root@master metrics]# docker push 192.168.1.100:5000/metrics-server:master
[root@master metrics]# vim deployment.yaml
29: image: 192.168.1.100:5000/metrics-server:master
[root@master metrics]# kubectl apply -f rbac.yaml
[root@master metrics]# kubectl apply -f pdb.yaml
[root@master metrics]# kubectl apply -f deployment.yaml
[root@master metrics]# kubectl apply -f service.yaml
[root@master metrics]# kubectl apply -f apiservice.yaml
#-------------------------------- 查詢驗證 ----------------------------------------------
[root@master metrics]# kubectl -n kube-system get pod
NAME? ? ? ? ? ? ? ? ? ? ? ? ? ? ? READY? ?STATUS? ? RESTARTS? ?AGE
metrics-server-78dfb54777-4dcjl? ?1/1? ? ?Running? ?0? ? ? ? ? 116s
[root@master metrics]# kubectl -n kube-system get apiservices
NAME? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?SERVICE? ? ? ? ? ? ? ? ? ? ? AVAILABLE? ?AGE
v1beta1.metrics.k8s.io? ? ? ? ? ? ? ? ?kube-system/metrics-server? ?True? ? ? ? 2m20s
[root@master metrics]# kubectl top node
NAME? ? ? ? CPU(cores)? ?CPU%? ?MEMORY(bytes)? ?MEMORY%? ?
master? ? ? 95m? ? ? ? ? 4%? ? ?840Mi? ? ? ? ? ?48%? ? ? ?
node-0001? ?24m? ? ? ? ? 1%? ? ?266Mi? ? ? ? ? ?15%? ? ? ?
node-0002? ?24m? ? ? ? ? 1%? ? ?270Mi? ? ? ? ? ?15%? ? ? ?
node-0003? ?26m? ? ? ? ? 1%? ? ?280Mi? ? ? ? ? ?16%? ?
[root@master ~]# kubectl apply -f myapache.yaml?
deployment.apps/myapache created
[root@master ~]# kubectl top pod
error: metrics not available yet
# 等待大約 30 秒
[root@master ~]# kubectl top pod
NAME? ? ? ? ? ? ? ? ? ? ? ?CPU(cores)? ?MEMORY(bytes)? ?
myapache-7d689bf8f-lfr5h? ?0m? ? ? ? ? ?0Mi? ?
[root@master ~]# curl http://10.244.2.17/info.php?id=5000000
<pre>
Array
(
? ? [REMOTE_ADDR] => 10.244.0.0
? ? [REQUEST_METHOD] => GET
? ? [HTTP_USER_AGENT] => curl/7.29.0
? ? [REQUEST_URI] => /info.php?id=5000000
? ? [id] => 5000000
)
php_host: myapache-7d689bf8f-lfr5h
[root@master ~]# kubectl top pod
NAME? ? ? ? ? ? ? ? ? ? ? ?CPU(cores)? ?MEMORY(bytes)? ?
myapache-7d689bf8f-w4rtt? ?1000m? ? ? ? 8Mi??
[root@master dashboard]# vim recommended.yaml
? ? ? ? ? # 190 行修改為
? ? ? ? ? image: 192.168.1.100:5000/dashboard:v2.0.0
? ? ? ? ? # 274 行修改為
? ? ? ? ? image: 192.168.1.100:5000/metrics-scraper:v1.0.4
[root@master dashboard]# kubectl apply -f recommended.yaml
# ---------------------------------- 查詢驗證 --------------------------------------
[root@master dashboard]# kubectl -n kubernetes-dashboard get pod
NAME? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?READY? ?STATUS? ? RESTARTS? ?AGE
dashboard-metrics-scraper-57bf85fcc9-vsz74? ?1/1? ? ?Running? ?0? ? ? ? ? 52s
kubernetes-dashboard-7b7f78bcf9-5k8vq? ? ? ? 1/1? ? ?Running? ?0? ? ? ? ? 52s
[root@master dashboard]# kubectl -n kubernetes-dashboard get service
NAME? ? ? ? ? ? ? ? ? ? ? ? TYPE? ? ? ? CLUSTER-IP? ? ? ?EXTERNAL-IP? ?PORT(S)
dashboard-metrics-scraper? ?ClusterIP? ?10.254.76.85? ? ?<none>? ? ? ? 8000/TCP
kubernetes-dashboard? ? ? ? ClusterIP? ?10.254.211.125? ?<none>? ? ? ? 443/TCP
# ---------------------------------- 對外發(fā)布服務(wù) -----------------------------------
[root@master dashboard]# vim service.yaml
---
kind: Service
apiVersion: v1
metadata:
? labels:
? ? k8s-app: kubernetes-dashboard
? name: kubernetes-dashboard
? namespace: kubernetes-dashboard
spec:
? ports:
? ? - port: 443
? ? ? nodePort: 30443? ? ? ? ? ? ? ? # 新添加
? ? ? targetPort: 8443
? selector:
? ? k8s-app: kubernetes-dashboard
? type: NodePort? ? ? ? ? ? ? ? ? ? ?# 新添加
[root@master dashboard]# kubectl apply -f service.yaml?
service/kubernetes-dashboard configured
[root@master dashboard]# kubectl -n kubernetes-dashboard get service
NAME? ? ? ? ? ? ? ? ? ? ? ? TYPE? ? ? ? CLUSTER-IP? ? ? ?EXTERNAL-IP? ?PORT(S)? ? ? ? ?AGE
dashboard-metrics-scraper? ?ClusterIP? ?10.254.66.25? ? ?<none>? ? ? ? 8000/TCP? ? ? ? 2m6s
kubernetes-dashboard? ? ? ? NodePort? ? 10.254.165.155? ?<none>? ? ? ? 443:30443/TCP? ?2m6s
[root@master dashboard]# kubectl apply -f admin-token.yaml
[root@master ~]# kubectl -n kubernetes-dashboard get secrets?
NAME? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?TYPE? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? DATA? ?AGE
admin-user-token-bxjlz? ? ? ? ? ? ?kubernetes.io/service-account-token? ?3? ? ? 23s
[root@master ~]# kubectl -n kubernetes-dashboard describe secrets admin-user-token-bxjlz
Name:? ? ? ? ?admin-user-token-bxjlz
... ...
ca.crt:? ? ?1025 bytes
namespace:? 20 bytes
token:? ? ? 這里這個很長的字符串就是你要找的認(rèn)證 token
[root@master images]# for i in *.gz;do docker load -i ${i};done
[root@master images]# img="prom/node-exporter v1.0.0
quay.io/coreos/prometheus-config-reloader v0.35.1
quay.io/coreos/prometheus-operator v0.35.1
quay.io/coreos/kube-state-metrics v1.9.2
grafana/grafana 6.4.3
jimmidyson/configmap-reload v0.3.0
quay.io/prometheus/prometheus v2.11.0
quay.io/prometheus/alertmanager v0.18.0
quay.io/coreos/k8s-prometheus-adapter-amd64 v0.5.0
quay.io/coreos/kube-rbac-proxy v0.4.1"
[root@master images]# while read _f _v;do?
? docker tag ${_f}:${_v} 192.168.1.100:5000/${_f##*/}:${_v}
? docker push 192.168.1.100:5000/${_f##*/}:${_v}
? docker rmi ${_f}:${_v}
done <<<"${img}"
[root@master images]# curl http://192.168.1.100:5000/v2/_catalog
{"repositories":["alertmanager","configmap-reload","coredns","dashboard","etcd","flannel","grafana","k8s-prometheus-adapter-amd64","kube-apiserver","kube-controller-manager","kube-proxy","kube-rbac-proxy","kube-scheduler","kube-state-metrics","metrics-scraper","metrics-server","myos","nginx-ingress-controller","node-exporter","pause","prometheus","prometheus-config-reloader","prometheus-operator"]}
[root@master prometheus]# curl http://192.168.1.100:5000/v2/configmap-reload/tags/list
{"name":"configmap-reload","tags":["v0.3.0"]}
[root@master prometheus]# curl http://192.168.1.100:5000/v2/prometheus-config-reloader/tags/list
{"name":"prometheus-config-reloader","tags":["v0.35.1"]}
[root@master prometheus]# curl http://192.168.1.100:5000/v2/prometheus-operator/tags/list
{"name":"prometheus-operator","tags":["v0.35.1"]}
[root@master prometheus]# vim setup/prometheus-operator-deployment.yaml
27:? ? ? ? - --config-reloader-image=192.168.1.100:5000/configmap-reload:v0.3.0
28:? ? ? ? - --prometheus-config-reloader=192.168.1.100:5000/prometheus-config-reloader:v0.35.1
29:? ? ? ? image: 192.168.1.100:5000/prometheus-operator:v0.35.1
# 驗證安裝
[root@master prometheus]# kubectl apply -f setup/
[root@master prometheus]# kubectl -n monitoring get pod
NAME? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?READY? ?STATUS? ? RESTARTS? ?AGE
prometheus-operator-75b4b59b74-72qhg? ?1/1? ? ?Running? ?0? ? ? ? ? 47s
[root@master prometheus]# curl http://192.168.1.100:5000/v2/prometheus/tags/list
{"name":"prometheus","tags":["v2.11.0"]}
[root@master prometheus]# vim prom-server/prometheus-prometheus.yaml
14: ? baseImage: 192.168.1.100:5000/prometheus
34: ? version: v2.11.0
[root@master prometheus]# kubectl apply -f prom-server/
[root@master prometheus]# kubectl -n monitoring get pod
NAME? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?READY? ?STATUS? ? RESTARTS? ?AGE
prometheus-k8s-0? ? ? ? ? ? ? ? ? ? ? ?3/3? ? ?Running? ?1? ? ? ? ? 45s
prometheus-k8s-1? ? ? ? ? ? ? ? ? ? ? ?3/3? ? ?Running? ?1? ? ? ? ? 45s
[root@master prometheus]# curl http://192.168.1.100:5000/v2/k8s-prometheus-adapter-amd64/tags/list
{"name":"k8s-prometheus-adapter-amd64","tags":["v0.5.0"]}
[root@master prometheus]# vim prom-adapter/prometheus-adapter-deployment.yaml
28:? ? ? image: 192.168.1.100:5000/k8s-prometheus-adapter-amd64:v0.5.0
[root@master prometheus]# kubectl apply -f prom-adapter
[root@master prometheus]# kubectl -n monitoring get pod
NAME? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?READY? ?STATUS? ? RESTARTS? ?AGE
prometheus-adapter-856854f9f6-knqtq? ? 1/1? ? ?Running? ?0? ? ? ? ? 6s
[root@master prometheus]# curl http://192.168.1.100:5000/v2/kube-state-metrics/tags/list
{"name":"kube-state-metrics","tags":["v1.9.2"]}
[root@master prometheus]# curl http://192.168.1.100:5000/v2/kube-rbac-proxy/tags/list
{"name":"kube-rbac-proxy","tags":["v0.4.1"]}
[root@master prometheus]# vim metrics-state/kube-state-metrics-deployment.yaml
24: ? ? ? ? image: 192.168.1.100:5000/kube-rbac-proxy:v0.4.1
41: ? ? ? ? image: 192.168.1.100:5000/kube-rbac-proxy:v0.4.1
58: ? ? ? ? image: 192.168.1.100:5000/kube-state-metrics:v1.9.2
[root@master prometheus]# kubectl apply -f metrics-state/
[root@master prometheus]# kubectl -n monitoring get pod
NAME? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?READY? ?STATUS? ? RESTARTS? ?AGE
kube-state-metrics-5894f64799-krvn6? ? 3/3? ? ?Running? ?0? ? ? ? ? 4s
[root@master prometheus]# curl http://192.168.1.100:5000/v2/node-exporter/tags/list
{"name":"node-exporter","tags":["v1.0.0"]}
[root@master prometheus]# curl http://192.168.1.100:5000/v2/kube-rbac-proxy/tags/list
{"name":"kube-rbac-proxy","tags":["v0.4.1"]}
[root@master prometheus]# vim node-exporter/node-exporter-daemonset.yaml
27: ? ? ? ? image: 192.168.1.100:5000/node-exporter:v1.0.0
57: ? ? ? ? image: 192.168.1.100:5000/kube-rbac-proxy:v0.4.1
[root@master prometheus]# kubectl apply -f node-exporter/
[root@master prometheus]# kubectl -n monitoring get pod
NAME? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?READY? ?STATUS? ? RESTARTS? ?AGE
node-exporter-7h4l9? ? ? ? ? ? ? ? ? ? 2/2? ? ?Running? ?0? ? ? ? ? 7s
node-exporter-7vxmx? ? ? ? ? ? ? ? ? ? 2/2? ? ?Running? ?0? ? ? ? ? 7s
node-exporter-mr6lw? ? ? ? ? ? ? ? ? ? 2/2? ? ?Running? ?0? ? ? ? ? 7s
node-exporter-zg2j8? ? ? ? ? ? ? ? ? ? 2/2? ? ?Running? ?0? ? ? ? ? 7s
[root@master prometheus]# curl http://192.168.1.100:5000/v2/alertmanager/tags/list
{"name":"alertmanager","tags":["v0.18.0"]}
[root@master prometheus]# vim alertmanager/alertmanager-alertmanager.yaml
09: baseImage: 192.168.1.100:5000/alertmanager
18: version: v0.18.0
[root@master prometheus]# kubectl apply -f alertmanager/
[root@master prometheus]# kubectl -n monitoring get pod
NAME? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?READY? ?STATUS? ? RESTARTS? ?AGE
alertmanager-main-0? ? ? ? ? ? ? ? ? ? 2/2? ? ?Running? ?0? ? ? ? ? 16s
alertmanager-main-1? ? ? ? ? ? ? ? ? ? 2/2? ? ?Running? ?0? ? ? ? ? 16s
alertmanager-main-2? ? ? ? ? ? ? ? ? ? 2/2? ? ?Running? ?0? ? ? ? ? 16s
[root@master prometheus]# curl http://192.168.1.100:5000/v2/grafana/tags/list
{"name":"grafana","tags":["6.4.3"]}
[root@master prometheus]# vim grafana/grafana-deployment.yaml
19:? ? ?- image: 192.168.1.100:5000/grafana:6.4.3
[root@master prometheus]# kubectl apply -f grafana/
[root@master prometheus]# kubectl -n monitoring get pod
NAME? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?READY? ?STATUS? ? RESTARTS? ?AGE
grafana-647d948b69-d2hv9? ? ? ? ? ? ? ?1/1? ? ?Running? ?0? ? ? ? ? 19s
[root@master prometheus]# cp grafana/grafana-service.yaml ./
[root@master prometheus]# vim grafana-service.yaml
apiVersion: v1
kind: Service
metadata:
? labels:
? ? app: grafana
? name: grafana
? namespace: monitoring
spec:
? type: NodePort? ? ? ? ? # 新添加
? ports:
? - name: http
? ? port: 3000
? ? nodePort: 30000? ? ? ?# 新添加
? ? targetPort: http
? selector:
? ? app: grafana
[root@master prometheus]# kubectl apply -f grafana-service.yaml
[root@master prometheus]# kubectl -n monitoring get service
NAME? ? ? ? ? ? ? ? ? ? TYPE? ? ? ? CLUSTER-IP? ? ? ?EXTERNAL-IP? ?PORT(S)
grafana? ? ? ? ? ? ? ? ?NodePort? ? 10.254.79.49? ? ?<none>? ? ? ? 3000:30000/TCP