From 0567bef2a563300dd36f7dbcab7697d936df1415 Mon Sep 17 00:00:00 2001 From: marcel-dempers Date: Tue, 7 Jan 2025 15:06:31 +1100 Subject: [PATCH 1/3] upgrade VPA to latest for kubernetes 1.30 --- ...etricserver-0.3.7.yaml => components.yaml} | 355 ++++++++++-------- .../vertical-pod-autoscaling/readme.md | 18 +- .../vertical-pod-autoscaling/vpa.yaml | 2 +- 3 files changed, 213 insertions(+), 162 deletions(-) rename kubernetes/autoscaling/components/metric-server/{metricserver-0.3.7.yaml => components.yaml} (58%) diff --git a/kubernetes/autoscaling/components/metric-server/metricserver-0.3.7.yaml b/kubernetes/autoscaling/components/metric-server/components.yaml similarity index 58% rename from kubernetes/autoscaling/components/metric-server/metricserver-0.3.7.yaml rename to kubernetes/autoscaling/components/metric-server/components.yaml index bdd960c..dae3931 100644 --- a/kubernetes/autoscaling/components/metric-server/metricserver-0.3.7.yaml +++ b/kubernetes/autoscaling/components/metric-server/components.yaml @@ -1,153 +1,202 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: system:aggregated-metrics-reader - labels: - rbac.authorization.k8s.io/aggregate-to-view: "true" - rbac.authorization.k8s.io/aggregate-to-edit: "true" - rbac.authorization.k8s.io/aggregate-to-admin: "true" -rules: -- apiGroups: ["metrics.k8s.io"] - resources: ["pods", "nodes"] - verbs: ["get", "list", "watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: metrics-server:system:auth-delegator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator -subjects: -- kind: ServiceAccount - name: metrics-server - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: metrics-server-auth-reader - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader -subjects: -- kind: ServiceAccount - name: metrics-server - namespace: kube-system ---- -apiVersion: apiregistration.k8s.io/v1beta1 -kind: APIService -metadata: - name: v1beta1.metrics.k8s.io -spec: - service: - name: metrics-server - namespace: kube-system - group: metrics.k8s.io - version: v1beta1 - insecureSkipTLSVerify: true - groupPriorityMinimum: 100 - versionPriority: 100 ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: metrics-server - namespace: kube-system ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: metrics-server - namespace: kube-system - labels: - k8s-app: metrics-server -spec: - selector: - matchLabels: - k8s-app: metrics-server - template: - metadata: - name: metrics-server - labels: - k8s-app: metrics-server - spec: - serviceAccountName: metrics-server - volumes: - # mount in tmp so we can safely use from-scratch images and/or read-only containers - - name: tmp-dir - emptyDir: {} - containers: - - name: metrics-server - image: k8s.gcr.io/metrics-server/metrics-server:v0.3.7 - imagePullPolicy: IfNotPresent - args: - - --cert-dir=/tmp - - --secure-port=4443 - - --kubelet-insecure-tls #remove these for production: only used for kind - - --kubelet-preferred-address-types="InternalIP" #remove these for production: only used for kind - ports: - - name: main-port - containerPort: 4443 - protocol: TCP - securityContext: - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 1000 - volumeMounts: - - name: tmp-dir - mountPath: /tmp - nodeSelector: - kubernetes.io/os: linux - kubernetes.io/arch: "amd64" ---- -apiVersion: v1 -kind: Service -metadata: - name: metrics-server - namespace: kube-system - labels: - kubernetes.io/name: "Metrics-server" - kubernetes.io/cluster-service: "true" -spec: - selector: - k8s-app: metrics-server - ports: - - port: 443 - protocol: TCP - targetPort: main-port ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: system:metrics-server -rules: -- apiGroups: - - "" - resources: - - pods - - nodes - - nodes/stats - - namespaces - - configmaps - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: system:metrics-server -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:metrics-server -subjects: -- kind: ServiceAccount - name: metrics-server - namespace: kube-system +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader +rules: +- apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server +rules: +- apiGroups: + - "" + resources: + - nodes/metrics + verbs: + - get +- apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + k8s-app: metrics-server + name: metrics-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: metrics-server + name: metrics-server:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + k8s-app: metrics-server +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: metrics-server + strategy: + rollingUpdate: + maxUnavailable: 0 + template: + metadata: + labels: + k8s-app: metrics-server + spec: + containers: + - args: + - --cert-dir=/tmp + - --secure-port=10250 + - --kubelet-insecure-tls #remove these for production: only used for kind + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --metric-resolution=15s + image: registry.k8s.io/metrics-server/metrics-server:v0.7.2 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + name: metrics-server + ports: + - containerPort: 10250 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /tmp + name: tmp-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: metrics-server + volumes: + - emptyDir: {} + name: tmp-dir +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + labels: + k8s-app: metrics-server + name: v1beta1.metrics.k8s.io +spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: metrics-server + namespace: kube-system + version: v1beta1 + versionPriority: 100 diff --git a/kubernetes/autoscaling/vertical-pod-autoscaling/readme.md b/kubernetes/autoscaling/vertical-pod-autoscaling/readme.md index 605398a..31c0a8c 100644 --- a/kubernetes/autoscaling/vertical-pod-autoscaling/readme.md +++ b/kubernetes/autoscaling/vertical-pod-autoscaling/readme.md @@ -7,7 +7,7 @@ Lets create a Kubernetes cluster to play with using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/) ``` -kind create cluster --name vpa --image kindest/node:v1.19.1 +kind create cluster --name vpa --image kindest/node:v1.30.4 ```
@@ -20,8 +20,8 @@ kind create cluster --name vpa --image kindest/node:v1.19.1 [Metric Server](https://github.com/kubernetes-sigs/metrics-server) provides container resource metrics for use in autoscaling pipelines
-Because I run K8s `1.19` in `kind`, the Metric Server version i need is `0.3.7`
-We will need to deploy Metric Server [0.3.7](https://github.com/kubernetes-sigs/metrics-server/releases/tag/v0.3.7)
+Because I run K8s `1.30` in `kind`, the Metric Server version i need is `0.7.2`
+We will need to deploy Metric Server [0.7.2](https://github.com/kubernetes-sigs/metrics-server/releases/tag/v0.7.2)
I used `components.yaml`from the release page link above.
Important Note : For Demo clusters (like `kind`), you will need to disable TLS
@@ -31,15 +31,13 @@ You can disable TLS by adding the following to the metrics-server container args ``` - --kubelet-insecure-tls -- --kubelet-preferred-address-types="InternalIP" - ``` Deployment:
``` cd kubernetes\autoscaling -kubectl -n kube-system apply -f .\components\metric-server\metricserver-0.3.7.yaml +kubectl -n kube-system apply -f .\components\metric-server\components.yaml #test kubectl -n kube-system get pods @@ -51,12 +49,12 @@ kubectl top nodes ## VPA -VPA docs [here]("https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler#install-command")
+VPA docs [here](https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler#install-command)
Let's install the VPA from a container that can access our cluster ``` cd kubernetes/autoscaling/vertical-pod-autoscaling -docker run -it --rm -v ${HOME}:/root/ -v ${PWD}:/work -w /work --net host debian:buster bash +docker run -it --rm -v ${HOME}:/root/ -v ${PWD}:/work -w /work --net host debian:bookworm bash # install git apt-get update && apt-get install -y git curl nano @@ -71,6 +69,10 @@ cd /tmp git clone https://github.com/kubernetes/autoscaler.git cd autoscaler/vertical-pod-autoscaler/ +# you may need to generate VPA certificates +bash ./pkg/admission-controller/gencerts.sh + +# deploy the VPA ./hack/vpa-up.sh # after few seconds, we can see the VPA components in: diff --git a/kubernetes/autoscaling/vertical-pod-autoscaling/vpa.yaml b/kubernetes/autoscaling/vertical-pod-autoscaling/vpa.yaml index 118da5f..a1d0dcf 100644 --- a/kubernetes/autoscaling/vertical-pod-autoscaling/vpa.yaml +++ b/kubernetes/autoscaling/vertical-pod-autoscaling/vpa.yaml @@ -8,4 +8,4 @@ spec: kind: Deployment name: application-cpu updatePolicy: - updateMode: "Off" \ No newline at end of file + updateMode: "Off" # Auto for automatic updates, Off for manual updates \ No newline at end of file From 6d217aad294fc9f39a2731252762931d4d15b69e Mon Sep 17 00:00:00 2001 From: marcel-dempers Date: Wed, 8 Jan 2025 14:49:36 +1100 Subject: [PATCH 2/3] fix kubectl exec --- kubernetes/autoscaling/vertical-pod-autoscaling/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/autoscaling/vertical-pod-autoscaling/readme.md b/kubernetes/autoscaling/vertical-pod-autoscaling/readme.md index 31c0a8c..56b0fe1 100644 --- a/kubernetes/autoscaling/vertical-pod-autoscaling/readme.md +++ b/kubernetes/autoscaling/vertical-pod-autoscaling/readme.md @@ -108,7 +108,7 @@ cd kubernetes\autoscaling\components\application kubectl apply -f .\traffic-generator.yaml # get a terminal to the traffic-generator -kubectl exec -it traffic-generator sh +kubectl exec -it traffic-generator -- sh # install wrk apk add --no-cache wrk From 83946594481ae43c081cf3bd89796bca24f25494 Mon Sep 17 00:00:00 2001 From: marceldempers Date: Wed, 8 Jan 2025 22:16:57 +1100 Subject: [PATCH 3/3] add sidecar pod example --- kubernetes/pods/pod-sidecar.yaml | 64 ++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 kubernetes/pods/pod-sidecar.yaml diff --git a/kubernetes/pods/pod-sidecar.yaml b/kubernetes/pods/pod-sidecar.yaml new file mode 100644 index 0000000..dcbfd6f --- /dev/null +++ b/kubernetes/pods/pod-sidecar.yaml @@ -0,0 +1,64 @@ +# kind create cluster --name pods --image kindest/node:v1.31.1 +apiVersion: v1 +kind: Pod +metadata: + name: example-pod + namespace: default + labels: + app: example-app +spec: + shareProcessNamespace: true + nodeSelector: + kubernetes.io/os: linux + containers: + - name: config-watcher + image: alpine:latest + command: ["/bin/sh", "-c"] + args: + - | + apk add --no-cache inotify-tools + while true; do + inotifywait -e modify /etc/nginx/nginx.conf + pkill -HUP nginx + done + volumeMounts: + - name: config-volume + mountPath: /etc/nginx/ + - name: nginx + image: nginx:latest + imagePullPolicy: Always + ports: + - containerPort: 80 + volumeMounts: + - name: config-volume + mountPath: /etc/nginx/ + volumes: + - name: config-volume + configMap: + name: nginx-config +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-config +data: + nginx.conf: | + events {} + http { + server { + listen 80; + server_name localhost; + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + location = /health { + access_log off; + default_type text/plain; + add_header Content-Type text/plain; + return 200 "ok"; + } + } + } \ No newline at end of file