first commit
This commit is contained in:
22
autoscaling/components/application/app.go
Normal file
22
autoscaling/components/application/app.go
Normal file
@@ -0,0 +1,22 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func main(){
|
||||
http.HandleFunc("/", useCPU)
|
||||
http.ListenAndServe(":80", nil)
|
||||
}
|
||||
|
||||
func useCPU(w http.ResponseWriter, r *http.Request) {
|
||||
count := 1
|
||||
|
||||
for i := 1; i <= 1000000; i++ {
|
||||
count = i
|
||||
}
|
||||
|
||||
fmt.Printf("count: %d", count)
|
||||
w.Write([]byte(string(count)))
|
||||
}
|
||||
50
autoscaling/components/application/deployment.yaml
Normal file
50
autoscaling/components/application/deployment.yaml
Normal file
@@ -0,0 +1,50 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: application-cpu
|
||||
labels:
|
||||
app: application-cpu
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: application-cpu
|
||||
ports:
|
||||
- protocol: TCP
|
||||
name: http
|
||||
port: 80
|
||||
targetPort: 80
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: application-cpu
|
||||
labels:
|
||||
app: application-cpu
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: application-cpu
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: application-cpu
|
||||
spec:
|
||||
containers:
|
||||
- name: application-cpu
|
||||
image: aimvector/application-cpu:v1.0.2
|
||||
imagePullPolicy: Always
|
||||
ports:
|
||||
- containerPort: 80
|
||||
resources:
|
||||
requests:
|
||||
memory: "50Mi"
|
||||
cpu: "500m"
|
||||
limits:
|
||||
memory: "500Mi"
|
||||
cpu: "2000m"
|
||||
15
autoscaling/components/application/dockerfile
Normal file
15
autoscaling/components/application/dockerfile
Normal file
@@ -0,0 +1,15 @@
|
||||
FROM golang:1.14-alpine as build
|
||||
|
||||
RUN apk add --no-cache git curl
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
COPY app.go /src
|
||||
|
||||
RUN go build app.go
|
||||
|
||||
FROM alpine as runtime
|
||||
|
||||
COPY --from=build /src/app /app/app
|
||||
|
||||
CMD [ "/app/app" ]
|
||||
11
autoscaling/components/application/traffic-generator.yaml
Normal file
11
autoscaling/components/application/traffic-generator.yaml
Normal file
@@ -0,0 +1,11 @@
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: traffic-generator
|
||||
spec:
|
||||
containers:
|
||||
- name: alpine
|
||||
image: alpine
|
||||
args:
|
||||
- sleep
|
||||
- "100000000"
|
||||
202
autoscaling/components/metric-server/components.yaml
Normal file
202
autoscaling/components/metric-server/components.yaml
Normal file
@@ -0,0 +1,202 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: metrics-server
|
||||
name: metrics-server
|
||||
namespace: kube-system
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: metrics-server
|
||||
rbac.authorization.k8s.io/aggregate-to-admin: "true"
|
||||
rbac.authorization.k8s.io/aggregate-to-edit: "true"
|
||||
rbac.authorization.k8s.io/aggregate-to-view: "true"
|
||||
name: system:aggregated-metrics-reader
|
||||
rules:
|
||||
- apiGroups:
|
||||
- metrics.k8s.io
|
||||
resources:
|
||||
- pods
|
||||
- nodes
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: metrics-server
|
||||
name: system:metrics-server
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- nodes/metrics
|
||||
verbs:
|
||||
- get
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
- nodes
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: metrics-server
|
||||
name: metrics-server-auth-reader
|
||||
namespace: kube-system
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: extension-apiserver-authentication-reader
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: metrics-server
|
||||
namespace: kube-system
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: metrics-server
|
||||
name: metrics-server:system:auth-delegator
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: system:auth-delegator
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: metrics-server
|
||||
namespace: kube-system
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: metrics-server
|
||||
name: system:metrics-server
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: system:metrics-server
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: metrics-server
|
||||
namespace: kube-system
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: metrics-server
|
||||
name: metrics-server
|
||||
namespace: kube-system
|
||||
spec:
|
||||
ports:
|
||||
- name: https
|
||||
port: 443
|
||||
protocol: TCP
|
||||
targetPort: https
|
||||
selector:
|
||||
k8s-app: metrics-server
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: metrics-server
|
||||
name: metrics-server
|
||||
namespace: kube-system
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
k8s-app: metrics-server
|
||||
strategy:
|
||||
rollingUpdate:
|
||||
maxUnavailable: 0
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: metrics-server
|
||||
spec:
|
||||
containers:
|
||||
- args:
|
||||
- --cert-dir=/tmp
|
||||
- --secure-port=10250
|
||||
- --kubelet-insecure-tls #remove these for production: only used for kind
|
||||
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
|
||||
- --kubelet-use-node-status-port
|
||||
- --metric-resolution=15s
|
||||
image: registry.k8s.io/metrics-server/metrics-server:v0.7.2
|
||||
imagePullPolicy: IfNotPresent
|
||||
livenessProbe:
|
||||
failureThreshold: 3
|
||||
httpGet:
|
||||
path: /livez
|
||||
port: https
|
||||
scheme: HTTPS
|
||||
periodSeconds: 10
|
||||
name: metrics-server
|
||||
ports:
|
||||
- containerPort: 10250
|
||||
name: https
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
failureThreshold: 3
|
||||
httpGet:
|
||||
path: /readyz
|
||||
port: https
|
||||
scheme: HTTPS
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 10
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 200Mi
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
volumeMounts:
|
||||
- mountPath: /tmp
|
||||
name: tmp-dir
|
||||
nodeSelector:
|
||||
kubernetes.io/os: linux
|
||||
priorityClassName: system-cluster-critical
|
||||
serviceAccountName: metrics-server
|
||||
volumes:
|
||||
- emptyDir: {}
|
||||
name: tmp-dir
|
||||
---
|
||||
apiVersion: apiregistration.k8s.io/v1
|
||||
kind: APIService
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: metrics-server
|
||||
name: v1beta1.metrics.k8s.io
|
||||
spec:
|
||||
group: metrics.k8s.io
|
||||
groupPriorityMinimum: 100
|
||||
insecureSkipTLSVerify: true
|
||||
service:
|
||||
name: metrics-server
|
||||
namespace: kube-system
|
||||
version: v1beta1
|
||||
versionPriority: 100
|
||||
180
autoscaling/readme.md
Normal file
180
autoscaling/readme.md
Normal file
@@ -0,0 +1,180 @@
|
||||
# Kubernetes Autoscaling Guide
|
||||
|
||||
## Cluster Autoscaling
|
||||
|
||||
Cluster autoscaler allows us to scale cluster nodes when they become full <br/>
|
||||
I would recommend to learn about scaling your cluster nodes before scaling pods. <br/>
|
||||
|
||||
## Horizontal Pod Autoscaling
|
||||
|
||||
HPA allows us to scale pods when their resource utilisation goes over a threshold <br/>
|
||||
|
||||
## Requirements
|
||||
|
||||
### A Cluster
|
||||
|
||||
* For both autoscaling guides, we'll need a cluster. <br/>
|
||||
* For `Cluster Autoscaler` You need a cloud based cluster that supports the cluster autoscaler <br/>
|
||||
* For `HPA` We'll use [kind](http://kind.sigs.k8s.io/)
|
||||
|
||||
### Cluster Autoscaling - Creating an AKS Cluster
|
||||
|
||||
```
|
||||
# azure example
|
||||
|
||||
NAME=aks-getting-started
|
||||
RESOURCEGROUP=aks-getting-started
|
||||
SERVICE_PRINCIPAL=
|
||||
SERVICE_PRINCIPAL_SECRET=
|
||||
|
||||
az aks create -n $NAME \
|
||||
--resource-group $RESOURCEGROUP \
|
||||
--location australiaeast \
|
||||
--kubernetes-version 1.16.10 \
|
||||
--nodepool-name default \
|
||||
--node-count 1 \
|
||||
--node-vm-size Standard_F4s_v2 \
|
||||
--node-osdisk-size 250 \
|
||||
--service-principal $SERVICE_PRINCIPAL \
|
||||
--client-secret $SERVICE_PRINCIPAL_SECRET \
|
||||
--output none \
|
||||
--enable-cluster-autoscaler \
|
||||
--min-count 1 \
|
||||
--max-count 5
|
||||
```
|
||||
|
||||
### Horizontal Pod Autocaling - Creating a Kind Cluster
|
||||
|
||||
My Node has 6 CPU cores for this demo <br/>
|
||||
|
||||
```
|
||||
kind create cluster --name hpa --image kindest/node:v1.18.4
|
||||
```
|
||||
|
||||
### Metric Server
|
||||
|
||||
* For `Cluster Autoscaler` - On cloud-based clusters, Metric server may already be installed. <br/>
|
||||
* For `HPA` - We're using kind
|
||||
|
||||
[Metric Server](https://github.com/kubernetes-sigs/metrics-server) provides container resource metrics for use in autoscaling pipelines <br/>
|
||||
|
||||
Because I run K8s `1.18` in `kind`, the Metric Server version i need is `0.3.7` <br/>
|
||||
We will need to deploy Metric Server [0.3.7](https://github.com/kubernetes-sigs/metrics-server/releases/tag/v0.3.7) <br/>
|
||||
I used `components.yaml`from the release page link above. <br/>
|
||||
|
||||
<b>Important Note</b> : For Demo clusters (like `kind`), you will need to disable TLS <br/>
|
||||
You can disable TLS by adding the following to the metrics-server container args <br/>
|
||||
|
||||
<b>For production, make sure you remove the following :</b> <br/>
|
||||
|
||||
```
|
||||
- --kubelet-insecure-tls
|
||||
- --kubelet-preferred-address-types="InternalIP"
|
||||
|
||||
```
|
||||
|
||||
Deployment: <br/>
|
||||
|
||||
|
||||
```
|
||||
cd kubernetes\autoscaling
|
||||
kubectl -n kube-system apply -f .\components\metric-server\metricserver-0.3.7.yaml
|
||||
|
||||
#test
|
||||
kubectl -n kube-system get pods
|
||||
|
||||
#note: wait for metrics to populate!
|
||||
kubectl top nodes
|
||||
|
||||
```
|
||||
|
||||
## Example Application
|
||||
|
||||
For all autoscaling guides, we'll need a simple app, that generates some CPU load <br/>
|
||||
|
||||
* Build the app
|
||||
* Push it to a registry
|
||||
* Ensure resource requirements are set
|
||||
* Deploy it to Kubernetes
|
||||
* Ensure metrics are visible for the app
|
||||
|
||||
```
|
||||
# build
|
||||
|
||||
cd kubernetes\autoscaling\components\application
|
||||
docker build . -t aimvector/application-cpu:v1.0.0
|
||||
|
||||
# push
|
||||
docker push aimvector/application-cpu:v1.0.0
|
||||
|
||||
# resource requirements
|
||||
resources:
|
||||
requests:
|
||||
memory: "50Mi"
|
||||
cpu: "500m"
|
||||
limits:
|
||||
memory: "500Mi"
|
||||
cpu: "2000m"
|
||||
|
||||
# deploy
|
||||
kubectl apply -f deployment.yaml
|
||||
|
||||
# metrics
|
||||
kubectl top pods
|
||||
|
||||
```
|
||||
|
||||
## Cluster Autoscaler
|
||||
|
||||
For cluster autoscaling, you should be able to scale the pods manually and watch the cluster scale. </br>
|
||||
Cluster autoscaling stops here. </br>
|
||||
For Pod Autoscaling (HPA), continue</br>
|
||||
|
||||
## Generate some traffic
|
||||
|
||||
Let's deploy a simple traffic generator pod
|
||||
|
||||
```
|
||||
cd kubernetes\autoscaling\components\application
|
||||
kubectl apply -f .\traffic-generator.yaml
|
||||
|
||||
# get a terminal to the traffic-generator
|
||||
kubectl exec -it traffic-generator sh
|
||||
|
||||
# install wrk
|
||||
apk add --no-cache wrk
|
||||
|
||||
# simulate some load
|
||||
wrk -c 5 -t 5 -d 99999 -H "Connection: Close" http://application-cpu
|
||||
|
||||
#you can scale to pods manually and see roughly 6-7 pods will satisfy resource requests.
|
||||
kubectl scale deploy/application-cpu --replicas 2
|
||||
```
|
||||
|
||||
## Deploy an autoscaler
|
||||
|
||||
```
|
||||
# scale the deployment back down to 2
|
||||
kubectl scale deploy/application-cpu --replicas 2
|
||||
|
||||
# deploy the autoscaler
|
||||
kubectl autoscale deploy/application-cpu --cpu-percent=95 --min=1 --max=10
|
||||
|
||||
# pods should scale to roughly 6-7 to match criteria of 95% of resource requests
|
||||
|
||||
kubectl get pods
|
||||
kubectl top pods
|
||||
kubectl get hpa/application-cpu -owide
|
||||
|
||||
kubectl describe hpa/application-cpu
|
||||
|
||||
```
|
||||
|
||||
## Vertical Pod Autoscaling
|
||||
|
||||
The vertical pod autoscaler allows us to automatically set request values on our pods <br/>
|
||||
based on recommendations.
|
||||
This helps us tune the request values based on actual CPU and Memory usage.<br/>
|
||||
|
||||
More [here](./vertical-pod-autoscaling/readme.md)
|
||||
|
||||
146
autoscaling/vertical-pod-autoscaling/readme.md
Normal file
146
autoscaling/vertical-pod-autoscaling/readme.md
Normal file
@@ -0,0 +1,146 @@
|
||||
# Vertical Pod Autoscaling
|
||||
|
||||
## We need a Kubernetes cluster
|
||||
|
||||
Lets create a Kubernetes cluster to play with using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/)
|
||||
|
||||
```
|
||||
kind create cluster --name vpa --image kindest/node:v1.30.4
|
||||
```
|
||||
<hr/>
|
||||
|
||||
## Metric Server
|
||||
|
||||
<br/>
|
||||
|
||||
* For `Cluster Autoscaler` - On cloud-based clusters, Metric server may already be installed. <br/>
|
||||
* For `HPA` - We're using kind
|
||||
|
||||
[Metric Server](https://github.com/kubernetes-sigs/metrics-server) provides container resource metrics for use in autoscaling pipelines <br/>
|
||||
|
||||
Because I run K8s `1.30` in `kind`, the Metric Server version i need is `0.7.2` <br/>
|
||||
We will need to deploy Metric Server [0.7.2](https://github.com/kubernetes-sigs/metrics-server/releases/tag/v0.7.2) <br/>
|
||||
I used `components.yaml`from the release page link above. <br/>
|
||||
|
||||
<b>Important Note</b> : For Demo clusters (like `kind`), you will need to disable TLS <br/>
|
||||
You can disable TLS by adding the following to the metrics-server container args <br/>
|
||||
|
||||
<b>For production, make sure you remove the following :</b> <br/>
|
||||
|
||||
```
|
||||
- --kubelet-insecure-tls
|
||||
```
|
||||
|
||||
Deployment: <br/>
|
||||
|
||||
```
|
||||
cd kubernetes\autoscaling
|
||||
kubectl -n kube-system apply -f .\components\metric-server\components.yaml
|
||||
|
||||
#test
|
||||
kubectl -n kube-system get pods
|
||||
|
||||
#note: wait for metrics to populate!
|
||||
kubectl top nodes
|
||||
|
||||
```
|
||||
|
||||
## VPA
|
||||
|
||||
VPA docs [here](https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler#install-command) <br/>
|
||||
Let's install the VPA from a container that can access our cluster
|
||||
|
||||
```
|
||||
cd kubernetes/autoscaling/vertical-pod-autoscaling
|
||||
docker run -it --rm -v ${HOME}:/root/ -v ${PWD}:/work -w /work --net host debian:bookworm bash
|
||||
|
||||
# install git
|
||||
apt-get update && apt-get install -y git curl nano
|
||||
|
||||
# install kubectl
|
||||
curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl
|
||||
chmod +x ./kubectl
|
||||
mv ./kubectl /usr/local/bin/kubectl
|
||||
|
||||
|
||||
cd /tmp
|
||||
git clone https://github.com/kubernetes/autoscaler.git
|
||||
cd autoscaler/vertical-pod-autoscaler/
|
||||
|
||||
# you may need to generate VPA certificates
|
||||
bash ./pkg/admission-controller/gencerts.sh
|
||||
|
||||
# deploy the VPA
|
||||
./hack/vpa-up.sh
|
||||
|
||||
# after few seconds, we can see the VPA components in:
|
||||
|
||||
kubectl -n kube-system get pods
|
||||
```
|
||||
|
||||
## Build and deploy example app
|
||||
|
||||
```
|
||||
# build
|
||||
|
||||
cd kubernetes\autoscaling\components\application
|
||||
docker build . -t aimvector/application-cpu:v1.0.0
|
||||
|
||||
# push
|
||||
docker push aimvector/application-cpu:v1.0.0
|
||||
|
||||
# deploy
|
||||
kubectl apply -f deployment.yaml
|
||||
|
||||
# metrics
|
||||
kubectl top pods
|
||||
|
||||
```
|
||||
|
||||
## Generate some traffic
|
||||
|
||||
Let's deploy a simple traffic generator pod
|
||||
|
||||
```
|
||||
cd kubernetes\autoscaling\components\application
|
||||
kubectl apply -f .\traffic-generator.yaml
|
||||
|
||||
# get a terminal to the traffic-generator
|
||||
kubectl exec -it traffic-generator -- sh
|
||||
|
||||
# install wrk
|
||||
apk add --no-cache wrk
|
||||
|
||||
# simulate some load
|
||||
wrk -c 5 -t 5 -d 99999 -H "Connection: Close" http://application-cpu
|
||||
|
||||
```
|
||||
|
||||
# Deploy an example VPA
|
||||
|
||||
```
|
||||
|
||||
kubectl apply -f .\vertical-pod-autoscaling\vpa.yaml
|
||||
|
||||
kubectl describe vpa application-cpu
|
||||
|
||||
```
|
||||
|
||||
# Deploy Goldilocks
|
||||
|
||||
```
|
||||
cd /tmp
|
||||
git clone https://github.com/FairwindsOps/goldilocks.git
|
||||
cd goldilocks/hack/manifests/
|
||||
|
||||
kubectl create namespace goldilocks
|
||||
kubectl -n goldilocks apply -f ./controller
|
||||
kubectl -n goldilocks apply -f ./dashboard
|
||||
|
||||
|
||||
kubectl label ns default goldilocks.fairwinds.com/enabled=true
|
||||
kubectl label ns default goldilocks.fairwinds.com/vpa-update-mode="off"
|
||||
|
||||
kubectl -n goldilocks port-forward svc/goldilocks-dashboard 80
|
||||
|
||||
```
|
||||
11
autoscaling/vertical-pod-autoscaling/vpa.yaml
Normal file
11
autoscaling/vertical-pod-autoscaling/vpa.yaml
Normal file
@@ -0,0 +1,11 @@
|
||||
apiVersion: autoscaling.k8s.io/v1
|
||||
kind: VerticalPodAutoscaler
|
||||
metadata:
|
||||
name: application-cpu
|
||||
spec:
|
||||
targetRef:
|
||||
apiVersion: "apps/v1"
|
||||
kind: Deployment
|
||||
name: application-cpu
|
||||
updatePolicy:
|
||||
updateMode: "Off" # Auto for automatic updates, Off for manual updates
|
||||
Reference in New Issue
Block a user