From 9b130c3b65cc1ecb067645bbbb18a68117b9dbb7 Mon Sep 17 00:00:00 2001 From: marcel-dempers Date: Mon, 7 Nov 2022 21:21:40 +1100 Subject: [PATCH 1/3] nginx wip --- kubernetes/ingress/controller/nginx/README.md | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 kubernetes/ingress/controller/nginx/README.md diff --git a/kubernetes/ingress/controller/nginx/README.md b/kubernetes/ingress/controller/nginx/README.md new file mode 100644 index 0000000..026c29a --- /dev/null +++ b/kubernetes/ingress/controller/nginx/README.md @@ -0,0 +1,46 @@ +# Introduction to NGINX Ingress Controller + +## Create a kubernetes cluster + +In this guide we we''ll need a Kubernetes cluster for testing. Let's create one using [kind](https://kind.sigs.k8s.io/)
+ +``` +kind create cluster --name nginx-ingress --image kindest/node:v1.23.5 +``` + +See cluster up and running: + +``` +kubectl get nodes +NAME STATUS ROLES AGE VERSION +nginx-ingress-control-plane Ready control-plane,master 2m12s v1.23.5 +``` + +## Run a container to work in + +### run Alpine Linux: +``` +docker run -it --rm -v ${HOME}:/root/ -v ${PWD}:/work -w /work --net host alpine sh +``` + +### install some tools + +``` +# install curl +apk add --no-cache curl + +# install kubectl +curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl +chmod +x ./kubectl +mv ./kubectl /usr/local/bin/kubectl + +# install helm + +``` + +### test cluster access: +``` +/work # kubectl get nodes +NAME STATUS ROLES AGE VERSION +nginx-ingress-control-plane Ready control-plane,master 3m26s v1.23.5 +``` From 4095c3050a516a635ce096bf2a8e3c98dc1b085f Mon Sep 17 00:00:00 2001 From: marcel-dempers Date: Fri, 11 Nov 2022 10:10:14 +1100 Subject: [PATCH 2/3] nginx ingress wip --- kubernetes/ingress/controller/nginx/README.md | 94 +++ .../nginx/manifests/nginx-ingress.1.5.1.yaml | 740 ++++++++++++++++++ 2 files changed, 834 insertions(+) create mode 100644 kubernetes/ingress/controller/nginx/manifests/nginx-ingress.1.5.1.yaml diff --git a/kubernetes/ingress/controller/nginx/README.md b/kubernetes/ingress/controller/nginx/README.md index 026c29a..097e9d1 100644 --- a/kubernetes/ingress/controller/nginx/README.md +++ b/kubernetes/ingress/controller/nginx/README.md @@ -36,6 +36,11 @@ mv ./kubectl /usr/local/bin/kubectl # install helm +curl -o /tmp/helm.tar.gz -LO https://get.helm.sh/helm-v3.10.1-linux-amd64.tar.gz +tar -C /tmp/ -zxvf /tmp/helm.tar.gz +mv /tmp/linux-amd64/helm /usr/local/bin/helm +chmod +x /usr/local/bin/helm + ``` ### test cluster access: @@ -44,3 +49,92 @@ mv ./kubectl /usr/local/bin/kubectl NAME STATUS ROLES AGE VERSION nginx-ingress-control-plane Ready control-plane,master 3m26s v1.23.5 ``` + +## NGINX Ingress Controller + +We'll start with the documentation as always
+You can find the [Kubernetes NGINX documentation here](https://kubernetes.github.io/ingress-nginx/)
+ +First thing we do is check the compatibility matrix to ensure we are deploying a compatible version of NGINX Ingress on our Kubernetes cluster
+ +The Documentation also has a link to the [GitHub Repo](https://github.com/kubernetes/ingress-nginx/) which has a compatibility matrix
+ +### Get the installation YAML + +The controller ships as a `helm` chart, so we can grab version `v1.5.1` as per the compatibility +matrix.
+ +From our container we can do this: + +``` +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm search repo ingress-nginx --versions +``` + +From the app version we select the version that matches the compatibility matrix.
+ +``` +NAME CHART VERSION APP VERSION DESCRIPTION +ingress-nginx/ingress-nginx 4.4.0 1.5.1 Ingress controller for Kubernetes using NGINX a... +``` + +Now we can use `helm` to install the chart directly if we want.
+Or we can use `helm` to grab the manifest and explore its content.
+We can also add that manifest to our git repo if we are using a GitOps workflow to deploy it.
+ +``` +CHART_VERSION="4.4.0" +APP_VERSION="1.5.1" + +mkdir ./kubernetes/ingress/controller/nginx/manifests/ + +helm template ingress-nginx ingress-nginx \ +--repo https://kubernetes.github.io/ingress-nginx \ +--version ${CHART_VERSION} \ +--namespace ingress-nginx \ +> ./kubernetes/ingress/controller/nginx/manifests/nginx-ingress.${APP_VERSION}.yaml +``` + +### Deploy the Ingress controller + +``` +kubectl create namespace ingress-nginx +kubectl apply -f ./kubernetes/ingress/controller/nginx/manifests/nginx-ingress.${APP_VERSION}.yaml +``` + + +### Check the installation + +``` +kubectl -n ingress-nginx get pods +``` +The traffic for our cluster will come in over the Ingress service
+Note that we dont have load balancer capability in `kind` by default, so our `LoadBalancer` is pending: + +``` +kubectl -n ingress-nginx get svc +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +ingress-nginx-controller LoadBalancer 10.96.130.21 80:31011/TCP,443:31772/TCP 26m +ingress-nginx-controller-admission ClusterIP 10.96.125.210 443/TCP 26m +``` + +For testing purposes, we will simply setup `port-forward`ing
+If you are running in the cloud, you will get a real IP address.
+ +``` +kubectl -n ingress-nginx port-forward svc/ingress-nginx-controller 443 +``` + +We can reach our controller on [https://localhost/](https://localhost/) + +## Features + +* Routing DOMAIN +* SSL terminating & passthrough + +* routing URL + +customization (configmap) + +* location snippet +* log formating \ No newline at end of file diff --git a/kubernetes/ingress/controller/nginx/manifests/nginx-ingress.1.5.1.yaml b/kubernetes/ingress/controller/nginx/manifests/nginx-ingress.1.5.1.yaml new file mode 100644 index 0000000..ccbd2cf --- /dev/null +++ b/kubernetes/ingress/controller/nginx/manifests/nginx-ingress.1.5.1.yaml @@ -0,0 +1,740 @@ +--- +# Source: ingress-nginx/templates/controller-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + helm.sh/chart: ingress-nginx-4.4.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.5.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +automountServiceAccountToken: true +--- +# Source: ingress-nginx/templates/controller-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + helm.sh/chart: ingress-nginx-4.4.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.5.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +data: + allow-snippet-annotations: "true" +--- +# Source: ingress-nginx/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + helm.sh/chart: ingress-nginx-4.4.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.5.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +rules: + - apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + - namespaces + verbs: + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get +--- +# Source: ingress-nginx/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-4.4.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.5.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: "ingress-nginx" +--- +# Source: ingress-nginx/templates/controller-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + helm.sh/chart: ingress-nginx-4.4.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.5.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +rules: + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch + # TODO(Jintao Zhang) + # Once we release a new version of the controller, + # we will be able to remove the configmap related permissions + # We have used the Lease API for selection + # ref: https://github.com/kubernetes/ingress-nginx/pull/8921 + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - ingress-nginx-leader + verbs: + - get + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - apiGroups: + - coordination.k8s.io + resources: + - leases + resourceNames: + - ingress-nginx-leader + verbs: + - get + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get +--- +# Source: ingress-nginx/templates/controller-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-4.4.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.5.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: "ingress-nginx" +--- +# Source: ingress-nginx/templates/controller-service-webhook.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + helm.sh/chart: ingress-nginx-4.4.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.5.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller-admission + namespace: ingress-nginx +spec: + type: ClusterIP + ports: + - name: https-webhook + port: 443 + targetPort: webhook + appProtocol: https + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller +--- +# Source: ingress-nginx/templates/controller-service.yaml +apiVersion: v1 +kind: Service +metadata: + annotations: + labels: + helm.sh/chart: ingress-nginx-4.4.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.5.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + type: LoadBalancer + ipFamilyPolicy: SingleStack + ipFamilies: + - IPv4 + ports: + - name: http + port: 80 + protocol: TCP + targetPort: http + appProtocol: http + - name: https + port: 443 + protocol: TCP + targetPort: https + appProtocol: https + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller +--- +# Source: ingress-nginx/templates/controller-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + helm.sh/chart: ingress-nginx-4.4.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.5.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + selector: + matchLabels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller + replicas: 1 + revisionHistoryLimit: 10 + minReadySeconds: 0 + template: + metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller + spec: + dnsPolicy: ClusterFirst + containers: + - name: controller + image: "registry.k8s.io/ingress-nginx/controller:v1.5.1@sha256:4ba73c697770664c1e00e9f968de14e08f606ff961c76e5d7033a4a9c593c629" + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + args: + - /nginx-ingress-controller + - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller + - --election-id=ingress-nginx-leader + - --controller-class=k8s.io/ingress-nginx + - --ingress-class=nginx + - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller + - --validating-webhook=:8443 + - --validating-webhook-certificate=/usr/local/certificates/cert + - --validating-webhook-key=/usr/local/certificates/key + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + runAsUser: 101 + allowPrivilegeEscalation: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - name: http + containerPort: 80 + protocol: TCP + - name: https + containerPort: 443 + protocol: TCP + - name: webhook + containerPort: 8443 + protocol: TCP + volumeMounts: + - name: webhook-cert + mountPath: /usr/local/certificates/ + readOnly: true + resources: + requests: + cpu: 100m + memory: 90Mi + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: ingress-nginx + terminationGracePeriodSeconds: 300 + volumes: + - name: webhook-cert + secret: + secretName: ingress-nginx-admission +--- +# Source: ingress-nginx/templates/controller-ingressclass.yaml +# We don't support namespaced ingressClass yet +# So a ClusterRole and a ClusterRoleBinding is required +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + labels: + helm.sh/chart: ingress-nginx-4.4.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.5.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: nginx +spec: + controller: k8s.io/ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/validating-webhook.yaml +# before changing this value, check the required kubernetes version +# https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + labels: + helm.sh/chart: ingress-nginx-4.4.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.5.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + name: ingress-nginx-admission +webhooks: + - name: validate.nginx.ingress.kubernetes.io + matchPolicy: Equivalent + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: + - v1 + clientConfig: + service: + namespace: "ingress-nginx" + name: ingress-nginx-controller-admission + path: /networking/v1/ingresses +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ingress-nginx-admission + namespace: ingress-nginx + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-4.4.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.5.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ingress-nginx-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-4.4.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.5.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - get + - update +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ingress-nginx-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-4.4.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.5.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: "ingress-nginx" +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ingress-nginx-admission + namespace: ingress-nginx + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-4.4.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.5.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ingress-nginx-admission + namespace: ingress-nginx + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-4.4.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.5.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: "ingress-nginx" +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-create + namespace: ingress-nginx + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-4.4.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.5.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +spec: + template: + metadata: + name: ingress-nginx-admission-create + labels: + helm.sh/chart: ingress-nginx-4.4.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.5.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: create + image: "registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20220916-gd32f8c343@sha256:39c5b2e3310dc4264d638ad28d9d1d96c4cbb2b2dcfb52368fe4e3c63f61e10f" + imagePullPolicy: IfNotPresent + args: + - create + - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name=ingress-nginx-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + securityContext: + allowPrivilegeEscalation: false + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + nodeSelector: + kubernetes.io/os: linux + securityContext: + fsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-patch + namespace: ingress-nginx + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-4.4.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.5.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +spec: + template: + metadata: + name: ingress-nginx-admission-patch + labels: + helm.sh/chart: ingress-nginx-4.4.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: "1.5.1" + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: patch + image: "registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20220916-gd32f8c343@sha256:39c5b2e3310dc4264d638ad28d9d1d96c4cbb2b2dcfb52368fe4e3c63f61e10f" + imagePullPolicy: IfNotPresent + args: + - patch + - --webhook-name=ingress-nginx-admission + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name=ingress-nginx-admission + - --patch-failure-policy=Fail + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + securityContext: + allowPrivilegeEscalation: false + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + nodeSelector: + kubernetes.io/os: linux + securityContext: + fsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 From dc163cf341f5d68ba4d944d6bbefd8f0b1f4e703 Mon Sep 17 00:00:00 2001 From: marcel-dempers Date: Mon, 14 Nov 2022 13:24:50 +1100 Subject: [PATCH 3/3] nginx demo and features --- kubernetes/ingress/controller/nginx/README.md | 277 +++++++++++++++++- .../controller/nginx/features/basic-auth.yaml | 43 +++ .../nginx/features/routing-by-domain.yaml | 36 +++ .../features/routing-by-path-rewrite.yaml | 40 +++ .../nginx/features/routing-by-path.yaml | 40 +++ .../controller/nginx/features/service-a.yaml | 92 ++++++ .../controller/nginx/features/service-b.yaml | 92 ++++++ .../nginx/manifests/nginx-ingress.1.5.1.yaml | 2 + 8 files changed, 615 insertions(+), 7 deletions(-) create mode 100644 kubernetes/ingress/controller/nginx/features/basic-auth.yaml create mode 100644 kubernetes/ingress/controller/nginx/features/routing-by-domain.yaml create mode 100644 kubernetes/ingress/controller/nginx/features/routing-by-path-rewrite.yaml create mode 100644 kubernetes/ingress/controller/nginx/features/routing-by-path.yaml create mode 100644 kubernetes/ingress/controller/nginx/features/service-a.yaml create mode 100644 kubernetes/ingress/controller/nginx/features/service-b.yaml diff --git a/kubernetes/ingress/controller/nginx/README.md b/kubernetes/ingress/controller/nginx/README.md index 097e9d1..a97ac54 100644 --- a/kubernetes/ingress/controller/nginx/README.md +++ b/kubernetes/ingress/controller/nginx/README.md @@ -125,16 +125,279 @@ If you are running in the cloud, you will get a real IP address.
kubectl -n ingress-nginx port-forward svc/ingress-nginx-controller 443 ``` -We can reach our controller on [https://localhost/](https://localhost/) +We can reach our controller on [https://localhost/](https://localhost/)
+ +It's important to understand that Ingress runs on two ports `80` and `443`
+NGINX Ingress creates a fake certificate which is served for default `HTTPS` traffic on port `443`.
+If you look in the browser you will notice the name of the certificate `Common Name (CN) Kubernetes Ingress Controller Fake Certificate` ## Features -* Routing DOMAIN -* SSL terminating & passthrough +Now before we take a look at the features we'll need two web applications that we can use as our test harness, `service-a` and `service-b`
-* routing URL +In this demo, i have a deployment that runs a pod and a service that exposes the pod on port 80.
+This is a typical scenario where you have a micrservice you want to expose publicly.
-customization (configmap) +### Deploy Service A & B -* location snippet -* log formating \ No newline at end of file +Will deploy these two apps to the default namespace: + +``` +kubectl apply -f ./kubernetes/ingress/controller/nginx/features/service-a.yaml +kubectl apply -f ./kubernetes/ingress/controller/nginx/features/service-b.yaml +``` + +Test our service : `kubectl port-forward svc/service-a 80` + +Our services accept traffic on: + +* `http://localhost/` which goes to the root `/` +* `http://localhost/path-a.html` which goes to the root `/path-a.html` +* `http://localhost/path-b.html` which goes to the root `/path-b.html` +* `http://localhost/.html` which goes to the root `404` + +### Routing by Domain + +The most common way to route traffic with ingress is by domain: + +* https://public.service-a.com/ --> Ingress --> k8s service --> http://service-a/ +* https://public.service-b.com/ --> Ingress --> k8s service --> http://service-b/ + +To showcase this, let's deploy an ingress for service-a and service-b that routes by domain.
+ +Note: we don't own public domain `public.service-a.com` so we're using a `/etc/hosts` file + +Example Ingress: + +``` +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: service-a +spec: + ingressClassName: nginx + rules: + - host: public.service-a.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: service-a + port: + number: 80 +``` + +Note: we don't own public domain `public.my-services.com` so we're using a `/etc/hosts` file + +Deploy our ingresses: + +``` +kubectl apply -f ./kubernetes/ingress/controller/nginx/features/routing-by-domain.yaml +``` + +Now we can access service-a and service-b on: + +* https://public.service-a.com/ +* https://public.service-b.com/ + + +### Routing by Path + +Another popular routing strategy is to use a shared domain and route based on the HTTP path. For example:
+ +* https://public.my-services.com/path-a --> Ingress --> k8s service --> http://service-a/path-a +* https://public.my-services.com/path-b --> Ingress --> k8s service --> http://service-b/path-b + +This way public path `/path-a` will hit our application on `/path-a`
+ +Example Ingress: + +``` +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: service-a + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / +spec: + ingressClassName: nginx + rules: + - host: public.my-services.com + http: + paths: + - path: /path-a + pathType: Prefix + backend: + service: + name: service-a + port: + number: 80 +``` +Deploy our ingresses: + +``` +kubectl apply -f ./kubernetes/ingress/controller/nginx/features/routing-by-path.yaml +``` +Now notice the following routing: + +* https://public.my-services.com/ --> Ingress (404) +* https://public.my-services.com/path-a --> Ingress --> k8s service --> http://service-a/ +* https://public.my-services.com/path-b --> Ingress --> k8s service --> http://service-b/ + +No matter what path you place on the front end, as long as the path matches `/path-a` or `/path-b` +it will be routed to the correct service on `/`
+It's important to note that no extra paths or querystrings will NOT be passed to the upstream
+ +We can see this by looking at our NGINX Ingress controller logs as the controller will write the path it sees as well as the upstream service where it sent the request +``` +kubectl -n ingress-nginx logs -l app.kubernetes.io/instance=ingress-nginx +``` + +### App Root + +Sometimes applications have different root paths and don't simply serve traffic on `/`
+For example, the base path may be `http://localhost/home`
+ +To tell the Ingress controller that our application root path is `/home`, we can set the annotation `nginx.ingress.kubernetes.io/app-root: /home`
+ +This means the controller will be aware that all traffic that matches `path-a` should go to `/home` on service-a.
+ +### URL Rewrite + +We saw earlier when we routed by path, that we could pass `/path-a` to service-a and `/path-b` to service-b.
+However, the traffic would always go to `/` so we lost any trailing URL, parameters and querystring.
+Not very useful.
+ +To allow the Ingress controller to pass paths to the upstream you need to look into [Rewrite Configuration](https://kubernetes.github.io/ingress-nginx/examples/rewrite/) + +Example Ingress: + +``` +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: service-a + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$2 +spec: + ingressClassName: nginx + rules: + - host: public.my-services.com + http: + paths: + - path: /path-a(/|$)(.*) + pathType: Prefix + backend: + service: + name: service-a + port: + number: 80 +``` +Deploy our ingresses: + +``` +kubectl apply -f ./kubernetes/ingress/controller/nginx/features/routing-by-path-rewrite.yaml +``` +Now notice the following routing: + +* https://public.my-services.com/ --> Ingress (404) +* https://public.my-services.com/path-a* --> Ingress --> k8s service --> http://service-a/* +* https://public.my-services.com/path-b* --> Ingress --> k8s service --> http://service-b/* + +``` +kubectl -n ingress-nginx logs -l app.kubernetes.io/instance=ingress-nginx +``` +It's important to study the logs of the Ingress Controller to learn what path it saw, where it routed to + +``` +127.0.0.1 - - [13/Nov/2022:02:17:47 +0000] "GET /path-a/path.html HTTP/2.0" 404 19 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36" 485 0.000 [default-service-a-80] [] 10.244.0.8:80 19 0.000 404 206ed4b88b712564fc073c3adb845dff +``` + +In the above case, the controller saw ` /path-a/path.html` , routed to service-a and we can see what our service-a saw, by looking at its logs: + +``` +kubectl logs -l app=service-a +10.244.0.7 - - [13/Nov/2022:02:28:36 +0000] "GET /path-a.html HTTP/1.1" 200 28 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36" +``` + + +### SSL terminating & passthrough + +As we noticed by logs, its default for the Ingress controller to offload SSL.
+We can see this because when it routes to upstreams, it routes to our service on port 80
+Ingress offloads the TLS connection and creates a new connection with its upstream.
+ +This is a common approach to offload TLS on the edge as internal traffic is generally unencrypted in private +networks especially in large microservice environments where security is tightened in other manners so TLS is not needed all the way through.
+ +We can enable SSL pass through with the annotation: `nginx.ingress.kubernetes.io/ssl-passthrough`.
+ +SSL Passthrough is disabled by default and requires starting the controller with the --enable-ssl-passthrough flag.
+ +### IP Whitelist + +We can add a layer of protection to our services that are exposed by an ingress.
+One popular way is IP whitelisting.
+ +This can be done with a [whitelist source range annotation](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#whitelist-source-range) for example:
+ +`nginx.ingress.kubernetes.io/whitelist-source-range: `
+ +You can set this globally if you want using the [Customization ConfigMap](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#whitelist-source-range).
+We'll take a look at this customization in a bit.
+ +### Authentication + +You can add a layer of protection to services exposed by ingress by several [Authentication methods](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#authentication).
+ +A simple example is basic Authentication where the client supplied a `username\password` to access our service.
+ +This is controlled by annotations: + +* `nginx.ingress.kubernetes.io/auth-type: basic` +* `nginx.ingress.kubernetes.io/auth-secret: server-a-secret` +* `nginx.ingress.kubernetes.io/auth-secret-type: auth-file` + +Create a username and password: + +``` +apk add apache2-utils + +htpasswd -c auth service-a-user + +kubectl create secret generic server-a-secret --from-file=auth +``` + +Deploy our ingresses: + +``` +kubectl apply -f ./kubernetes/ingress/controller/nginx/features/basic-auth.yaml +``` + +### Server snippet + +Every ingress is technically an NGINX server block with a NGINX proxy pass.
+We can even customise this server block with a [Server Snippet annotation](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#server-snippet) + + +### Customization + +As mentioned before, the NGINX Ingress controller can be customized quite heavily with the [ConfigMap](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/) + +We can customize log format to JSON as well for example: + +``` +log-format-escape-json: "true" + log-format-upstream: '{"time":"$time_iso8601","remote_addr":"$remote_addr","proxy_protocol_addr":"$proxy_protocol_addr","proxy_protocol_port":"$proxy_protocol_port","x_forward_for":"$proxy_add_x_forwarded_for","remote_user":"$remote_user","host":"$host","request_method":"$request_method","request_uri":"$request_uri","server_protocol":"$server_protocol","status":$status,"request_time":$request_time,"request_length":$request_length,"bytes_sent":$bytes_sent,"upstream_name":"$proxy_upstream_name","upstream_addr":"$upstream_addr","upstream_uri":"$uri","upstream_response_length":$upstream_response_length,"upstream_response_time":$upstream_response_time,"upstream_status":$upstream_status,"http_referrer":"$http_referer","http_user_agent":"$http_user_agent","http_cookie":"$http_cookie","http_device_id":"$http_x_device_id","http_customer_id":"$http_x_customer_id"}' + +``` + +Apply the changes and restart Ingress: + +``` +kubectl apply -f ./kubernetes/ingress/controller/nginx/manifests/nginx-ingress.${APP_VERSION}.yaml +``` + +kubectl -n ingress-nginx logs -l app.kubernetes.io/instance=ingress-nginx \ No newline at end of file diff --git a/kubernetes/ingress/controller/nginx/features/basic-auth.yaml b/kubernetes/ingress/controller/nginx/features/basic-auth.yaml new file mode 100644 index 0000000..2feebfb --- /dev/null +++ b/kubernetes/ingress/controller/nginx/features/basic-auth.yaml @@ -0,0 +1,43 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: service-a + annotations: + nginx.ingress.kubernetes.io/auth-type: basic + nginx.ingress.kubernetes.io/auth-secret: server-a-secret + nginx.ingress.kubernetes.io/auth-secret-type: auth-file + nginx.ingress.kubernetes.io/rewrite-target: /$2 +spec: + ingressClassName: nginx + rules: + - host: public.my-services.com + http: + paths: + - path: /path-a(/|$)(.*) + pathType: Prefix + backend: + service: + name: service-a + port: + number: 80 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: service-b + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$2 +spec: + ingressClassName: nginx + rules: + - host: public.my-services.com + http: + paths: + - path: /path-b(/|$)(.*) + pathType: Prefix + backend: + service: + name: service-b + port: + number: 80 +--- \ No newline at end of file diff --git a/kubernetes/ingress/controller/nginx/features/routing-by-domain.yaml b/kubernetes/ingress/controller/nginx/features/routing-by-domain.yaml new file mode 100644 index 0000000..286b457 --- /dev/null +++ b/kubernetes/ingress/controller/nginx/features/routing-by-domain.yaml @@ -0,0 +1,36 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: service-a +spec: + ingressClassName: nginx + rules: + - host: public.service-a.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: service-a + port: + number: 80 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: service-b +spec: + ingressClassName: nginx + rules: + - host: public.service-b.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: service-b + port: + number: 80 +--- \ No newline at end of file diff --git a/kubernetes/ingress/controller/nginx/features/routing-by-path-rewrite.yaml b/kubernetes/ingress/controller/nginx/features/routing-by-path-rewrite.yaml new file mode 100644 index 0000000..d74e1a8 --- /dev/null +++ b/kubernetes/ingress/controller/nginx/features/routing-by-path-rewrite.yaml @@ -0,0 +1,40 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: service-a + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$2 +spec: + ingressClassName: nginx + rules: + - host: public.my-services.com + http: + paths: + - path: /path-a(/|$)(.*) + pathType: Prefix + backend: + service: + name: service-a + port: + number: 80 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: service-b + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$2 +spec: + ingressClassName: nginx + rules: + - host: public.my-services.com + http: + paths: + - path: /path-b(/|$)(.*) + pathType: Prefix + backend: + service: + name: service-b + port: + number: 80 +--- \ No newline at end of file diff --git a/kubernetes/ingress/controller/nginx/features/routing-by-path.yaml b/kubernetes/ingress/controller/nginx/features/routing-by-path.yaml new file mode 100644 index 0000000..3a42f4d --- /dev/null +++ b/kubernetes/ingress/controller/nginx/features/routing-by-path.yaml @@ -0,0 +1,40 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: service-a + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / +spec: + ingressClassName: nginx + rules: + - host: public.my-services.com + http: + paths: + - path: /path-a + pathType: Prefix + backend: + service: + name: service-a + port: + number: 80 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: service-b + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / +spec: + ingressClassName: nginx + rules: + - host: public.my-services.com + http: + paths: + - path: /path-b + pathType: Prefix + backend: + service: + name: service-b + port: + number: 80 +--- \ No newline at end of file diff --git a/kubernetes/ingress/controller/nginx/features/service-a.yaml b/kubernetes/ingress/controller/nginx/features/service-a.yaml new file mode 100644 index 0000000..a7e96b8 --- /dev/null +++ b/kubernetes/ingress/controller/nginx/features/service-a.yaml @@ -0,0 +1,92 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: service-a +data: + path-a.html: | + "/path-a.html" on service-a + path-b.html: | + "/path-b.html" on service-a + index.html: | + "/" on service-a + 404.html: | + service-a 404 page +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: service-a-nginx.conf +data: + nginx.conf: | + user nginx; + worker_processes 1; + error_log /var/log/nginx/error.log warn; + pid /var/run/nginx.pid; + events { + worker_connections 1024; + } + + http { + sendfile on; + server { + listen 80; + server_name localhost; + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + error_page 404 /404.html; + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + } + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: service-a + labels: + app: service-a +spec: + replicas: 1 + selector: + matchLabels: + app: service-a + template: + metadata: + labels: + app: service-a + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 + volumeMounts: + - name: html + mountPath: "/usr/share/nginx/html/" + - name: config + mountPath: "/etc/nginx/" + volumes: + - name: html + configMap: + name: service-a + - name: config + configMap: + name: service-a-nginx.conf +--- +apiVersion: v1 +kind: Service +metadata: + name: service-a +spec: + selector: + app: service-a + ports: + - protocol: TCP + port: 80 + targetPort: 80 \ No newline at end of file diff --git a/kubernetes/ingress/controller/nginx/features/service-b.yaml b/kubernetes/ingress/controller/nginx/features/service-b.yaml new file mode 100644 index 0000000..a39c4fe --- /dev/null +++ b/kubernetes/ingress/controller/nginx/features/service-b.yaml @@ -0,0 +1,92 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: service-b +data: + path-a.html: | + "/path-a.html" on service-b + path-b.html: | + "/path-b.html" on service-b + index.html: | + "/" on service-b + 404.html: | + service-b 404 page +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: service-b-nginx.conf +data: + nginx.conf: | + user nginx; + worker_processes 1; + error_log /var/log/nginx/error.log warn; + pid /var/run/nginx.pid; + events { + worker_connections 1024; + } + + http { + sendfile on; + server { + listen 80; + server_name localhost; + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + error_page 404 /404.html; + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + } + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: service-b + labels: + app: service-b +spec: + replicas: 1 + selector: + matchLabels: + app: service-b + template: + metadata: + labels: + app: service-b + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 + volumeMounts: + - name: html + mountPath: "/usr/share/nginx/html/" + - name: config + mountPath: "/etc/nginx/" + volumes: + - name: html + configMap: + name: service-b + - name: config + configMap: + name: service-b-nginx.conf +--- +apiVersion: v1 +kind: Service +metadata: + name: service-b +spec: + selector: + app: service-b + ports: + - protocol: TCP + port: 80 + targetPort: 80 \ No newline at end of file diff --git a/kubernetes/ingress/controller/nginx/manifests/nginx-ingress.1.5.1.yaml b/kubernetes/ingress/controller/nginx/manifests/nginx-ingress.1.5.1.yaml index ccbd2cf..8c65d1d 100644 --- a/kubernetes/ingress/controller/nginx/manifests/nginx-ingress.1.5.1.yaml +++ b/kubernetes/ingress/controller/nginx/manifests/nginx-ingress.1.5.1.yaml @@ -31,6 +31,8 @@ metadata: namespace: ingress-nginx data: allow-snippet-annotations: "true" + log-format-escape-json: "true" + log-format-upstream: '{"time":"$time_iso8601","remote_addr":"$remote_addr","proxy_protocol_addr":"$proxy_protocol_addr","proxy_protocol_port":"$proxy_protocol_port","x_forward_for":"$proxy_add_x_forwarded_for","remote_user":"$remote_user","host":"$host","request_method":"$request_method","request_uri":"$request_uri","server_protocol":"$server_protocol","status":$status,"request_time":$request_time,"request_length":$request_length,"bytes_sent":$bytes_sent,"upstream_name":"$proxy_upstream_name","upstream_addr":"$upstream_addr","upstream_uri":"$uri","upstream_response_length":$upstream_response_length,"upstream_response_time":$upstream_response_time,"upstream_status":$upstream_status,"http_referrer":"$http_referer","http_user_agent":"$http_user_agent","http_cookie":"$http_cookie","http_device_id":"$http_x_device_id","http_customer_id":"$http_x_customer_id"}' --- # Source: ingress-nginx/templates/clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1