From 655764b074734acc9d65c6cea54697af99b9ed50 Mon Sep 17 00:00:00 2001 From: marcel-dempers Date: Mon, 7 Nov 2022 15:52:54 +1100 Subject: [PATCH 1/3] sealedsecrets prototype --- kubernetes/secrets/sealed-secrets/README.md | 231 ++++++++++++ .../sealed-secrets/controller-v0.19.1.yaml | 354 ++++++++++++++++++ .../secrets/sealed-secrets/sealed-secret.yaml | 16 + 3 files changed, 601 insertions(+) create mode 100644 kubernetes/secrets/sealed-secrets/README.md create mode 100644 kubernetes/secrets/sealed-secrets/controller-v0.19.1.yaml create mode 100644 kubernetes/secrets/sealed-secrets/sealed-secret.yaml diff --git a/kubernetes/secrets/sealed-secrets/README.md b/kubernetes/secrets/sealed-secrets/README.md new file mode 100644 index 0000000..2e927d0 --- /dev/null +++ b/kubernetes/secrets/sealed-secrets/README.md @@ -0,0 +1,231 @@ +# Introduction to Sealed Secrets + +Checkout the [Sealed Secrets GitHub Repo](https://github.com/bitnami-labs/sealed-secrets)
+ +## Create a kubernetes cluster + +In this guide we we''ll need a Kubernetes cluster for testing. Let's create one using [kind](https://kind.sigs.k8s.io/)
+ +``` +kind create cluster --name sealedsecrets --image kindest/node:v1.23.5 +``` + +See cluster up and running: + +``` +kubectl get nodes +NAME STATUS ROLES AGE VERSION +sealedsecrets-control-plane Ready control-plane,master 2m12s v1.23.5 +``` + +## Run a container to work in + +### run Alpine Linux: +``` +docker run -it --rm -v ${HOME}:/root/ -v ${PWD}:/work -w /work --net host alpine sh +``` + +### install kubectl + +``` +apk add --no-cache curl +curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl +chmod +x ./kubectl +mv ./kubectl /usr/local/bin/kubectl +``` + +### test cluster access: +``` +/work # kubectl get nodes +NAME STATUS ROLES AGE VERSION +sealedsecrets-control-plane Ready control-plane,master 3m26s v1.23.5 +``` + +## Install Sealed Secret Controller + +### download the YAML + +In this demo we'll use version [0.19.1](https://github.com/bitnami-labs/sealed-secrets/releases/download/v0.19.1/controller.yaml) of the sealed secrets controller downloaded from the +[Github releases](https://github.com/bitnami-labs/sealed-secrets/releases) page + +``` +curl -L -o ./kubernetes/secrets/sealed-secrets/controller-v0.19.1.yaml https://github.com/bitnami-labs/sealed-secrets/releases/download/v0.19.1/controller.yaml + +``` + +### alternative install Helm + +TODO: cover helm https://github.com/bitnami-labs/sealed-secrets#helm-chart + +### install the controller + +``` +kubectl apply -f kubernetes/secrets/sealed-secrets/controller-v0.19.1.yaml +``` + +### Check the install + +``` +kubectl -n kube-system get pods +``` + +TODO: check the logs with `kubectl -n kube-system logs` command + +TODO: important logs + +``` +2022/11/05 21:38:20 New key written to kube-system/sealed-secrets-keymwzn9 +2022/11/05 21:38:20 Certificate is +-----BEGIN CERTIFICATE----- + < cert content > + -----END CERTIFICATE----- + +2022/11/05 21:38:20 HTTP server serving on :808 +``` + +TODO: check our secret + +``` +kubectl get secret -n kube-system sealed-secrets-keymwzn9 -o yaml +``` + +## Download KubeSeal + +The same way we downloaded the sealed secrets controller from the [GitHub releases](https://github.com/bitnami-labs/sealed-secrets/releases) page, +we'll want to download kubeseal from the assets section +``` + +curl -L -o /tmp/kubeseal.tar.gz \ +https://github.com/bitnami-labs/sealed-secrets/releases/download/v0.19.1/kubeseal-0.19.1-linux-amd64.tar.gz + +apk add tar +tar -xzf /tmp/kubeseal.tar.gz -C /tmp/ + +chmod +x /tmp/kubeseal +mv /tmp/kubeseal /usr/local/bin/ +``` + +### run kubeseal + +We can now run `kubeseal --help` + +## Sealing a basic Kubernetes Secret + +Looks at our existing Kubernetes secret YAML + +``` +cat kubernetes/secrets/secret.yaml +``` + +Create a sealed secret using `stdin` + +``` + cat kubernetes/secrets/secret.yaml | kubeseal -o yaml > kubernetes/secrets/sealed-secrets/sealed-secret.yaml +``` + +Create a sealed secret using file + +``` +kubeseal -f kubernetes/secrets/secret.yaml -o yaml > kubernetes/secrets/sealed-secrets/sealed-secret.yaml +``` + +Deploy the sealed secret + +``` +kubectl apply -f kubernetes/secrets/sealed-secrets/sealed-secret.yaml +sealedsecret.bitnami.com/mysecret created +``` + +Now few seconds later, see the secret + +``` +kubectl -n default get secret +NAME TYPE DATA AGE +mysecret Opaque 1 25s +``` + +## How the encryption key is managed + +TODO: How the encryption key is managed and stored + +TODO: Set a duration to test `--key-renew-period=` + +``` +apk add nano +export KUBE_EDITOR=nano +``` + +Set the flag on the command like so to add a new key every 5 min for testing: + +``` +spec: + containers: + - command: + - controller + - --key-renew-period=5m + +kubectl edit deployment/sealed-secrets-controller --namespace=kube-system +``` + +You should see a new key created under secrets in the `kube-system` namespace + +``` +kubectl -n kube-system get secrets +``` + + +## Backup your encryption keys + +TODO: * backup encryption keys +TODO: migrating kubernetes clusters + +``` +kubectl get secret -n kube-system \ + -l sealedsecrets.bitnami.com/sealed-secrets-key \ + -o yaml \ + > kubernetes/secrets/sealed-secrets/sealed-secret-keys.key +``` + +## Migrate your encryption keys to a new cluster + +Delete & Deploy a new Kubernetes cluster + +``` +kind delete cluster --name sealedsecrets +kind create cluster --name sealedsecrets --image kindest/node:v1.23.5 + +# check the cluster +kubectl get nodes + +# redeploy sealed-secrets controller +kubectl apply -f kubernetes/secrets/sealed-secrets/controller-v0.19.1.yaml + +kubectl -n kube-system get pods + +``` + +### restore our encryption keys + +``` +kubectl apply -f kubernetes/secrets/sealed-secrets/sealed-secret-keys.key +``` + +restart the controller: +``` +kubectl delete pod -n kube-system -l name=sealed-secrets-controller +``` + +### apply our old sealed secret + +``` +kubectl apply -f kubernetes/secrets/sealed-secrets/sealed-secret.yaml +``` + +TODO: Encrypted sealed secrets across namespaces https://github.com/bitnami-labs/sealed-secrets#scopes + +## Re-encrypting secrets with the latest key + +``` +kubeseal --re-encrypt tmp.json \ + && mv tmp.json my_sealed_secret.json +``` \ No newline at end of file diff --git a/kubernetes/secrets/sealed-secrets/controller-v0.19.1.yaml b/kubernetes/secrets/sealed-secrets/controller-v0.19.1.yaml new file mode 100644 index 0000000..9fd9775 --- /dev/null +++ b/kubernetes/secrets/sealed-secrets/controller-v0.19.1.yaml @@ -0,0 +1,354 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: {} + labels: + name: sealed-secrets-controller + name: sealed-secrets-controller + namespace: kube-system +spec: + minReadySeconds: 30 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: sealed-secrets-controller + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + annotations: {} + labels: + name: sealed-secrets-controller + spec: + containers: + - args: [] + command: + - controller + env: [] + image: docker.io/bitnami/sealed-secrets-controller:v0.19.1 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthz + port: http + name: sealed-secrets-controller + ports: + - containerPort: 8080 + name: http + readinessProbe: + httpGet: + path: /healthz + port: http + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + stdin: false + tty: false + volumeMounts: + - mountPath: /tmp + name: tmp + imagePullSecrets: [] + initContainers: [] + securityContext: + fsGroup: 65534 + serviceAccountName: sealed-secrets-controller + terminationGracePeriodSeconds: 30 + volumes: + - emptyDir: {} + name: tmp +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: sealedsecrets.bitnami.com +spec: + group: bitnami.com + names: + kind: SealedSecret + listKind: SealedSecretList + plural: sealedsecrets + singular: sealedsecret + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: SealedSecret is the K8s representation of a "sealed Secret" - + a regular k8s Secret that has been sealed (encrypted) using the controller's + key. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SealedSecretSpec is the specification of a SealedSecret + properties: + data: + description: Data is deprecated and will be removed eventually. Use + per-value EncryptedData instead. + format: byte + type: string + encryptedData: + additionalProperties: + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + template: + description: Template defines the structure of the Secret that will + be created from this sealed secret. + properties: + data: + additionalProperties: + type: string + description: Keys that should be templated using decrypted data + nullable: true + type: object + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: + description: Used to facilitate programmatic handling of secret + data. + type: string + type: object + required: + - encryptedData + type: object + status: + description: SealedSecretStatus is the most recently observed status of + the SealedSecret. + properties: + conditions: + description: Represents the latest available observations of a sealed + secret's current state. + items: + description: SealedSecretCondition describes the state of a sealed + secret at a certain point. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + format: date-time + type: string + lastUpdateTime: + description: The last time this condition was updated. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: 'Status of the condition for a sealed secret. Valid + values for "Synced": "True", "False", or "Unknown".' + type: string + type: + description: 'Type of condition for a sealed secret. Valid value: + "Synced"' + type: string + required: + - status + - type + type: object + type: array + observedGeneration: + description: ObservedGeneration reflects the generation most recently + observed by the sealed-secrets controller. + format: int64 + type: integer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: {} + labels: + name: sealed-secrets-controller + name: sealed-secrets-controller + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: sealed-secrets-key-admin +subjects: +- kind: ServiceAccount + name: sealed-secrets-controller + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: {} + labels: + name: sealed-secrets-controller + name: sealed-secrets-controller + namespace: kube-system +--- +apiVersion: v1 +kind: Service +metadata: + annotations: {} + labels: + name: sealed-secrets-controller + name: sealed-secrets-controller + namespace: kube-system +spec: + ports: + - port: 8080 + targetPort: 8080 + selector: + name: sealed-secrets-controller + type: ClusterIP +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: {} + labels: + name: sealed-secrets-service-proxier + name: sealed-secrets-service-proxier + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: sealed-secrets-service-proxier +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:authenticated +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: {} + labels: + name: sealed-secrets-service-proxier + name: sealed-secrets-service-proxier + namespace: kube-system +rules: +- apiGroups: + - "" + resourceNames: + - sealed-secrets-controller + resources: + - services + verbs: + - get +- apiGroups: + - "" + resourceNames: + - 'http:sealed-secrets-controller:' + - http:sealed-secrets-controller:http + - sealed-secrets-controller + resources: + - services/proxy + verbs: + - create + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: {} + labels: + name: sealed-secrets-key-admin + name: sealed-secrets-key-admin + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: {} + labels: + name: sealed-secrets-controller + name: sealed-secrets-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: secrets-unsealer +subjects: +- kind: ServiceAccount + name: sealed-secrets-controller + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: {} + labels: + name: secrets-unsealer + name: secrets-unsealer +rules: +- apiGroups: + - bitnami.com + resources: + - sealedsecrets + verbs: + - get + - list + - watch +- apiGroups: + - bitnami.com + resources: + - sealedsecrets/status + verbs: + - update +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - create + - update + - delete + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get diff --git a/kubernetes/secrets/sealed-secrets/sealed-secret.yaml b/kubernetes/secrets/sealed-secrets/sealed-secret.yaml new file mode 100644 index 0000000..fc45d09 --- /dev/null +++ b/kubernetes/secrets/sealed-secrets/sealed-secret.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: mysecret + namespace: default +spec: + encryptedData: + secret.json: AgBic1vT0+Y9WcUY3GqOlX+Zjstl7IDEY/U/XOykbRXGtUo6kgAG7Bkq/CyoXqDaNKD+4UhRpd9LZUw/6pvMpAjqsgSQw0ZBaALivxyZ7xSJ3/CpfsLlatWQoVWsnIoY8s0VW0eKhKPq0t30sNJSghH9yni3dgvBI3cyS1Xe2s9A7m/eUqUJqa82UE5y1UuRTRNiSUPmAVCaaW1md6K17DB9V7TE4mQn23IYF/xxSjjEhYFwsra2d7AXvSJiY63mHKRltQ6NxzJGA7xpvJTPV2MY1YIz8wsyKk8gQ2WZnnJrNGCoRlkE3hw2ZUFJGrivlWogcVYDL8EW8ngEEGdrhAbbTwzV17GDVt4rqgk6ag37yXL5rc01YD6BVTDGLglb59sRtRNkHAFr8GZmawVosswjRRDNHhzemgmpI2sOAn/ftvM5bjBNStnLBXAvOrh6sKDAiC2hERYegSjc/dwa9sdKI095lOduNXk55HorlyfiJ6bSV7h9Qc/A6P/whjL1YkZ+3iZA2b2DLEOWPSbaRhUYv4vfwkk3Abis3olXfI1dKwRec039AibhSeE/8FcrAik21XScWXT/QEH80yq6fzOUTMQRXLRzx7XRHfawEt6Oz5WV8mmR/XwueYx6Hw/SG8hxx6UUQqPaS66xVJgyL0g2EwonjLEtNQc+5/xrhaWAqyyRoeCB+2da5ZSUYWVhYdw99XNeg9HQ5mQpdPjMBW0/83gG2ZIr8XPgoo3OalYD5vwL4Q8drw== + template: + metadata: + creationTimestamp: null + name: mysecret + namespace: default + type: Opaque + From 84890e84c3de89adc464213fc20c6ff7c0c074b7 Mon Sep 17 00:00:00 2001 From: marcel-dempers Date: Tue, 8 Nov 2022 16:58:51 +1100 Subject: [PATCH 2/3] updates to sealed secrets --- kubernetes/secrets/sealed-secrets/README.md | 65 ++-- .../controller-helm-v0.19.1.yaml | 278 ++++++++++++++++++ 2 files changed, 321 insertions(+), 22 deletions(-) create mode 100644 kubernetes/secrets/sealed-secrets/controller-helm-v0.19.1.yaml diff --git a/kubernetes/secrets/sealed-secrets/README.md b/kubernetes/secrets/sealed-secrets/README.md index 2e927d0..d0ee5e6 100644 --- a/kubernetes/secrets/sealed-secrets/README.md +++ b/kubernetes/secrets/sealed-secrets/README.md @@ -4,7 +4,7 @@ Checkout the [Sealed Secrets GitHub Repo](https://github.com/bitnami-labs/sealed ## Create a kubernetes cluster -In this guide we we''ll need a Kubernetes cluster for testing. Let's create one using [kind](https://kind.sigs.k8s.io/)
+In this guide we we'll need a Kubernetes cluster for testing. Let's create one using [kind](https://kind.sigs.k8s.io/)
``` kind create cluster --name sealedsecrets --image kindest/node:v1.23.5 @@ -34,6 +34,15 @@ chmod +x ./kubectl mv ./kubectl /usr/local/bin/kubectl ``` +### install helm + +``` +curl -o /tmp/helm.tar.gz -LO https://get.helm.sh/helm-v3.10.1-linux-amd64.tar.gz +tar -C /tmp/ -zxvf /tmp/helm.tar.gz +mv /tmp/linux-amd64/helm /usr/local/bin/helm +chmod +x /usr/local/bin/helm +``` + ### test cluster access: ``` /work # kubectl get nodes @@ -53,40 +62,52 @@ curl -L -o ./kubernetes/secrets/sealed-secrets/controller-v0.19.1.yaml https://g ``` -### alternative install Helm +### install using Helm -TODO: cover helm https://github.com/bitnami-labs/sealed-secrets#helm-chart +You can also install the controller using `helm` -### install the controller +``` +helm repo add sealed-secrets https://bitnami-labs.github.io/sealed-secrets +helm search repo sealed-secrets --versions +helm template sealed-secrets --version 2.7.0 -n kube-system sealed-secrets/sealed-secrets \ +> ./kubernetes/secrets/sealed-secrets/controller-helm-v0.19.1.yaml + +``` +With `helm template` we can explore the YAML and then replace the `helm template` with `helm install` +to install the chart + +### install using YAML manifest ``` kubectl apply -f kubernetes/secrets/sealed-secrets/controller-v0.19.1.yaml ``` -### Check the install +### Check the installation + +The controller deploys to the `kube-system` namespace by default. ``` kubectl -n kube-system get pods ``` -TODO: check the logs with `kubectl -n kube-system logs` command +Check the logs of the sealed secret controller -TODO: important logs +``` +kubectl -n kube-system logs -l name=sealed-secrets-controller --tail -1 +``` + +From the logs we can see that it writes the encryption key its going to use as a kubernetes secret
+Example log: ``` 2022/11/05 21:38:20 New key written to kube-system/sealed-secrets-keymwzn9 -2022/11/05 21:38:20 Certificate is ------BEGIN CERTIFICATE----- - < cert content > - -----END CERTIFICATE----- - -2022/11/05 21:38:20 HTTP server serving on :808 ``` -TODO: check our secret +## Encryption keys ``` -kubectl get secret -n kube-system sealed-secrets-keymwzn9 -o yaml +kubectl -n kube-system get secrets +kubectl -n kube-system get secret sealed-secrets-keygxlvg -o yaml ``` ## Download KubeSeal @@ -97,16 +118,11 @@ we'll want to download kubeseal from the assets section curl -L -o /tmp/kubeseal.tar.gz \ https://github.com/bitnami-labs/sealed-secrets/releases/download/v0.19.1/kubeseal-0.19.1-linux-amd64.tar.gz - -apk add tar tar -xzf /tmp/kubeseal.tar.gz -C /tmp/ - chmod +x /tmp/kubeseal mv /tmp/kubeseal /usr/local/bin/ ``` -### run kubeseal - We can now run `kubeseal --help` ## Sealing a basic Kubernetes Secret @@ -117,13 +133,18 @@ Looks at our existing Kubernetes secret YAML cat kubernetes/secrets/secret.yaml ``` -Create a sealed secret using `stdin` +If you run `kubeseal` you will see it pause and expect input from `stdin`.
+You can paste your secret YAML and press CTRL+D to terminate `stdin`.
+You will notive it writes a `sealedSecret` to `stdout`.
+We can then automate this using `|` characters.
+ +Create a sealed secret using `stdin` : ``` cat kubernetes/secrets/secret.yaml | kubeseal -o yaml > kubernetes/secrets/sealed-secrets/sealed-secret.yaml ``` -Create a sealed secret using file +Create a sealed secret using a YAML file: ``` kubeseal -f kubernetes/secrets/secret.yaml -o yaml > kubernetes/secrets/sealed-secrets/sealed-secret.yaml diff --git a/kubernetes/secrets/sealed-secrets/controller-helm-v0.19.1.yaml b/kubernetes/secrets/sealed-secrets/controller-helm-v0.19.1.yaml new file mode 100644 index 0000000..1418edb --- /dev/null +++ b/kubernetes/secrets/sealed-secrets/controller-helm-v0.19.1.yaml @@ -0,0 +1,278 @@ +--- +# Source: sealed-secrets/templates/service-account.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sealed-secrets + namespace: kube-system + labels: + app.kubernetes.io/name: sealed-secrets + helm.sh/chart: sealed-secrets-2.7.0 + app.kubernetes.io/instance: sealed-secrets + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/version: v0.19.1 +--- +# Source: sealed-secrets/templates/cluster-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: secrets-unsealer + labels: + app.kubernetes.io/name: sealed-secrets + helm.sh/chart: sealed-secrets-2.7.0 + app.kubernetes.io/instance: sealed-secrets + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/version: v0.19.1 +rules: + - apiGroups: + - bitnami.com + resources: + - sealedsecrets + verbs: + - get + - list + - watch + - apiGroups: + - bitnami.com + resources: + - sealedsecrets/status + verbs: + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - create + - update + - delete + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +# Source: sealed-secrets/templates/cluster-role-binding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: sealed-secrets + labels: + app.kubernetes.io/name: sealed-secrets + helm.sh/chart: sealed-secrets-2.7.0 + app.kubernetes.io/instance: sealed-secrets + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/version: v0.19.1 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: secrets-unsealer +subjects: + - apiGroup: "" + kind: ServiceAccount + name: sealed-secrets + namespace: kube-system +--- +# Source: sealed-secrets/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: sealed-secrets-key-admin + namespace: kube-system + labels: + app.kubernetes.io/name: sealed-secrets + helm.sh/chart: sealed-secrets-2.7.0 + app.kubernetes.io/instance: sealed-secrets + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/version: v0.19.1 +rules: + - apiGroups: + - "" + resourceNames: + - sealed-secrets-key + resources: + - secrets + verbs: + - get + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - list +--- +# Source: sealed-secrets/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: sealed-secrets-service-proxier + namespace: kube-system + labels: + app.kubernetes.io/name: sealed-secrets + helm.sh/chart: sealed-secrets-2.7.0 + app.kubernetes.io/instance: sealed-secrets + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/version: v0.19.1 +rules: + - apiGroups: + - "" + resourceNames: + - sealed-secrets + resources: + - services + verbs: + - get + - apiGroups: + - "" + resourceNames: + - 'http:sealed-secrets:' + - 'http:sealed-secrets:http' + - sealed-secrets + resources: + - services/proxy + verbs: + - create + - get +--- +# Source: sealed-secrets/templates/role-binding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: sealed-secrets-key-admin + namespace: kube-system + labels: + app.kubernetes.io/name: sealed-secrets + helm.sh/chart: sealed-secrets-2.7.0 + app.kubernetes.io/instance: sealed-secrets + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/version: v0.19.1 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: sealed-secrets-key-admin +subjects: + - apiGroup: "" + kind: ServiceAccount + name: sealed-secrets + namespace: kube-system +--- +# Source: sealed-secrets/templates/role-binding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: sealed-secrets-service-proxier + namespace: kube-system + labels: + app.kubernetes.io/name: sealed-secrets + helm.sh/chart: sealed-secrets-2.7.0 + app.kubernetes.io/instance: sealed-secrets + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/version: v0.19.1 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: sealed-secrets-service-proxier +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:authenticated +--- +# Source: sealed-secrets/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: sealed-secrets + namespace: kube-system + labels: + app.kubernetes.io/name: sealed-secrets + helm.sh/chart: sealed-secrets-2.7.0 + app.kubernetes.io/instance: sealed-secrets + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/version: v0.19.1 +spec: + type: ClusterIP + ports: + - name: http + port: 8080 + targetPort: http + nodePort: null + selector: + app.kubernetes.io/name: sealed-secrets + app.kubernetes.io/instance: sealed-secrets +--- +# Source: sealed-secrets/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sealed-secrets + namespace: kube-system + labels: + app.kubernetes.io/name: sealed-secrets + helm.sh/chart: sealed-secrets-2.7.0 + app.kubernetes.io/instance: sealed-secrets + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/version: v0.19.1 +spec: + selector: + matchLabels: + app.kubernetes.io/name: sealed-secrets + app.kubernetes.io/instance: sealed-secrets + template: + metadata: + labels: + app.kubernetes.io/name: sealed-secrets + app.kubernetes.io/instance: sealed-secrets + spec: + securityContext: + fsGroup: 65534 + serviceAccountName: sealed-secrets + containers: + - name: controller + command: + - controller + args: + - --update-status + - --key-prefix + - "sealed-secrets-key" + image: docker.io/bitnami/sealed-secrets-controller:v0.19.1 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + name: http + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + httpGet: + path: /healthz + port: http + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + httpGet: + path: /healthz + port: http + resources: + limits: {} + requests: {} + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + volumeMounts: + - mountPath: /tmp + name: tmp + volumes: + - name: tmp + emptyDir: {} From ee246f3e9f1e1f48b539972de9f2028886e2d4b4 Mon Sep 17 00:00:00 2001 From: marcel-dempers Date: Tue, 8 Nov 2022 19:44:56 +1100 Subject: [PATCH 3/3] sealed secret prototyping --- kubernetes/secrets/sealed-secrets/README.md | 67 +++-- .../controller-helm-v0.19.1.yaml | 278 ------------------ .../secrets/sealed-secrets/sealed-secret.yaml | 2 +- 3 files changed, 51 insertions(+), 296 deletions(-) delete mode 100644 kubernetes/secrets/sealed-secrets/controller-helm-v0.19.1.yaml diff --git a/kubernetes/secrets/sealed-secrets/README.md b/kubernetes/secrets/sealed-secrets/README.md index d0ee5e6..62f905b 100644 --- a/kubernetes/secrets/sealed-secrets/README.md +++ b/kubernetes/secrets/sealed-secrets/README.md @@ -2,6 +2,16 @@ Checkout the [Sealed Secrets GitHub Repo](https://github.com/bitnami-labs/sealed-secrets)
+There are a number of use-cases where this is a really great concept.
+ +1) GitOps - Storing your YAML manifests in Git and using GitOps tools to sync the manifests to your clusters (For example Flux and ArgoCD!) + +2) Giving a team access to secrets without revealing the secret material. + +developer: "I want to confirm my deployed secret value is X in the cluster"
+ +developer can compare `sealedSecret` YAML in Git, with the `sealedSecret` in the cluster and confirm the value is the same.
+ ## Create a kubernetes cluster In this guide we we'll need a Kubernetes cluster for testing. Let's create one using [kind](https://kind.sigs.k8s.io/)
@@ -135,7 +145,7 @@ cat kubernetes/secrets/secret.yaml If you run `kubeseal` you will see it pause and expect input from `stdin`.
You can paste your secret YAML and press CTRL+D to terminate `stdin`.
-You will notive it writes a `sealedSecret` to `stdout`.
+You will notice it writes a `sealedSecret` to `stdout`.
We can then automate this using `|` characters.
Create a sealed secret using `stdin` : @@ -154,7 +164,6 @@ Deploy the sealed secret ``` kubectl apply -f kubernetes/secrets/sealed-secrets/sealed-secret.yaml -sealedsecret.bitnami.com/mysecret created ``` Now few seconds later, see the secret @@ -167,15 +176,22 @@ mysecret Opaque 1 25s ## How the encryption key is managed -TODO: How the encryption key is managed and stored +By default the controller generates a key as we saw earlier and stores it in a Kubernetes secret.
+By default, the controller will generate a new active key every 30 days. +It keeps old keys so it can decrypt previous encrypted sealed secrets and will use the active key with new encryption.
-TODO: Set a duration to test `--key-renew-period=` +It's important to keep these keys secured.
+When the controller starts it consumes all the secrets and will start using them
+This means we can backup these keys in a Vault and use them to migrate our clusters if we wanted to.
+ +We can also override the renewal period to increase or decrease the value. `0` turns it off
+ +To showcase this I can set `--key-renew-period=` to 5min to watch how it works.
``` apk add nano export KUBE_EDITOR=nano ``` - Set the flag on the command like so to add a new key every 5 min for testing: ``` @@ -194,11 +210,9 @@ You should see a new key created under secrets in the `kube-system` namespace kubectl -n kube-system get secrets ``` - ## Backup your encryption keys -TODO: * backup encryption keys -TODO: migrating kubernetes clusters +To get your keys out for backup purpose, it's as simple as grabbing a secret by label using `kubectl` : ``` kubectl get secret -n kube-system \ @@ -206,10 +220,11 @@ kubectl get secret -n kube-system \ -o yaml \ > kubernetes/secrets/sealed-secrets/sealed-secret-keys.key ``` +This can be used when migrating from one cluster to another, or simply for keeping backups.
## Migrate your encryption keys to a new cluster -Delete & Deploy a new Kubernetes cluster +To test this, lets delete our cluster and recreate it.
``` kind delete cluster --name sealedsecrets @@ -231,22 +246,40 @@ kubectl -n kube-system get pods kubectl apply -f kubernetes/secrets/sealed-secrets/sealed-secret-keys.key ``` -restart the controller: -``` -kubectl delete pod -n kube-system -l name=sealed-secrets-controller -``` - ### apply our old sealed secret ``` kubectl apply -f kubernetes/secrets/sealed-secrets/sealed-secret.yaml ``` -TODO: Encrypted sealed secrets across namespaces https://github.com/bitnami-labs/sealed-secrets#scopes +### see sealed secret status + +To troubleshoot the secret, you can use the popular `kubectl describe` command.
+Note that we're unable to decrypt the secret.
+Why is that ?
+ +We'll this is because the encryption key secrets are read when the controller starts.
+So we will need to restart the controller to that it can read ingest the encryption keys: + +``` +kubectl delete pod -n kube-system -l name=sealed-secrets-controller +``` ## Re-encrypting secrets with the latest key +We can also use `kubeseal --re-encrypt` to encrypt a secret again.
+Let's say we want to encrypt with the latest key.
+This will re-encrypt the sealed secret without having to pull the actual secret to the client
+ ``` -kubeseal --re-encrypt tmp.json \ - && mv tmp.json my_sealed_secret.json +cat ./kubernetes/secrets/sealed-secrets/sealed-secret.yaml \ +| kubeseal --re-encrypt -o yaml +``` + +I can then save this to override the original old local sealed secret file: + +``` +cat ./kubernetes/secrets/sealed-secrets/sealed-secret.yaml \ +| kubeseal --re-encrypt -o yaml \ +> tmp.yaml && mv tmp.yaml ./kubernetes/secrets/sealed-secrets/sealed-secret.yaml ``` \ No newline at end of file diff --git a/kubernetes/secrets/sealed-secrets/controller-helm-v0.19.1.yaml b/kubernetes/secrets/sealed-secrets/controller-helm-v0.19.1.yaml deleted file mode 100644 index 1418edb..0000000 --- a/kubernetes/secrets/sealed-secrets/controller-helm-v0.19.1.yaml +++ /dev/null @@ -1,278 +0,0 @@ ---- -# Source: sealed-secrets/templates/service-account.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: sealed-secrets - namespace: kube-system - labels: - app.kubernetes.io/name: sealed-secrets - helm.sh/chart: sealed-secrets-2.7.0 - app.kubernetes.io/instance: sealed-secrets - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/version: v0.19.1 ---- -# Source: sealed-secrets/templates/cluster-role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: secrets-unsealer - labels: - app.kubernetes.io/name: sealed-secrets - helm.sh/chart: sealed-secrets-2.7.0 - app.kubernetes.io/instance: sealed-secrets - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/version: v0.19.1 -rules: - - apiGroups: - - bitnami.com - resources: - - sealedsecrets - verbs: - - get - - list - - watch - - apiGroups: - - bitnami.com - resources: - - sealedsecrets/status - verbs: - - update - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - create - - update - - delete - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch ---- -# Source: sealed-secrets/templates/cluster-role-binding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: sealed-secrets - labels: - app.kubernetes.io/name: sealed-secrets - helm.sh/chart: sealed-secrets-2.7.0 - app.kubernetes.io/instance: sealed-secrets - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/version: v0.19.1 -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: secrets-unsealer -subjects: - - apiGroup: "" - kind: ServiceAccount - name: sealed-secrets - namespace: kube-system ---- -# Source: sealed-secrets/templates/role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: sealed-secrets-key-admin - namespace: kube-system - labels: - app.kubernetes.io/name: sealed-secrets - helm.sh/chart: sealed-secrets-2.7.0 - app.kubernetes.io/instance: sealed-secrets - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/version: v0.19.1 -rules: - - apiGroups: - - "" - resourceNames: - - sealed-secrets-key - resources: - - secrets - verbs: - - get - - apiGroups: - - "" - resources: - - secrets - verbs: - - create - - list ---- -# Source: sealed-secrets/templates/role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: sealed-secrets-service-proxier - namespace: kube-system - labels: - app.kubernetes.io/name: sealed-secrets - helm.sh/chart: sealed-secrets-2.7.0 - app.kubernetes.io/instance: sealed-secrets - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/version: v0.19.1 -rules: - - apiGroups: - - "" - resourceNames: - - sealed-secrets - resources: - - services - verbs: - - get - - apiGroups: - - "" - resourceNames: - - 'http:sealed-secrets:' - - 'http:sealed-secrets:http' - - sealed-secrets - resources: - - services/proxy - verbs: - - create - - get ---- -# Source: sealed-secrets/templates/role-binding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: sealed-secrets-key-admin - namespace: kube-system - labels: - app.kubernetes.io/name: sealed-secrets - helm.sh/chart: sealed-secrets-2.7.0 - app.kubernetes.io/instance: sealed-secrets - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/version: v0.19.1 -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: sealed-secrets-key-admin -subjects: - - apiGroup: "" - kind: ServiceAccount - name: sealed-secrets - namespace: kube-system ---- -# Source: sealed-secrets/templates/role-binding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: sealed-secrets-service-proxier - namespace: kube-system - labels: - app.kubernetes.io/name: sealed-secrets - helm.sh/chart: sealed-secrets-2.7.0 - app.kubernetes.io/instance: sealed-secrets - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/version: v0.19.1 -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: sealed-secrets-service-proxier -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:authenticated ---- -# Source: sealed-secrets/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: sealed-secrets - namespace: kube-system - labels: - app.kubernetes.io/name: sealed-secrets - helm.sh/chart: sealed-secrets-2.7.0 - app.kubernetes.io/instance: sealed-secrets - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/version: v0.19.1 -spec: - type: ClusterIP - ports: - - name: http - port: 8080 - targetPort: http - nodePort: null - selector: - app.kubernetes.io/name: sealed-secrets - app.kubernetes.io/instance: sealed-secrets ---- -# Source: sealed-secrets/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: sealed-secrets - namespace: kube-system - labels: - app.kubernetes.io/name: sealed-secrets - helm.sh/chart: sealed-secrets-2.7.0 - app.kubernetes.io/instance: sealed-secrets - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/version: v0.19.1 -spec: - selector: - matchLabels: - app.kubernetes.io/name: sealed-secrets - app.kubernetes.io/instance: sealed-secrets - template: - metadata: - labels: - app.kubernetes.io/name: sealed-secrets - app.kubernetes.io/instance: sealed-secrets - spec: - securityContext: - fsGroup: 65534 - serviceAccountName: sealed-secrets - containers: - - name: controller - command: - - controller - args: - - --update-status - - --key-prefix - - "sealed-secrets-key" - image: docker.io/bitnami/sealed-secrets-controller:v0.19.1 - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8080 - name: http - livenessProbe: - failureThreshold: 3 - initialDelaySeconds: 0 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - httpGet: - path: /healthz - port: http - readinessProbe: - failureThreshold: 3 - initialDelaySeconds: 0 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - httpGet: - path: /healthz - port: http - resources: - limits: {} - requests: {} - securityContext: - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 1001 - volumeMounts: - - mountPath: /tmp - name: tmp - volumes: - - name: tmp - emptyDir: {} diff --git a/kubernetes/secrets/sealed-secrets/sealed-secret.yaml b/kubernetes/secrets/sealed-secrets/sealed-secret.yaml index fc45d09..f3e1d28 100644 --- a/kubernetes/secrets/sealed-secrets/sealed-secret.yaml +++ b/kubernetes/secrets/sealed-secrets/sealed-secret.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: encryptedData: - secret.json: AgBic1vT0+Y9WcUY3GqOlX+Zjstl7IDEY/U/XOykbRXGtUo6kgAG7Bkq/CyoXqDaNKD+4UhRpd9LZUw/6pvMpAjqsgSQw0ZBaALivxyZ7xSJ3/CpfsLlatWQoVWsnIoY8s0VW0eKhKPq0t30sNJSghH9yni3dgvBI3cyS1Xe2s9A7m/eUqUJqa82UE5y1UuRTRNiSUPmAVCaaW1md6K17DB9V7TE4mQn23IYF/xxSjjEhYFwsra2d7AXvSJiY63mHKRltQ6NxzJGA7xpvJTPV2MY1YIz8wsyKk8gQ2WZnnJrNGCoRlkE3hw2ZUFJGrivlWogcVYDL8EW8ngEEGdrhAbbTwzV17GDVt4rqgk6ag37yXL5rc01YD6BVTDGLglb59sRtRNkHAFr8GZmawVosswjRRDNHhzemgmpI2sOAn/ftvM5bjBNStnLBXAvOrh6sKDAiC2hERYegSjc/dwa9sdKI095lOduNXk55HorlyfiJ6bSV7h9Qc/A6P/whjL1YkZ+3iZA2b2DLEOWPSbaRhUYv4vfwkk3Abis3olXfI1dKwRec039AibhSeE/8FcrAik21XScWXT/QEH80yq6fzOUTMQRXLRzx7XRHfawEt6Oz5WV8mmR/XwueYx6Hw/SG8hxx6UUQqPaS66xVJgyL0g2EwonjLEtNQc+5/xrhaWAqyyRoeCB+2da5ZSUYWVhYdw99XNeg9HQ5mQpdPjMBW0/83gG2ZIr8XPgoo3OalYD5vwL4Q8drw== + secret.json: AgBGAxpwLeMnCxvuGAY0tipvPVexoqucs3qKUNwD8MNy9PXw+SwLnUzMGSCX1Q/I8/EvNcDOY7vkenHts4Rzexva6m0MNMn60223QEm1394HRQQS5xELnCyAsjpSeHuN4TizxmWU/O8IBPvAMYINFapA0Hc3VADArKaToECyJJxAxBmUFUuQTVdWS8E/yX1ZMibpwYdPQt0PK7i/8eA6Qln7KfvNxp4KaWz4enL9uScIztXLfeHaDgE7CazaINL/vmkUIB0J7tbAuhLwPpy86pcL+MioeXTk4Kze7ikPn8dXSdYgSzVR85CaaFwueXWFujqUDEgu6ROGH0CyLZ8aK/Z8X3NAnIxupQLl26xbRDONXwjLsEFAOHLmX+J/gpkDJBU6F+Nt/Unc7q2KBPPuQgdozr3e7ZtCRnUO7kU75UccwgqHF9o/8n4j4+/WjkepTuKy6hBP16sFRTdTWg+iP79xBPLAKyJ1urRVYRhLkzgRf15Wyk28XeGMw0Ip8hEJnDIzTc9IepsMdXDHQppHIUUfp8LIgeWZUxsb//Z7+28qwuffEr436cgKI6jf4g8eIaIFNyk1JRU2cmcDgEQEhTYaskqsEedaDfBWzjxhtqGYh2SJIDvpZkL43rZQLM68EhApFkefo/6goNRupXAEUY8xb9XP3Z6nMiIlGZMU5pl4A6NPpRSEgyipHaM8E8MqN9dyQVMgcnRBWYjk8WZIQHWrKiw0BtNTgVuVBxKkpQkqQHO7FXLdQw== template: metadata: creationTimestamp: null