From f9b9a3f1fed517fd4860a0f9cd2918f9028f4057 Mon Sep 17 00:00:00 2001 From: marcel-dempers Date: Sun, 8 Nov 2020 16:00:38 +1100 Subject: [PATCH] fluent-k8s wip --- .../configurations/containers-fluent.conf | 2 +- .../configurations/file-fluent.conf | 2 +- .../logging/fluentd/introduction/dockerfile | 1 - .../logging/fluentd/kubernetes/README.md | 100 +++++++++++++++++- .../{ => elastic}/elastic-demo.yaml | 26 +++-- .../kubernetes/{ => elastic}/kibana-demo.yaml | 12 +-- .../fluentd/kubernetes/fluentd-configmap.yaml | 39 +++++++ .../fluentd/kubernetes/fluentd-rbac.yaml | 34 ++++++ .../logging/fluentd/kubernetes/fluentd.yaml | 52 +++++++++ 9 files changed, 247 insertions(+), 21 deletions(-) rename monitoring/logging/fluentd/kubernetes/{ => elastic}/elastic-demo.yaml (55%) rename monitoring/logging/fluentd/kubernetes/{ => elastic}/kibana-demo.yaml (72%) create mode 100644 monitoring/logging/fluentd/kubernetes/fluentd-configmap.yaml create mode 100644 monitoring/logging/fluentd/kubernetes/fluentd-rbac.yaml create mode 100644 monitoring/logging/fluentd/kubernetes/fluentd.yaml diff --git a/monitoring/logging/fluentd/introduction/configurations/containers-fluent.conf b/monitoring/logging/fluentd/introduction/configurations/containers-fluent.conf index 49084ff..30cacf8 100644 --- a/monitoring/logging/fluentd/introduction/configurations/containers-fluent.conf +++ b/monitoring/logging/fluentd/introduction/configurations/containers-fluent.conf @@ -1,4 +1,4 @@ - + @type tail format json read_from_head true diff --git a/monitoring/logging/fluentd/introduction/configurations/file-fluent.conf b/monitoring/logging/fluentd/introduction/configurations/file-fluent.conf index a0cf07a..0053a0b 100644 --- a/monitoring/logging/fluentd/introduction/configurations/file-fluent.conf +++ b/monitoring/logging/fluentd/introduction/configurations/file-fluent.conf @@ -1,4 +1,4 @@ - + @type tail format json read_from_head true diff --git a/monitoring/logging/fluentd/introduction/dockerfile b/monitoring/logging/fluentd/introduction/dockerfile index 2ab00f0..b00f600 100644 --- a/monitoring/logging/fluentd/introduction/dockerfile +++ b/monitoring/logging/fluentd/introduction/dockerfile @@ -2,4 +2,3 @@ FROM fluent/fluentd:v1.11-debian USER root RUN gem install fluent-plugin-elasticsearch -USER fluent diff --git a/monitoring/logging/fluentd/kubernetes/README.md b/monitoring/logging/fluentd/kubernetes/README.md index bbe6422..2b38b06 100644 --- a/monitoring/logging/fluentd/kubernetes/README.md +++ b/monitoring/logging/fluentd/kubernetes/README.md @@ -1,5 +1,13 @@ # Introduction to Fluentd on Kubernetes +## Prerequisites + +You will need a basic understanding of Fluentd before you attempt to run it on Kubernetes.
+Fluentd and Kubernetes have a bunch of moving parts.
+To understand the basics of Fluentd, I highly recommend you start with this video:
+ +Fluentd + ## We need a Kubernetes cluster Lets create a Kubernetes cluster to play with using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/) @@ -10,8 +18,96 @@ kind create cluster --name fluentd --image kindest/node:v1.19.1 ## Fluentd Manifests -I would highly recommend to use manifests from the official fluentd [github repo](https://github.com/fluent/fluentd-kubernetes-daemonset)
+I would highly recommend to use manifests from the official fluentd [github repo](https://github.com/fluent/fluentd-kubernetes-daemonset) for production usage
The manifests found here are purely for demo purpose.
+The manifests in this repo are broken down and simplified for educational purpose.
+
+In this example I will use the most common use case and we'll break it down to get an understanding of each component. -In this example I will use the most common use case and we'll break it down to get an understanding of each component. \ No newline at end of file +## Fluentd Docker + +I would recommend to start with the official [fluentd](https://hub.docker.com/r/fluent/fluentd/) +docker image.
+You may want to build your own image if you want to install plugins. +In this demo I will be using the `fluentd` elasticsearch plugin
+It's pretty simple to adjust `fluentd` to send logs to any other destination in case you are not an `elasticsearch` user.
+ +
+ +Let's build our [docker image](https://github.com/marcel-dempers/docker-development-youtube-series/blob/master/monitoring/logging/fluentd/introduction/dockerfile) in the introduction folder: + + +``` +cd monitoring\logging\fluentd\introduction + +#note: use your own tag! +docker build . -t aimvector/fluentd-demo + +#note: use your own tag! +docker push aimvector/fluentd-demo + +``` + +## Fluentd Namespace + +I like to run certain infrastructure components in their own namespaces.
+If you are using the official manifests, they may be using the `kube-system` namespace instead.
+You may want to carefully adjust it based on your preference
+Let's create a `fluentd` namespace:
+ +``` +kubectl create ns fluentd + +``` +## Fluentd Configmap + +In my [fluentd introduction video](https://youtu.be/Gp0-7oVOtPw), I talk about how `fluentd` allows us to simplify our configs using the `include` statement.
+This helps us prevent having a large complex file. + +
+ +We have 3 files in our `fluentd-configmap.yaml` : +* fluent.conf: Our main config which includes all other configurations +* pods-fluent.conf: `tail` config that sources all pod logs on the `kubernetes` host +* file-fluent.conf: `match` config to capture all logs and write it to file for testing log collection +* elastic-fluent.conf: `match` config that captures all logs and sends it to `elasticseach` + +Let's deploy our `configmap`: + +``` +kubectl apply -f .\monitoring\logging\fluentd\kubernetes\fluentd-configmap.yaml + +``` + +## Fluentd Daemonset + +Let's deploy our `daemonset`: + +``` +kubectl apply -f .\monitoring\logging\fluentd\kubernetes\fluentd-rbac.yaml +kubectl apply -f .\monitoring\logging\fluentd\kubernetes\fluentd.yaml + +kubectl -n fluentd get pods +``` + + +NOT message:("pattern not matched") and NOT message:("/var/log/containers/") + + + + +## Demo ElasticSearch and Kibana + +``` +kubectl create ns elastic-kibana + +kubectl -n elastic-kibana apply -f .\monitoring\logging\fluentd\kubernetes\elastic\elastic-demo.yaml +kubectl -n elastic-kibana apply -f .\monitoring\logging\fluentd\kubernetes\elastic\kibana-demo.yaml +``` + +## Kibana + +``` +kubectl -n elastic-kibana port-forward svc/kibana 5601 +``` \ No newline at end of file diff --git a/monitoring/logging/fluentd/kubernetes/elastic-demo.yaml b/monitoring/logging/fluentd/kubernetes/elastic/elastic-demo.yaml similarity index 55% rename from monitoring/logging/fluentd/kubernetes/elastic-demo.yaml rename to monitoring/logging/fluentd/kubernetes/elastic/elastic-demo.yaml index d4cc2b2..e4064b7 100644 --- a/monitoring/logging/fluentd/kubernetes/elastic-demo.yaml +++ b/monitoring/logging/fluentd/kubernetes/elastic/elastic-demo.yaml @@ -14,21 +14,27 @@ spec: labels: app: elasticsearch spec: + initContainers: + - name: vm-max-fix + image: busybox + command: ["sysctl", "-w", "vm.max_map_count=262144"] + securityContext: + privileged: true containers: - name: elasticsearch image: elasticsearch:7.9.1 - imagePullPolicy: IfNotExists + imagePullPolicy: IfNotPresent ports: - containerPort: 9200 - env: - - name: node.name - value: "elasticsearch" - - name: cluster.initial_master_nodes - value: "elasticsearch" - - name: bootstrap.memory_lock - value: "true" - - name: ES_JAVA_OPTS - value: "-Xms512m -Xmx512m" + env: + - name: node.name + value: "elasticsearch" + - name: cluster.initial_master_nodes + value: "elasticsearch" + - name: bootstrap.memory_lock + value: "false" + - name: ES_JAVA_OPTS + value: "-Xms512m -Xmx512m" --- apiVersion: v1 kind: Service diff --git a/monitoring/logging/fluentd/kubernetes/kibana-demo.yaml b/monitoring/logging/fluentd/kubernetes/elastic/kibana-demo.yaml similarity index 72% rename from monitoring/logging/fluentd/kubernetes/kibana-demo.yaml rename to monitoring/logging/fluentd/kubernetes/elastic/kibana-demo.yaml index 8f63c9d..c58524b 100644 --- a/monitoring/logging/fluentd/kubernetes/kibana-demo.yaml +++ b/monitoring/logging/fluentd/kubernetes/elastic/kibana-demo.yaml @@ -17,14 +17,14 @@ spec: containers: - name: kibana image: kibana:7.9.1 - imagePullPolicy: IfNotExists + imagePullPolicy: IfNotPresent ports: - containerPort: 5601 - env: - - name: ELASTICSEARCH_URL - value: "http://elasticsearch:9200" - - name: ELASTICSEARCH_HOSTS - value: "http://elasticsearch:9200" + env: + - name: ELASTICSEARCH_URL + value: "http://elasticsearch:9200" + - name: ELASTICSEARCH_HOSTS + value: "http://elasticsearch:9200" --- apiVersion: v1 kind: Service diff --git a/monitoring/logging/fluentd/kubernetes/fluentd-configmap.yaml b/monitoring/logging/fluentd/kubernetes/fluentd-configmap.yaml new file mode 100644 index 0000000..18bbbb5 --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/fluentd-configmap.yaml @@ -0,0 +1,39 @@ +#@include file-fluent.conf + +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluentd-config + namespace: fluentd +data: + fluent.conf: |- + ################################################################ + # This source gets all logs from local docker host + @include pods-fluent.conf + @include elastic-fluent.conf + pods-fluent.conf: |- + + @type tail + read_from_head true + tag kubernetes.* + path /var/log/containers/*.log + pos_file /var/log/fluentd-containers.log.pos + exclude_path ["/var/log/containers/fluent*"] + + @type json + time_format %Y-%m-%dT%H:%M:%S.%NZ + + + file-fluent.conf: |- + + @type file + path /tmp/file-test.log + + elastic-fluent.conf: |- + + @type elasticsearch + host "#{ENV['FLUENT_ELASTICSEARCH_HOST'] || 'elasticsearch.elastic-kibana'}" + port "#{ENV['FLUENT_ELASTICSEARCH_PORT'] || '9200'}" + index_name fluentd-k8s + type_name fluentd + \ No newline at end of file diff --git a/monitoring/logging/fluentd/kubernetes/fluentd-rbac.yaml b/monitoring/logging/fluentd/kubernetes/fluentd-rbac.yaml new file mode 100644 index 0000000..2dff202 --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/fluentd-rbac.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fluentd + namespace: fluentd +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fluentd + namespace: fluentd +rules: +- apiGroups: + - "" + resources: + - pods + - namespaces + verbs: + - get + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: fluentd +roleRef: + kind: ClusterRole + name: fluentd + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: fluentd + namespace: fluentd \ No newline at end of file diff --git a/monitoring/logging/fluentd/kubernetes/fluentd.yaml b/monitoring/logging/fluentd/kubernetes/fluentd.yaml new file mode 100644 index 0000000..2a99992 --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/fluentd.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fluentd + namespace: fluentd + labels: + k8s-app: fluentd-logging + version: v1 +spec: + selector: + matchLabels: + k8s-app: fluentd-logging + version: v1 + template: + metadata: + labels: + k8s-app: fluentd-logging + version: v1 + spec: + serviceAccount: fluentd + serviceAccountName: fluentd + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: fluentd + imagePullPolicy: "Always" + image: aimvector/fluentd-demo + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: "elasticsearch.elastic-kibana" + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 200Mi + volumeMounts: + - name: fluentd-config + mountPath: /fluentd/etc + - name: varlog + mountPath: /var/log + terminationGracePeriodSeconds: 30 + volumes: + - name: fluentd-config + configMap: + name: fluentd-config + - name: varlog + hostPath: + path: /var/log \ No newline at end of file