From 2b4df899b11e9e418884d9f28b9978de91eb4217 Mon Sep 17 00:00:00 2001 From: marcel-dempers Date: Sun, 8 Nov 2020 09:10:40 +1100 Subject: [PATCH 1/8] fluent-k8s-wip --- .../logging/fluentd/kubernetes/README.md | 17 +++++++ .../fluentd/kubernetes/elastic-demo.yaml | 47 +++++++++++++++++++ .../fluentd/kubernetes/kibana-demo.yaml | 43 +++++++++++++++++ 3 files changed, 107 insertions(+) create mode 100644 monitoring/logging/fluentd/kubernetes/README.md create mode 100644 monitoring/logging/fluentd/kubernetes/elastic-demo.yaml create mode 100644 monitoring/logging/fluentd/kubernetes/kibana-demo.yaml diff --git a/monitoring/logging/fluentd/kubernetes/README.md b/monitoring/logging/fluentd/kubernetes/README.md new file mode 100644 index 0000000..bbe6422 --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/README.md @@ -0,0 +1,17 @@ +# Introduction to Fluentd on Kubernetes + +## We need a Kubernetes cluster + +Lets create a Kubernetes cluster to play with using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/) + +``` +kind create cluster --name fluentd --image kindest/node:v1.19.1 +``` + +## Fluentd Manifests + +I would highly recommend to use manifests from the official fluentd [github repo](https://github.com/fluent/fluentd-kubernetes-daemonset)
+ +The manifests found here are purely for demo purpose.
+ +In this example I will use the most common use case and we'll break it down to get an understanding of each component. \ No newline at end of file diff --git a/monitoring/logging/fluentd/kubernetes/elastic-demo.yaml b/monitoring/logging/fluentd/kubernetes/elastic-demo.yaml new file mode 100644 index 0000000..d4cc2b2 --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/elastic-demo.yaml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elasticsearch + labels: + app: elasticsearch +spec: + selector: + matchLabels: + app: elasticsearch + replicas: 1 + template: + metadata: + labels: + app: elasticsearch + spec: + containers: + - name: elasticsearch + image: elasticsearch:7.9.1 + imagePullPolicy: IfNotExists + ports: + - containerPort: 9200 + env: + - name: node.name + value: "elasticsearch" + - name: cluster.initial_master_nodes + value: "elasticsearch" + - name: bootstrap.memory_lock + value: "true" + - name: ES_JAVA_OPTS + value: "-Xms512m -Xmx512m" +--- +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch + labels: + app: elasticsearch +spec: + type: ClusterIP + selector: + app: elasticsearch + ports: + - protocol: TCP + name: http + port: 9200 + targetPort: 9200 \ No newline at end of file diff --git a/monitoring/logging/fluentd/kubernetes/kibana-demo.yaml b/monitoring/logging/fluentd/kubernetes/kibana-demo.yaml new file mode 100644 index 0000000..8f63c9d --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/kibana-demo.yaml @@ -0,0 +1,43 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kibana + labels: + app: kibana +spec: + selector: + matchLabels: + app: kibana + replicas: 1 + template: + metadata: + labels: + app: kibana + spec: + containers: + - name: kibana + image: kibana:7.9.1 + imagePullPolicy: IfNotExists + ports: + - containerPort: 5601 + env: + - name: ELASTICSEARCH_URL + value: "http://elasticsearch:9200" + - name: ELASTICSEARCH_HOSTS + value: "http://elasticsearch:9200" +--- +apiVersion: v1 +kind: Service +metadata: + name: kibana + labels: + app: kibana +spec: + type: ClusterIP + selector: + app: kibana + ports: + - protocol: TCP + name: http + port: 5601 + targetPort: 5601 \ No newline at end of file From f9b9a3f1fed517fd4860a0f9cd2918f9028f4057 Mon Sep 17 00:00:00 2001 From: marcel-dempers Date: Sun, 8 Nov 2020 16:00:38 +1100 Subject: [PATCH 2/8] fluent-k8s wip --- .../configurations/containers-fluent.conf | 2 +- .../configurations/file-fluent.conf | 2 +- .../logging/fluentd/introduction/dockerfile | 1 - .../logging/fluentd/kubernetes/README.md | 100 +++++++++++++++++- .../{ => elastic}/elastic-demo.yaml | 26 +++-- .../kubernetes/{ => elastic}/kibana-demo.yaml | 12 +-- .../fluentd/kubernetes/fluentd-configmap.yaml | 39 +++++++ .../fluentd/kubernetes/fluentd-rbac.yaml | 34 ++++++ .../logging/fluentd/kubernetes/fluentd.yaml | 52 +++++++++ 9 files changed, 247 insertions(+), 21 deletions(-) rename monitoring/logging/fluentd/kubernetes/{ => elastic}/elastic-demo.yaml (55%) rename monitoring/logging/fluentd/kubernetes/{ => elastic}/kibana-demo.yaml (72%) create mode 100644 monitoring/logging/fluentd/kubernetes/fluentd-configmap.yaml create mode 100644 monitoring/logging/fluentd/kubernetes/fluentd-rbac.yaml create mode 100644 monitoring/logging/fluentd/kubernetes/fluentd.yaml diff --git a/monitoring/logging/fluentd/introduction/configurations/containers-fluent.conf b/monitoring/logging/fluentd/introduction/configurations/containers-fluent.conf index 49084ff..30cacf8 100644 --- a/monitoring/logging/fluentd/introduction/configurations/containers-fluent.conf +++ b/monitoring/logging/fluentd/introduction/configurations/containers-fluent.conf @@ -1,4 +1,4 @@ - + @type tail format json read_from_head true diff --git a/monitoring/logging/fluentd/introduction/configurations/file-fluent.conf b/monitoring/logging/fluentd/introduction/configurations/file-fluent.conf index a0cf07a..0053a0b 100644 --- a/monitoring/logging/fluentd/introduction/configurations/file-fluent.conf +++ b/monitoring/logging/fluentd/introduction/configurations/file-fluent.conf @@ -1,4 +1,4 @@ - + @type tail format json read_from_head true diff --git a/monitoring/logging/fluentd/introduction/dockerfile b/monitoring/logging/fluentd/introduction/dockerfile index 2ab00f0..b00f600 100644 --- a/monitoring/logging/fluentd/introduction/dockerfile +++ b/monitoring/logging/fluentd/introduction/dockerfile @@ -2,4 +2,3 @@ FROM fluent/fluentd:v1.11-debian USER root RUN gem install fluent-plugin-elasticsearch -USER fluent diff --git a/monitoring/logging/fluentd/kubernetes/README.md b/monitoring/logging/fluentd/kubernetes/README.md index bbe6422..2b38b06 100644 --- a/monitoring/logging/fluentd/kubernetes/README.md +++ b/monitoring/logging/fluentd/kubernetes/README.md @@ -1,5 +1,13 @@ # Introduction to Fluentd on Kubernetes +## Prerequisites + +You will need a basic understanding of Fluentd before you attempt to run it on Kubernetes.
+Fluentd and Kubernetes have a bunch of moving parts.
+To understand the basics of Fluentd, I highly recommend you start with this video:
+ +Fluentd + ## We need a Kubernetes cluster Lets create a Kubernetes cluster to play with using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/) @@ -10,8 +18,96 @@ kind create cluster --name fluentd --image kindest/node:v1.19.1 ## Fluentd Manifests -I would highly recommend to use manifests from the official fluentd [github repo](https://github.com/fluent/fluentd-kubernetes-daemonset)
+I would highly recommend to use manifests from the official fluentd [github repo](https://github.com/fluent/fluentd-kubernetes-daemonset) for production usage
The manifests found here are purely for demo purpose.
+The manifests in this repo are broken down and simplified for educational purpose.
+
+In this example I will use the most common use case and we'll break it down to get an understanding of each component. -In this example I will use the most common use case and we'll break it down to get an understanding of each component. \ No newline at end of file +## Fluentd Docker + +I would recommend to start with the official [fluentd](https://hub.docker.com/r/fluent/fluentd/) +docker image.
+You may want to build your own image if you want to install plugins. +In this demo I will be using the `fluentd` elasticsearch plugin
+It's pretty simple to adjust `fluentd` to send logs to any other destination in case you are not an `elasticsearch` user.
+ +
+ +Let's build our [docker image](https://github.com/marcel-dempers/docker-development-youtube-series/blob/master/monitoring/logging/fluentd/introduction/dockerfile) in the introduction folder: + + +``` +cd monitoring\logging\fluentd\introduction + +#note: use your own tag! +docker build . -t aimvector/fluentd-demo + +#note: use your own tag! +docker push aimvector/fluentd-demo + +``` + +## Fluentd Namespace + +I like to run certain infrastructure components in their own namespaces.
+If you are using the official manifests, they may be using the `kube-system` namespace instead.
+You may want to carefully adjust it based on your preference
+Let's create a `fluentd` namespace:
+ +``` +kubectl create ns fluentd + +``` +## Fluentd Configmap + +In my [fluentd introduction video](https://youtu.be/Gp0-7oVOtPw), I talk about how `fluentd` allows us to simplify our configs using the `include` statement.
+This helps us prevent having a large complex file. + +
+ +We have 3 files in our `fluentd-configmap.yaml` : +* fluent.conf: Our main config which includes all other configurations +* pods-fluent.conf: `tail` config that sources all pod logs on the `kubernetes` host +* file-fluent.conf: `match` config to capture all logs and write it to file for testing log collection +* elastic-fluent.conf: `match` config that captures all logs and sends it to `elasticseach` + +Let's deploy our `configmap`: + +``` +kubectl apply -f .\monitoring\logging\fluentd\kubernetes\fluentd-configmap.yaml + +``` + +## Fluentd Daemonset + +Let's deploy our `daemonset`: + +``` +kubectl apply -f .\monitoring\logging\fluentd\kubernetes\fluentd-rbac.yaml +kubectl apply -f .\monitoring\logging\fluentd\kubernetes\fluentd.yaml + +kubectl -n fluentd get pods +``` + + +NOT message:("pattern not matched") and NOT message:("/var/log/containers/") + + + + +## Demo ElasticSearch and Kibana + +``` +kubectl create ns elastic-kibana + +kubectl -n elastic-kibana apply -f .\monitoring\logging\fluentd\kubernetes\elastic\elastic-demo.yaml +kubectl -n elastic-kibana apply -f .\monitoring\logging\fluentd\kubernetes\elastic\kibana-demo.yaml +``` + +## Kibana + +``` +kubectl -n elastic-kibana port-forward svc/kibana 5601 +``` \ No newline at end of file diff --git a/monitoring/logging/fluentd/kubernetes/elastic-demo.yaml b/monitoring/logging/fluentd/kubernetes/elastic/elastic-demo.yaml similarity index 55% rename from monitoring/logging/fluentd/kubernetes/elastic-demo.yaml rename to monitoring/logging/fluentd/kubernetes/elastic/elastic-demo.yaml index d4cc2b2..e4064b7 100644 --- a/monitoring/logging/fluentd/kubernetes/elastic-demo.yaml +++ b/monitoring/logging/fluentd/kubernetes/elastic/elastic-demo.yaml @@ -14,21 +14,27 @@ spec: labels: app: elasticsearch spec: + initContainers: + - name: vm-max-fix + image: busybox + command: ["sysctl", "-w", "vm.max_map_count=262144"] + securityContext: + privileged: true containers: - name: elasticsearch image: elasticsearch:7.9.1 - imagePullPolicy: IfNotExists + imagePullPolicy: IfNotPresent ports: - containerPort: 9200 - env: - - name: node.name - value: "elasticsearch" - - name: cluster.initial_master_nodes - value: "elasticsearch" - - name: bootstrap.memory_lock - value: "true" - - name: ES_JAVA_OPTS - value: "-Xms512m -Xmx512m" + env: + - name: node.name + value: "elasticsearch" + - name: cluster.initial_master_nodes + value: "elasticsearch" + - name: bootstrap.memory_lock + value: "false" + - name: ES_JAVA_OPTS + value: "-Xms512m -Xmx512m" --- apiVersion: v1 kind: Service diff --git a/monitoring/logging/fluentd/kubernetes/kibana-demo.yaml b/monitoring/logging/fluentd/kubernetes/elastic/kibana-demo.yaml similarity index 72% rename from monitoring/logging/fluentd/kubernetes/kibana-demo.yaml rename to monitoring/logging/fluentd/kubernetes/elastic/kibana-demo.yaml index 8f63c9d..c58524b 100644 --- a/monitoring/logging/fluentd/kubernetes/kibana-demo.yaml +++ b/monitoring/logging/fluentd/kubernetes/elastic/kibana-demo.yaml @@ -17,14 +17,14 @@ spec: containers: - name: kibana image: kibana:7.9.1 - imagePullPolicy: IfNotExists + imagePullPolicy: IfNotPresent ports: - containerPort: 5601 - env: - - name: ELASTICSEARCH_URL - value: "http://elasticsearch:9200" - - name: ELASTICSEARCH_HOSTS - value: "http://elasticsearch:9200" + env: + - name: ELASTICSEARCH_URL + value: "http://elasticsearch:9200" + - name: ELASTICSEARCH_HOSTS + value: "http://elasticsearch:9200" --- apiVersion: v1 kind: Service diff --git a/monitoring/logging/fluentd/kubernetes/fluentd-configmap.yaml b/monitoring/logging/fluentd/kubernetes/fluentd-configmap.yaml new file mode 100644 index 0000000..18bbbb5 --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/fluentd-configmap.yaml @@ -0,0 +1,39 @@ +#@include file-fluent.conf + +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluentd-config + namespace: fluentd +data: + fluent.conf: |- + ################################################################ + # This source gets all logs from local docker host + @include pods-fluent.conf + @include elastic-fluent.conf + pods-fluent.conf: |- + + @type tail + read_from_head true + tag kubernetes.* + path /var/log/containers/*.log + pos_file /var/log/fluentd-containers.log.pos + exclude_path ["/var/log/containers/fluent*"] + + @type json + time_format %Y-%m-%dT%H:%M:%S.%NZ + + + file-fluent.conf: |- + + @type file + path /tmp/file-test.log + + elastic-fluent.conf: |- + + @type elasticsearch + host "#{ENV['FLUENT_ELASTICSEARCH_HOST'] || 'elasticsearch.elastic-kibana'}" + port "#{ENV['FLUENT_ELASTICSEARCH_PORT'] || '9200'}" + index_name fluentd-k8s + type_name fluentd + \ No newline at end of file diff --git a/monitoring/logging/fluentd/kubernetes/fluentd-rbac.yaml b/monitoring/logging/fluentd/kubernetes/fluentd-rbac.yaml new file mode 100644 index 0000000..2dff202 --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/fluentd-rbac.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fluentd + namespace: fluentd +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fluentd + namespace: fluentd +rules: +- apiGroups: + - "" + resources: + - pods + - namespaces + verbs: + - get + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: fluentd +roleRef: + kind: ClusterRole + name: fluentd + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: fluentd + namespace: fluentd \ No newline at end of file diff --git a/monitoring/logging/fluentd/kubernetes/fluentd.yaml b/monitoring/logging/fluentd/kubernetes/fluentd.yaml new file mode 100644 index 0000000..2a99992 --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/fluentd.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fluentd + namespace: fluentd + labels: + k8s-app: fluentd-logging + version: v1 +spec: + selector: + matchLabels: + k8s-app: fluentd-logging + version: v1 + template: + metadata: + labels: + k8s-app: fluentd-logging + version: v1 + spec: + serviceAccount: fluentd + serviceAccountName: fluentd + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: fluentd + imagePullPolicy: "Always" + image: aimvector/fluentd-demo + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: "elasticsearch.elastic-kibana" + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 200Mi + volumeMounts: + - name: fluentd-config + mountPath: /fluentd/etc + - name: varlog + mountPath: /var/log + terminationGracePeriodSeconds: 30 + volumes: + - name: fluentd-config + configMap: + name: fluentd-config + - name: varlog + hostPath: + path: /var/log \ No newline at end of file From 992d78042f65073f3ebcf0106ec99f54c2e04b86 Mon Sep 17 00:00:00 2001 From: marcel-dempers Date: Wed, 11 Nov 2020 18:42:16 +1100 Subject: [PATCH 3/8] fluent wip --- kubernetes/cloud/azure/getting-started.md | 2 +- .../logging/fluentd/kubernetes/README.md | 24 +-- .../logging/fluentd/kubernetes/counter.yaml | 10 ++ .../fluentd/kubernetes/dockerfiles/Gemfile | 22 +++ .../kubernetes/dockerfiles/Gemfile.lock | 152 ++++++++++++++++++ .../fluentd/kubernetes/dockerfiles/dockerfile | 42 +++++ .../kubernetes/dockerfiles/entrypoint.sh | 3 + .../dockerfiles/plugins/parser_kubernetes.rb | 68 ++++++++ .../plugins/parser_multiline_kubernetes.rb | 69 ++++++++ .../fluentd/kubernetes/fluentd-configmap.yaml | 34 +++- .../logging/fluentd/kubernetes/fluentd.yaml | 8 +- 11 files changed, 421 insertions(+), 13 deletions(-) create mode 100644 monitoring/logging/fluentd/kubernetes/counter.yaml create mode 100644 monitoring/logging/fluentd/kubernetes/dockerfiles/Gemfile create mode 100644 monitoring/logging/fluentd/kubernetes/dockerfiles/Gemfile.lock create mode 100644 monitoring/logging/fluentd/kubernetes/dockerfiles/dockerfile create mode 100644 monitoring/logging/fluentd/kubernetes/dockerfiles/entrypoint.sh create mode 100644 monitoring/logging/fluentd/kubernetes/dockerfiles/plugins/parser_kubernetes.rb create mode 100644 monitoring/logging/fluentd/kubernetes/dockerfiles/plugins/parser_multiline_kubernetes.rb diff --git a/kubernetes/cloud/azure/getting-started.md b/kubernetes/cloud/azure/getting-started.md index fd6e452..4035f80 100644 --- a/kubernetes/cloud/azure/getting-started.md +++ b/kubernetes/cloud/azure/getting-started.md @@ -20,7 +20,7 @@ az login az account list -o table SUBSCRIPTION= -az account set --subscription +az account set --subscription $SUBSCRIPTION ``` diff --git a/monitoring/logging/fluentd/kubernetes/README.md b/monitoring/logging/fluentd/kubernetes/README.md index 2b38b06..c5b6868 100644 --- a/monitoring/logging/fluentd/kubernetes/README.md +++ b/monitoring/logging/fluentd/kubernetes/README.md @@ -8,6 +8,9 @@ To understand the basics of Fluentd, I highly recommend you start with this vide Fluentd +The most important components to understand is the fluentd `tail` plugin.
+This plugin is used to read logs from containers and pods on the file system and collect them. + ## We need a Kubernetes cluster Lets create a Kubernetes cluster to play with using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/) @@ -39,7 +42,7 @@ Let's build our [docker image](https://github.com/marcel-dempers/docker-developm ``` -cd monitoring\logging\fluentd\introduction +cd .\monitoring\logging\fluentd\kubernetes\ #note: use your own tag! docker build . -t aimvector/fluentd-demo @@ -69,8 +72,12 @@ This helps us prevent having a large complex file. We have 3 files in our `fluentd-configmap.yaml` : * fluent.conf: Our main config which includes all other configurations -* pods-fluent.conf: `tail` config that sources all pod logs on the `kubernetes` host -* file-fluent.conf: `match` config to capture all logs and write it to file for testing log collection +* pods-kind-fluent.conf: `tail` config that sources all pod logs on the `kind` cluster. + Note: `kind` cluster writes its log in a different format +* pods-fluent.conf: `tail` config that sources all pod logs on the `kubernetes` host in the cloud.
+ Note: When running K8s in the cloud, logs may go into JSON format. +* file-fluent.conf: `match` config to capture all logs and write it to file for testing log collection
+ Note: This is great to test if collection of logs works * elastic-fluent.conf: `match` config that captures all logs and sends it to `elasticseach` Let's deploy our `configmap`: @@ -91,19 +98,18 @@ kubectl apply -f .\monitoring\logging\fluentd\kubernetes\fluentd.yaml kubectl -n fluentd get pods ``` - -NOT message:("pattern not matched") and NOT message:("/var/log/containers/") - - - - ## Demo ElasticSearch and Kibana ``` kubectl create ns elastic-kibana +# deploy elastic search kubectl -n elastic-kibana apply -f .\monitoring\logging\fluentd\kubernetes\elastic\elastic-demo.yaml +kubectl -n elastic-kibana get pods + +# deploy kibana kubectl -n elastic-kibana apply -f .\monitoring\logging\fluentd\kubernetes\elastic\kibana-demo.yaml +kubectl -n elastic-kibana get pods ``` ## Kibana diff --git a/monitoring/logging/fluentd/kubernetes/counter.yaml b/monitoring/logging/fluentd/kubernetes/counter.yaml new file mode 100644 index 0000000..0dc4deb --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/counter.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Pod +metadata: + name: counter +spec: + containers: + - name: count + image: busybox + args: [/bin/sh, -c, + 'i=0; while true; do echo "$i: $(date)"; i=$((i+1)); sleep 1; done'] diff --git a/monitoring/logging/fluentd/kubernetes/dockerfiles/Gemfile b/monitoring/logging/fluentd/kubernetes/dockerfiles/Gemfile new file mode 100644 index 0000000..1d6417c --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/dockerfiles/Gemfile @@ -0,0 +1,22 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.11.5" +gem "oj", "3.8.1" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.4.0" +gem "fluent-plugin-grok-parser", "~> 2.6.0" +gem "fluent-plugin-prometheus", "~> 1.6.1" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.0.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.12" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.2.0" +gem "elasticsearch", "~> 7.0" +gem "fluent-plugin-elasticsearch", "~> 4.1.1" +gem "elasticsearch-xpack", "~> 7.0" +gem "fluent-plugin-dedot_filter", "~> 1.0" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 2.5.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.0.1" \ No newline at end of file diff --git a/monitoring/logging/fluentd/kubernetes/dockerfiles/Gemfile.lock b/monitoring/logging/fluentd/kubernetes/dockerfiles/Gemfile.lock new file mode 100644 index 0000000..6cba863 --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/dockerfiles/Gemfile.lock @@ -0,0 +1,152 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.7.0) + public_suffix (>= 2.0.2, < 5.0) + concurrent-ruby (1.1.7) + cool.io (1.7.0) + domain_name (0.5.20190701) + unf (>= 0.0.5, < 1.0.0) + elasticsearch (7.9.0) + elasticsearch-api (= 7.9.0) + elasticsearch-transport (= 7.9.0) + elasticsearch-api (7.9.0) + multi_json + elasticsearch-transport (7.9.0) + faraday (~> 1) + multi_json + elasticsearch-xpack (7.9.0) + elasticsearch-api (>= 6) + excon (0.78.0) + faraday (1.1.0) + multipart-post (>= 1.2, < 3) + ruby2_keywords + ffi (1.13.1) + ffi-compiler (1.0.1) + ffi (>= 1.0.0) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.4.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-dedot_filter (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.13) + fluentd (>= 0.10) + fluent-plugin-elasticsearch (4.1.4) + elasticsearch + excon + fluentd (>= 0.14.22) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (2.5.2) + fluentd (>= 0.14.0, < 1.12) + kubeclient (< 5) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-prometheus (1.6.1) + fluentd (>= 0.14.20, < 2) + prometheus-client (< 0.10) + fluent-plugin-record-modifier (2.0.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.2.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.0.2) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 1.3.2) + fluentd (1.11.5) + cool.io (>= 1.4.5, < 2.0.0) + http_parser.rb (>= 0.5.1, < 0.7.0) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.2.2, < 3.0.0) + sigdump (~> 0.2.2) + strptime (>= 0.2.2, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + yajl-ruby (~> 1.0) + http (4.4.1) + addressable (~> 2.3) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + http-parser (~> 1.2.0) + http-accept (1.7.0) + http-cookie (1.0.3) + domain_name (~> 0.5) + http-form_data (2.3.0) + http-parser (1.2.1) + ffi-compiler (>= 1.0, < 2.0) + http_parser.rb (0.6.0) + jsonpath (1.0.5) + multi_json + to_regexp (~> 0.2.1) + kubeclient (4.9.1) + http (>= 3.0, < 5.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + lru_redux (1.1.0) + mime-types (3.3.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2020.1104) + msgpack (1.3.3) + multi_json (1.15.0) + multipart-post (2.1.1) + netrc (0.11.0) + oj (3.8.1) + prometheus-client (0.9.0) + quantile (~> 0.2.1) + public_suffix (4.0.6) + quantile (0.2.1) + rake (13.0.1) + recursive-open-struct (1.1.3) + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + ruby2_keywords (0.0.2) + serverengine (2.2.2) + sigdump (~> 0.2.2) + sigdump (0.2.4) + strptime (0.2.5) + systemd-journal (1.3.3) + ffi (~> 1.9) + to_regexp (0.2.1) + tzinfo (2.0.3) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2020.4) + tzinfo (>= 1.0.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.7) + yajl-ruby (1.4.1) + +PLATFORMS + ruby + +DEPENDENCIES + elasticsearch (~> 7.0) + elasticsearch-xpack (~> 7.0) + ffi + fluent-plugin-concat (~> 2.4.0) + fluent-plugin-dedot_filter (~> 1.0) + fluent-plugin-detect-exceptions (~> 0.0.12) + fluent-plugin-elasticsearch (~> 4.1.1) + fluent-plugin-grok-parser (~> 2.6.0) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 2.5.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-prometheus (~> 1.6.1) + fluent-plugin-record-modifier (~> 2.0.0) + fluent-plugin-rewrite-tag-filter (~> 2.2.0) + fluent-plugin-systemd (~> 1.0.1) + fluentd (= 1.11.5) + oj (= 3.8.1) + +BUNDLED WITH + 2.1.4 \ No newline at end of file diff --git a/monitoring/logging/fluentd/kubernetes/dockerfiles/dockerfile b/monitoring/logging/fluentd/kubernetes/dockerfiles/dockerfile new file mode 100644 index 0000000..9d9e135 --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/dockerfiles/dockerfile @@ -0,0 +1,42 @@ +FROM fluent/fluentd:v1.11-debian + +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/2.6.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/2.6.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/2.6.0 + +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.1.4 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem + +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] \ No newline at end of file diff --git a/monitoring/logging/fluentd/kubernetes/dockerfiles/entrypoint.sh b/monitoring/logging/fluentd/kubernetes/dockerfiles/entrypoint.sh new file mode 100644 index 0000000..575c288 --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/dockerfiles/entrypoint.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins \ No newline at end of file diff --git a/monitoring/logging/fluentd/kubernetes/dockerfiles/plugins/parser_kubernetes.rb b/monitoring/logging/fluentd/kubernetes/dockerfiles/plugins/parser_kubernetes.rb new file mode 100644 index 0000000..a1ec3dd --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/dockerfiles/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# +# @type kubernetes +# +# +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?\w)(?