From 992d78042f65073f3ebcf0106ec99f54c2e04b86 Mon Sep 17 00:00:00 2001 From: marcel-dempers Date: Wed, 11 Nov 2020 18:42:16 +1100 Subject: [PATCH] fluent wip --- kubernetes/cloud/azure/getting-started.md | 2 +- .../logging/fluentd/kubernetes/README.md | 24 +-- .../logging/fluentd/kubernetes/counter.yaml | 10 ++ .../fluentd/kubernetes/dockerfiles/Gemfile | 22 +++ .../kubernetes/dockerfiles/Gemfile.lock | 152 ++++++++++++++++++ .../fluentd/kubernetes/dockerfiles/dockerfile | 42 +++++ .../kubernetes/dockerfiles/entrypoint.sh | 3 + .../dockerfiles/plugins/parser_kubernetes.rb | 68 ++++++++ .../plugins/parser_multiline_kubernetes.rb | 69 ++++++++ .../fluentd/kubernetes/fluentd-configmap.yaml | 34 +++- .../logging/fluentd/kubernetes/fluentd.yaml | 8 +- 11 files changed, 421 insertions(+), 13 deletions(-) create mode 100644 monitoring/logging/fluentd/kubernetes/counter.yaml create mode 100644 monitoring/logging/fluentd/kubernetes/dockerfiles/Gemfile create mode 100644 monitoring/logging/fluentd/kubernetes/dockerfiles/Gemfile.lock create mode 100644 monitoring/logging/fluentd/kubernetes/dockerfiles/dockerfile create mode 100644 monitoring/logging/fluentd/kubernetes/dockerfiles/entrypoint.sh create mode 100644 monitoring/logging/fluentd/kubernetes/dockerfiles/plugins/parser_kubernetes.rb create mode 100644 monitoring/logging/fluentd/kubernetes/dockerfiles/plugins/parser_multiline_kubernetes.rb diff --git a/kubernetes/cloud/azure/getting-started.md b/kubernetes/cloud/azure/getting-started.md index fd6e452..4035f80 100644 --- a/kubernetes/cloud/azure/getting-started.md +++ b/kubernetes/cloud/azure/getting-started.md @@ -20,7 +20,7 @@ az login az account list -o table SUBSCRIPTION= -az account set --subscription +az account set --subscription $SUBSCRIPTION ``` diff --git a/monitoring/logging/fluentd/kubernetes/README.md b/monitoring/logging/fluentd/kubernetes/README.md index 2b38b06..c5b6868 100644 --- a/monitoring/logging/fluentd/kubernetes/README.md +++ b/monitoring/logging/fluentd/kubernetes/README.md @@ -8,6 +8,9 @@ To understand the basics of Fluentd, I highly recommend you start with this vide Fluentd +The most important components to understand is the fluentd `tail` plugin.
+This plugin is used to read logs from containers and pods on the file system and collect them. + ## We need a Kubernetes cluster Lets create a Kubernetes cluster to play with using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/) @@ -39,7 +42,7 @@ Let's build our [docker image](https://github.com/marcel-dempers/docker-developm ``` -cd monitoring\logging\fluentd\introduction +cd .\monitoring\logging\fluentd\kubernetes\ #note: use your own tag! docker build . -t aimvector/fluentd-demo @@ -69,8 +72,12 @@ This helps us prevent having a large complex file. We have 3 files in our `fluentd-configmap.yaml` : * fluent.conf: Our main config which includes all other configurations -* pods-fluent.conf: `tail` config that sources all pod logs on the `kubernetes` host -* file-fluent.conf: `match` config to capture all logs and write it to file for testing log collection +* pods-kind-fluent.conf: `tail` config that sources all pod logs on the `kind` cluster. + Note: `kind` cluster writes its log in a different format +* pods-fluent.conf: `tail` config that sources all pod logs on the `kubernetes` host in the cloud.
+ Note: When running K8s in the cloud, logs may go into JSON format. +* file-fluent.conf: `match` config to capture all logs and write it to file for testing log collection
+ Note: This is great to test if collection of logs works * elastic-fluent.conf: `match` config that captures all logs and sends it to `elasticseach` Let's deploy our `configmap`: @@ -91,19 +98,18 @@ kubectl apply -f .\monitoring\logging\fluentd\kubernetes\fluentd.yaml kubectl -n fluentd get pods ``` - -NOT message:("pattern not matched") and NOT message:("/var/log/containers/") - - - - ## Demo ElasticSearch and Kibana ``` kubectl create ns elastic-kibana +# deploy elastic search kubectl -n elastic-kibana apply -f .\monitoring\logging\fluentd\kubernetes\elastic\elastic-demo.yaml +kubectl -n elastic-kibana get pods + +# deploy kibana kubectl -n elastic-kibana apply -f .\monitoring\logging\fluentd\kubernetes\elastic\kibana-demo.yaml +kubectl -n elastic-kibana get pods ``` ## Kibana diff --git a/monitoring/logging/fluentd/kubernetes/counter.yaml b/monitoring/logging/fluentd/kubernetes/counter.yaml new file mode 100644 index 0000000..0dc4deb --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/counter.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Pod +metadata: + name: counter +spec: + containers: + - name: count + image: busybox + args: [/bin/sh, -c, + 'i=0; while true; do echo "$i: $(date)"; i=$((i+1)); sleep 1; done'] diff --git a/monitoring/logging/fluentd/kubernetes/dockerfiles/Gemfile b/monitoring/logging/fluentd/kubernetes/dockerfiles/Gemfile new file mode 100644 index 0000000..1d6417c --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/dockerfiles/Gemfile @@ -0,0 +1,22 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.11.5" +gem "oj", "3.8.1" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.4.0" +gem "fluent-plugin-grok-parser", "~> 2.6.0" +gem "fluent-plugin-prometheus", "~> 1.6.1" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.0.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.12" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.2.0" +gem "elasticsearch", "~> 7.0" +gem "fluent-plugin-elasticsearch", "~> 4.1.1" +gem "elasticsearch-xpack", "~> 7.0" +gem "fluent-plugin-dedot_filter", "~> 1.0" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 2.5.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.0.1" \ No newline at end of file diff --git a/monitoring/logging/fluentd/kubernetes/dockerfiles/Gemfile.lock b/monitoring/logging/fluentd/kubernetes/dockerfiles/Gemfile.lock new file mode 100644 index 0000000..6cba863 --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/dockerfiles/Gemfile.lock @@ -0,0 +1,152 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.7.0) + public_suffix (>= 2.0.2, < 5.0) + concurrent-ruby (1.1.7) + cool.io (1.7.0) + domain_name (0.5.20190701) + unf (>= 0.0.5, < 1.0.0) + elasticsearch (7.9.0) + elasticsearch-api (= 7.9.0) + elasticsearch-transport (= 7.9.0) + elasticsearch-api (7.9.0) + multi_json + elasticsearch-transport (7.9.0) + faraday (~> 1) + multi_json + elasticsearch-xpack (7.9.0) + elasticsearch-api (>= 6) + excon (0.78.0) + faraday (1.1.0) + multipart-post (>= 1.2, < 3) + ruby2_keywords + ffi (1.13.1) + ffi-compiler (1.0.1) + ffi (>= 1.0.0) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.4.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-dedot_filter (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.13) + fluentd (>= 0.10) + fluent-plugin-elasticsearch (4.1.4) + elasticsearch + excon + fluentd (>= 0.14.22) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (2.5.2) + fluentd (>= 0.14.0, < 1.12) + kubeclient (< 5) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-prometheus (1.6.1) + fluentd (>= 0.14.20, < 2) + prometheus-client (< 0.10) + fluent-plugin-record-modifier (2.0.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.2.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.0.2) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 1.3.2) + fluentd (1.11.5) + cool.io (>= 1.4.5, < 2.0.0) + http_parser.rb (>= 0.5.1, < 0.7.0) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.2.2, < 3.0.0) + sigdump (~> 0.2.2) + strptime (>= 0.2.2, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + yajl-ruby (~> 1.0) + http (4.4.1) + addressable (~> 2.3) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + http-parser (~> 1.2.0) + http-accept (1.7.0) + http-cookie (1.0.3) + domain_name (~> 0.5) + http-form_data (2.3.0) + http-parser (1.2.1) + ffi-compiler (>= 1.0, < 2.0) + http_parser.rb (0.6.0) + jsonpath (1.0.5) + multi_json + to_regexp (~> 0.2.1) + kubeclient (4.9.1) + http (>= 3.0, < 5.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + lru_redux (1.1.0) + mime-types (3.3.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2020.1104) + msgpack (1.3.3) + multi_json (1.15.0) + multipart-post (2.1.1) + netrc (0.11.0) + oj (3.8.1) + prometheus-client (0.9.0) + quantile (~> 0.2.1) + public_suffix (4.0.6) + quantile (0.2.1) + rake (13.0.1) + recursive-open-struct (1.1.3) + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + ruby2_keywords (0.0.2) + serverengine (2.2.2) + sigdump (~> 0.2.2) + sigdump (0.2.4) + strptime (0.2.5) + systemd-journal (1.3.3) + ffi (~> 1.9) + to_regexp (0.2.1) + tzinfo (2.0.3) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2020.4) + tzinfo (>= 1.0.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.7) + yajl-ruby (1.4.1) + +PLATFORMS + ruby + +DEPENDENCIES + elasticsearch (~> 7.0) + elasticsearch-xpack (~> 7.0) + ffi + fluent-plugin-concat (~> 2.4.0) + fluent-plugin-dedot_filter (~> 1.0) + fluent-plugin-detect-exceptions (~> 0.0.12) + fluent-plugin-elasticsearch (~> 4.1.1) + fluent-plugin-grok-parser (~> 2.6.0) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 2.5.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-prometheus (~> 1.6.1) + fluent-plugin-record-modifier (~> 2.0.0) + fluent-plugin-rewrite-tag-filter (~> 2.2.0) + fluent-plugin-systemd (~> 1.0.1) + fluentd (= 1.11.5) + oj (= 3.8.1) + +BUNDLED WITH + 2.1.4 \ No newline at end of file diff --git a/monitoring/logging/fluentd/kubernetes/dockerfiles/dockerfile b/monitoring/logging/fluentd/kubernetes/dockerfiles/dockerfile new file mode 100644 index 0000000..9d9e135 --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/dockerfiles/dockerfile @@ -0,0 +1,42 @@ +FROM fluent/fluentd:v1.11-debian + +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/2.6.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/2.6.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/2.6.0 + +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.1.4 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem + +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] \ No newline at end of file diff --git a/monitoring/logging/fluentd/kubernetes/dockerfiles/entrypoint.sh b/monitoring/logging/fluentd/kubernetes/dockerfiles/entrypoint.sh new file mode 100644 index 0000000..575c288 --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/dockerfiles/entrypoint.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins \ No newline at end of file diff --git a/monitoring/logging/fluentd/kubernetes/dockerfiles/plugins/parser_kubernetes.rb b/monitoring/logging/fluentd/kubernetes/dockerfiles/plugins/parser_kubernetes.rb new file mode 100644 index 0000000..a1ec3dd --- /dev/null +++ b/monitoring/logging/fluentd/kubernetes/dockerfiles/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# +# @type kubernetes +# +# +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?\w)(?