fluent wip

kubernetes/cloud/azure/getting-started.md

@@ -20,7 +20,7 @@ az login
 
 az account list -o table
 SUBSCRIPTION=<id>
-az account set --subscription <SubscriptionId-id-here>
+az account set --subscription $SUBSCRIPTION
 
 ```
 

monitoring/logging/fluentd/kubernetes/README.md

@@ -8,6 +8,9 @@ To understand the basics of Fluentd, I highly recommend you start with this vide
 
 <a href="https://youtu.be/Gp0-7oVOtPw" title="Fluentd"><img src="https://i.ytimg.com/vi/Gp0-7oVOtPw/hqdefault.jpg" width="50%" height="50%" alt="Fluentd" /></a>
 
+The most important components to understand is the fluentd `tail` plugin. <br/>
+This plugin is used to read logs from containers and pods on the file system and collect them.
+
 ## We need a Kubernetes cluster
 
 Lets create a Kubernetes cluster to play with using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/)
@@ -39,7 +42,7 @@ Let's build our [docker image](https://github.com/marcel-dempers/docker-developm
 
 
 ```
-cd monitoring\logging\fluentd\introduction
+cd .\monitoring\logging\fluentd\kubernetes\
 
 #note: use your own tag!
 docker build . -t aimvector/fluentd-demo
@@ -69,8 +72,12 @@ This helps us prevent having a large complex file.
 
 We have 3 files in our `fluentd-configmap.yaml` :
 * fluent.conf: Our main config which includes all other configurations
-* pods-fluent.conf: `tail` config that sources all pod logs on the `kubernetes` host
+* pods-kind-fluent.conf: `tail` config that sources all pod logs on the `kind` cluster.
-* file-fluent.conf: `match` config to capture all logs and write it to file for testing log collection
+  Note: `kind` cluster writes its log in a different format
+* pods-fluent.conf: `tail` config that sources all pod logs on the `kubernetes` host in the cloud. <br/>
+  Note: When running K8s in the cloud, logs may go into JSON format.
+* file-fluent.conf: `match` config to capture all logs and write it to file for testing log collection </br>
+  Note: This is great to test if collection of logs works
 * elastic-fluent.conf: `match` config that captures all logs and sends it to `elasticseach`
 
 Let's deploy our `configmap`:
@@ -91,19 +98,18 @@ kubectl apply -f .\monitoring\logging\fluentd\kubernetes\fluentd.yaml
 kubectl -n fluentd get pods
 ```
 
-
-NOT message:("pattern not matched") and NOT message:("/var/log/containers/")
-
-
-
-
 ## Demo ElasticSearch and Kibana
 
 ```
 kubectl create ns elastic-kibana
 
+# deploy elastic search
 kubectl -n elastic-kibana apply -f .\monitoring\logging\fluentd\kubernetes\elastic\elastic-demo.yaml
+kubectl -n elastic-kibana get pods
+
+# deploy kibana
 kubectl -n elastic-kibana apply -f .\monitoring\logging\fluentd\kubernetes\elastic\kibana-demo.yaml
+kubectl -n elastic-kibana get pods
 ```
 
 ## Kibana

monitoring/logging/fluentd/kubernetes/counter.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: counter
+spec:
+  containers:
+  - name: count
+    image: busybox
+    args: [/bin/sh, -c,
+            'i=0; while true; do echo "$i: $(date)"; i=$((i+1)); sleep 1; done']

monitoring/logging/fluentd/kubernetes/dockerfiles/Gemfile

@@ -0,0 +1,22 @@
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb
+
+source "https://rubygems.org"
+
+gem "fluentd", "1.11.5"
+gem "oj", "3.8.1"
+gem "fluent-plugin-multi-format-parser", "~> 1.0.0"
+gem "fluent-plugin-concat", "~> 2.4.0"
+gem "fluent-plugin-grok-parser", "~> 2.6.0"
+gem "fluent-plugin-prometheus", "~> 1.6.1"
+gem 'fluent-plugin-json-in-json-2', ">= 1.0.2"
+gem "fluent-plugin-record-modifier", "~> 2.0.0"
+gem "fluent-plugin-detect-exceptions", "~> 0.0.12"
+gem "fluent-plugin-rewrite-tag-filter", "~> 2.2.0"
+gem "elasticsearch", "~> 7.0"
+gem "fluent-plugin-elasticsearch", "~> 4.1.1"
+gem "elasticsearch-xpack", "~> 7.0"
+gem "fluent-plugin-dedot_filter", "~> 1.0"
+gem "fluent-plugin-kubernetes_metadata_filter", "~> 2.5.0"
+gem "ffi"
+gem "fluent-plugin-systemd", "~> 1.0.1"

monitoring/logging/fluentd/kubernetes/dockerfiles/Gemfile.lock

@@ -0,0 +1,152 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    concurrent-ruby (1.1.7)
+    cool.io (1.7.0)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    elasticsearch (7.9.0)
+      elasticsearch-api (= 7.9.0)
+      elasticsearch-transport (= 7.9.0)
+    elasticsearch-api (7.9.0)
+      multi_json
+    elasticsearch-transport (7.9.0)
+      faraday (~> 1)
+      multi_json
+    elasticsearch-xpack (7.9.0)
+      elasticsearch-api (>= 6)
+    excon (0.78.0)
+    faraday (1.1.0)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords
+    ffi (1.13.1)
+    ffi-compiler (1.0.1)
+      ffi (>= 1.0.0)
+      rake
+    fluent-config-regexp-type (1.0.0)
+      fluentd (> 1.0.0, < 2)
+    fluent-plugin-concat (2.4.0)
+      fluentd (>= 0.14.0, < 2)
+    fluent-plugin-dedot_filter (1.0.0)
+      fluentd (>= 0.14.0, < 2)
+    fluent-plugin-detect-exceptions (0.0.13)
+      fluentd (>= 0.10)
+    fluent-plugin-elasticsearch (4.1.4)
+      elasticsearch
+      excon
+      fluentd (>= 0.14.22)
+    fluent-plugin-grok-parser (2.6.2)
+      fluentd (>= 0.14.6, < 2)
+    fluent-plugin-json-in-json-2 (1.0.2)
+      fluentd (>= 0.14.0, < 2)
+      yajl-ruby (~> 1.0)
+    fluent-plugin-kubernetes_metadata_filter (2.5.2)
+      fluentd (>= 0.14.0, < 1.12)
+      kubeclient (< 5)
+      lru_redux
+    fluent-plugin-multi-format-parser (1.0.0)
+      fluentd (>= 0.14.0, < 2)
+    fluent-plugin-prometheus (1.6.1)
+      fluentd (>= 0.14.20, < 2)
+      prometheus-client (< 0.10)
+    fluent-plugin-record-modifier (2.0.1)
+      fluentd (>= 1.0, < 2)
+    fluent-plugin-rewrite-tag-filter (2.2.0)
+      fluent-config-regexp-type
+      fluentd (>= 0.14.2, < 2)
+    fluent-plugin-systemd (1.0.2)
+      fluentd (>= 0.14.11, < 2)
+      systemd-journal (~> 1.3.2)
+    fluentd (1.11.5)
+      cool.io (>= 1.4.5, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      msgpack (>= 1.3.1, < 2.0.0)
+      serverengine (>= 2.2.2, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.2.2, < 1.0.0)
+      tzinfo (>= 1.0, < 3.0)
+      tzinfo-data (~> 1.0)
+      yajl-ruby (~> 1.0)
+    http (4.4.1)
+      addressable (~> 2.3)
+      http-cookie (~> 1.0)
+      http-form_data (~> 2.2)
+      http-parser (~> 1.2.0)
+    http-accept (1.7.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    http-form_data (2.3.0)
+    http-parser (1.2.1)
+      ffi-compiler (>= 1.0, < 2.0)
+    http_parser.rb (0.6.0)
+    jsonpath (1.0.5)
+      multi_json
+      to_regexp (~> 0.2.1)
+    kubeclient (4.9.1)
+      http (>= 3.0, < 5.0)
+      jsonpath (~> 1.0)
+      recursive-open-struct (~> 1.1, >= 1.1.1)
+      rest-client (~> 2.0)
+    lru_redux (1.1.0)
+    mime-types (3.3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2020.1104)
+    msgpack (1.3.3)
+    multi_json (1.15.0)
+    multipart-post (2.1.1)
+    netrc (0.11.0)
+    oj (3.8.1)
+    prometheus-client (0.9.0)
+      quantile (~> 0.2.1)
+    public_suffix (4.0.6)
+    quantile (0.2.1)
+    rake (13.0.1)
+    recursive-open-struct (1.1.3)
+    rest-client (2.1.0)
+      http-accept (>= 1.7.0, < 2.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
+    ruby2_keywords (0.0.2)
+    serverengine (2.2.2)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.2.5)
+    systemd-journal (1.3.3)
+      ffi (~> 1.9)
+    to_regexp (0.2.1)
+    tzinfo (2.0.3)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2020.4)
+      tzinfo (>= 1.0.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.7)
+    yajl-ruby (1.4.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  elasticsearch (~> 7.0)
+  elasticsearch-xpack (~> 7.0)
+  ffi
+  fluent-plugin-concat (~> 2.4.0)
+  fluent-plugin-dedot_filter (~> 1.0)
+  fluent-plugin-detect-exceptions (~> 0.0.12)
+  fluent-plugin-elasticsearch (~> 4.1.1)
+  fluent-plugin-grok-parser (~> 2.6.0)
+  fluent-plugin-json-in-json-2 (>= 1.0.2)
+  fluent-plugin-kubernetes_metadata_filter (~> 2.5.0)
+  fluent-plugin-multi-format-parser (~> 1.0.0)
+  fluent-plugin-prometheus (~> 1.6.1)
+  fluent-plugin-record-modifier (~> 2.0.0)
+  fluent-plugin-rewrite-tag-filter (~> 2.2.0)
+  fluent-plugin-systemd (~> 1.0.1)
+  fluentd (= 1.11.5)
+  oj (= 3.8.1)
+
+BUNDLED WITH
+   2.1.4

monitoring/logging/fluentd/kubernetes/dockerfiles/dockerfile

@@ -0,0 +1,42 @@
+FROM fluent/fluentd:v1.11-debian
+
+USER root
+WORKDIR /home/fluent
+ENV PATH /fluentd/vendor/bundle/ruby/2.6.0/bin:$PATH
+ENV GEM_PATH /fluentd/vendor/bundle/ruby/2.6.0
+ENV GEM_HOME /fluentd/vendor/bundle/ruby/2.6.0
+
+# skip runtime bundler installation
+ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1
+
+COPY Gemfile* /fluentd/
+RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \
+  runtimeDeps="" \
+      && apt-get update \
+     && apt-get upgrade -y \
+     && apt-get install \
+     -y --no-install-recommends \
+     $buildDeps $runtimeDeps net-tools \
+    && gem install bundler --version 2.1.4 \
+    && bundle config silence_root_warning true \
+    && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \
+    && SUDO_FORCE_REMOVE=yes \
+    apt-get purge -y --auto-remove \
+                  -o APT::AutoRemove::RecommendsImportant=false \
+                  $buildDeps \
+ && rm -rf /var/lib/apt/lists/* \
+    && gem sources --clear-all \
+    && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
+
+RUN touch /fluentd/etc/disable.conf
+
+# Copy plugins
+COPY plugins /fluentd/plugins/
+COPY entrypoint.sh /fluentd/entrypoint.sh
+
+# Environment variables
+ENV FLUENTD_OPT=""
+ENV FLUENTD_CONF="fluent.conf"
+
+# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib
+ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"]

monitoring/logging/fluentd/kubernetes/dockerfiles/entrypoint.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env sh
+
+exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins

monitoring/logging/fluentd/kubernetes/dockerfiles/plugins/parser_kubernetes.rb

@@ -0,0 +1,68 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+# The following Fluentd parser plugin, aims to simplify the parsing of multiline
+# logs found in Kubernetes nodes. Since many log files shared the same format and
+# in order to simplify the configuration, this plugin provides a 'kubernetes' format
+# parser (built on top of MultilineParser).
+#
+# When tailing files, this 'kubernetes' format should be applied to the following
+# log file sources:
+#
+#  - /var/log/kubelet.log
+#  - /var/log/kube-proxy.log
+#  - /var/log/kube-apiserver.log
+#  - /var/log/kube-controller-manager.log
+#  - /var/log/kube-scheduler.log
+#  - /var/log/rescheduler.log
+#  - /var/log/glbc.log
+#  - /var/log/cluster-autoscaler.log
+#
+# Usage:
+#
+# ---- fluentd.conf ----
+#
+# <source>
+#   @type tail
+#   path ./kubelet.log
+#   read_from_head yes
+#   tag kubelet
+#   <parse>
+#     @type kubernetes
+#   </parse>
+# </source>
+#
+# ----   EOF       ---
+
+require 'fluent/plugin/parser_regexp'
+
+module Fluent
+  module Plugin
+    class KubernetesParser < RegexpParser
+      Fluent::Plugin.register_parser("kubernetes", self)
+
+      CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/}
+      CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m}
+      CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N"
+
+      def configure(conf)
+        conf['expression'] = CONF_FORMAT1
+        conf['time_format'] = CONF_TIME_FORMAT
+        super
+      end
+    end
+  end
+end

monitoring/logging/fluentd/kubernetes/dockerfiles/plugins/parser_multiline_kubernetes.rb

@@ -0,0 +1,69 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+# The following Fluentd parser plugin, aims to simplify the parsing of multiline
+# logs found in Kubernetes nodes. Since many log files shared the same format and
+# in order to simplify the configuration, this plugin provides a 'kubernetes' format
+# parser (built on top of MultilineParser).
+#
+# When tailing files, this 'kubernetes' format should be applied to the following
+# log file sources:
+#
+#  - /var/log/kubelet.log
+#  - /var/log/kube-proxy.log
+#  - /var/log/kube-apiserver.log
+#  - /var/log/kube-controller-manager.log
+#  - /var/log/kube-scheduler.log
+#  - /var/log/rescheduler.log
+#  - /var/log/glbc.log
+#  - /var/log/cluster-autoscaler.log
+#
+# Usage:
+#
+# ---- fluentd.conf ----
+#
+# <source>
+#   @type tail
+#   path ./kubelet.log
+#   read_from_head yes
+#   tag kubelet
+#   <parse>
+#     @type multiline_kubernetes
+#   </parse>
+# </source>
+#
+# ----   EOF       ---
+
+require 'fluent/plugin/parser_multiline'
+
+module Fluent
+  module Plugin
+    class MultilineKubernetesParser < MultilineParser
+      Fluent::Plugin.register_parser("multiline_kubernetes", self)
+
+      CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/}
+      CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/}
+      CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N"
+
+      def configure(conf)
+        conf['format_firstline'] = CONF_FORMAT_FIRSTLINE
+        conf['format1'] = CONF_FORMAT1
+        conf['time_format'] = CONF_TIME_FORMAT
+        super
+      end
+    end
+  end
+end

monitoring/logging/fluentd/kubernetes/fluentd-configmap.yaml

@@ -9,8 +9,37 @@ data:
   fluent.conf: |-
     ################################################################
     # This source gets all logs from local docker host
-    @include pods-fluent.conf
+    @include pods-kind-fluent.conf
+    #@include pods-fluent.conf
+    #@include file-fluent.conf
     @include elastic-fluent.conf
+  pods-kind-fluent.conf: |-
+    <source>
+      @type tail
+      read_from_head true
+      tag kubernetes.*
+      path /var/log/containers/*.log
+      pos_file /var/log/fluentd-containers.log.pos
+      exclude_path ["/var/log/containers/fluent*"]
+      <parse>
+        @type regexp
+        #https://regex101.com/r/ZkOBTI/1
+        expression ^(?<time>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.[^Z]*Z)\s(?<stream>[^\s]+)\s(?<character>[^\s])\s(?<message>.*)$
+        #time_format %Y-%m-%dT%H:%M:%S.%NZ
+      </parse>
+    </source>
+
+    <filter kubernetes.**>
+      @type kubernetes_metadata
+      @id filter_kube_metadata
+      kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
+      verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
+      ca_file "#{ENV['KUBERNETES_CA_FILE']}"
+      skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
+      skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
+      skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
+      skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
+    </filter>
   pods-fluent.conf: |-
     <source>
       @type tail
@@ -20,7 +49,8 @@ data:
       pos_file /var/log/fluentd-containers.log.pos
       exclude_path ["/var/log/containers/fluent*"]
       <parse>
-        @type json
+        @type kubernetes
+        @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}"
         time_format %Y-%m-%dT%H:%M:%S.%NZ
       </parse>
     </source>

monitoring/logging/fluentd/kubernetes/fluentd.yaml

@@ -42,6 +42,9 @@ spec:
           mountPath: /fluentd/etc
         - name: varlog
           mountPath: /var/log
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+          readOnly: true
       terminationGracePeriodSeconds: 30
       volumes:
       - name: fluentd-config
@@ -50,3 +53,6 @@ spec:
       - name: varlog
         hostPath:
           path: /var/log
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers