fluentd intro wip

monitoring/logging/fluentd/introduction/configurations/fluent.conf

@@ -0,0 +1,46 @@
+# This source reads tail of a file
+ <source>
+  @type tail
+  format json
+  read_from_head true
+  tag file-myapp.log
+  path /app/example-log.log
+  pos_file /tmp/example-log.log.pos
+</source>
+
+<filter file-myapp.log>
+  @type record_transformer
+  <record>
+    host_param "#{Socket.gethostname}"
+  </record>
+</filter>
+
+<match file-myapp.log>
+  @type file
+  path /output/file-myapp.log
+</match>
+
+################################################################
+
+# This source gets incoming logs over HTTP
+<source>
+  @type http
+  port 9880
+  bind 0.0.0.0
+  body_size_limit 32m
+  keepalive_timeout 10s
+</source>
+
+<filter http-*.log>
+  @type record_transformer
+  <record>
+    host_param "#{Socket.gethostname}"
+  </record>
+</filter>
+
+<match http-*.log>
+  @type file
+  path /output/http.log
+</match>
+
+################################################################

monitoring/logging/fluentd/introduction/docker-compose.yaml

@@ -5,8 +5,24 @@ services:
     user: root
     image: fluent/fluentd:v1.11-debian
     volumes:
-    - /var/lib/docker/containers:/fluentd/log/containers
-    - ./fluent.conf:/fluentd/etc/fluent.conf
+    - /var/lib/docker/containers:/fluentd/log/containers # Example: Reading docker logs
+    - ./file:/app/ #Example: Reading logs from a file
+    - ./configurations:/fluentd/etc/
     - ./logs:/output/
     logging:
-      driver: "local"
+      driver: "local"
+  # This app sends logs to Fluentd via HTTP
+  http-myapp:
+    container_name: http-myapp
+    image: alpine
+    volumes:
+    - ./http:/app
+    command: [ /bin/sh , -c , "apk add --no-cache curl && chmod +x /app/send-http-log.sh && ./app/send-http-log.sh"]
+  # This app writes logs to a local file
+  file-myapp:
+    container_name: file-myapp
+    image: alpine
+    volumes:
+    - ./file:/app
+    command: [ /bin/sh , -c , "apk add --no-cache curl && chmod +x /app/write-file-log.sh && ./app/write-file-log.sh"]
+  

monitoring/logging/fluentd/introduction/file/example-log.log

@@ -0,0 +1 @@
+This is a log

monitoring/logging/fluentd/introduction/file/write-file-log.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+while true
+do
+	echo "Writing log to a file"
+  echo '{"app":"file-myapp"}' >> /app/example-log.log
+	sleep 5
+done

monitoring/logging/fluentd/introduction/fluent.conf

@@ -1,14 +0,0 @@
-
- <source>
-  @type tail
-  format json
-  read_from_head true
-  tag docker.logs
-  path /fluentd/log/containers/*/*-json.log
-  pos_file /tmp/container-logs.pos
-</source>
-
-<match docker.logs>
-  @type file
-  path /output/test.log
-</match>

monitoring/logging/fluentd/introduction/http/send-http-log.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+while true
+do
+	echo "Sending logs to FluentD"
+  curl -X POST -d 'json={"foo":"bar"}' http://fluentd:9880/http-myapp.log
+	sleep 5
+done

monitoring/logging/fluentd/introduction/logs/readme.txt

@@ -0,0 +1,2 @@
+fluentd will collect all container logs and write them
+in this folder