wip

monitoring/logging/fluentd/basic-demo/readme.md

@@ -1,5 +1,6 @@
 # Fluentd basic demo
 
+Check out the [video](https://youtu.be/MMVdkzeQ848)
 In my video: Introduction to logging <br/>
 I run fluentd locally <br/>
 I collect all local container logs into the `./logs` folder <br/>

monitoring/logging/fluentd/introduction/configurations/elastic-fluent.conf

@@ -1,56 +0,0 @@
-
-# incoming http --> elastic search
-<source>
-  @type http
-  port 9880
-  bind 0.0.0.0
-  body_size_limit 32m
-  keepalive_timeout 10s
-</source>
-
-#container logs --> elastic search
- <source>
-  @type tail
-  format json
-  read_from_head true
-  tag docker.log
-  path /fluentd/log/containers/*/*-json.log
-  pos_file /tmp/container-logs.pos
-</source>
-
-#local file --> elastic search
- <source>
-  @type tail
-  format json
-  read_from_head true
-  tag file-myapp.log
-  path /app/example-log.log
-  pos_file /tmp/example-log.log.pos
-</source>
-
-# where to send http logs
-<match http-*.log>
-  @type elasticsearch
-  host elasticsearch
-  port 9200
-  index_name fluentd-http
-  type_name fluentd
-</match>
-
-#where to send file logs
-<match file-myapp.log>
-  @type elasticsearch
-  host elasticsearch
-  port 9200
-  index_name fluentd-file
-  type_name fluentd
-</match>
-
-#where to send docker logs
-<match docker.log>
-  @type elasticsearch
-  host elasticsearch
-  port 9200
-  index_name fluentd-docker
-  type_name fluentd
-</match>

monitoring/logging/fluentd/introduction/configurations/fluent.conf

@@ -5,17 +5,10 @@
   format json
   read_from_head true
   tag file-myapp.log
-  path /app/example-log.log
+  path /fluentd/log/files/example-log.log
   pos_file /tmp/example-log.log.pos
 </source>
 
-<filter file-myapp.log>
-  @type record_transformer
-  <record>
-    host_param "#{Socket.gethostname}"
-  </record>
-</filter>
-
 <match file-myapp.log>
   @type file
   path /output/file-myapp.log
@@ -32,13 +25,6 @@
   keepalive_timeout 10s
 </source>
 
-<filter http-*.log>
-  @type record_transformer
-  <record>
-    host_param "#{Socket.gethostname}"
-  </record>
-</filter>
-
 <match http-*.log>
   @type file
   path /output/http.log

monitoring/logging/fluentd/introduction/docker-compose.yaml

@@ -8,9 +8,9 @@ services:
     image: fluentd
     volumes:
     - /var/lib/docker/containers:/fluentd/log/containers # Example: Reading docker logs
-    - ./file:/app/ #Example: Reading logs from a file
+    - ./file:/fluentd/log/files/ #Example: Reading logs from a file
     - ./configurations:/fluentd/etc/
-    - ./logs:/output/
+    - ./logs:/output/ # Example: Fluentd will collect logs and store it here for demo
     logging:
       driver: "local"
   # This app sends logs to Fluentd via HTTP
@@ -19,14 +19,14 @@ services:
     image: alpine
     volumes:
     - ./http:/app
-    command: [ /bin/sh , -c , "apk add --no-cache curl && chmod +x /app/send-http-log.sh && ./app/send-http-log.sh"]
+    command: [ /bin/sh , -c , "apk add --no-cache curl && chmod +x /app/app.sh && ./app/app.sh"]
   # This app writes logs to a local file
   file-myapp:
     container_name: file-myapp
     image: alpine
     volumes:
     - ./file:/app
-    command: [ /bin/sh , -c , "apk add --no-cache curl && chmod +x /app/write-file-log.sh && ./app/write-file-log.sh"]
+    command: [ /bin/sh , -c , "chmod +x /app/app.sh && ./app/app.sh"]
   elasticsearch: # port 9200
     image: elasticsearch:7.9.1
     container_name: elasticsearch

monitoring/logging/fluentd/introduction/readme.md

@@ -0,0 +1,28 @@
+# Introduction to Fluentd
+
+## Collecting logs from files
+
+Reading logs from a file we need an application that writes logs to a file. <br/>
+Lets start one:
+
+```
+cd monitoring\logging\fluentd\introduction\
+
+docker-compose up -d file-myapp
+
+```
+
+To collect the logs, lets start fluentd
+
+```
+docker-compose up -d fluentd
+```
+
+## Collecting logs over HTTP (incoming)
+
+```
+cd monitoring\logging\fluentd\introduction\
+
+docker-compose up -d http-myapp
+
+```

monitoring/logging/fluentd/readme.md

@@ -1,8 +0,0 @@
-# Logging
-
-## Logging Basics
-
-* Standardised Logging
-* Centralised Logging 
-
-[Demo](./basic-demo/readme.md)

monitoring/logging/readme.md

@@ -0,0 +1,17 @@
+# Logging
+
+## Logging Basics
+
+* Standardised Logging
+* Centralised Logging 
+
+[Check it out](./fluentd/basic-demo/readme.md)
+
+## Introduction to Fluentd
+
+* What is fluentd
+* Configuration
+* Plugins
+* Demos
+
+[Check if out](./fluentd/introduction/readme.md)