From b0a6b7af40a5abd08a2b38182af62ea0eb16d7f6 Mon Sep 17 00:00:00 2001 From: marcel-dempers Date: Fri, 25 Sep 2020 19:06:21 +1000 Subject: [PATCH] wip --- .../logging/fluentd/basic-demo/readme.md | 1 + .../configurations/elastic-fluent.conf | 56 ------------------- .../configurations/file-fluent.conf | 0 .../introduction/configurations/fluent.conf | 18 +----- .../configurations/http-fluent.conf | 0 .../fluentd/introduction/docker-compose.yaml | 8 +-- .../file/{write-file-log.sh => app.sh} | 0 .../http/{send-http-log.sh => app.sh} | 0 .../logging/fluentd/introduction/readme.md | 28 ++++++++++ monitoring/logging/fluentd/readme.md | 8 --- monitoring/logging/readme.md | 17 ++++++ 11 files changed, 52 insertions(+), 84 deletions(-) delete mode 100644 monitoring/logging/fluentd/introduction/configurations/elastic-fluent.conf delete mode 100644 monitoring/logging/fluentd/introduction/configurations/file-fluent.conf delete mode 100644 monitoring/logging/fluentd/introduction/configurations/http-fluent.conf rename monitoring/logging/fluentd/introduction/file/{write-file-log.sh => app.sh} (100%) rename monitoring/logging/fluentd/introduction/http/{send-http-log.sh => app.sh} (100%) create mode 100644 monitoring/logging/fluentd/introduction/readme.md delete mode 100644 monitoring/logging/fluentd/readme.md create mode 100644 monitoring/logging/readme.md diff --git a/monitoring/logging/fluentd/basic-demo/readme.md b/monitoring/logging/fluentd/basic-demo/readme.md index f69d349..6915e65 100644 --- a/monitoring/logging/fluentd/basic-demo/readme.md +++ b/monitoring/logging/fluentd/basic-demo/readme.md @@ -1,5 +1,6 @@ # Fluentd basic demo +Check out the [video](https://youtu.be/MMVdkzeQ848) In my video: Introduction to logging
I run fluentd locally
I collect all local container logs into the `./logs` folder
diff --git a/monitoring/logging/fluentd/introduction/configurations/elastic-fluent.conf b/monitoring/logging/fluentd/introduction/configurations/elastic-fluent.conf deleted file mode 100644 index 89a61b4..0000000 --- a/monitoring/logging/fluentd/introduction/configurations/elastic-fluent.conf +++ /dev/null @@ -1,56 +0,0 @@ - -# incoming http --> elastic search - - @type http - port 9880 - bind 0.0.0.0 - body_size_limit 32m - keepalive_timeout 10s - - -#container logs --> elastic search - - @type tail - format json - read_from_head true - tag docker.log - path /fluentd/log/containers/*/*-json.log - pos_file /tmp/container-logs.pos - - -#local file --> elastic search - - @type tail - format json - read_from_head true - tag file-myapp.log - path /app/example-log.log - pos_file /tmp/example-log.log.pos - - -# where to send http logs - - @type elasticsearch - host elasticsearch - port 9200 - index_name fluentd-http - type_name fluentd - - -#where to send file logs - - @type elasticsearch - host elasticsearch - port 9200 - index_name fluentd-file - type_name fluentd - - -#where to send docker logs - - @type elasticsearch - host elasticsearch - port 9200 - index_name fluentd-docker - type_name fluentd - \ No newline at end of file diff --git a/monitoring/logging/fluentd/introduction/configurations/file-fluent.conf b/monitoring/logging/fluentd/introduction/configurations/file-fluent.conf deleted file mode 100644 index e69de29..0000000 diff --git a/monitoring/logging/fluentd/introduction/configurations/fluent.conf b/monitoring/logging/fluentd/introduction/configurations/fluent.conf index 6ecbf2a..ef5bd2a 100644 --- a/monitoring/logging/fluentd/introduction/configurations/fluent.conf +++ b/monitoring/logging/fluentd/introduction/configurations/fluent.conf @@ -5,17 +5,10 @@ format json read_from_head true tag file-myapp.log - path /app/example-log.log + path /fluentd/log/files/example-log.log pos_file /tmp/example-log.log.pos - - @type record_transformer - - host_param "#{Socket.gethostname}" - - - @type file path /output/file-myapp.log @@ -32,16 +25,9 @@ keepalive_timeout 10s - - @type record_transformer - - host_param "#{Socket.gethostname}" - - - @type file path /output/http.log -################################################################ \ No newline at end of file +################################################################ diff --git a/monitoring/logging/fluentd/introduction/configurations/http-fluent.conf b/monitoring/logging/fluentd/introduction/configurations/http-fluent.conf deleted file mode 100644 index e69de29..0000000 diff --git a/monitoring/logging/fluentd/introduction/docker-compose.yaml b/monitoring/logging/fluentd/introduction/docker-compose.yaml index 6c00516..9a06952 100644 --- a/monitoring/logging/fluentd/introduction/docker-compose.yaml +++ b/monitoring/logging/fluentd/introduction/docker-compose.yaml @@ -8,9 +8,9 @@ services: image: fluentd volumes: - /var/lib/docker/containers:/fluentd/log/containers # Example: Reading docker logs - - ./file:/app/ #Example: Reading logs from a file + - ./file:/fluentd/log/files/ #Example: Reading logs from a file - ./configurations:/fluentd/etc/ - - ./logs:/output/ + - ./logs:/output/ # Example: Fluentd will collect logs and store it here for demo logging: driver: "local" # This app sends logs to Fluentd via HTTP @@ -19,14 +19,14 @@ services: image: alpine volumes: - ./http:/app - command: [ /bin/sh , -c , "apk add --no-cache curl && chmod +x /app/send-http-log.sh && ./app/send-http-log.sh"] + command: [ /bin/sh , -c , "apk add --no-cache curl && chmod +x /app/app.sh && ./app/app.sh"] # This app writes logs to a local file file-myapp: container_name: file-myapp image: alpine volumes: - ./file:/app - command: [ /bin/sh , -c , "apk add --no-cache curl && chmod +x /app/write-file-log.sh && ./app/write-file-log.sh"] + command: [ /bin/sh , -c , "chmod +x /app/app.sh && ./app/app.sh"] elasticsearch: # port 9200 image: elasticsearch:7.9.1 container_name: elasticsearch diff --git a/monitoring/logging/fluentd/introduction/file/write-file-log.sh b/monitoring/logging/fluentd/introduction/file/app.sh similarity index 100% rename from monitoring/logging/fluentd/introduction/file/write-file-log.sh rename to monitoring/logging/fluentd/introduction/file/app.sh diff --git a/monitoring/logging/fluentd/introduction/http/send-http-log.sh b/monitoring/logging/fluentd/introduction/http/app.sh similarity index 100% rename from monitoring/logging/fluentd/introduction/http/send-http-log.sh rename to monitoring/logging/fluentd/introduction/http/app.sh diff --git a/monitoring/logging/fluentd/introduction/readme.md b/monitoring/logging/fluentd/introduction/readme.md new file mode 100644 index 0000000..53631ac --- /dev/null +++ b/monitoring/logging/fluentd/introduction/readme.md @@ -0,0 +1,28 @@ +# Introduction to Fluentd + +## Collecting logs from files + +Reading logs from a file we need an application that writes logs to a file.
+Lets start one: + +``` +cd monitoring\logging\fluentd\introduction\ + +docker-compose up -d file-myapp + +``` + +To collect the logs, lets start fluentd + +``` +docker-compose up -d fluentd +``` + +## Collecting logs over HTTP (incoming) + +``` +cd monitoring\logging\fluentd\introduction\ + +docker-compose up -d http-myapp + +``` \ No newline at end of file diff --git a/monitoring/logging/fluentd/readme.md b/monitoring/logging/fluentd/readme.md deleted file mode 100644 index 82606de..0000000 --- a/monitoring/logging/fluentd/readme.md +++ /dev/null @@ -1,8 +0,0 @@ -# Logging - -## Logging Basics - -* Standardised Logging -* Centralised Logging - -[Demo](./basic-demo/readme.md) \ No newline at end of file diff --git a/monitoring/logging/readme.md b/monitoring/logging/readme.md new file mode 100644 index 0000000..5fbe28d --- /dev/null +++ b/monitoring/logging/readme.md @@ -0,0 +1,17 @@ +# Logging + +## Logging Basics + +* Standardised Logging +* Centralised Logging + +[Check it out](./fluentd/basic-demo/readme.md) + +## Introduction to Fluentd + +* What is fluentd +* Configuration +* Plugins +* Demos + +[Check if out](./fluentd/introduction/readme.md) \ No newline at end of file