From 17394dde983811a2a3478c791c7366205caf7497 Mon Sep 17 00:00:00 2001 From: marcel-dempers Date: Mon, 26 Aug 2019 22:43:46 +1000 Subject: [PATCH] add prometheus integration for node and netcore --- prometheus-monitoring/dashboard.json | 758 ++++++++++++++++++ prometheus-monitoring/docker-compose.yaml | 8 +- .../src/Pages/Index.cshtml.cs | 20 +- .../dotnet-application/src/Startup.cs | 3 +- .../dotnet-application/src/work.csproj | 1 + .../go-application/dockerfile | 2 +- prometheus-monitoring/go-application/main.go | 2 +- .../nodejs-application/src/package.json | 3 +- .../nodejs-application/src/server.js | 36 + prometheus-monitoring/prometheus.yaml | 2 +- .../python-application/dockerfile | 17 +- .../python-application/src/requirements.txt | 3 +- .../python-application/src/server.py | 29 +- prometheus-monitoring/readme.md | 53 +- 14 files changed, 910 insertions(+), 27 deletions(-) create mode 100644 prometheus-monitoring/dashboard.json diff --git a/prometheus-monitoring/dashboard.json b/prometheus-monitoring/dashboard.json new file mode 100644 index 0000000..58a070a --- /dev/null +++ b/prometheus-monitoring/dashboard.json @@ -0,0 +1,758 @@ +{ + "__inputs": [], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.0.4" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 20, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 9, + "w": 11, + "x": 0, + "y": 1 + }, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(dotnet_request_operations_total[2m])", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Request per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 9, + "w": 11, + "x": 11, + "y": 1 + }, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(dotnet_request_duration_seconds_sum[2m]) / rate(dotnet_request_duration_seconds_count[2m])", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Request Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": ".NET Core", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 14, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 9, + "w": 11, + "x": 0, + "y": 2 + }, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_request_operations_total[2m])", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Request per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 9, + "w": 11, + "x": 11, + "y": 2 + }, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_request_duration_seconds_sum[2m]) / rate(node_request_duration_seconds_count[2m])", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Request Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "NodeJS", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 8, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 9, + "w": 11, + "x": 0, + "y": 2 + }, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(go_request_operations_total[2m])", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Request per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 9, + "w": 11, + "x": 11, + "y": 2 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(go_request_duration_seconds_sum[2m]) / rate(go_request_duration_seconds_count[2m])", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Request Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Golang", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 6, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 8, + "w": 11, + "x": 0, + "y": 3 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(python_request_duration_seconds_sum[2m]) / rate(python_request_duration_seconds_count[2m])", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Request Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 8, + "w": 11, + "x": 11, + "y": 3 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(python_request_operations_total[2m])", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Request per Second (Python)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "Python", + "type": "row" + } + ], + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Application Telemetry (Prometheus)", + "uid": "pvXvovKWz", + "version": 2 + } \ No newline at end of file diff --git a/prometheus-monitoring/docker-compose.yaml b/prometheus-monitoring/docker-compose.yaml index 97d97a5..9ee4cd9 100644 --- a/prometheus-monitoring/docker-compose.yaml +++ b/prometheus-monitoring/docker-compose.yaml @@ -6,28 +6,28 @@ services: container_name: go-application image: go-application ports: - - "80:80" + - "80:5000" python-application: build: context: ./python-application container_name: python-application image: python-application ports: - - "81:80" + - "81:5000" dotnet-application: build: context: ./dotnet-application container_name: dotnet-application image: dotnet-application ports: - - "82:80" + - "82:5000" nodejs-application: build: context: ./nodejs-application container_name: nodejs-application image: nodejs-application ports: - - "83:80" + - "83:5000" prometheus: container_name: prometheus-svc image: prom/prometheus diff --git a/prometheus-monitoring/dotnet-application/src/Pages/Index.cshtml.cs b/prometheus-monitoring/dotnet-application/src/Pages/Index.cshtml.cs index 53e6589..9a959d1 100644 --- a/prometheus-monitoring/dotnet-application/src/Pages/Index.cshtml.cs +++ b/prometheus-monitoring/dotnet-application/src/Pages/Index.cshtml.cs @@ -1,16 +1,34 @@ using System; +using System.Diagnostics; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc.RazorPages; - +using Prometheus; namespace work.Pages { public class IndexModel : PageModel { + private static readonly Counter ProcessedJobCount = Metrics + .CreateCounter("dotnet_request_operations_total", "The total number of processed requests"); public void OnGet() { + var sw = Stopwatch.StartNew(); + + sw.Stop(); + ProcessedJobCount.Inc(); + var histogram = + Metrics + .CreateHistogram( + "dotnet_request_duration_seconds", + "Histogram for the duration in seconds.", + new[] { 0.02, 0.05, 0.1, 0.15, 0.2, 0.5, 0.8, 1 }, + "GET", + "/"); + + histogram + .Observe(sw.Elapsed.TotalSeconds); } } diff --git a/prometheus-monitoring/dotnet-application/src/Startup.cs b/prometheus-monitoring/dotnet-application/src/Startup.cs index 550b66b..7b38743 100644 --- a/prometheus-monitoring/dotnet-application/src/Startup.cs +++ b/prometheus-monitoring/dotnet-application/src/Startup.cs @@ -9,6 +9,7 @@ using Microsoft.AspNetCore.HttpsPolicy; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Prometheus; namespace work { @@ -52,7 +53,7 @@ namespace work app.UseHttpsRedirection(); app.UseStaticFiles(); app.UseCookiePolicy(); - + app.UseMetricServer(); app.UseMvc(); } } diff --git a/prometheus-monitoring/dotnet-application/src/work.csproj b/prometheus-monitoring/dotnet-application/src/work.csproj index 8cae578..869f886 100644 --- a/prometheus-monitoring/dotnet-application/src/work.csproj +++ b/prometheus-monitoring/dotnet-application/src/work.csproj @@ -10,6 +10,7 @@ + diff --git a/prometheus-monitoring/go-application/dockerfile b/prometheus-monitoring/go-application/dockerfile index 11e3ef9..d3d7b52 100644 --- a/prometheus-monitoring/go-application/dockerfile +++ b/prometheus-monitoring/go-application/dockerfile @@ -25,6 +25,6 @@ RUN apk update && apk upgrade && \ WORKDIR /go/src/app COPY --from=builder /go/src/app/myapp /go/src/app/myapp -EXPOSE 80 +EXPOSE 5000 CMD ["./myapp"] \ No newline at end of file diff --git a/prometheus-monitoring/go-application/main.go b/prometheus-monitoring/go-application/main.go index 460a459..3052401 100644 --- a/prometheus-monitoring/go-application/main.go +++ b/prometheus-monitoring/go-application/main.go @@ -46,6 +46,6 @@ func main() { }) http.Handle("/metrics", promhttp.Handler()) - http.ListenAndServe(":80", nil) + http.ListenAndServe(":5000", nil) } diff --git a/prometheus-monitoring/nodejs-application/src/package.json b/prometheus-monitoring/nodejs-application/src/package.json index 6af247d..8f93e8d 100644 --- a/prometheus-monitoring/nodejs-application/src/package.json +++ b/prometheus-monitoring/nodejs-application/src/package.json @@ -8,6 +8,7 @@ "start": "node server.js" }, "dependencies": { - "express": "^4.16.1" + "express": "^4.16.1", + "prom-client" : "11.5.3" } } diff --git a/prometheus-monitoring/nodejs-application/src/server.js b/prometheus-monitoring/nodejs-application/src/server.js index cea6861..3f09462 100644 --- a/prometheus-monitoring/nodejs-application/src/server.js +++ b/prometheus-monitoring/nodejs-application/src/server.js @@ -7,10 +7,46 @@ const PORT = 5000; const HOST = '0.0.0.0'; // App +const client = require('prom-client'); +const collectDefaultMetrics = client.collectDefaultMetrics; +// Probe every 5th second. +collectDefaultMetrics({ timeout: 5000 }); + +const counter = new client.Counter({ + name: 'node_request_operations_total', + help: 'The total number of processed requests' +}); + +const histogram = new client.Histogram({ + name: 'node_request_duration_seconds', + help: 'Histogram for the duration in seconds.', + buckets: [1, 2, 5, 6, 10] +}); + const app = express(); app.get('/', (req, res) => { + + //Simulate a sleep + var start = new Date() + var simulateTime = 1000 + + setTimeout(function(argument) { + // execution time simulated with setTimeout function + var end = new Date() - start + histogram.observe(end / 1000); //convert to seconds + }, simulateTime) + + counter.inc(); + res.send('Hello world\n'); }); + +// Metrics endpoint +app.get('/metrics', (req, res) => { + res.set('Content-Type', client.register.contentType) + res.end(client.register.metrics()) +}) + app.listen(PORT, HOST); console.log(`Running on http://${HOST}:${PORT}`); \ No newline at end of file diff --git a/prometheus-monitoring/prometheus.yaml b/prometheus-monitoring/prometheus.yaml index 368b68b..fbee3ed 100644 --- a/prometheus-monitoring/prometheus.yaml +++ b/prometheus-monitoring/prometheus.yaml @@ -5,4 +5,4 @@ scrape_configs: - job_name: my-application honor_labels: true static_configs: - - targets: ['go-application','python-application','dotnet-application','nodejs-application'] + - targets: ['go-application:5000','python-application:5000','dotnet-application:5000','nodejs-application:5000'] \ No newline at end of file diff --git a/prometheus-monitoring/python-application/dockerfile b/prometheus-monitoring/python-application/dockerfile index 5763282..66ed99b 100644 --- a/prometheus-monitoring/python-application/dockerfile +++ b/prometheus-monitoring/python-application/dockerfile @@ -1,21 +1,12 @@ -FROM python:3.7.3-alpine3.9 as dev - -RUN mkdir /work/ -WORKDIR /work/ - -COPY ./src/requirements.txt /work/requirements.txt -RUN pip install -r requirements.txt - -COPY ./src/ /work/ - -###########START NEW IMAGE################### - FROM python:3.7.3-alpine3.9 as prod RUN mkdir /app/ WORKDIR /app/ -COPY --from=dev /work/ /app/ +COPY ./src/requirements.txt /app/requirements.txt RUN pip install -r requirements.txt + +COPY ./src/ /app/ + ENV FLASK_APP=server.py CMD flask run -h 0.0.0 -p 5000 diff --git a/prometheus-monitoring/python-application/src/requirements.txt b/prometheus-monitoring/python-application/src/requirements.txt index 9614ae3..e1e23f7 100644 --- a/prometheus-monitoring/python-application/src/requirements.txt +++ b/prometheus-monitoring/python-application/src/requirements.txt @@ -1 +1,2 @@ -Flask == 1.0.3 \ No newline at end of file +Flask == 1.0.3 +prometheus_client == 0.7.1 \ No newline at end of file diff --git a/prometheus-monitoring/python-application/src/server.py b/prometheus-monitoring/python-application/src/server.py index 6c8be82..e8b3966 100644 --- a/prometheus-monitoring/python-application/src/server.py +++ b/prometheus-monitoring/python-application/src/server.py @@ -1,6 +1,31 @@ -from flask import Flask +from flask import Response, Flask, request +import prometheus_client +from prometheus_client.core import CollectorRegistry +from prometheus_client import Summary, Counter, Histogram, Gauge +import time + app = Flask(__name__) +_INF = float("inf") + +graphs = {} +graphs['c'] = Counter('python_request_operations_total', 'The total number of processed requests') +graphs['h'] = Histogram('python_request_duration_seconds', 'Histogram for the duration in seconds.', buckets=(1, 2, 5, 6, 10, _INF)) + @app.route("/") def hello(): - return "Hello World!" \ No newline at end of file + start = time.time() + graphs['c'].inc() + + time.sleep(0.600) + end = time.time() + graphs['h'].observe(end - start) + return "Hello World!" + +@app.route("/metrics") +def requests_count(): + res = [] + for k,v in graphs.items(): + res.append(prometheus_client.generate_latest(v)) + return Response(res, mimetype="text/plain") + diff --git a/prometheus-monitoring/readme.md b/prometheus-monitoring/readme.md index a80d17f..b851ef1 100644 --- a/prometheus-monitoring/readme.md +++ b/prometheus-monitoring/readme.md @@ -1,4 +1,55 @@ # Prometheus Application Monitoring ## a Video reference guide -coming soon! \ No newline at end of file +To run any of the commands, please ensure you open a terminal and navigate to the path where this readme is located. + +## Start Prometheus + +``` +docker-compose up -d prometheus +docker-compose up -d grafana +``` + +Wait for Grafana to start up +Import the dashboards +``` +TODO +``` + +You should see all application targets un `UNKNOWN` or `DOWN` status. +```http://localhost:9090/targets``` + +## Start the example app you prefer + +``` +docker-compose up -d golang-application +docker-compose up -d python-application +docker-compose up -d dotnet-application +docker-compose up -d nodejs-application +``` + +## Generate some requests by opening the application in the browser + +``` +http://localhost:80 #Golang +http://localhost:81 #Python +http://localhost:82 #Dotnet +http://localhost:83 #NodeJS +``` + +## Check Dashboards +``` +http://localhost:3000 + +``` +## Prometheus Queries +### Golang Examples + +Requests per Second over 2minutes +``` +irate(go_request_operations_total[2m]) +``` +Request duration +``` +rate(go_request_duration_seconds_sum[2m]) / rate(go_request_duration_seconds_total[2m]) +``` \ No newline at end of file