diff --git a/docker/application/Dockerfile_Application b/docker/application/Dockerfile_Application new file mode 100644 index 0000000..2e3f186 --- /dev/null +++ b/docker/application/Dockerfile_Application @@ -0,0 +1,11 @@ +FROM amazoncorretto:17-alpine-jdk + +COPY docker/jmx/jmx_prometheus_javaagent-0.3.1.jar /opt/jmx_exporter/jmx_prometheus_javaagent-0.3.1.jar +COPY docker/jmx/client-metrics.yml /opt/jmx_exporter/client-metrics.yml +COPY build/libs/consumer-ssl-0.0.1-SNAPSHOT.jar /app/consumer-ssl-0.0.1-SNAPSHOT.jar +COPY ssl /ssl + +EXPOSE 8080 +EXPOSE 9400 + +ENTRYPOINT ["java", "-javaagent:/opt/jmx_exporter/jmx_prometheus_javaagent-0.3.1.jar=9400:/opt/jmx_exporter/client-metrics.yml", "-jar", "/app/consumer-ssl-0.0.1-SNAPSHOT.jar"] \ No newline at end of file diff --git a/docker/docker-compose-application.yml b/docker/docker-compose-application.yml new file mode 100644 index 0000000..4749746 --- /dev/null +++ b/docker/docker-compose-application.yml @@ -0,0 +1,10 @@ +services: + application: + container_name: client-app + image: client-application:latest + ports: + - "8080:8080" + - "9400:9400" + environment: + RUN_ENVIRONMENT: ${RUN_ENVIRONMENT} + CONSUMER_PROTOCOL: ${CONSUMER_PROTOCOL} \ No newline at end of file diff --git a/docker/docker-compose-kafka-cluster.yml b/docker/docker-compose-kafka-cluster.yml new file mode 100644 index 0000000..ac48967 --- /dev/null +++ b/docker/docker-compose-kafka-cluster.yml @@ -0,0 +1,111 @@ +services: + controller: + image: apache/kafka:latest + container_name: controller + environment: + KAFKA_NODE_ID: 0 + KAFKA_PROCESS_ROLES: controller + KAFKA_LISTENERS: CONTROLLER://:9093 + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_CONTROLLER_QUORUM_VOTERS: 0@controller:9093 + + broker-1: + image: apache/kafka:latest + container_name: broker-1 + privileged: true + ports: + - "29092:9092" + - "29093:9093" + - "7071:7071" + environment: + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker + KAFKA_LISTENERS: PLAINTEXT://:19092,PLAINTEXT_HOST://:9092,SSL://:19093,SSL_HOST://:9093 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-1:19092,PLAINTEXT_HOST://localhost:29092,SSL://broker-1:19093,SSL_HOST://localhost:29093 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,SSL:SSL,SSL_HOST:SSL + KAFKA_SSL_TRUSTSTORE_FILENAME: kafka.broker.truststore.jks + KAFKA_SSL_TRUSTSTORE_CREDENTIALS: password + KAFKA_SSL_KEYSTORE_FILENAME: kafka.broker.keystore.jks + KAFKA_SSL_KEYSTORE_CREDENTIALS: password + KAFKA_SSL_KEY_CREDENTIALS: password + KAFKA_NUM_PARTITIONS: 3 + KAFKA_DEFAULT_REPLICATION_FACTOR: 3 + KAFKA_MIN_INSYNC_REPLICAS: 2 + KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: 5 + KAFKA_CONTROLLER_QUORUM_VOTERS: 0@controller:9093 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_OPTS: -javaagent:/opt/jmx_exporter/jmx_prometheus_javaagent-0.3.1.jar=7071:/opt/jmx_exporter/server-metrics.yml + depends_on: + - controller + volumes: + - ./jmx/server-metrics.yml:/opt/jmx_exporter/server-metrics.yml + - ./jmx/jmx_prometheus_javaagent-0.3.1.jar:/opt/jmx_exporter/jmx_prometheus_javaagent-0.3.1.jar + - ../ssl:/etc/kafka/secrets + + broker-2: + image: apache/kafka:latest + container_name: broker-2 + privileged: true + ports: + - "39092:9092" + - "39093:9093" + - "7072:7071" + environment: + KAFKA_NODE_ID: 2 + KAFKA_PROCESS_ROLES: broker + KAFKA_LISTENERS: PLAINTEXT://:19092,PLAINTEXT_HOST://:9092,SSL://:19093,SSL_HOST://:9093 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-2:19092,PLAINTEXT_HOST://localhost:39092,SSL://broker-2:19093,SSL_HOST://localhost:39093 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,SSL:SSL,SSL_HOST:SSL + KAFKA_SSL_TRUSTSTORE_FILENAME: kafka.broker.truststore.jks + KAFKA_SSL_TRUSTSTORE_CREDENTIALS: password + KAFKA_SSL_KEYSTORE_FILENAME: kafka.broker.keystore.jks + KAFKA_SSL_KEYSTORE_CREDENTIALS: password + KAFKA_SSL_KEY_CREDENTIALS: password + KAFKA_NUM_PARTITIONS: 3 + KAFKA_DEFAULT_REPLICATION_FACTOR: 3 + KAFKA_MIN_INSYNC_REPLICAS: 2 + KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: 5 + KAFKA_CONTROLLER_QUORUM_VOTERS: 0@controller:9093 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_OPTS: -javaagent:/opt/jmx_exporter/jmx_prometheus_javaagent-0.3.1.jar=7071:/opt/jmx_exporter/server-metrics.yml + depends_on: + - controller + volumes: + - ./jmx/server-metrics.yml:/opt/jmx_exporter/server-metrics.yml + - ./jmx/jmx_prometheus_javaagent-0.3.1.jar:/opt/jmx_exporter/jmx_prometheus_javaagent-0.3.1.jar + - ../ssl:/etc/kafka/secrets + + broker-3: + image: apache/kafka:latest + container_name: broker-3 + privileged: true + ports: + - "49092:9092" + - "49093:9093" + - "7073:7071" + environment: + KAFKA_NODE_ID: 3 + KAFKA_PROCESS_ROLES: broker + KAFKA_LISTENERS: PLAINTEXT://:19092,PLAINTEXT_HOST://:9092,SSL://:19093,SSL_HOST://:9093 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-3:19092,PLAINTEXT_HOST://localhost:49092,SSL://broker-3:19093,SSL_HOST://localhost:49093 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,SSL:SSL,SSL_HOST:SSL + KAFKA_SSL_TRUSTSTORE_FILENAME: kafka.broker.truststore.jks + KAFKA_SSL_TRUSTSTORE_CREDENTIALS: password + KAFKA_SSL_KEYSTORE_FILENAME: kafka.broker.keystore.jks + KAFKA_SSL_KEYSTORE_CREDENTIALS: password + KAFKA_SSL_KEY_CREDENTIALS: password + KAFKA_NUM_PARTITIONS: 3 + KAFKA_DEFAULT_REPLICATION_FACTOR: 3 + KAFKA_MIN_INSYNC_REPLICAS: 2 + KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: 5 + KAFKA_CONTROLLER_QUORUM_VOTERS: 0@controller:9093 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_OPTS: -javaagent:/opt/jmx_exporter/jmx_prometheus_javaagent-0.3.1.jar=7071:/opt/jmx_exporter/server-metrics.yml + depends_on: + - controller + volumes: + - ./jmx/server-metrics.yml:/opt/jmx_exporter/server-metrics.yml + - ./jmx/jmx_prometheus_javaagent-0.3.1.jar:/opt/jmx_exporter/jmx_prometheus_javaagent-0.3.1.jar + - ../ssl:/etc/kafka/secrets + diff --git a/docker/docker-compose-monitoring.yml b/docker/docker-compose-monitoring.yml new file mode 100644 index 0000000..cea511e --- /dev/null +++ b/docker/docker-compose-monitoring.yml @@ -0,0 +1,38 @@ +services: + kafka-ui: + container_name: kafka-ui + image: provectuslabs/kafka-ui:latest + ports: + - "8000:8080" + environment: + KAFKA_CLUSTERS_0_NAME: local + KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: broker-1:19092,broker-2:19092,broker-3:19092 + + prometheus: + image: prom/prometheus:latest + container_name: prometheus + ports: + - "9090:9090" + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + command: + - '--config.file=/etc/prometheus/prometheus.yml' + depends_on: + - kafka-ui + + grafana: + image: grafana/grafana:latest + container_name: grafana + ports: + - "3000:3000" + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning + - ./grafana/provisioning/dashboards/dashboards:/var/lib/grafana/dashboards + - grafana-storage:/var/lib/grafana + depends_on: + - prometheus + +volumes: + grafana-storage: + external: true + diff --git a/docker/grafana/provisioning/dashboards/dashboard.yml b/docker/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 0000000..f1dd055 --- /dev/null +++ b/docker/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +providers: + - name: 'default' + orgId: 1 + type: file + allowUiUpdates: true + options: + path: "/var/lib/grafana/dashboards" + editable: true diff --git a/docker/grafana/provisioning/dashboards/dashboards/application.json b/docker/grafana/provisioning/dashboards/dashboards/application.json new file mode 100644 index 0000000..41c0870 --- /dev/null +++ b/docker/grafana/provisioning/dashboards/dashboards/application.json @@ -0,0 +1,2615 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Dashboard for Spring Boot2.1 Statistics(based on Spring Boot2 Statistic by micrometer-prometheus).", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 54, + "panels": [], + "title": "Basic Statistics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 0, + "y": 1 + }, + "id": 52, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "process_uptime_seconds{application=\"$application\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 14400 + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 70 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 5, + "y": 1 + }, + "id": 58, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum(jvm_memory_used_bytes{application=\"$application\", instance=\"$instance\", area=\"heap\"})*100/sum(jvm_memory_max_bytes{application=\"$application\",instance=\"$instance\", area=\"heap\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 14400 + } + ], + "title": "Heap Used", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + }, + { + "options": { + "from": -1e+32, + "result": { + "text": "N/A" + }, + "to": 0 + }, + "type": "range" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 70 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 10, + "y": 1 + }, + "id": 60, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum(jvm_memory_used_bytes{application=\"$application\", instance=\"$instance\", area=\"nonheap\"})*100/sum(jvm_memory_max_bytes{application=\"$application\",instance=\"$instance\", area=\"nonheap\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 14400 + } + ], + "title": "Non-Heap Used", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 9, + "x": 15, + "y": 1 + }, + "id": 66, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "process_files_open_files{application=\"$application\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Open Files", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "process_files_max_files{application=\"$application\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Max Files", + "refId": "B" + } + ], + "title": "Process Open Files", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "dateTimeAsIso" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 0, + "y": 4 + }, + "id": 56, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "process_start_time_seconds{application=\"$application\", instance=\"$instance\"}*1000", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 14400 + } + ], + "title": "Start time", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 95, + "options": { + "dataLinks": [], + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "system_cpu_usage{instance=\"$instance\", application=\"$application\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "System CPU Usage", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "process_cpu_usage{instance=\"$instance\", application=\"$application\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Process CPU Usage", + "refId": "B" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 96, + "options": { + "dataLinks": [], + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "system_load_average_1m{instance=\"$instance\", application=\"$application\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Load Average [1m]", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "system_cpu_count{instance=\"$instance\", application=\"$application\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "CPU Core Size", + "refId": "B" + } + ], + "title": "Load Average", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 48, + "panels": [], + "title": "JVM Statistics - Heaps", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 85, + "options": { + "dataLinks": [] + }, + "repeat": "memory_pool_heap", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "jvm_memory_used_bytes{instance=\"$instance\", application=\"$application\", id=\"$memory_pool_heap\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "jvm_memory_committed_bytes{instance=\"$instance\", application=\"$application\", id=\"$memory_pool_heap\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Commited", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "jvm_memory_max_bytes{instance=\"$instance\", application=\"$application\", id=\"$memory_pool_heap\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Max", + "refId": "B" + } + ], + "title": "$memory_pool_heap (heap)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 88, + "options": { + "dataLinks": [] + }, + "repeat": "memory_pool_nonheap", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "jvm_memory_used_bytes{instance=\"$instance\", application=\"$application\", id=\"$memory_pool_nonheap\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "jvm_memory_committed_bytes{instance=\"$instance\", application=\"$application\", id=\"$memory_pool_nonheap\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Commited", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "jvm_memory_max_bytes{instance=\"$instance\", application=\"$application\", id=\"$memory_pool_nonheap\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Max", + "refId": "B" + } + ], + "title": "$memory_pool_nonheap (non-heap)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 117, + "panels": [], + "title": "JVM Statistics Threads/Buffers", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 68, + "options": { + "dataLinks": [], + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "jvm_threads_daemon_threads{instance=\"$instance\", application=\"$application\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Daemon", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "jvm_threads_live_threads{instance=\"$instance\", application=\"$application\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Live", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "jvm_threads_peak_threads{instance=\"$instance\", application=\"$application\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Peak", + "refId": "C" + } + ], + "title": "Threads", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 78, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(jvm_gc_memory_allocated_bytes_total{instance=\"$instance\", application=\"$application\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "allocated", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(jvm_gc_memory_promoted_bytes_total{instance=\"$instance\", application=\"$application\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "promoted", + "refId": "B" + } + ], + "title": "Memory Allocate/Promote", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 50, + "options": { + "dataLinks": [], + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "jvm_classes_loaded_classes{instance=\"$instance\", application=\"$application\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Classes Loaded", + "refId": "A" + } + ], + "title": "Classes Loaded", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 80, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(jvm_classes_unloaded_classes_total{instance=\"$instance\", application=\"$application\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Classes Unloaded", + "refId": "A" + } + ], + "title": "Classes Unloaded", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 54 + }, + "id": 82, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "jvm_buffer_memory_used_bytes{instance=\"$instance\", application=\"$application\", id=\"direct\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used Bytes", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "jvm_buffer_total_capacity_bytes{instance=\"$instance\", application=\"$application\", id=\"direct\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Capacity Bytes", + "refId": "B" + } + ], + "title": "Direct Buffers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 54 + }, + "id": 83, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "jvm_buffer_memory_used_bytes{instance=\"$instance\", application=\"$application\", id=\"mapped\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used Bytes", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "jvm_buffer_total_capacity_bytes{instance=\"$instance\", application=\"$application\", id=\"mapped\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Capacity Bytes", + "refId": "B" + } + ], + "title": "Mapped Buffers", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 72, + "panels": [], + "title": "JVM Statistics - GC", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 62 + }, + "id": 74, + "options": { + "dataLinks": [], + "legend": { + "calcs": [ + "mean", + "max", + "min", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(jvm_gc_pause_seconds_count{instance=\"$instance\", application=\"$application\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} [{{cause}}]", + "refId": "A" + } + ], + "title": "GC Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 62 + }, + "id": 76, + "options": { + "dataLinks": [], + "legend": { + "calcs": [ + "mean", + "max", + "min", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(jvm_gc_pause_seconds_sum{instance=\"$instance\", application=\"$application\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{action}} [{{cause}}]", + "refId": "A" + } + ], + "title": "GC Stop the World Duration", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 8, + "panels": [], + "title": "Logback Statistics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 71 + }, + "id": 6, + "options": { + "dataLinks": [], + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "alias": "", + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(logback_events_total{instance=\"$instance\", application=\"$application\", level=\"info\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "info", + "rawSql": "SELECT\n $__time(time_column),\n value1\nFROM\n metric_table\nWHERE\n $__timeFilter(time_column)\n", + "refId": "A" + } + ], + "title": "INFO logs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 71 + }, + "id": 10, + "options": { + "dataLinks": [], + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "alias": "", + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(logback_events_total{instance=\"$instance\", application=\"$application\", level=\"error\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "error", + "rawSql": "SELECT\n $__time(time_column),\n value1\nFROM\n metric_table\nWHERE\n $__timeFilter(time_column)\n", + "refId": "A" + } + ], + "title": "ERROR logs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 78 + }, + "id": 14, + "options": { + "dataLinks": [], + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "alias": "", + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(logback_events_total{instance=\"$instance\", application=\"$application\", level=\"warn\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "warn", + "rawSql": "SELECT\n $__time(time_column),\n value1\nFROM\n metric_table\nWHERE\n $__timeFilter(time_column)\n", + "refId": "A" + } + ], + "title": "WARN logs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 78 + }, + "id": 16, + "options": { + "dataLinks": [], + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "alias": "", + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(logback_events_total{instance=\"$instance\", application=\"$application\", level=\"debug\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "debug", + "rawSql": "SELECT\n $__time(time_column),\n value1\nFROM\n metric_table\nWHERE\n $__timeFilter(time_column)\n", + "refId": "A" + } + ], + "title": "DEBUG logs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 78 + }, + "id": 20, + "options": { + "dataLinks": [], + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "alias": "", + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(logback_events_total{instance=\"$instance\", application=\"$application\", level=\"trace\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "trace", + "rawSql": "SELECT\n $__time(time_column),\n value1\nFROM\n metric_table\nWHERE\n $__timeFilter(time_column)\n", + "refId": "A" + } + ], + "title": "TRACE logs", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "", + "value": "" + }, + "datasource": "PBFA97CFB590B2093", + "definition": "label_values(application)", + "includeAll": false, + "label": "Application", + "name": "application", + "options": [], + "query": "label_values(application)", + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "application:8080", + "value": "application:8080" + }, + "datasource": "PBFA97CFB590B2093", + "definition": "label_values(jvm_classes_loaded_classes{application=\"$application\"}, instance)", + "includeAll": false, + "label": "Instance", + "name": "instance", + "options": [], + "query": "label_values(jvm_classes_loaded_classes{application=\"$application\"}, instance)", + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "", + "value": "" + }, + "datasource": "PBFA97CFB590B2093", + "definition": "", + "includeAll": false, + "label": "HikariCP-Pool", + "name": "hikaricp", + "options": [], + "query": "label_values(hikaricp_connections{instance=\"$instance\", application=\"$application\"}, pool)", + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "PBFA97CFB590B2093", + "definition": "", + "includeAll": true, + "label": "Memory Pool (heap)", + "name": "memory_pool_heap", + "options": [], + "query": "label_values(jvm_memory_used_bytes{application=\"$application\", instance=\"$instance\", area=\"heap\"},id)", + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "PBFA97CFB590B2093", + "definition": "", + "includeAll": true, + "label": "Memory Pool (nonheap)", + "name": "memory_pool_nonheap", + "options": [], + "query": "label_values(jvm_memory_used_bytes{application=\"$application\", instance=\"$instance\", area=\"nonheap\"},id)", + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Spring Boot 2.1 System Monitor", + "uid": "spring_boot_21", + "version": 3 +} \ No newline at end of file diff --git a/docker/grafana/provisioning/dashboards/dashboards/consumer.json b/docker/grafana/provisioning/dashboards/dashboards/consumer.json new file mode 100644 index 0000000..74eb8b2 --- /dev/null +++ b/docker/grafana/provisioning/dashboards/dashboards/consumer.json @@ -0,0 +1,2852 @@ +{ + "annotations": { + "list": [ + { + "$$hashKey": "object:1134", + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Kafka Consumer Java client metrics", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 2, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 69, + "panels": [], + "title": "Row title", + "type": "row" + }, + { + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 58, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "# Note\nThis dashboard requires a JMX agent reporting statistics from a Kafka Java consumer into Prometheus.", + "mode": "markdown" + }, + "pluginVersion": "12.0.2", + "title": "", + "type": "text" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 60, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The number of commit calls per second .", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(kafka_consumer_coordinator_commit_rate{client_id=~'[[client_id]]',instance=~'[[instance]]'} )", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Commit rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The average time in ms a request was throttled by a broker.", + "fieldConfig": { + "defaults": { + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 10, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "kafka_producer_produce_throttle_time_avg", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Average fetch throttle time", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of failed authentication attempts\n", + "fieldConfig": { + "defaults": { + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 5 + }, + "id": 13, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "kafka_admin_client_admin_client_failed_authentication_rate{instance=~'[[instance]]'}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Failed auth rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The number of total rebalance events per hour, both successful and unsuccessful rebalance attempts.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 5 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(kafka_consumer_coordinator_rebalance_rate_per_hour{client_id=~'[[client_id]]'}) by ([[aggr_criteria]])", + "interval": "", + "legendFormat": "{{client_id}} {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Rebalance rate per hour", + "type": "timeseries" + } + ], + "title": "Key metrics", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 77, + "panels": [], + "title": "Selector Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 73, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "kafka_consumer_io_wait_ratio", + "legendFormat": "{{clientId}}", + "range": true, + "refId": "A" + } + ], + "title": "Consumer I/O Wait ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 74, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "kafka_consumer_io_ratio", + "legendFormat": "{{clientId}}", + "range": true, + "refId": "A" + } + ], + "title": "Consumer I/O Ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 75, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "kafka_consumer_select_rate", + "legendFormat": "{{clientId}}", + "range": true, + "refId": "A" + } + ], + "title": "Consumer Select Ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 76, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "kafka_consumer_network_io_rate", + "legendFormat": "{{clientId}}", + "range": true, + "refId": "A" + } + ], + "title": "Consumer Network I/O Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 79, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "avg by (clientId, nodeId) (kafka_consumer_node_incoming_byte_rate)", + "legendFormat": "{{nodeId}}", + "range": true, + "refId": "A" + } + ], + "title": "Node incoming byte rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "blue", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 80, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "elapsed_time / 1000", + "legendFormat": "{{protocol}}", + "range": true, + "refId": "A" + } + ], + "title": "μ†Œμš” μ‹œκ°„(초)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 70, + "panels": [], + "title": "Lag", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 27 + }, + "id": 71, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(kafka_consumer_fetch_manager_records_lag) by (topic)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{instance}} {{topic}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Consumer Lag", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 8, + "y": 27 + }, + "id": 72, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(kafka_consumer_fetch_manager_records_lag_avg) by (topic)", + "legendFormat": "{{instance}} {{topic}}", + "range": true, + "refId": "A" + } + ], + "title": "Consumer Lag Average", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 16, + "panels": [], + "title": "System", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 39 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(process_cpu_seconds_total [$__rate_interval])", + "interval": "", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 39 + }, + "id": 67, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(jvm_memory_bytes_max) by (area, [[aggr_criteria]])", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "JVM Memory Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 39 + }, + "id": 68, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(rate(jvm_gc_collection_seconds_sum[5m])) by (gc,[[aggr_criteria]])", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "JVM GC time", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 24, + "panels": [], + "title": "Throughput", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The average number of bytes consumed per second\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(kafka_consumer_fetch_manager_bytes_consumed_total{client_id=~'[[client_id]]',instance=~'[[instance]]',topic=~'[[topic]]'}[1m])) by ([[aggr_criteria]])", + "interval": "", + "legendFormat": "{{client_id}} {{instance}} {{topic}}", + "range": true, + "refId": "A" + } + ], + "title": "Bytes consumed rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The average number of records consumed per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(kafka_consumer_fetch_manager_records_consumed_total{client_id=~'[[client_id]]',instance=~'[[instance]]',topic=~'[[topic]]'}[1m])) by ([[aggr_criteria]])", + "interval": "", + "legendFormat": "{{client_id}} {{instance}} {{topic}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate of records consumed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The average number of bytes consumed per topic per second.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(kafka_consumer_fetch_manager_bytes_consumed_total{client_id=~'[[client_id]]',instance=~'[[instance]]',topic=~'[[topic]]'}[1m])) by (topic)", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "{{topic}}", + "range": true, + "refId": "A" + } + ], + "title": "Bytes consumed rate per topic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The average number of records consumed per second per topic.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "consumer-sangdol-group-1", + "consumer-sangdol-group-2", + "consumer-sangdol-group-3" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(kafka_consumer_fetch_manager_records_consumed_rate{client_id=~'[[client_id]]',instance=~'[[instance]]',topic=~'[[topic]]'}[1m])) by (clientId, topic)", + "interval": "", + "legendFormat": "{{topic}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate of records consumed per topic", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 30, + "panels": [], + "title": "Consumer Fetch Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The number of fetch requests per second.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 65 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(kafka_consumer_fetch_manager_fetch_rate) by ([[aggr_criteria]])", + "interval": "", + "legendFormat": "Request Rate {{instance}} {{client_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Fetch request rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The number of bytes fetched per request for a topic", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Average " + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 65 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "kafka_consumer_fetch_manager_fetch_size_avg", + "interval": "", + "legendFormat": "Average {{topic}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "", + "hide": false, + "legendFormat": "Max {{instance}} {{client_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Fetch size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The average time taken for a fetch request.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 65 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(kafka_consumer_fetch_manager_fetch_latency_avg) by ([[aggr_criteria]])", + "interval": "", + "legendFormat": "Average {{instance}} {{client_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max(kafka_consumer_fetch_manager_fetch_latency_max) by ([[aggr_criteria]])", + "hide": false, + "interval": "", + "legendFormat": "Max {{instance}} {{client_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Fetch latency average", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 42, + "panels": [], + "title": "Consumer Commit Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The average time taken for a commit request", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 73 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(kafka_consumer_coordinator_commit_latency_avg) by ([[aggr_criteria]])", + "interval": "", + "legendFormat": "Average {{instance}} {{client_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max(kafka_consumer_coordinator_commit_latency_max) by ([[aggr_criteria]])", + "hide": false, + "interval": "", + "legendFormat": "Max {{instance}} {{client_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Commit latency average", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The number of commit calls per second .", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 73 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(kafka_consumer_coordinator_commit_rate) by ([[aggr_criteria]])", + "interval": "", + "legendFormat": "{{instance}} {{client_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Commit rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 81 + }, + "id": 26, + "panels": [], + "title": "Connections", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of simultaneous connections\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 82 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(kafka_consumer_connection_count{instance=~'[[instance]]', client_id=~'[[client_id]]'}) by ([[aggr_criteria]])", + "hide": false, + "interval": "", + "legendFormat": "{{instance}} {{client_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Current connection count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "response rate per node\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 82 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(kafka_consumer_node_response_rate{instance=~'[[instance]]', client_id=~'[[client_id]]'}) by ([[aggr_criteria]])", + "interval": "", + "legendFormat": "{{instance}} {{client_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Node reponse rate", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(kafka_consumer_app_info_starttimems,id)", + "includeAll": true, + "label": "Client ID", + "multi": true, + "name": "client_id", + "options": [], + "query": { + "query": "label_values(kafka_consumer_app_info_starttimems,id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(kafka_producer_topic_record_send_total,topic)", + "includeAll": true, + "label": "topic", + "multi": true, + "name": "topic", + "options": [], + "query": { + "query": "label_values(kafka_producer_topic_record_send_total,topic)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(kafka_consumer_app_info_starttimems,instance)", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values(kafka_consumer_app_info_starttimems,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "client_id,instance", + "value": "client_id,instance" + }, + "includeAll": false, + "label": "Aggregate By", + "name": "aggr_criteria", + "options": [ + { + "selected": false, + "text": "cluster", + "value": "cluster" + }, + { + "selected": false, + "text": "client_id", + "value": "client_id" + }, + { + "selected": true, + "text": "client_id,instance", + "value": "client_id,instance" + }, + { + "selected": false, + "text": "instance", + "value": "instance" + } + ], + "query": "cluster,client_id,client_id\\,instance,instance", + "type": "custom" + }, + { + "current": { + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "includeAll": false, + "label": "Data Source", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Redpanda Kafka Java Consumer", + "uid": "-C-IEldWk2", + "version": 2 +} \ No newline at end of file diff --git a/docker/grafana/provisioning/dashboards/dashboards/producer.json b/docker/grafana/provisioning/dashboards/dashboards/producer.json new file mode 100644 index 0000000..87ed969 --- /dev/null +++ b/docker/grafana/provisioning/dashboards/dashboards/producer.json @@ -0,0 +1,2222 @@ +{ + "annotations": { + "list": [ + { + "$$hashKey": "object:1134", + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Kafka Producer Java client metrics", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 2, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 69, + "panels": [], + "title": "io", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 0, + "y": 1 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "editorMode": "code", + "expr": "kafka_producer_io_ratio", + "legendFormat": "{{clientId}}", + "range": true, + "refId": "A" + } + ], + "title": "Producer I/O Ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 7, + "y": 1 + }, + "id": 71, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "editorMode": "code", + "expr": "kafka_producer_io_wait_ratio", + "legendFormat": "{{clientId}}", + "range": true, + "refId": "A" + } + ], + "title": "Producer I/O Wait", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 13, + "y": 1 + }, + "id": 72, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "editorMode": "code", + "expr": "kafka_producer_network_io_rate", + "legendFormat": "{{clientId}}", + "range": true, + "refId": "A" + } + ], + "title": "Producer Network I/O", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 68, + "panels": [ + { + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 58, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "# Note\nThis dashboard requires a JMX agent reporting statistics from a Kafka Java producer into Prometheus.\n\n⚠️ This dashboard has some sample thresholds, this example is not meant to fit all use cases nor is it meant for production. Think of it as a learning tool to help you become comfortable with the metrics and thresholding.\n", + "mode": "markdown" + }, + "pluginVersion": "12.0.1", + "title": "", + "type": "text" + } + ], + "title": "Row title", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 16, + "panels": [], + "title": "System", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 11 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aeoysn6f6kg00a" + }, + "expr": "process_cpu_seconds_total ", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 11 + }, + "id": 64, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aeoysn6f6kg00a" + }, + "editorMode": "code", + "expr": "sum(jvm_memory_bytes_max) by (area, [[aggr_criteria]])", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "JVM Memory Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 11 + }, + "id": 66, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aeoysn6f6kg00a" + }, + "editorMode": "code", + "expr": "avg(rate(jvm_gc_collection_seconds_sum[5m])) by (gc,[[aggr_criteria]])", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "JVM GC time", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 60, + "panels": [], + "title": "Key metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The number of bytes produced", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 20 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aeoysn6f6kg00a" + }, + "editorMode": "code", + "expr": "sum(kafka_producer_topic_byte_rate{client_id=~'[[client_id]]',instance=~'[[instance]]',topic=~'[[topic]]'}) ", + "interval": "", + "legendFormat": "Produce Rate", + "range": true, + "refId": "A" + } + ], + "title": "Produce rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The average time in ms a request was throttled by a broker.", + "fieldConfig": { + "defaults": { + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 6, + "y": 20 + }, + "id": 10, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aeoysn6f6kg00a" + }, + "editorMode": "code", + "expr": "kafka_producer_produce_throttle_time_avg ", + "hide": false, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Average produce throttle time", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of failed authentication attempts\n", + "fieldConfig": { + "defaults": { + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 10, + "y": 20 + }, + "id": 13, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aeoysn6f6kg00a" + }, + "editorMode": "code", + "expr": "kafka_producer_failed_authentication_rate{instance=~'[[instance]]'}", + "interval": "", + "legendFormat": "{{clientId}} {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Failed auth rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Producer error rate", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 14, + "y": 20 + }, + "id": 67, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aeoysn6f6kg00a" + }, + "editorMode": "code", + "expr": "sum(kafka_producer_record_error_rate)", + "interval": "", + "legendFormat": "Error Rate", + "range": true, + "refId": "A" + } + ], + "title": "Producer Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Average compression rate across all producers", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 19, + "y": 20 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aeoysn6f6kg00a" + }, + "editorMode": "code", + "expr": "avg(kafka_producer_compression_rate_avg{client_id=~\"[[client_id]]\",instance=~\"[[instance]]\"}) by (clientId, instance)", + "interval": "", + "legendFormat": "{{clientId}} {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Average Compression Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 24, + "panels": [], + "title": "Throughput", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The average number of bytes produced per second\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aeoysn6f6kg00a" + }, + "editorMode": "code", + "expr": "sum(rate(kafka_producer_topic_byte_total{client_id=~'[[client_id]]',instance=~'[[instance]]',topic=~'[[topic]]'}[1m])) by (clientId, topic)", + "interval": "", + "legendFormat": "{{clientId}}-{{instance}}-{{topic}}", + "range": true, + "refId": "A" + } + ], + "title": "Bytes produced rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The average number of records produced per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aeoysn6f6kg00a" + }, + "editorMode": "code", + "expr": "sum(rate(kafka_producer_topic_record_send_total{client_id=~'[[client_id]]',instance=~'[[instance]]',topic=~'[[topic]]'}[1m])) by (clientId, topic)", + "interval": "", + "legendFormat": "{{clientId}} {{instance}} {{topic}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate of records produced", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The average number of bytes produced per topic per second.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aeoysn6f6kg00a" + }, + "editorMode": "code", + "expr": "sum(rate(kafka_producer_topic_byte_total{client_id=~'[[client_id]]',instance=~'[[instance]]',topic=~'[[topic]]'}[1m])) by (topic)", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Bytes produced rate per topic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The average number of records produced per second per topic.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aeoysn6f6kg00a" + }, + "editorMode": "code", + "expr": "sum(rate(kafka_producer_topic_record_send_total{client_id=~'[[client_id]]',instance=~'[[instance]]',topic=~'[[topic]]'}[1m])) by (clientId, topic)", + "interval": "", + "legendFormat": "{{clientId}}-{{topic}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate of records produced per topic", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 30, + "panels": [], + "title": "Producer Batch Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The number of produce requests per second.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 42 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aeoysn6f6kg00a" + }, + "editorMode": "code", + "expr": "sum(kafka_producer_node_request_rate) by (clientId)", + "interval": "", + "legendFormat": "Request Rate {{instance}} {{clientId}}", + "range": true, + "refId": "A" + } + ], + "title": "Produce request rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Batch sizes produced", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 42 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aeoysn6f6kg00a" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg(kafka_producer_batch_size_avg) by (clientId)", + "hide": false, + "interval": "", + "legendFormat": "Average {{instance}} {{clientId}}", + "range": true, + "refId": "A" + } + ], + "title": "Produce batch size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The average time taken for a produce request.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 42 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aeoysn6f6kg00a" + }, + "editorMode": "code", + "expr": "avg(kafka_producer_request_latency_avg) by (clientId)", + "hide": false, + "interval": "", + "legendFormat": "Average {{instance}} {{clientId}}", + "range": true, + "refId": "A" + } + ], + "title": "Produce latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 26, + "panels": [], + "title": "Connections", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Number of simultaneous connections\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 50 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aeoysn6f6kg00a" + }, + "editorMode": "code", + "expr": "sum(kafka_producer_connection_count{instance=~'[[instance]]'}) by (clientId)", + "hide": false, + "interval": "", + "legendFormat": "{{instance}} {{clientId}}", + "range": true, + "refId": "A" + } + ], + "title": "Current connection count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "response rate per node\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 50 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aeoysn6f6kg00a" + }, + "editorMode": "code", + "expr": "sum(kafka_producer_response_rate{instance=~'[[instance]]'}) by (instance, clientId)", + "hide": false, + "interval": "", + "legendFormat": "{{instance}} {{clientId}}", + "range": true, + "refId": "A" + } + ], + "title": "Node reponse rate", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "aeoysn6f6kg00a" + }, + "definition": "label_values(kafka_producer_app_info_starttimems,id)", + "includeAll": true, + "label": "Client ID", + "multi": true, + "name": "client_id", + "options": [], + "query": { + "query": "label_values(kafka_producer_app_info_starttimems,id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "aeoysn6f6kg00a" + }, + "definition": "label_values(topic)", + "includeAll": true, + "label": "topic", + "multi": true, + "name": "topic", + "options": [], + "query": { + "query": "label_values(topic)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "aeoysn6f6kg00a" + }, + "definition": "label_values(kafka_producer_app_info_start_time_ms,instance)", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values(kafka_producer_app_info_start_time_ms,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "cluster", + "value": "cluster" + }, + "includeAll": false, + "label": "Aggregate By", + "name": "aggr_criteria", + "options": [ + { + "selected": true, + "text": "cluster", + "value": "cluster" + }, + { + "selected": false, + "text": "client_id", + "value": "client_id" + }, + { + "selected": false, + "text": "client_id,instance", + "value": "client_id,instance" + }, + { + "selected": false, + "text": "instance", + "value": "instance" + } + ], + "query": "cluster,client_id,client_id\\,instance,instance", + "type": "custom" + }, + { + "current": { + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "includeAll": false, + "label": "Data Source", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Redpanda Kafka Producer Metrics", + "uid": "asd3tgdf1", + "version": 2 +} \ No newline at end of file diff --git a/docker/grafana/provisioning/datasources/datasource.yml b/docker/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 0000000..f88db84 --- /dev/null +++ b/docker/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true \ No newline at end of file diff --git a/docker/jmx/client-metrics.yml b/docker/jmx/client-metrics.yml new file mode 100644 index 0000000..da24e99 --- /dev/null +++ b/docker/jmx/client-metrics.yml @@ -0,0 +1,107 @@ +# source: https://blog.voidmainvoid.net/476 +lowercaseOutputName: true +rules: + #kafka.connect:type=app-info,client-id="{clientid}" + #kafka.consumer:type=app-info,client-id="{clientid}" + #kafka.producer:type=app-info,client-id="{clientid}" + - pattern: 'kafka.(.+)<>start-time-ms' + name: kafka_$1_start_time_seconds + labels: + clientId: "$2" + help: "Kafka $1 JMX metric start time seconds" + type: GAUGE + valueFactor: 0.001 + - pattern: 'kafka.(.+)<>(commit-id|version): (.+)' + name: kafka_$1_$3_info + value: 1 + labels: + clientId: "$2" + $3: "$4" + help: "Kafka $1 JMX metric info version and commit-id" + type: GAUGE + + #kafka.producer:type=producer-topic-metrics,client-id="{clientid}",topic="{topic}"", partition="{partition}" + #kafka.consumer:type=consumer-fetch-manager-metrics,client-id="{clientid}",topic="{topic}"", partition="{partition}" + - pattern: kafka.(.+)<>(.+-total|compression-rate|.+-avg|.+-replica|.+-lag|.+-lead) + name: kafka_$2_$6 + labels: + clientId: "$3" + topic: "$4" + partition: "$5" + help: "Kafka $1 JMX metric type $2" + type: GAUGE + + #kafka.producer:type=producer-topic-metrics,client-id="{clientid}",topic="{topic}" + #kafka.consumer:type=consumer-fetch-manager-metrics,client-id="{clientid}",topic="{topic}"", partition="{partition}" + - pattern: kafka.(.+)<>(.+-total|compression-rate|.+-avg) + name: kafka_$2_$5 + labels: + clientId: "$3" + topic: "$4" + help: "Kafka $1 JMX metric type $2" + type: GAUGE + + #kafka.connect:type=connect-node-metrics,client-id="{clientid}",node-id="{nodeid}" + #kafka.consumer:type=consumer-node-metrics,client-id=consumer-1,node-id="{nodeid}" + - pattern: kafka.(.+)<>(.+-total|.+-avg) + name: kafka_$2_$5 + labels: + clientId: "$3" + nodeId: "$4" + help: "Kafka $1 JMX metric type $2" + type: UNTYPED + + #kafka.connect:type=kafka-metrics-count,client-id="{clientid}" + #kafka.consumer:type=consumer-fetch-manager-metrics,client-id="{clientid}" + #kafka.consumer:type=consumer-coordinator-metrics,client-id="{clientid}" + #kafka.consumer:type=consumer-metrics,client-id="{clientid}" + - pattern: kafka.(.+)<>(.+-total|.+-avg|.+-bytes|.+-count|.+-rate|.+-ratio|.+-age|.+-flight|.+-threads|.+-connectors|.+-tasks|.+-ago) + name: kafka_$2_$4 + labels: + clientId: "$3" + help: "Kafka $1 JMX metric type $2" + type: GAUGE + + #kafka.connect:type=connector-task-metrics,connector="{connector}",task="{task}<> status" + - pattern: 'kafka.connect<>status: ([a-z-]+)' + name: kafka_connect_connector_status + value: 1 + labels: + connector: "$1" + task: "$2" + status: "$3" + help: "Kafka Connect JMX Connector status" + type: GAUGE + + #kafka.connect:type=task-error-metrics,connector="{connector}",task="{task}" + #kafka.connect:type=source-task-metrics,connector="{connector}",task="{task}" + #kafka.connect:type=sink-task-metrics,connector="{connector}",task="{task}" + #kafka.connect:type=connector-task-metrics,connector="{connector}",task="{task}" + - pattern: kafka.connect<>(.+-total|.+-count|.+-ms|.+-ratio|.+-avg|.+-failures|.+-requests|.+-timestamp|.+-logged|.+-errors|.+-retries|.+-skipped) + name: kafka_connect_$1_$4 + labels: + connector: "$2" + task: "$3" + help: "Kafka Connect JMX metric type $1" + type: GAUGE + + #kafka.connect:type=connector-metrics,connector="{connector}" + #kafka.connect:type=connect-worker-metrics,connector="{connector}" + - pattern: kafka.connect<>([a-z-]+) + name: kafka_connect_worker_$2 + labels: + connector: "$1" + help: "Kafka Connect JMX metric $1" + type: GAUGE + + #kafka.connect:type=connect-worker-metrics + - pattern: kafka.connect<>([a-z-]+) + name: kafka_connect_worker_$1 + help: "Kafka Connect JMX metric worker" + type: GAUGE + + #kafka.connect:type=connect-worker-rebalance-metrics + - pattern: kafka.connect<>([a-z-]+) + name: kafka_connect_worker_rebalance_$1 + help: "Kafka Connect JMX metric rebalance information" + type: GAUGE \ No newline at end of file diff --git a/docker/jmx/jmx_prometheus_javaagent-0.3.1.jar b/docker/jmx/jmx_prometheus_javaagent-0.3.1.jar new file mode 100644 index 0000000..1408b9f Binary files /dev/null and b/docker/jmx/jmx_prometheus_javaagent-0.3.1.jar differ diff --git a/docker/jmx/server-metrics.yml b/docker/jmx/server-metrics.yml new file mode 100644 index 0000000..505279b --- /dev/null +++ b/docker/jmx/server-metrics.yml @@ -0,0 +1,247 @@ +# source: https://github.com/oded-dd/prometheus-jmx-kafka/tree/master +lowercaseOutputName: true +rules: + - pattern: kafka.server<>Value + name: kafka_server_under_replicated_partitions + help: Number of under-replicated partitions (| ISR | < | all replicas |). Alert if value is greater than 0 + type: GAUGE + + - pattern: kafka.controller<>Value + name: kafka_controller_offline_partitions_count + help: Number of partitions that do not have an active leader and are hence not writable or readable. Alert if value is greater than 0 + type: GAUGE + + - pattern: kafka.controller<>Value + name: kafka_controller_active_controller_count + help: Number of active controllers in the cluster. Alert if the aggregated sum across all brokers in the cluster is anything other than 1 (there should be exactly one controller per cluster) + type: GAUGE + + - pattern: kafka.server<>Count + name: kafka_server_total_bytes_in_per_sec + help: Aggregate incoming byte rate + type: UNTYPED + + - pattern: kafka.server<>Count + name: kafka_server_total_bytes_in_per_sec_per_topic + help: Aggregate incoming byte rate per topic + labels: + topic: "$1" + type: UNTYPED + + - pattern: kafka.server<>Count + name: kafka_server_total_bytes_out_per_sec + help: Aggregate outgoing byte rate + type: UNTYPED + + - pattern: kafka.server<>Count + name: kafka_server_total_bytes_out_per_sec_per_topic + help: Aggregate outgoing byte rate per topic + labels: + topic: "$1" + type: UNTYPED + + - pattern: kafka.network<>Count + name: kafka_network_requests_per_sec + help: Request rate (Produce, FetchConsumer, FetchFollower) + labels: + request: "$1" + type: UNTYPED + + - pattern: kafka.server<>Count + name: kafka_server_total_produce_requests_per_sec + help: Produce request rate + type: UNTYPED + + - pattern: kafka.server<>Count + name: kafka_server_total_produce_requests_per_sec_per_topic + help: Produce request rate per topic + labels: + topic: "$1" + type: UNTYPED + + - pattern: kafka.server<>Count + name: kafka_server_total_fetch_requests_per_sec + help: Fetch request rate + type: UNTYPED + + - pattern: kafka.server<>Count + name: kafka_server_total_fetch_requests_per_sec_per_topic + help: Fetch request rate per topic + labels: + topic: "$1" + type: UNTYPED + + - pattern: kafka.server<>Count + name: kafka_server_failed_produce_requests_per_sec + help: Produce request rate for requests that failed + type: UNTYPED + + - pattern: kafka.server<>Count + name: kafka_server_failed_produce_requests_per_sec_per_topic + help: Produce request rate for requests that failed per topic + labels: + topic: "$1" + type: UNTYPED + + - pattern: kafka.server<>Count + name: kafka_server_failed_fetch_requests_per_sec + help: Fetch request rate for requests that failed + type: UNTYPED + + - pattern: kafka.server<>Count + name: kafka_server_failed_fetch_requests_per_sec_per_topic + help: Fetch request rate for requests that failed per topic + labels: + topic: "$1" + type: UNTYPED + + - pattern: kafka.controller<>Count + name: kafka_controller_leader_election_rate_time + help: Leader election rate and latency (rate in seconds, latency|time in ms) + type: UNTYPED + + - pattern: kafka.controller<>Count + name: kafka_controller_unclean_leader_elections_per_sec + help: Unclean leader election rate + type: UNTYPED + + - pattern: kafka.server<>Value + name: kafka_server_partition_count + help: Number of partitions on this broker (This should be mostly even across all brokers) + type: GAUGE + + - pattern: kafka.server<>Value + name: kafka_server_leader_count + help: Number of leaders on this broker (This should be mostly even across all brokers) + type: GAUGE + + - pattern: kafka.server<>Value + name: kafka_server_max_lag_in_replica + help: Maximum lag in messages between the follower and leader replicas + type: GAUGE + + - pattern: kafka.server<>Count + name: kafka_server_request_handler_avg_idle_precent + help: Average fraction of time the request handler threads are idle. Values are between 0 (all resources are used) and 1 (all resources are available) + type: GAUGE + + - pattern: kafka.network<>Value + name: kafka_network_network_processor_avg_idle_precent + help: Average fraction of time the network processor threads are idle. Values are between 0 (all resources are used) and 1 (all resources are available) + type: GAUGE + + - pattern: kafka.network<>Value + name: kafka_network_request_queue_size + help: Size of the request queue. A congested request queue will not be able to process incoming or outgoing requests + type: GAUGE + + - pattern: kafka.network<>Count + name: kafka_network_total_time_ms + help: Total time in ms to serve the specified request (Produce, FetchConsumer, FetchFollower) + labels: + request: "$1" + type: COUNTER + + - pattern: kafka.network<>Count + name: kafka_network_request_queue_time_ms + help: Time the request waits in the request queue (Produce, FetchConsumer, FetchFollower) + labels: + request: "$1" + type: COUNTER + + - pattern: kafka.network<>Count + name: kafka_network_local_time_ms + help: Time the request is processed at the leader (Produce, FetchConsumer, FetchFollower) + labels: + request: "$1" + type: COUNTER + + - pattern: kafka.network<>Count + name: kafka_network_remote_time_ms + help: Time the request waits for the follower (Produce, FetchConsumer, FetchFollower) + labels: + request: "$1" + type: COUNTER + + - pattern: kafka.network<>Count + name: kafka_network_response_queue_time_ms + help: Time the request waits in the response queue (Produce, FetchConsumer, FetchFollower) + labels: + request: "$1" + type: COUNTER + + - pattern: kafka.network<>Count + name: kafka_network_response_send_time_ms + help: Time to send the response (Produce, FetchConsumer, FetchFollower) + labels: + request: "$1" + type: COUNTER + + - pattern: kafka.server<>Count + name: kafka_server_messages_in_per_sec + help: Aggregate incoming message rate + type: COUNTER + + - pattern: kafka.server<>Count + name: kafka_server_messages_in_per_sec_per_topic + help: Aggregate incoming message rate per_topic + labels: + topic: "$1" + type: COUNTER + + - pattern: kafka.log<>Count + name: kafka_log_log_flush_rate_time + help: Log flush rate and time (rate in seconds, latency|time in ms) + type: UNTYPED + + - pattern: kafka.server<>Count + name: kafka_server_isr_shrinks_per_sec + help: If a broker goes down, ISR for some of the partitions will shrink. When that broker is up again, ISR will be expanded once the replicas are fully caught up. Other than that, the expected value for both ISR shrink rate and expansion rate is 0 + type: GAUGE + + - pattern: kafka.server<>Count + name: kafka_server_isr_expands_per_sec + help: When a broker is brought up after a failure, it starts catching up by reading from the leader. Once it is caught up, it gets added back to the ISR. + type: GAUGE + + - pattern: kafka.server<>Value + name: kafka_server_purgatory_size + help: Number of requests waiting in the producer purgatory. This should be non-zero when acks=all is used on the producer | Number of requests waiting in the fetch purgatory. This is high if consumers use a large value for fetch.wait.max.ms + labels: + delayed_operation: "$1" + type: GAUGE + + - pattern: kafka.server<>Count + name: kafka_server_zookeeper_disconnects_per_sec + help: Zookeeper client is currently disconnected from the ensemble. The client lost its previous connection to a server and it is currently trying to reconnect. The session is not necessarily expired + type: UNTYPED + + - pattern: kafka.server<>Count + name: kafka_server_zookeeper_expires_per_sec + help: The ZooKeeper session has expired. When a session expires, we can have leader changes and even a new controller. Alert if value of such events across a Kafka cluster and if the overall number is high + type: UNTYPED + + - pattern: kafka.server<>Count + name: kafka_server_zookeeper_sync_connects_per_sec + help: ZooKeeper client is connected to the ensemble and ready to execute operations + type: UNTYPED + + - pattern: kafka.server<>Count + name: kafka_server_zookeeper_auth_failure_per_sec + help: An attempt to connect to the ensemble failed because the client has not provided correct credentials + type: UNTYPED + + - pattern: kafka.server<>Count + name: kafka_server_zookeeper_readonly_connects_per_sec + help: The server the client is connected to is currently LOOKING, which means that it is neither FOLLOWING nor LEADING. Consequently, the client can only read the ZooKeeper state, but not make any changes (create, delete, or set the data of znodes) + type: UNTYPED + + - pattern: kafka.server<>Count + name: kafka_server_zookeeper_sasl_auth_per_sec + help: Client has successfully authenticated + type: UNTYPED + + - pattern: kafka.server<>Count + name: kafka_server_zookeeper_expired_per_sec + help: The ZooKeeper session has expired. When a session expires, we can have leader changes and even a new controller. Alert if value of such events across a Kafka cluster and if the overall number is high + type: UNTYPED \ No newline at end of file diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml new file mode 100644 index 0000000..52963a0 --- /dev/null +++ b/docker/prometheus/prometheus.yml @@ -0,0 +1,22 @@ +global: + scrape_interval: 15s + scrape_timeout: 10s + +scrape_configs: + - job_name: 'kafka-server' + static_configs: + - targets: + - 'broker-1:7071' + - 'broker-2:7071' + - 'broker-3:7071' + + - job_name: 'kafka-application-jmx' + static_configs: + - targets: + - 'application:9400' + + - job_name: 'kafka-application-actuator' + metrics_path: '/actuator/prometheus' + static_configs: + - targets: + - 'application:8080' \ No newline at end of file diff --git a/keygen.sh b/keygen.sh new file mode 100755 index 0000000..ad1f6fe --- /dev/null +++ b/keygen.sh @@ -0,0 +1,46 @@ +#!/bin/bash +PASSWORD="abcd1234" +VALIDITY_DAYS=365 +KEYS_DIR="ssl" +DNAME="CN=localhost, OU=Dev, O=MyCompany, L=Seoul, C=KR" +CA_DNAME="CN=My Kafka CA, OU=Dev, O=MyCompany, L=Seoul, C=KR" + +rm -rf ./${KEYS_DIR} +mkdir ${KEYS_DIR} +cd ${KEYS_DIR} + +keytool -genkeypair -alias ca -keyalg RSA -keysize 2048 -validity ${VALIDITY_DAYS} \ + -keystore kafka.ca.keystore.jks \ + -storepass ${PASSWORD} -keypass ${PASSWORD} -dname "${CA_DNAME}" \ + -ext "BasicConstraints:critical=ca:true" + +keytool -exportcert -alias ca -file ca.crt \ + -keystore kafka.ca.keystore.jks -storepass ${PASSWORD} + +keytool -genkeypair -alias broker -keyalg RSA -keysize 2048 -validity ${VALIDITY_DAYS} \ + -keystore kafka.broker.keystore.jks \ + -storepass ${PASSWORD} -keypass ${PASSWORD} -dname "${DNAME}" + +keytool -certreq -alias broker -file broker.csr \ + -keystore kafka.broker.keystore.jks -storepass ${PASSWORD} + +keytool -gencert -alias ca -infile broker.csr -outfile broker.crt \ + -keystore kafka.ca.keystore.jks -storepass ${PASSWORD} -validity ${VALIDITY_DAYS} \ + -ext "SAN=dns:broker-1,dns:broker-2,dns:broker-3,dns:localhost" \ + -ext "ExtendedKeyUsage=serverAuth,clientAuth" + +keytool -importcert -alias ca -file ca.crt \ + -keystore kafka.broker.keystore.jks -storepass ${PASSWORD} -noprompt + +keytool -importcert -alias broker -file broker.crt \ + -keystore kafka.broker.keystore.jks -storepass ${PASSWORD} -noprompt + +keytool -importcert -alias ca -file ca.crt \ + -keystore kafka.broker.truststore.jks -storepass ${PASSWORD} -noprompt + +echo ${PASSWORD} > password + +rm ca.crt broker.csr broker.crt + +echo "" +echo "πŸŽ‰ SSL/TLS ν‚€ 파일 생성 μ™„λ£Œ" \ No newline at end of file diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..d2d2d9d --- /dev/null +++ b/run.sh @@ -0,0 +1,89 @@ +#!/bin/bash +echo "0. μ‹€ν–‰ 쀑인 λͺ¨λ“  μ»¨ν…Œμ΄λ„ˆ μ’…λ£Œ" +find docker/ -name 'docker-compose*.yml' -exec docker compose -f {} down --remove-orphans \; + +echo "0. SSL μΈμ¦μ„œ 생성" +./keygen.sh + +echo " " +echo "======================" +echo " " + +echo "1. κ·ΈλΌνŒŒλ‚˜ λ³Όλ₯¨ 생성" +GRAFANA_VOLUME="grafana-storage" +if [ -z "$(docker volume ls --filter name=${GRAFANA_VOLUME} --format '{{ .Name }}')" ]; then + echo "κ·ΈλΌνŒŒλ‚˜μ—μ„œ μ‚¬μš©ν•  ${GRAFANA_VOLUME} λ³Όλ₯¨μ„ μƒμ„±ν•©λ‹ˆλ‹€." + docker volume create ${GRAFANA_VOLUME} +fi + +echo " " +echo "======================" +echo " " + +echo "2. μΉ΄ν”„μΉ΄ & λͺ¨λ‹ˆν„°λ§ 툴 μ‹€ν–‰" +docker compose -f docker/docker-compose-kafka-cluster.yml up -d +docker compose -f docker/docker-compose-monitoring.yml up -d + +echo " " +echo "======================" +echo " " + +echo "3. μ• ν”Œλ¦¬μΌ€μ΄μ…˜ 이미지 λΉŒλ“œ" +./gradlew bootjar +docker buildx build -t client-application:latest -f docker/application/Dockerfile_Application . + +echo " " +echo "======================" +echo " " + +echo "4. μ• ν”Œλ¦¬μΌ€μ΄μ…˜ μ‹€ν–‰ 및 ν…ŒμŠ€νŠΈ" + +echo "[4-1] PLAINTEXT λͺ¨λ“œ" +rm docker/.env +echo "RUN_ENVIRONMENT=docker" > docker/.env +echo "CONSUMER_PROTOCOL=plaintext" >> docker/.env + +docker compose -f docker/docker-compose-application.yml up -d +echo "[4-1] μ‹€ν–‰ ν›„ 30초 λŒ€κΈ°..." +sleep 30 +echo "[4-1] ν…ŒμŠ€νŠΈ μ‹œμž‘" +./gradlew clean test --tests "org.study.consumerssl.data.TestUtils.produceRecords" +echo "[4-1] ν…ŒμŠ€νŠΈ μ™„λ£Œ. SSL λͺ¨λ“œ μ‹€ν–‰ μ „ 2λΆ„ λŒ€κΈ°.." +sleep 120 + +echo "[4-1] ν† ν”½ μ΄ˆκΈ°ν™”.." +./gradlew clean test --tests "org.study.consumerssl.data.TestUtils.deleteAllExistTopics" + +echo " " +echo "---------------------" +echo " " + +echo "[4-2] SSL λͺ¨λ“œ" +rm docker/.env +echo "RUN_ENVIRONMENT=docker" > docker/.env +echo "CONSUMER_PROTOCOL=ssl" >> docker/.env + +echo "[4-2] μ• ν”Œλ¦¬μΌ€μ΄μ…˜ μž¬μ‹œμž‘" +docker compose -f docker/docker-compose-application.yml down +docker compose -f docker/docker-compose-application.yml up -d + +echo "[4-2] μ‹€ν–‰ ν›„ 30초 λŒ€κΈ°..." +sleep 30 + +echo "[4-2] ν…ŒμŠ€νŠΈ μ‹œμž‘" +./gradlew clean test --tests "org.study.consumerssl.data.TestUtils.produceRecords" + +echo "[4-2] 이전과 λ™μΌν•˜κ²Œ 2λΆ„ λŒ€κΈ° ν›„ μ• ν”Œλ¦¬μΌ€μ΄μ…˜ μ’…λ£Œ.." +sleep 120 +docker compose -f docker/docker-compose-application.yml down + +echo " " +echo "======================" +echo " " + +echo "ν…ŒμŠ€νŠΈ μ™„λ£Œ. λŒ€μ‹œλ³΄λ“œ ν™•μΈν•˜κΈ° πŸ‘‡" +echo "ID: admin / PW: admin" +echo "Consumer: http://localhost:3000/d/-C-IEldWk2/redpanda-kafka-java-consumer" +echo "Springboot: http://localhost:3000/d/spring_boot_21/spring-boot-2-1-system-monitor" +echo "Kafka UI: http://localhost:8000" +echo "확인이 끝났닀면 ./close-all.sh λ₯Ό μ‹€ν–‰ν•˜μ—¬ μ΄ˆκΈ°ν™” κ°€λŠ₯ν•©λ‹ˆλ‹€." \ No newline at end of file