diff --git a/backend/.gitignore b/backend/.gitignore index e2fece8..3df90f2 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -24,6 +24,15 @@ __pycache__/ # logging *.log +logs # security -*.pem \ No newline at end of file +*.pem + +# loki +chunks +compactor +rules +tsdb-shipper-active +tsdb-shipper-cache +wal \ No newline at end of file diff --git a/backend/alembic/env.py b/backend/alembic/env.py index 88fa485..0bb89b2 100644 --- a/backend/alembic/env.py +++ b/backend/alembic/env.py @@ -18,7 +18,7 @@ DATABASE_URL = get_env("DATABASE_URL") if not DATABASE_URL: - raise ValueError(f"No database URL configured for the {get_env("ENVIRONMENT")} environment.") + raise ValueError(f"No database URL configured for the {get_env('ENVIRONMENT')} environment.") config = context.config config.set_main_option("sqlalchemy.url", DATABASE_URL) diff --git a/backend/app/db/database.py b/backend/app/db/database.py index e7bcc03..a3fb870 100644 --- a/backend/app/db/database.py +++ b/backend/app/db/database.py @@ -1,3 +1,4 @@ +import asyncpg from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession from sqlalchemy.orm import sessionmaker from sqlalchemy.ext.declarative import declarative_base @@ -33,3 +34,14 @@ async def create_db_session(): async def close_db_session(session): await session.close() + + +async def get_connection(): + """PostgreSQL 데이터베이스에 직접 연결하고 연결 객체를 반환""" + conn = await asyncpg.connect(DATABASE_URL.replace('postgresql+asyncpg', 'postgresql')) + return conn + + +async def close_connection(conn): + """PostgreSQL 연결 종료""" + await conn.close() \ No newline at end of file diff --git a/backend/app/llm/langchain.py b/backend/app/llm/langchain.py index 6f2b992..4bf567a 100644 --- a/backend/app/llm/langchain.py +++ b/backend/app/llm/langchain.py @@ -16,6 +16,7 @@ from langchain_text_splitters import RecursiveCharacterTextSplitter from app.llm.message_task import store +from app.utils.logging import logger def get_rag_chain(document_text: str): @@ -115,6 +116,7 @@ def get_langchain_response(user_message: str, chat_room_id: int): ) except Exception as e: + logger.error("Error generating answer: ", exc_info={e}) raise RuntimeError(f"Error generating answer: {e}") diff --git a/backend/app/main.py b/backend/app/main.py index 2ed3111..f3a7276 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,10 +1,12 @@ +import time from fastapi import FastAPI, HTTPException, Request from fastapi.responses import JSONResponse from fastapi.middleware.cors import CORSMiddleware from contextlib import asynccontextmanager from app.utils.logging import log_exception -from app.routers import auth, me, folders, chatrooms, messages, links, rating - +from app.routers import auth, me, folders, chatrooms, messages, links, rating, metrics +from app.metrics.prometheus_metrics import REQUEST_COUNT, REQUEST_DURATION, RESPONSE_STATUS +from prometheus_fastapi_instrumentator import Instrumentator @asynccontextmanager async def lifespan(app: FastAPI): @@ -17,6 +19,7 @@ async def lifespan(app: FastAPI): origins = [ "http://localhost:3000", + "http://localhost:3001", ] app.add_middleware( @@ -27,6 +30,35 @@ async def lifespan(app: FastAPI): allow_headers=["*"], ) +instrumentator = Instrumentator().instrument(app) +instrumentator.expose(app) + +@app.middleware("http") +async def metrics_middleware(request: Request, call_next): + start_time = time.time() + response = await call_next(request) + process_time = time.time() - start_time + + # 요청 처리 시간 기록 + response.headers["X-Process-Time"] = str(process_time) + + # 요청 메트릭을 수집 (HTTP 메서드, 요청 URL) + method = request.method + path = request.url.path + + # 메트릭 수집 + method = request.method + path = request.url.path + status_code = response.status_code + + REQUEST_COUNT.labels(method=method, path=path).inc() + REQUEST_DURATION.labels(method=method, path=path).observe(process_time) + RESPONSE_STATUS.labels(method=method, path=path, status_code=status_code).inc() + + response.headers["X-Process-Time"] = str(process_time) + + return response + @app.exception_handler(HTTPException) async def custom_http_exception_handler(req: Request, exc: HTTPException): @@ -46,3 +78,4 @@ async def custom_http_exception_handler(req: Request, exc: HTTPException): app.include_router(messages.router) app.include_router(links.router) app.include_router(rating.router) +app.include_router(metrics.router) diff --git a/backend/app/metrics/pg_metrics.py b/backend/app/metrics/pg_metrics.py new file mode 100644 index 0000000..98eb39f --- /dev/null +++ b/backend/app/metrics/pg_metrics.py @@ -0,0 +1,108 @@ +from prometheus_client import Gauge +from app.db.database import get_connection, close_connection +from app.metrics.prometheus_metrics import * + +async def record_pg_metrics(): + # Connect to PostgreSQL + conn = await get_connection() + try: + # Total transactions + commit_result = await conn.fetchval("SELECT xact_commit FROM pg_stat_database WHERE datname = current_database();") + COMMIT_COUNT.set(commit_result) + + rollback_result = await conn.fetchval("SELECT xact_rollback FROM pg_stat_database WHERE datname = current_database();") + ROLLBACK_COUNT.set(rollback_result) + + # Active connections + active_conn_result = await conn.fetchval("SELECT count(*) FROM pg_stat_activity WHERE state = 'active';") + ACTIVE_CONNECTIONS.set(active_conn_result) + + # Total cache hit ration + cache_hit_result = await conn.fetchval(""" + SELECT + CASE + WHEN SUM(heap_blks_hit + heap_blks_read) = 0 THEN 0 + ELSE 100 * SUM(heap_blks_hit) / SUM(heap_blks_hit + heap_blks_read) + END AS overall_cache_hit_ratio + FROM pg_statio_user_tables + WHERE (heap_blks_hit + heap_blks_read) > 0; + """) + if cache_hit_result is not None: + TOTAL_CACHE_HIT_RATIO.set(cache_hit_result) + else: + TOTAL_CACHE_HIT_RATIO.set(0) + + # Cache hit ratio + cache_hit_results = await conn.fetch(""" + SELECT relname, + heap_blks_hit, + heap_blks_read, + CASE + WHEN (heap_blks_hit + heap_blks_read) = 0 THEN 0 + ELSE 100 * heap_blks_hit / (heap_blks_hit + heap_blks_read) + END AS cache_hit_ratio + FROM pg_statio_user_tables; + """) + + for result in cache_hit_results: + table_name = result['relname'] + cache_hit_ratio = result['cache_hit_ratio'] + + # If the metric for this table doesn't exist yet, create it + if table_name not in CACHE_HIT_RATIO_METRICS: + CACHE_HIT_RATIO_METRICS[table_name] = Gauge(f'cache_hit_ratio_{table_name}', f'Cache hit ratio for {table_name} table') + + # Set the value of the metric to the cache hit ratio + CACHE_HIT_RATIO_METRICS[table_name].set(cache_hit_ratio) + + # Index scan ratio (with division by zero and None protection) + index_scan_result = await conn.fetchval(""" + SELECT CASE + WHEN (sum(seq_scan) + sum(idx_scan)) = 0 THEN 0 + ELSE sum(idx_scan) / (sum(seq_scan) + sum(idx_scan)) + END AS index_scan_ratio + FROM pg_stat_user_tables; + """) + + # Handle None case for index scan ratio + INDEX_SCAN_RATIO.set(index_scan_result) + + # Fetch, Insert, Update, Delete rates + rates_result = await conn.fetchrow(""" + SELECT tup_fetched, tup_inserted, tup_updated, tup_deleted + FROM pg_stat_database WHERE datname = current_database(); + """) + FETCH_RATE.set(rates_result['tup_fetched']) + INSERT_RATE.set(rates_result['tup_inserted']) + UPDATE_RATE.set(rates_result['tup_updated']) + DELETE_RATE.set(rates_result['tup_deleted']) + + # Deadlock count + deadlock_result = await conn.fetchval(""" + SELECT deadlocks FROM pg_stat_database WHERE datname = current_database(); + """) + DEADLOCK_COUNT.set(deadlock_result) + + # Replication lag (in bytes) + replication_lag_result = await conn.fetchval(""" + SELECT pg_current_wal_lsn() - replay_lsn AS replication_lag_bytes + FROM pg_stat_replication; + """) + + # Handle None case for replication lag + if replication_lag_result is not None: + REPLICATION_LAG_BYTES.set(replication_lag_result) + else: + REPLICATION_LAG_BYTES.set(0) # Set to 0 if no + + + # Data transfer volume + disk_result = await conn.fetchval("SELECT blks_read FROM pg_stat_database WHERE datname = current_database();") + READ_DISK.set(disk_result) + + cache_result = await conn.fetchval("SELECT blks_hit FROM pg_stat_database WHERE datname = current_database();") + READ_CACHE.set(cache_result) + + + finally: + await close_connection(conn) diff --git a/backend/app/metrics/prometheus_metrics.py b/backend/app/metrics/prometheus_metrics.py new file mode 100644 index 0000000..2678f11 --- /dev/null +++ b/backend/app/metrics/prometheus_metrics.py @@ -0,0 +1,23 @@ +from prometheus_client import Counter, Histogram, Gauge + +# Prometheus 메트릭 정의 (backend) +REQUEST_COUNT = Counter("request_count", "Total request count", ["method", "path"]) +REQUEST_DURATION = Histogram("request_duration_seconds", "Request duration", ["method", "path"]) +RESPONSE_STATUS = Counter('http_response_status', 'HTTP Response Status Codes', ['method', 'path', 'status_code']) + +# Prometheus 메트릭 정의 (postgres) +ACTIVE_CONNECTIONS = Gauge('pg_active_connections', 'Number of active connections') +TOTAL_CACHE_HIT_RATIO = Gauge('pg_total_cache_hit_ratio', 'Total cache hit ratio') +INDEX_SCAN_RATIO = Gauge('pg_index_scan_ratio', 'Index scan ratio') +FETCH_RATE = Gauge('pg_fetch_rate', 'Fetch throughput') +INSERT_RATE = Gauge('pg_insert_rate', 'Insert throughput') +UPDATE_RATE = Gauge('pg_update_rate', 'Update throughput') +DELETE_RATE = Gauge('pg_delete_rate', 'Delete throughput') +DEADLOCK_COUNT = Gauge('pg_deadlock_count', 'Number of deadlocks') +REPLICATION_LAG_BYTES = Gauge('pg_replication_lag_bytes', 'Replication lag in bytes') +COMMIT_COUNT = Gauge('pg_stat_database_xact_commit', 'Number of commit transactions') +ROLLBACK_COUNT = Gauge('pg_stat_database_xact_rollback', 'Number of rollback transactions') +READ_DISK = Gauge('pg_stat_database_blks_read', 'Number of blocks read from disk') +READ_CACHE = Gauge('pg_stat_database_blks_hit', 'Number of blocks read from cache') + +CACHE_HIT_RATIO_METRICS = {} diff --git a/backend/app/monitoring/grafana/provisioning/dashboards/back_observability.json b/backend/app/monitoring/grafana/provisioning/dashboards/back_observability.json new file mode 100644 index 0000000..57640ed --- /dev/null +++ b/backend/app/monitoring/grafana/provisioning/dashboards/back_observability.json @@ -0,0 +1,645 @@ +{ + "id": null, + "uid": "Backend-Observability", + "title": "Backend Observability", + "tags": ["Prometheus", "Loki"], + "timezone": "browser", + "weekStart": "monday", + "fiscalYearStartMonth": 10, + "timepicker": { + "refresh_intervals": ["30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d", "7d"] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "editable": true, + "graphTooltip": 2, + "panels": [ + { + "id": 8, + "title": "HTTP Request", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + } + }, + { + "id": 14, + "title": "HTTP Status code", + "type": "piechart", + "gridPos": { + "x": 0, + "y": 1, + "w": 6, + "h": 7 + }, + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "http_response_status_total{job=\"backend\", path!~\"/(metrics/backend|metrics/postgres)\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{status_code}} {{path}}", + "range": true, + "refId": "A", + "useBackend": false + } + ] + }, + { + "id": 10, + "title": "HTTP Request Duration", + "type": "timeseries", + "gridPos": { + "x": 6, + "y": 1, + "w": 9, + "h": 7 + }, + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "http_request_duration_seconds_sum{job=\"backend\", handler!=\"/metrics\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{method}} {{handler}}", + "range": true, + "refId": "A", + "useBackend": false + } + ] + }, + { + "id": 7, + "title": "Total Request Count", + "type": "bargauge", + "gridPos": { + "x": 15, + "y": 1, + "w": 9, + "h": 7 + }, + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "valueMode": "color" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "request_count_total{job=\"backend\", path!~\"/(metrics/backend|metrics/postgres)\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{method}} {{path}} {{status}}", + "range": true, + "refId": "A", + "useBackend": false + } + ] + }, + { + "id": 4, + "title": "Metrics", + "type": "row", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 9, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 6, + "title": "CPU usage", + "type": "timeseries", + "gridPos": { + "x": 0, + "y": 10, + "w": 10, + "h": 8 + }, + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)", + "instant": false, + "legendFormat": "CPU", + "range": true, + "refId": "A" + } + ] + }, + { + "id": 5, + "title": "Memory usage", + "type": "timeseries", + "gridPos": { + "x": 10, + "y": 10, + "w": 9, + "h": 8 + }, + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "(node_memory_MemTotal_bytes - node_memory_MemFree_bytes - node_memory_Buffers_bytes - node_memory_Cached_bytes) / node_memory_MemTotal_bytes * 100", + "legendFormat": "RAM", + "range": true, + "refId": "A" + } + ] + }, + { + "id": 1, + "title": "All logs", + "type": "row", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 18, + "w": 24, + "h": 1 + } + }, + { + "id": 3, + "title": "FastAPI Logs", + "type": "logs", + "datasource": { + "default": false, + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "gridPos": { + "x": 0, + "y": 19, + "w": 24, + "h": 9 + }, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "builder", + "expr": "{job=\"backend_logs\"} |= ``", + "queryType": "range", + "refId": "A" + } + ] + } + ], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "uid": "Loki" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Log Time", + "multi": true, + "name": "LogTime", + "options": [], + "query": "label_values(time)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "uid": "Prometheus" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "File Path", + "multi": true, + "name": "FilePath", + "options": [], + "query": "label_values(file_path)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "uid": "Loki" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Log Level", + "multi": true, + "name": "LogLevel", + "options": [], + "query": "label_values(level)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "refresh": "5s", + "schemaVersion": 38, + "version": 1, + "style": "light", + "links": [] +} \ No newline at end of file diff --git a/backend/app/monitoring/grafana/provisioning/dashboards/dashboards.yaml b/backend/app/monitoring/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..7eeab7b --- /dev/null +++ b/backend/app/monitoring/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 +providers: +- name: 'Observability' + orgId: 1 + folder: '' + type: 'file' + disableDeletion: true + editable: true + options: + path: '/etc/grafana/provisioning/dashboards' diff --git a/backend/app/monitoring/grafana/provisioning/dashboards/front_observability.json b/backend/app/monitoring/grafana/provisioning/dashboards/front_observability.json new file mode 100644 index 0000000..d33b809 --- /dev/null +++ b/backend/app/monitoring/grafana/provisioning/dashboards/front_observability.json @@ -0,0 +1,1653 @@ +{ + "id": null, + "uid": "Frontend-Observability", + "title": "Frontend Observability", + "tags": ["Prometheus", "Loki"], + "timezone": "browser", + "weekStart": "monday", + "fiscalYearStartMonth": 10, + "timepicker": { + "refresh_intervals": ["30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d", "7d"] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "editable": true, + "graphTooltip": 2, + "panels": [ + { + "id": 1, + "panels": [], + "title": "CPU status", + "type": "row", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + } + }, + { + "id": 2, + "title": "1 minute load average ", + "type": "timeseries", + "gridPos": { + "x": 0, + "y": 1, + "w": 8, + "h": 8 + }, + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "CPU가 처리해야 할 작업의 양을 나타내는 숫자 \nnode_load1 = 1 => 하나의 코어가 지속적으로 작업을 처리하고 있는 상태 \nnode_load1 > 1 => 작업 요청을 처리하기 위해 대기 중인 작업이 있는 상태 \nnode_load1 < 1 => 작업 요청을 처리하기에 여유가 있는 상태 ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "node_load1", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "", + "hide": false, + "instant": false, + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "", + "hide": false, + "instant": false, + "range": true, + "refId": "C" + } + ] + }, + { + "id": 3, + "title": "CPU usage", + "type": "table", + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "mode": "lcd", + "type": "gauge", + "valueDisplayMode": "color" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "cpu" + }, + "properties": [ + { + "id": "custom.width", + "value": 20 + }, + { + "id": "unit", + "value": "none" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "iowait (lastNotNull)" + }, + "properties": [ + { + "id": "custom.width", + "value": 142 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "irq (lastNotNull)" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + } + ] + } + ] + }, + "gridPos": { + "x": 8, + "y": 1, + "w": 16, + "h": 8 + }, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(rate(node_cpu_seconds_total{mode!~\"(idle|nice|steal|irq)\"}[5m])) by (mode, cpu)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "transformations": [ + { + "id": "joinByLabels", + "options": { + "join": [ + "cpu" + ], + "value": "mode" + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "cpu": { + "aggregations": [], + "operation": "groupby" + }, + "iowait": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "irq": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "nice": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "softirq": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "steal": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "system": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "user": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + } + } + } + } + ] + }, + { + "id": 4, + "title": "Memory & Disk status", + "type": "row", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 9, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 5, + "title": "Memory usage", + "type": "bargauge", + "gridPos": { + "x": 0, + "y": 10, + "w": 10, + "h": 9 + }, + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "node_memory_MemAvailable_bytes{job=\"node_exporter\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "node_memory_MemFree_bytes{job=\"node_exporter\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "node_memory_Cached_bytes{job=\"node_exporter\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "node_memory_Buffers_bytes{job=\"node_exporter\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "D", + "useBackend": false + } + ] + }, + { + "id": 6, + "title": "Disk I/O (Read)", + "type": "gauge", + "gridPos": { + "x": 10, + "y": 10, + "w": 5, + "h": 9 + }, + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "node_disk_read_bytes_total{job=\"node_exporter\",device=~\"vda|vdb\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{device}}", + "range": true, + "refId": "A", + "useBackend": false + } + ] + }, + { + "id": 7, + "title": "Disk I/O (Write)", + "type": "gauge", + "gridPos": { + "x": 15, + "y": 10, + "w": 5, + "h": 9 + }, + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "node_disk_written_bytes_total{job=\"node_exporter\",device=~\"vda|vdb\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{device}}", + "range": true, + "refId": "A", + "useBackend": false + } + ] + }, + { + "id": 8, + "title": "Network Status", + "type": "row", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 19, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 9, + "title": "Network usage", + "type": "stat", + "gridPos": { + "x": 0, + "y": 20, + "w": 9, + "h": 6 + }, + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "{__name__=\"node_network_transmit_bytes_total\", device=\"eth0\", instance=\"node_exporter:9100\", job=\"node_exporter\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [] + } + ] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "node_network_transmit_bytes_total{job=\"node_exporter\", device=\"eth0\"}", + "hide": false, + "instant": false, + "legendFormat": "Transmit", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "node_network_receive_bytes_total{job=\"node_exporter\", device=\"eth0\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Receive", + "range": true, + "refId": "B", + "useBackend": false + } + ] + }, + { + "id": 10, + "title": "Network error", + "type": "stat", + "gridPos": { + "x": 9, + "y": 20, + "w": 8, + "h": 6 + }, + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "node_network_transmit_errs_total{job=\"node_exporter\", device=\"eth0\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Transmit", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "node_network_receive_errs_total{job=\"node_exporter\", device=\"eth0\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Receive", + "range": true, + "refId": "A", + "useBackend": false + } + ] + }, + { + "id": 11, + "title": "Page Performance", + "type": "row", + "gridPos": { + "x": 0, + "y": 26, + "w": 24, + "h": 1 + } + }, + { + "id": 12, + "title": "Web vitals", + "type": "table", + "gridPos": { + "x": 0, + "y": 27, + "w": 24, + "h": 6 + }, + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "applyToRow": false, + "mode": "gradient", + "type": "color-background", + "wrapText": false + }, + "inspect": false + }, + "mappings": [ + { + "type": "value", + "options": { + "web_vitals_ttfb (mean)": { + "text": "TTFB (mean)" + }, + "web_vitals_fcp (mean)": { + "text": "FCP (mean)" + }, + "web_vitals_lcp (mean)": { + "text": "LCP (mean)" + }, + "web_vitals_cls (mean)": { + "text": "CLS (mean)" + }, + "web_vitals_inp (mean)": { + "text": "INP (mean)" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 800 + }, + { + "color": "red", + "value": 1800 + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "web_vitals_ttfb (mean)" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 800 + }, + { + "color": "red", + "value": 1800 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "web_vitals_fcp (mean)" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1800 + }, + { + "color": "red", + "value": 3000 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "web_vitals_lcp (mean)" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 2500 + }, + { + "color": "red", + "value": 4000 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "web_vitals_cls (mean)" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 0.25 + } + ] + } + }, + { + "id": "unit", + "value": "short" + } + ] + } + ] + }, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true + }, + "pluginVersion": "11.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by (path, __name__) (web_vitals_ttfb{job=\"frontend\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{path}} - TTFB", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (path, __name__) (web_vitals_fcp{job=\"frontend\"})", + "hide": false, + "instant": false, + "legendFormat": "{{path}} - FCP", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (path, __name__) (web_vitals_lcp{job=\"frontend\"})", + "hide": false, + "instant": false, + "legendFormat": "{{path}} - LCP", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (path, __name__) (web_vitals_cls{job=\"frontend\"})", + "hide": false, + "instant": false, + "legendFormat": "{{path}} - CLS", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (path, __name__) (web_vitals_inp{job=\"frontend\"})", + "hide": false, + "instant": false, + "legendFormat": "{{path}} - INP", + "range": true, + "refId": "E" + } + ], + "transformations": [ + { + "id": "joinByLabels", + "options": { + "join": [ + "path" + ], + "value": "__name__" + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "path": { + "aggregations": [], + "operation": "groupby" + }, + "web_vitals_cls": { + "aggregations": [ + "mean" + ], + "operation": "aggregate" + }, + "web_vitals_fcp": { + "aggregations": [ + "mean" + ], + "operation": "aggregate" + }, + "web_vitals_inp": { + "aggregations": [ + "mean" + ], + "operation": "aggregate" + }, + "web_vitals_lcp": { + "aggregations": [ + "mean" + ], + "operation": "aggregate" + }, + "web_vitals_ttfb": { + "aggregations": [ + "mean" + ], + "operation": "aggregate" + } + } + } + } + ] + }, + { + "id": 13, + "title": "Page Load (TTFB, FCP, LCP) - p75", + "type": "timeseries", + "gridPos": { + "x": 0, + "y": 33, + "w": 8, + "h": 8 + }, + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "p75" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "avg by (__name__) (web_vitals_ttfb{job=\"frontend\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "avg by (__name__) (web_vitals_fcp{job=\"frontend\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "avg by (__name__) (web_vitals_lcp{job=\"frontend\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "C", + "useBackend": false + } + ] + }, + { + "id": 14, + "title": "Cumulative Layout Shift(CLS) - p75", + "type": "timeseries", + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "x": 8, + "y": 33, + "w": 8, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "avg by (__name__) (web_vitals_cls{job=\"frontend\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ] + }, + { + "id": 15, + "title": "Interaction to Next Paint", + "type": "timeseries", + "gridPos": { + "x": 16, + "y": 33, + "w": 8, + "h": 8 + }, + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "avg by (__name__) (web_vitals_inp{job=\"frontend\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ] + }, + { + "id": 16, + "title": "ALL Logs", + "type": "row", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 41, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 17, + "title": "Panel Title", + "type": "logs", + "gridPos": { + "x": 0, + "y": 42, + "w": 24, + "h": 8 + }, + "datasource": { + "default": false, + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "builder", + "expr": "{job=\"frontend_logs\"} |= ``", + "queryType": "range", + "refId": "A" + } + ] + } + ], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "uid": "Loki" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Log Time", + "multi": true, + "name": "LogTime", + "options": [], + "query": "label_values(time)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "uid": "Prometheus" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "File Path", + "multi": true, + "name": "FilePath", + "options": [], + "query": "label_values(file_path)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "uid": "Loki" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Log Level", + "multi": true, + "name": "LogLevel", + "options": [], + "query": "label_values(level)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "refresh": "5s", + "schemaVersion": 38, + "version": 1, + "style": "light", + "links": [], + "pluginVersion": "7.0.0" +} \ No newline at end of file diff --git a/backend/app/monitoring/grafana/provisioning/dashboards/pg_observability.json b/backend/app/monitoring/grafana/provisioning/dashboards/pg_observability.json new file mode 100644 index 0000000..7dbdbad --- /dev/null +++ b/backend/app/monitoring/grafana/provisioning/dashboards/pg_observability.json @@ -0,0 +1,588 @@ +{ + "id": null, + "uid": "Postgres-Observability", + "title": "Postgres Observability", + "tags": ["Prometheus"], + "timezone": "browser", + "weekStart": "monday", + "fiscalYearStartMonth": 10, + "timepicker": { + "refresh_intervals": ["30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d", "7d"] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "editable": true, + "graphTooltip": 2, + "panels": [ + { + "id": 1, + "title": "Database Stats", + "type": "row", + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + } + }, + { + "id": 2, + "title": "Concurrent user", + "type": "gauge", + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 1, + "h": 6, + "w": 5 + }, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "pg_active_connections{job=\"postgres\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ] + }, + { + "id": 3, + "title": "Total cache hit ratio", + "type": "gauge", + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "x": 5, + "y": 1, + "w": 6, + "h": 6 + }, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "pg_total_cache_hit_ratio{job=\"postgres\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ] + }, + { + "id": 4, + "title": "Cache hit ratio per table", + "type": "stat", + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "x": 11, + "y": 1, + "w": 7, + "h": 6 + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "label_replace(cache_hit_ratio_user_info{job=\"postgres\"}, \"last_part\", \"$1\", \"__name__\", \"cache_hit_ratio_(.*)\")", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{last_part}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "label_replace(cache_hit_ratio_folder{job=\"postgres\"}, \"last_part\", \"$1\", \"__name__\", \"cache_hit_ratio_(.*)\")", + "hide": false, + "instant": false, + "legendFormat": "{{last_part}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "label_replace(cache_hit_ratio_link{job=\"postgres\"}, \"last_part\", \"$1\", \"__name__\", \"cache_hit_ratio_(.*)\")", + "hide": false, + "instant": false, + "legendFormat": "{{last_part}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "label_replace(cache_hit_ratio_link_stat{job=\"postgres\"}, \"last_part\", \"$1\", \"__name__\", \"cache_hit_ratio_(.*)\")", + "hide": false, + "instant": false, + "legendFormat": "{{last_part}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "label_replace(cache_hit_ratio_link_summary_embedding{job=\"postgres\"}, \"last_part\", \"$1\", \"__name__\", \"cache_hit_ratio_(.*)\")", + "hide": false, + "instant": false, + "legendFormat": "{{last_part}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "label_replace(cache_hit_ratio_chat_room{job=\"postgres\"}, \"last_part\", \"$1\", \"__name__\", \"cache_hit_ratio_(.*)\")", + "hide": false, + "instant": false, + "legendFormat": "{{last_part}}", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "label_replace(cache_hit_ratio_chat_room_link{job=\"postgres\"}, \"last_part\", \"$1\", \"__name__\", \"cache_hit_ratio_(.*)\")", + "hide": false, + "instant": false, + "legendFormat": "{{last_part}}", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "label_replace(cache_hit_ratio_message{job=\"postgres\"}, \"last_part\", \"$1\", \"__name__\", \"cache_hit_ratio_(.*)\")", + "hide": false, + "instant": false, + "legendFormat": "{{last_part}}", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "label_replace(cache_hit_ratio_rating{job=\"postgres\"}, \"last_part\", \"$1\", \"__name__\", \"cache_hit_ratio_(.*)\")", + "hide": false, + "instant": false, + "legendFormat": "{{last_part}}", + "range": true, + "refId": "I" + } + ] + }, + { + "id": 6, + "title": "Deadlock count", + "type": "gauge", + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "x": 18, + "y": 1, + "w": 6, + "h": 6 + }, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "pg_deadlock_count{job=\"postgres\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ] + }, + { + "id": 5, + "title": "Insert/Update/Delete", + "type": "timeseries", + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 7, + "w": 11, + "h": 6 + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "label_replace(pg_insert_rate{job=\"postgres\"}, \"DML\", \"$1\", \"__name__\", \"pg_(.*)_rate\")", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{DML}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "label_replace(pg_update_rate{job=\"postgres\"}, \"DML\", \"$1\", \"__name__\", \"pg_(.*)_rate\")", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{DML}}", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "label_replace(pg_delete_rate{job=\"postgres\"}, \"DML\", \"$1\", \"__name__\", \"pg_(.*)_rate\")", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{DML}}", + "range": true, + "refId": "C", + "useBackend": false + } + ] + } + ], + "templating": { + }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "refresh": "5s", + "schemaVersion": 38, + "version": 1, + "style": "light", + "links": [] +} \ No newline at end of file diff --git a/backend/app/monitoring/grafana/provisioning/datasources/datasources.yaml b/backend/app/monitoring/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..89f53b8 --- /dev/null +++ b/backend/app/monitoring/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,27 @@ +apiVersion: 1 + +datasources: +- name: Loki + access: proxy + type: loki + url: http://loki:3100 + isDefault: false + database: '' + user: '' + password: '' + basicAuth: false + id: 1 + orgId: 1 + readOnly: false + jsonData: + keepCookies: [] + typeLogoUrl: public/app/plugins/datasource/loki/img/loki_icon.svg +- name: Prometheus + access: proxy + type: prometheus + url: http://prometheus:9090 + isDefault: true + orgId: 1 + editable: false + jsonData: + timeInterval: 10s \ No newline at end of file diff --git a/backend/app/monitoring/loki/local-config.yaml b/backend/app/monitoring/loki/local-config.yaml new file mode 100644 index 0000000..2df8a70 --- /dev/null +++ b/backend/app/monitoring/loki/local-config.yaml @@ -0,0 +1,33 @@ +# 사용자 인증 활성화 여부 +auth_enabled: false + +# Loki 서버의 포트번호 +server: + http_listen_port: 3100 + +# Loki 인스턴스의 주소 +common: + instance_addr: 127.0.0.1 + path_prefix: /workdir/backend/app/monitoring/loki + storage: + filesystem: # 파일시스템 기반 저장소를 사용 + chunks_directory: /workdir/backend/app/monitoring/loki/chunks + rules_directory: /workdir/backend/app/monitoring/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +# Loki의 스키마 설정 +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v12 + index: + prefix: index_ + period: 24h + +ruler: + alertmanager_url: http://localhost:9093 diff --git a/backend/app/monitoring/prometheus/prometheus.yml b/backend/app/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000..6604a1d --- /dev/null +++ b/backend/app/monitoring/prometheus/prometheus.yml @@ -0,0 +1,43 @@ +global: + scrape_interval: 15s # 15초마다 데이터 가지고옴 / default = 1m + scrape_timeout: 15s # 데이터를 가지고 오는 작업을 15초간 기다릴 수 있음 / default = 10s + evaluation_interval: 2m # 알람 규칙이 평가되는 빈도 (2분에 한번) / default = 1m + + external_labels: + monitor: 'backend-monitor' + +scrape_configs: + - job_name: 'backend' + scrape_interval: 15s + scrape_timeout: 10s + honor_labels: false + honor_timestamps: false + scheme: 'http' + metrics_path: '/metrics/backend' + static_configs: + - targets: ['backend:8000'] + + - job_name: 'frontend' + scrape_interval: 15s + scrape_timeout: 10s + honor_labels: false + honor_timestamps: false + scheme: 'http' + metrics_path: '/api/metrics' + static_configs: + - targets: ['frontend:3000'] + + - job_name: 'node_exporter' + scrape_interval: 5s + static_configs: + - targets: ['node_exporter:9100'] + + - job_name: 'postgres' + scrape_interval: 15s + scrape_timeout: 10s + honor_labels: false + honor_timestamps: false + scheme: 'http' + metrics_path: '/metrics/postgres' + static_configs: + - targets: ['backend:8000'] \ No newline at end of file diff --git a/backend/app/monitoring/promtail/config.yml b/backend/app/monitoring/promtail/config.yml new file mode 100644 index 0000000..d427af1 --- /dev/null +++ b/backend/app/monitoring/promtail/config.yml @@ -0,0 +1,48 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /workdir/backend/app/monitoring/promtail/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: backend + static_configs: + - targets: + - backend:8000 + labels: + job: backend_logs + __path__: /workdir/backend/app/monitoring/logs/backend/**/**/*.log + pipeline_stages: + - multiline: + firstline: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}' + max_wait_time: 3s + - regex: + expression: '^(?P