Metrics Collection: Track App Performance with Prometheus

Problem

Modern applications need visibility into their performance, health, and behavior in production. Without proper metrics collection, teams are blind to:

• Application latency and response times
• Error rates and failure patterns
• Resource utilization (CPU, memory, disk)
• Business metrics (requests, transactions, conversions)
• System bottlenecks and degradation

This leads to slow incident detection, poor capacity planning, and inability to optimize performance.

Solution

Prometheus is an open-source monitoring and alerting toolkit that solves this by:

• Pull-based metrics collection: Prometheus scrapes metrics from application endpoints
• Time-series database: Stores metrics with timestamps for historical analysis
• PromQL: Powerful query language for analyzing metrics
• Alerting: Rule-based alerts for anomalies and thresholds
• Visualization: Integration with Grafana for dashboards
• Client libraries: Easy instrumentation across languages

Key Concepts

1. Metrics Types:

   • Counter: Monotonically increasing value
   • Gauge: Value that can go up or down
   • Histogram: Distribution of observations
   • Summary: Quantiles of observations

2. Labels: Key-value pairs for dimensionality (service, endpoint, status)

3. Scraping: Prometheus periodically fetches metrics from /metrics endpoint

Code Implementation

1. Python Flask Application with Prometheus

# app.py
from flask import Flask, jsonify, request
from prometheus_client import Counter, Histogram, Gauge, generate_latest
import time
import random

app = Flask(__name__)

# Define metrics
request_count = Counter(
    'app_requests_total',
    'Total HTTP requests',
    ['method', 'endpoint', 'status']
)

request_duration = Histogram(
    'app_request_duration_seconds',
    'HTTP request latency',
    ['method', 'endpoint'],
    buckets=(0.1, 0.5, 1.0, 2.0, 5.0)
)

active_connections = Gauge(
    'app_active_connections',
    'Number of active connections'
)

database_query_time = Histogram(
    'app_db_query_duration_seconds',
    'Database query duration',
    ['query_type']
)

cache_hits = Counter(
    'app_cache_hits_total',
    'Total cache hits',
    ['cache_name']
)

cache_misses = Counter(
    'app_cache_misses_total',
    'Total cache misses',
    ['cache_name']
)

# Middleware for request tracking
@app.before_request
def before_request():
    request.start_time = time.time()
    active_connections.inc()

@app.after_request
def after_request(response):
    duration = time.time() - request.start_time

    request_duration.labels(
        method=request.method,
        endpoint=request.path
    ).observe(duration)

    request_count.labels(
        method=request.method,
        endpoint=request.path,
        status=response.status_code
    ).inc()

    active_connections.dec()
    return response

# Routes
@app.route('/metrics')
def metrics():
    """Prometheus metrics endpoint"""
    return generate_latest()

@app.route('/api/users/<int:user_id>')
def get_user(user_id):
    """Simulate user lookup with cache"""
    cache_hit = random.random() > 0.3

    if cache_hit:
        cache_hits.labels(cache_name='user_cache').inc()
        return jsonify({'id': user_id, 'name': 'John Doe', 'source': 'cache'})
    else:
        cache_misses.labels(cache_name='user_cache').inc()

        # Simulate database query
        with database_query_time.labels(query_type='select').time():
            time.sleep(random.uniform(0.1, 0.5))

        return jsonify({'id': user_id, 'name': 'John Doe', 'source': 'database'})

@app.route('/api/products')
def list_products():
    """List products endpoint"""
    with database_query_time.labels(query_type='select').time():
        time.sleep(random.uniform(0.05, 0.3))

    return jsonify({'products': ['Product A', 'Product B', 'Product C']})

@app.route('/health')
def health():
    """Health check endpoint"""
    return jsonify({'status': 'healthy'})

if __name__ == '__main__':
    app.run(debug=True, port=5000)

2. Prometheus Configuration

# prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s
  external_labels:
    monitor: 'app-monitor'

alerting:
  alertmanagers:
    - static_configs:
        - targets:
            - localhost:9093

rule_files:
  - 'alert_rules.yml'

scrape_configs:
  - job_name: 'flask-app'
    static_configs:
      - targets: ['localhost:5000']
    metrics_path: '/metrics'
    scrape_interval: 10s
    scrape_timeout: 5s

  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

3. Alert Rules

# alert_rules.yml
groups:
  - name: app_alerts
    interval: 30s
    rules:
      - alert: HighErrorRate
        expr: |
          (sum(rate(app_requests_total{status=~"5.."}[5m])) /
           sum(rate(app_requests_total[5m]))) > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "High error rate detected"
          description: "Error rate is {{ $value | humanizePercentage }}"

      - alert: HighLatency
        expr: |
          histogram_quantile(0.95, 
            rate(app_request_duration_seconds_bucket[5m])) > 2
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High request latency"
          description: "P95 latency is {{ $value }}s"

      - alert: HighActiveConnections
        expr: app_active_connections > 100
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "High number of active connections"
          description: "Active connections: {{ $value }}"

      - alert: LowCacheHitRate
        expr: |
          (sum(rate(app_cache_hits_total[5m])) /
           (sum(rate(app_cache_hits_total[5m])) + 
            sum(rate(app_cache_misses_total[5m])))) < 0.7
        for: 10m
        labels:
          severity: info
        annotations:
          summary: "Low cache hit rate"
          description: "Cache hit rate is {{ $value | humanizePercentage }}"

4. Docker Compose Setup

# docker-compose.yml
version: '3.8'

services:
  flask-app:
    build: .
    ports:
      - "5000:5000"
    environment:
      - FLASK_ENV=production
    networks:
      - monitoring

  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - ./alert_rules.yml:/etc/prometheus/alert_rules.yml
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
    networks:
      - monitoring

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
    volumes:
      - grafana_data:/var/lib/grafana
    networks:
      - monitoring

  alertmanager:
    image: prom/alertmanager:latest
    ports:
      - "9093:9093"
    volumes:
      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml
    networks:
      - monitoring

volumes:
  prometheus_data:
  grafana_data:

networks:
  monitoring:
    driver: bridge

5. Custom Metrics Decorator

# metrics_decorator.py
from functools import wraps
from prometheus_client import Histogram, Counter
import time

def track_metrics(endpoint_name):
    """Decorator to automatically track metrics for any function"""
    duration_metric = Histogram(
        f'{endpoint_name}_duration_seconds',
        f'Duration of {endpoint_name}',
        buckets=(0.01, 0.05, 0.1, 0.5, 1.0)
    )

    error_metric = Counter(
        f'{endpoint_name}_errors_total',
        f'Errors in {endpoint_name}'
    )

    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            start = time.time()
            try:
                result = func(*args, **kwargs)
                return result
            except Exception as e:
                error_metric.inc()
                raise
            finally:
                duration = time.time() - start
                duration_metric.observe(duration)
        return wrapper
    return decorator

# Usage
@track_metrics('expensive_operation')
def expensive_operation():
    time.sleep(0.5)
    return "Done"

6. PromQL Query Examples

# Request rate (requests per second)
rate(app_requests_total[5m])

# Error rate percentage
(sum(rate(app_requests_total{status=~"5.."}[5m])) / 
 sum(rate(app_requests_total[5m]))) * 100

# P95 latency
histogram_quantile(0.95, rate(app_request_duration_seconds_bucket[5m]))

# Cache hit ratio
sum(rate(app_cache_hits_total[5m])) / 
(sum(rate(app_cache_hits_total[5m])) + sum(rate(app_cache_misses_total[5m])))

# Top 5 slowest endpoints
topk(5, histogram_quantile(0.99, rate(app_request_duration_seconds_bucket[5m])))

Benefits

✅ Real-time visibility into application performance
✅ Proactive alerting before issues impact users
✅ Historical analysis for capacity planning
✅ Debugging aid for identifying bottlenecks
✅ Business insights through custom metrics
✅ Scalable architecture for growing systems

This setup provides comprehensive metrics collection enabling data-driven decisions and rapid incident response.