Observability & Monitoring

# In your values.yaml
app:
  telemetry:
    metrics:
      enabled: true
      apiKey:
        value: "your-metrics-api-key"  # Authenticates scrape requests

prometheus:
  chartManaged: true
  server:
    retention: 30d
    persistentVolume:
      size: 20Gi

kubectl -n langwatch port-forward svc/langwatch-prometheus-server 9090:9090
# Open http://localhost:9090

prometheus:
  chartManaged: false
  external:
    existingSecret: prometheus-credentials
    secretKeys:
      host: "host"
      port: "port"
      username: "username"
      password: "password"

prometheus.io/scrape: "true"
prometheus.io/port: "5560"    # or 2999 for workers
prometheus.io/path: "/metrics"

# Trace ingestion rate (per minute)
rate(langwatch_traces_ingested_total[5m]) * 60

# Worker queue depth
langwatch_worker_queue_depth

# HTTP error rate
rate(http_requests_total{status=~"5.."}[5m])
  / rate(http_requests_total[5m])

# ClickHouse query p95 latency
histogram_quantile(0.95, rate(clickhouse_query_duration_seconds_bucket[5m]))

# Startup probe (allows time for migrations)
startupProbe:
  httpGet:
    path: /api/health
    port: 5560
  periodSeconds: 5
  failureThreshold: 30    # Up to 150s for startup

# Liveness probe (restarts unhealthy pods)
livenessProbe:
  httpGet:
    path: /api/health
    port: 5560
  periodSeconds: 10
  failureThreshold: 5

# Readiness probe (removes from service if unhealthy)
readinessProbe:
  httpGet:
    path: /api/health
    port: 5560
  periodSeconds: 5
  failureThreshold: 3

# Check app health
kubectl -n langwatch exec deploy/langwatch-app -- \
  curl -s http://localhost:5560/api/health

# Check worker health
kubectl -n langwatch exec deploy/langwatch-workers -- \
  curl -s http://localhost:2999/healthz

groups:
  - name: langwatch
    rules:
      - alert: WorkerQueueBacklog
        expr: langwatch_worker_queue_depth > 10000
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Worker queue backlog is high"
          description: "Queue depth is {{ $value }} — workers may need scaling."