Stack PLG: Prometheus + Loki + Grafana no Kubernetes do jeito certo

1
2
3
4
5
6
7
8

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update

helm upgrade --install kube-prometheus-stack \
  prometheus-community/kube-prometheus-stack \
  --namespace monitoring \
  --create-namespace \
  --values prometheus-values.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

# prometheus-values.yaml
grafana:
  enabled: true
  adminPassword: "TROCA_ISSO_POR_SECRET"
  ingress:
    enabled: false    # usamos IngressRoute do Traefik
  persistence:
    enabled: true
    size: 10Gi

prometheus:
  prometheusSpec:
    retention: 15d
    retentionSize: "40GB"
    storageSpec:
      volumeClaimTemplate:
        spec:
          storageClassName: oci-bv-high-perf
          resources:
            requests:
              storage: 50Gi
    resources:
      requests:
        cpu: 500m
        memory: 2Gi
      limits:
        cpu: 2
        memory: 4Gi

alertmanager:
  alertmanagerSpec:
    storage:
      volumeClaimTemplate:
        spec:
          storageClassName: oci-bv-high-perf
          resources:
            requests:
              storage: 5Gi

# ServiceMonitors automáticos para todos os namespaces
prometheus:
  prometheusSpec:
    serviceMonitorSelectorNilUsesHelmValues: false
    podMonitorSelectorNilUsesHelmValues: false

1
2
3
4
5

helm repo add grafana https://grafana.github.io/helm-charts

helm upgrade --install loki grafana/loki \
  --namespace monitoring \
  --values loki-values.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

# loki-values.yaml
loki:
  auth_enabled: false
  commonConfig:
    replication_factor: 1     # single node; use 3 em HA

  storage:
    type: s3
    s3:
      endpoint: https://<namespace>.compat.objectstorage.<region>.oraclecloud.com
      bucketnames: loki-chunks
      region: sa-saopaulo-1
      access_key_id: "${OCI_ACCESS_KEY}"
      secret_access_key: "${OCI_SECRET_KEY}"
      s3forcepathstyle: true

  schemaConfig:
    configs:
      - from: "2024-01-01"
        store: tsdb
        object_store: s3
        schema: v13
        index:
          prefix: loki_index_
          period: 24h

  limits_config:
    retention_period: 744h   # 31 dias
    ingestion_rate_mb: 16
    ingestion_burst_size_mb: 32

singleBinary:
  replicas: 1
  resources:
    requests:
      cpu: 200m
      memory: 512Mi
    limits:
      cpu: 1
      memory: 1Gi

promtail:
  enabled: true
  config:
    clients:
      - url: http://loki:3100/loki/api/v1/push

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

# grafana-ingressroute.yaml
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
  name: grafana
  namespace: monitoring
spec:
  entryPoints:
    - websecure
  routes:
    - match: Host(`grafana.castroti.com.br`)
      kind: Rule
      services:
        - name: kube-prometheus-stack-grafana
          port: 80
  tls:
    certResolver: le

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

# custom-alerts.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: infra-alerts
  namespace: monitoring
spec:
  groups:
    - name: infra.critical
      rules:
        - alert: NodeDiskSpaceCritical
          expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) < 0.10
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Disco {{ $labels.instance }} com menos de 10% livre"

        - alert: PodCrashLooping
          expr: rate(kube_pod_container_status_restarts_total[15m]) > 0
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Pod {{ $labels.pod }} em CrashLoop"

        - alert: NodeMemoryPressure
          expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) < 0.15
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Memória disponível abaixo de 15% em {{ $labels.instance }}"