Grafana Loki の simple scalable deployment mode が READ と WRITE と BACKEND の3つになったので試してみた (Loki 3.0.0)

Loki 2.8.0 で simple scalable deployment mode に backend が追加されたのと Loki 3.0.0 がリリースされたので試してみました。パフォーマンス的にはクエリが 504 Gateway Time-out になる事が少なくなったと感じているので simple scalable deployment mode の Read, Write, Backend で運用するメリットはあると感じています。

Loki 2.8.0 以前の simple scalable deployment mode では Read と Wirte の 2つで構成されていて Read には ruler のコンポーネントがあり、Grafana から アラートの状態が確認できたのですが、Loki 2.8.0 以降では ruler のコンポーネントが Backend に移動したので、 Grafana から Alert rules の状態が見えなくなりました。(Datasource を追加する際は Read を登録するため)
個人的には Grafana から ログ監視の状態が見えるのはありがたかったので Read に ruler を追加して運用しています。

Loki を運用しているリポジトリ: https://github.com/AbeYuki/monitoring-k8s

Loki 3.0.0 での注意点は BoltDB store やいくつか廃止された設定があるので以前の設定を流用する場合は確認しておく必要があります。

• 非推奨のストレージ オプション
• 非推奨の構成オプション
• API エンドポイントの非推奨

BoltDB の設定については以前から tsdb に変更していたため影響はありませんでしたが、Loki 3.0.0 にアップデートするにあたって以下の設定を削除しました。

@@ -40,7 +40,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /data/tsdb-index
     cache_location: /data/tsdb-cache
-    shared_store: s3
 
 schema_config:
   configs:
@@ -52,9 +51,6 @@ schema_config:
       prefix: index_
       period: 24h
 
-chunk_store_config:
-  max_look_back_period: 336h
-
 limits_config:
   max_cache_freshness_per_query: '10m'
   reject_old_samples: true

Simple Scalable Deploymode Component

Read target	Query Frontend, Querier
Write target	Distributer, Ingester
Backend target	Compactor, Index Gateway, Query Scheduler, Ruler

Configure

auth_enabled: false

server:
  http_listen_address: 0.0.0.0
  grpc_listen_address: 0.0.0.0
  http_listen_port: 3100
  grpc_listen_port: 9095
  http_server_read_timeout: 600s
  http_server_write_timeout: 600s
  grpc_server_max_recv_msg_size: 33554432
  grpc_server_max_send_msg_size: 33554432
  log_level: info

memberlist:
  join_members: 
    - ${NAMESPACE}-frontend-loki-memberlist01-headless-001.${NAMESPACE}.svc.cluster.local:7946
  dead_node_reclaim_time: 30s
  gossip_to_dead_nodes_time: 15s
  left_ingesters_timeout: 30s
  bind_addr: ['0.0.0.0']
  bind_port: 7946
  gossip_interval: 2s

common:
  path_prefix: /data
  compactor_address: http://${NAMESPACE}-frontend-loki-write01-headless-001.${NAMESPACE}.svc.cluster.local:3100
  storage:
    s3:
      endpoint: ${NAMESPACE}-minio-loki-01-001.${NAMESPACE}.svc.cluster.local:9000
      insecure: true
      bucketnames: loki-data
      access_key_id: ${MINIO_ROOT_USER}
      secret_access_key: ${MINIO_ROOT_PASSWORD}
      s3forcepathstyle: true
  ring:
    kvstore:
      store: memberlist

storage_config:
  tsdb_shipper:
    active_index_directory: /data/tsdb-index
    cache_location: /data/tsdb-cache
  named_stores:
    aws:
      minio:
        s3: http://${MINIO_ROOT_USER}:${MINIO_ROOT_PASSWORD}@${NAMESPACE}-minio-loki-01-001.${NAMESPACE}.svc.cluster.local:9000/loki-data
        s3forcepathstyle: true

schema_config:
  configs:
  - from: 2023-07-11
    store: tsdb
    object_store: s3
    schema: v13
    index:
      prefix: index_
      period: 24h

limits_config:
  max_cache_freshness_per_query: '10m'
  reject_old_samples: true
  reject_old_samples_max_age: 30m
  ingestion_rate_mb: 10
  ingestion_burst_size_mb: 20
  split_queries_by_interval: 15m
  volume_enabled: true
  max_label_names_per_series: 20
  retention_period: 720h 

frontend:
  log_queries_longer_than: 5s
  compress_responses: true
  max_outstanding_per_tenant: 2048

querier:
  query_ingesters_within: 2h

query_range:
  align_queries_with_step: true
  max_retries: 5
  parallelise_shardable_queries: true
  cache_results: true

query_scheduler:
  max_outstanding_requests_per_tenant: 4096

compactor:
  working_directory: /data/compactor
  retention_enabled: true
  retention_delete_delay: 2h
  retention_delete_worker_count: 150
  delete_request_store: "minio"

ingester:
  lifecycler:
    join_after: 10s
    observe_period: 5s
    ring:
      replication_factor: 3
      kvstore:
        store: memberlist
    final_sleep: 0s
  chunk_idle_period: 1m
  wal:
    enabled: true
    dir: /data/wal
  max_chunk_age: 1m
  chunk_retain_period: 30s
  chunk_encoding: snappy
  chunk_target_size: 1.572864e+06
  chunk_block_size: 262144
  flush_op_timeout: 10s

ruler:
  alertmanager_url: "http://${NAMESPACE}-backend-alertmanager01-001.${NAMESPACE}.svc.cluster.local:9093"
  enable_alertmanager_v2: true
  enable_api: true
  enable_sharding: true
  wal:
    dir: /data/ruler-wal
  storage:
    type: local
    local:
      directory: /data/rules
  rule_path: /data/rules-temp
  remote_write:
    enabled: true
    clients:
      local:
         url: http://${NAMESPACE}-backend-prometheus-db01-001.${NAMESPACE}.svc.cluster.local:9090/api/v1/write
         queue_config:
           capacity: 1
           batch_send_deadline: 0s

Manifest

apiVersion: v1
kind: Service
metadata:
  name: frontend-loki-read01-headless
  labels:
    app.kubernetes.io/name: loki
    app.kubernetes.io/component: loki-read
spec:
  ports:
  - name: tcp01
    port: 3100
    protocol: TCP
    targetPort: 3100
  - name: tcp02
    port: 9095
    protocol: TCP
    targetPort: 9095
  selector:
    app.kubernetes.io/name: loki
    app.kubernetes.io/component: loki-read
  sessionAffinity: None
  clusterIP: None
  type: ClusterIP
  publishNotReadyAddresses: true
---
apiVersion: v1
kind: Service
metadata:
  name: frontend-loki-write01-headless
  labels:
    app.kubernetes.io/name: loki
    app.kubernetes.io/component: loki-write
spec:
  ports:
  - name: tcp01
    port: 3100
    protocol: TCP
    targetPort: 3100
  - name: tcp02
    port: 9095
    protocol: TCP
    targetPort: 9095
  selector:
    app.kubernetes.io/name: loki
    app.kubernetes.io/component: loki-write
  sessionAffinity: None
  clusterIP: None
  type: ClusterIP
  publishNotReadyAddresses: true
---
apiVersion: v1
kind: Service
metadata:
  name: frontend-loki-backend01-headless
  labels:
    app.kubernetes.io/name: loki
    app.kubernetes.io/component: loki-backend
spec:
  ports:
  - name: tcp01
    port: 3100
    protocol: TCP
    targetPort: 3100
  - name: tcp02
    port: 9095
    protocol: TCP
    targetPort: 9095
  selector:
    app.kubernetes.io/name: loki
    app.kubernetes.io/component: loki-backend
  sessionAffinity: None
  clusterIP: None
  type: ClusterIP
  publishNotReadyAddresses: true
---
apiVersion: v1
kind: Service
metadata:
  name: frontend-loki-memberlist01-headless
  labels:
    app.kubernetes.io/component: loki
spec:
  ports:
  - name: tcp03
    port: 7946
    protocol: TCP
    targetPort: 7946
  selector:
    app.kubernetes.io/name: loki
  sessionAffinity: None
  clusterIP: None
  type: ClusterIP
  publishNotReadyAddresses: true

read については ruler を追加して args を設定しています。

apiVersion: v1
kind: Service
metadata:
  name: frontend-loki-read01
  labels:
    app.kubernetes.io/name: loki
    app.kubernetes.io/component: loki-read
spec:
  ports:
  - name: tcp01
    port: 3100
    protocol: TCP
    targetPort: 3100
  - name: tcp02
    port: 9095
    protocol: TCP
    targetPort: 9095
  selector:
    app.kubernetes.io/name: loki
    app.kubernetes.io/component: loki-read
  type: ClusterIP
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: frontend-loki-read01
  labels:
    app.kubernetes.io/name: loki
    app.kubernetes.io/component: loki-read
spec:
  serviceName: "frontend-loki-read01"
  replicas: 3
  selector:
    matchLabels:
      app.kubernetes.io/name: loki
      app.kubernetes.io/component: loki-read
  template:
    metadata:
      labels:
        app.kubernetes.io/name: loki
        app.kubernetes.io/component: loki-read
    spec:
      containers:
      - args:
        - -config.file=/etc/loki/config-loki.yaml
        - -config.expand-env
        - -target=read,ruler
        - -legacy-read-mode=false
        name: frontend-loki-read01
        image: frontend-loki-app01
        imagePullPolicy: Always
        resources:
          requests:
            memory: "200Mi"
            cpu: "200m"
          limits:
            memory: "1Gi"
            cpu: "1000m"
        ports:
        - containerPort: 3100
          name: tcp01
          protocol: TCP
        - containerPort: 9095
          name: tcp02
          protocol: TCP
        - containerPort: 7946
          name: tcp03
          protocol: TCP
        env:
        - name: NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: MINIO_ROOT_USER
          valueFrom:
            secretKeyRef: 
              name: kustomize-secret-minio
              key: minio_root_user
        - name: MINIO_ROOT_PASSWORD
          valueFrom:
            secretKeyRef:
              name: kustomize-secret-minio
              key: minio_root_password
        volumeMounts:
        - mountPath: /etc/loki
          name: configmap-loki-config01
        - mountPath: /data/rules/fake
          name: configmap-loki-config04
        - mountPath: /data
          name: loki-read
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
      restartPolicy: Always
      terminationGracePeriodSeconds: 4800
      volumes:
      - name: configmap-loki-config01
        configMap:
          name: configmap-loki-config01
          defaultMode: 420
      - name: configmap-loki-config04
        configMap:
          name: configmap-loki-config04
          defaultMode: 420
  volumeClaimTemplates:
  - metadata:
      name: loki-read
    spec:
      accessModes: ["ReadWriteOnce"]
      resources:
        requests:
          storage: 1Gi
      storageClassName: longhorn

apiVersion: v1
kind: Service
metadata:
  name: frontend-loki-write01
  labels:
    app.kubernetes.io/name: loki
    app.kubernetes.io/component: loki-write
spec:
  ports:
  - name: tcp01
    port: 3100
    protocol: TCP
    targetPort: 3100
  - name: tcp02
    port: 9095
    protocol: TCP
    targetPort: 9095
  selector:
    app.kubernetes.io/name: loki
    app.kubernetes.io/component: loki-write
  type: ClusterIP
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: frontend-loki-write01
  labels:
    app.kubernetes.io/name: loki
    app.kubernetes.io/component: loki-write
spec:
  serviceName: "frontend-loki-write01"
  replicas: 3
  selector:
    matchLabels:
      app.kubernetes.io/name: loki
      app.kubernetes.io/component: loki-write
  template:
    metadata:
      labels:
        app.kubernetes.io/name: loki
        app.kubernetes.io/component: loki-write
    spec:
      containers:
      - args:
        - -config.file=/etc/loki/config-loki.yaml
        - -config.expand-env
        - -target=write
        - -legacy-read-mode=false
        name: frontend-loki-write01
        image: frontend-loki-app01
        imagePullPolicy: Always
        resources:
          requests:
            memory: "200Mi"
            cpu: "200m"
          limits:
            memory: "1Gi"
            cpu: "1000m"
        ports:
        - containerPort: 3100
          name: tcp01
          protocol: TCP
        - containerPort: 9095
          name: tcp02
          protocol: TCP
        - containerPort: 7946
          name: tcp03
          protocol: TCP
        env:
        - name: NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: MINIO_ROOT_USER
          valueFrom:
            secretKeyRef: 
              name: kustomize-secret-minio
              key: minio_root_user
        - name: MINIO_ROOT_PASSWORD
          valueFrom:
            secretKeyRef:
              name: kustomize-secret-minio
              key: minio_root_password
        volumeMounts:
        - mountPath: /etc/loki
          name: configmap-loki-config01
        - mountPath: /data
          name: loki-write
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
      restartPolicy: Always
      terminationGracePeriodSeconds: 4800
      volumes:
      - name: configmap-loki-config01
        configMap:
          name: configmap-loki-config01
          defaultMode: 420
  volumeClaimTemplates:
  - metadata:
      name: loki-write
    spec:
      accessModes: ["ReadWriteOnce"]
      resources:
        requests:
          storage: 5Gi
      storageClassName: longhorn

apiVersion: v1
kind: Service
metadata:
  name: frontend-loki-backend01
  labels:
    app.kubernetes.io/name: loki
    app.kubernetes.io/component: loki-backend
spec:
  ports:
  - name: tcp01
    port: 3100
    protocol: TCP
    targetPort: 3100
  - name: tcp02
    port: 9095
    protocol: TCP
    targetPort: 9095
  selector:
    app.kubernetes.io/name: loki
    app.kubernetes.io/component: loki-backend
  type: ClusterIP
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: frontend-loki-backend01
  labels:
    app.kubernetes.io/name: loki
    app.kubernetes.io/component: loki-backend
spec:
  serviceName: "frontend-loki-backend01"
  replicas: 3
  selector:
    matchLabels:
      app.kubernetes.io/name: loki
      app.kubernetes.io/component: loki-backend
  template:
    metadata:
      labels:
        app.kubernetes.io/name: loki
        app.kubernetes.io/component: loki-backend
    spec:
      containers:
      - args:
        - -config.file=/etc/loki/config-loki.yaml
        - -config.expand-env
        - -target=backend
        - -legacy-read-mode=false
        name: frontend-loki-backend01
        image: frontend-loki-app01
        imagePullPolicy: Always
        resources:
          requests:
            memory: "200Mi"
            cpu: "200m"
          limits:
            memory: "1Gi"
            cpu: "1000m"
        ports:
        - containerPort: 3100
          name: tcp01
          protocol: TCP
        - containerPort: 9095
          name: tcp02
          protocol: TCP
        - containerPort: 7946
          name: tcp03
          protocol: TCP
        env:
        - name: NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: MINIO_ROOT_USER
          valueFrom:
            secretKeyRef: 
              name: kustomize-secret-minio
              key: minio_root_user
        - name: MINIO_ROOT_PASSWORD
          valueFrom:
            secretKeyRef:
              name: kustomize-secret-minio
              key: minio_root_password
        volumeMounts:
        - mountPath: /etc/loki
          name: configmap-loki-config01
        - mountPath: /data/rules/fake
          name: configmap-loki-config04
        - mountPath: /data
          name: loki-backend
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
      restartPolicy: Always
      terminationGracePeriodSeconds: 4800
      volumes:
      - name: configmap-loki-config01
        configMap:
          name: configmap-loki-config01
          defaultMode: 420
      - name: configmap-loki-config04
        configMap:
          name: configmap-loki-config04
          defaultMode: 420
  volumeClaimTemplates:
  - metadata:
      name: loki-backend
    spec:
      accessModes: ["ReadWriteOnce"]
      resources:
        requests:
          storage: 1Gi
      storageClassName: longhorn