Kubernetes 集群应用日志采集最佳实践

loki:
  auth_enabled: false
  commonConfig:
    replication_factor: 2
  storage:
    type: s3
    s3:
      region: ap-southeast-1
      bucketnames: mycompany-loki-logs
  schemaConfig:
    configs:
      - from: "2024-01-01"
        store: tsdb
        object_store: s3
        schema: v13
        index:
          prefix: index_
          period: 24h
  limits_config:
    retention_period: 720h     # 日志保留 30 天
    ingestion_rate_mb: 64
    ingestion_burst_size_mb: 128

distributor:
  replicas: 2

ingester:
  replicas: 3
  persistence:
    enabled: true
    size: 50Gi

querier:
  replicas: 2
  persistence:
    enabled: true
    size: 10Gi

queryFrontend:
  replicas: 2

gateway:
  enabled: true
  replicas: 2

helm repo add grafana https://grafana.github.io/helm-charts
helm repo update

helm upgrade --install loki grafana/loki \
  --namespace monitoring \
  --create-namespace \
  -f helm-values.yaml

# fluent-bit-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: fluent-bit-config
  namespace: monitoring
data:
  fluent-bit.conf: |
    [SERVICE]
        Flush             5
        Daemon            Off
        Log_Level         info
        Parsers_File      parsers.conf
        HTTP_Server       On
        HTTP_Listen       0.0.0.0
        HTTP_Port         2020
        storage.type      filesystem          # 启用持久化缓冲，防止重启丢日志
        storage.path      /var/fluent-bit/state

    [INPUT]
        Name              tail
        Tag               kube.*
        Path              /var/log/containers/*.log
        Parser            containerd
        Refresh_Interval  5
        Mem_Buf_Limit     50MB
        Skip_Long_Lines   On
        DB                /var/fluent-bit/state/tail.db   # 记录读取位置，防止重复采集

    [FILTER]
        Name                kubernetes
        Match               kube.*
        Kube_URL            https://kubernetes.default.svc:443
        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
        Merge_Log           On
        Keep_Log            Off
        K8S-Logging.Parser  On
        K8S-Logging.Exclude On
        Annotations         Off
        Labels              On

    [FILTER]
        Name                nest
        Match               kube.*
        Operation           lift
        Nested_under        kubernetes
        Add_prefix          k8s_

    [FILTER]
        Name                modify
        Match               kube.*
        Rename              k8s_pod_name        pod
        Rename              k8s_namespace_name  namespace
        Rename              k8s_container_name  container
        Rename              k8s_node_name       node
        Add                 cluster             prod-k8s-01
        Remove_regex        k8s_.*

    [OUTPUT]
        Name            loki
        Match           kube.*
        Host            loki-gateway.monitoring.svc
        Port            80
        Labels          job=fluentbit,cluster=$cluster,namespace=$namespace,pod=$pod,container=$container
        Line_Format     json
        Retry_Limit     5
        Batch_Wait      1s
        Batch_Size      1MB

  parsers.conf: |
    [PARSER]
        Name        containerd
        Format      regex
        Regex       ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L%z
        Time_Keep   On

# fluent-bit-daemonset.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: fluent-bit
  namespace: monitoring
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: fluent-bit
rules:
  - apiGroups: [""]
    resources: [pods, namespaces, nodes]
    verbs: [get, list, watch]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: fluent-bit
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: fluent-bit
subjects:
  - kind: ServiceAccount
    name: fluent-bit
    namespace: monitoring
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluent-bit
  namespace: monitoring
  labels:
    app: fluent-bit
spec:
  selector:
    matchLabels:
      app: fluent-bit
  template:
    metadata:
      labels:
        app: fluent-bit
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "2020"
        prometheus.io/path: "/api/v1/metrics/prometheus"
    spec:
      serviceAccountName: fluent-bit
      tolerations:
        - operator: Exists   # 确保在所有节点（包括 master）上运行
      containers:
        - name: fluent-bit
          image: cr.fluentbit.io/fluent/fluent-bit:3.1
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 2020
              name: http-metrics
          resources:
            limits:
              cpu: 500m
              memory: 256Mi
            requests:
              cpu: 100m
              memory: 64Mi
          volumeMounts:
            - name: varlog
              mountPath: /var/log
            - name: varlibdockercontainers
              mountPath: /var/lib/docker/containers
              readOnly: true
            - name: fluent-bit-config
              mountPath: /fluent-bit/etc/
            - name: fluent-bit-state
              mountPath: /var/fluent-bit/state
      volumes:
        - name: varlog
          hostPath:
            path: /var/log
        - name: varlibdockercontainers
          hostPath:
            path: /var/lib/docker/containers
        - name: fluent-bit-config
          configMap:
            name: fluent-bit-config
        - name: fluent-bit-state
          hostPath:
            path: /var/fluent-bit/state
            type: DirectoryOrCreate

kubectl apply -f fluent-bit-config.yaml
kubectl apply -f fluent-bit-daemonset.yaml

# 验证部署状态
kubectl get daemonset fluent-bit -n monitoring
kubectl logs -n monitoring -l app=fluent-bit --tail=50

# grafana-loki-datasource.yaml
apiVersion: grafana.integreatly.org/v1beta1
kind: GrafanaDatasource
metadata:
  name: loki
  namespace: monitoring
spec:
  datasource:
    name: Loki
    type: loki
    url: http://loki-gateway.monitoring.svc:80
    access: proxy
    isDefault: false
    jsonData:
      maxLines: 1000
      derivedFields:
        - name: TraceID
          matcherRegex: '"trace_id":"(\w+)"'
          url: "${__value.raw}"
          datasourceUid: tempo   # 关联 Tempo 实现日志 → Trace 跳转

[INPUT]
    Name              tail
    Tag               java.*
    Path              /var/log/containers/*-java-*.log
    Multiline.Parser  java   # 使用内置 Java 多行解析器

# 如需自定义规则
[MULTILINE_PARSER]
    Name           java-custom
    Type           regex
    Flush_Timeout  5000
    # 以日期时间开头为新日志条目的起始行
    Rule           "start_state"  "/^\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}/"   "cont"
    Rule           "cont"         "/^\s+at.*/"                                  "cont"
    Rule           "cont"         "/^Caused by:.*/"                             "cont"
    Rule           "cont"         "/^\s+\.\.\. \d+ more/"                       "cont"

[FILTER]
    Name    lua
    Match   kube.*
    script  /fluent-bit/etc/sanitize.lua
    call    sanitize_log

function sanitize_log(tag, timestamp, record)
    local log = record["log"]
    if log then
        -- 手机号脱敏：138****8888
        log = string.gsub(log, "([1][3-9]%d%d)%d%d%d%d(%d%d%d%d)", "%1****%2")
        -- 身份证号脱敏：110101********1234
        log = string.gsub(log, "(%d%d%d%d%d%d)%d%d%d%d%d%d%d%d(%d%d%d[%dXx])", "%1********%2")
        -- 银行卡号脱敏（保留前4后4位）
        log = string.gsub(log, "(%d%d%d%d)%d%d%d%d%d%d%d%d(%d%d%d%d)", "%1 **** **** %2")
        record["log"] = log
    end
    return 1, timestamp, record
end

# 禁用指定 Pod 的日志采集
apiVersion: apps/v1
kind: Deployment
metadata:
  name: debug-app
spec:
  template:
    metadata:
      annotations:
        fluentbit.io/exclude: "true"

# 为 Pod 指定自定义日志解析器
---
metadata:
  annotations:
    fluentbit.io/parser: "nginx"

[FILTER]
    Name                grep
    Match               kube.*
    Exclude             $kubernetes['annotations']['fluentbit.io/exclude'] true

[SERVICE]
    storage.type            filesystem    # 持久化缓冲，防止 OOM 导致日志丢失
    storage.max_chunks_up   128           # 内存中最大 chunk 数

[INPUT]
    Buffer_Chunk_Size  2MB      # 单文件读取 chunk 大小
    Buffer_Max_Size    10MB     # 单条日志最大长度（防止超长日志截断）
    Refresh_Interval   2        # 文件监听刷新频率（秒）

[OUTPUT]
    Batch_Wait    500ms     # 批次等待时间，减少请求数
    Batch_Size    4MB       # 批次大小，提升吞吐量
    Retry_Limit   10        # 重试次数，保障可靠性
    Workers       2         # 并发输出 worker 数

# Loki 分级存储配置示例
loki:
  storage_config:
    aws:
      s3: s3://mybucket/loki
  compactor:
    retention_enabled: true
  limits_config:
    retention_period: 720h    # 热/温存储保留 30 天

# Fluent Bit：输出重试率（过高说明 Loki 端有压力）
rate(fluentbit_output_retried_records_total[5m])

# Fluent Bit：各输出目标的错误计数
fluentbit_output_errors_total

# Loki：日志摄入速率（lines/s）
sum(rate(loki_distributor_lines_received_total[5m]))

# Loki：摄入延迟 P99
histogram_quantile(0.99,
  sum(rate(loki_distributor_ingester_appends_latency_seconds_bucket[5m])) by (le)
)

# Loki：查询延迟 P99
histogram_quantile(0.99,
  sum(rate(loki_querier_request_duration_seconds_bucket[5m])) by (le)
)