[Service]
    Http_Listen    0.0.0.0
    Http_Port    2020
    Http_Server    true
    Parsers_File    /fluent-bit/etc/parsers.conf
    storage.path    /fluent-bit/tail/
    storage.backlog.mem_limit    128M
    storage.metrics    on
    storage.max_chunks_up    2048
[Input]
    Name    tail
    Buffer_Chunk_Size    5M
    Buffer_Max_Size    200M
    Path    /var/log/containers/*.log
    Skip_Long_Lines    true
    DB    /fluent-bit/tail/pos.db
    Mem_Buf_Limit    512MB
    Parser    cri
    Key    message
    Tag    kube.*
    storage.type    filesystem
    Read_from_Head    true
    Rotate_Wait    60
    Skip_Empty_Lines    true
    Threaded    true
[Input]
    Name    fluentbit_metrics
    Tag    fluentbit.metrics
    scrape_interval    5
    scrape_on_start    true
[Filter]
    Name    modify
    Match    kube.*
    Rename    log    message
[Filter]
    Name    lua
    Match    kube.*
    code    function drop_no_message_records(tag, timestamp, record) if record["message"] == nil then return -1, timestamp, record; end return 0, timestamp, record; end
    call    drop_no_message_records
[Filter]
    Name    kubernetes
    Match    kube.*
    Buffer_Size    128k
    Kube_URL    https://kubernetes.default.svc:443
    Kube_CA_File    /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    Kube_Token_File    /var/run/secrets/kubernetes.io/serviceaccount/token
    Kube_Tag_Prefix    kube.var.log.containers.
    Merge_Log    false
    K8S-Logging.Exclude    true
    Labels    false
    Annotations    false
    Use_Kubelet    false
    Namespace_Labels    true
    Namespace_Annotations    false
[Filter]
    Name    lua
    Match    kube.*
    code    function add_env_label(tag, timestamp, record) if record["kubernetes_namespace"] and record["kubernetes_namespace"]["labels"] then if record["kubernetes_namespace"]["labels"]["env"] == nil then local new_record = record; new_record["kubernetes_namespace"]["labels"]["env"] = "infra"; return 2, timestamp, new_record; end end return 0, timestamp, record; end
    call    add_env_label
[Filter]
    Name    rewrite_tag
    Match    kube.*
    Rule    $logtag .* re_emitted.$kubernetes_namespace['labels']['env'].$kubernetes['namespace_name'].$kubernetes['pod_name'].$kubernetes['container_name'] false
    Emitter_Name    re_emitted
    Emitter_Storage.type    filesystem
[Output]
    Name    prometheus_exporter
    Match    fluentbit.metrics
    Host    0.0.0.0
    Port    2021
    add_label     app    fluent-bit
    add_label     cluster    xxx
[Output]
    Name    s3
    Match    re_emitted.*
    region    eu-west-1
    bucket    xxx
    total_file_size    256M
    upload_chunk_size    50M
    upload_timeout    60m
    store_dir    /host/fluent-bit-buffer/
    s3_key_format    /archive/logs/xxx/$TAG[1]/$TAG[2]/%Y/%m/%d/%H/$TAG[3]_$TAG[4]_$INDEX.gz
    s3_key_format_tag_delimiters    .
    compression    gzip
    auto_retry_requests    true
    log_key    message
    retry_limit    10