Prometheus + Grafana + Loki with docker-compose

I'm currently using the combination of Grafana, Prometheus and Loki as logging and monitoring stack and built a docker-compose file for the combination which I'd like to share with you.

I am using the following config on a server with a single user (me) that may be destroyed at any time - if you have higher security or availability requirements, please check additional setup strategies like non-root docker or kubernetes or similar.

Parts

docker-compose.yml for Grafana/Prometheus/Loki/Promtail + Docker Container Monitoring

version: '3'

services:
  mon_prometheus:
    image: prom/prometheus:latest
    container_name: mon_prometheus
    restart: always
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - /etc/timezone:/etc/timezone:ro
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
    depends_on:
      - mon_node-exporter
      - mon_cadvisor
      - mon_loki
      - mon_caddy
      - mon_promtail
    networks:
      - monitoring

  mon_node-exporter:
    image: prom/node-exporter:latest
    container_name: mon_node-exporter
    restart: always
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - /etc/timezone:/etc/timezone:ro
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.sysfs=/host/sys'
      - '--path.rootfs=/rootfs'
      - '--collector.filesystem.ignored-mount-points="^(/rootfs|/host|)/(sys|proc|dev|host|etc)($$|/)"'
      - '--collector.filesystem.ignored-fs-types="^(sys|proc|auto|cgroup|devpts|ns|au|fuse\.lxc|mqueue)(fs|)$$"'
    networks:
      - monitoring

  mon_cadvisor:
    image: google/cadvisor:latest
    container_name: mon_cadvisor
    restart: always
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - /etc/timezone:/etc/timezone:ro
      - /:/rootfs:ro
      - /var/run:/var/run:rw
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
    networks:
      - monitoring

  mon_grafana:
    image: grafana/grafana:latest
    container_name: mon_grafana
    restart: unless-stopped
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - /etc/timezone:/etc/timezone:ro
      - ./grafana/grafana.ini:/etc/grafana/grafana.ini
      - grafana:/var/lib/grafana
    ports:
      - 127.0.0.1:3000:3000
    environment:
      GF_SERVER_DOMAIN: "{{ gf_server_domain }}"
      GF_SERVER_ROOT_URL: "{{ gf_server_root_url }}"
    user: "1000"
    depends_on:
      - mon_prometheus
    networks:
      - monitoring

  mon_loki:
    image: grafana/loki:latest
    container_name: mon_loki
    ports:
      - 127.0.0.1:3100:3100
    command: -config.file=/etc/loki/local-config.yaml
    volumes:
      - ./loki/config.yaml:/etc/loki/config.yaml
      - loki:/data/loki
    networks:
      - monitoring

  mon_promtail:
    image: grafana/promtail:latest
    container_name: mon_promtail
    volumes:
      - /var/log:/var/log
      - /var/lib/docker/containers:/var/lib/docker/containers
      - ./promtail:/etc/promtail-config/
    command: -config.file=/etc/promtail-config/promtail.yml
    networks:
      - monitoring

Note that you should use your server vars at {{ gf_server_domain }} and {{ gf_server_root_url }}

Necessary config files

./prometheus/prometheus.yml

global:
  scrape_interval: 15s
  evaluation_interval: 15s
rule_files:
scrape_configs:
  - job_name: 'node-exporter'
    static_configs:
      - targets: [ 'mon_node-exporter:9100' ]

  - job_name: 'cadvisor'
    static_configs:
      - targets: [ 'mon_cadvisor:8080' ]

  - job_name: 'caddy'
    static_configs:
      - targets: [ 'mon_caddy:3210' ]

./loki/config.yaml

# (default configuration)
auth_enabled: false

server:
  http_listen_port: 3100

ingester:
  lifecycler:
    address: 127.0.0.1
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
    final_sleep: 0s
  chunk_idle_period: 1h       # Any chunk not receiving new logs in this time will be flushed
  max_chunk_age: 1h           # All chunks will be flushed when they hit this age, default is 1h
  chunk_target_size: 1048576  # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
  chunk_retain_period: 30s    # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
  max_transfer_retries: 0     # Chunk transfers disabled

schema_config:
  configs:
    - from: 2020-10-24
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

storage_config:
  boltdb_shipper:
    active_index_directory: /loki/boltdb-shipper-active
    cache_location: /loki/boltdb-shipper-cache
    cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
    shared_store: filesystem
  filesystem:
    directory: /loki/chunks

compactor:
  working_directory: /loki/boltdb-shipper-compactor
  shared_store: filesystem

limits_config:
  reject_old_samples: true
  reject_old_samples_max_age: 168h
  ingestion_burst_size_mb: 16
  ingestion_rate_mb: 16

chunk_store_config:
  max_look_back_period: 0s

table_manager:
  retention_deletes_enabled: false
  retention_period: 0s

ruler:
  storage:
    type: local
    local:
      directory: /loki/rules
  rule_path: /loki/rules-temp
  alertmanager_url: localhost
  ring:
    kvstore:
      store: inmemory
  enable_api: true

./grafana/grafana.ini

Overwrite default grafana settings here - I'm not using anything other than the default currently.

./promtail/promtail.yml

server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

clients:
  - url: mon_loki/loki/api/v1/push

Execute docker-compose up -d and enjoy a full monitoring stack you can use and configure.