chore: added loki logs to example docker-compose (#385)

sygmaprotocol · Nov 7, 2024 · 2f772c2 · 2f772c2
1 parent eb43206
commit 2f772c2
Show file tree

Hide file tree

Showing 5 changed files with 203 additions and 11 deletions.
diff --git a/example/docker-compose.yml b/example/docker-compose.yml
@@ -1,20 +1,36 @@
 # Copyright 2020 ChainSafe Systems
 # SPDX-License-Identifier: LGPL-3.0-only
 
+x-logging:
+  &default-logging
+  driver: "json-file"
+  options:
+    max-size: "1m"
+    max-file: "1"
+    tag: "{{.Name}}"
+
 services:
   mempool-stub:
     image: ghcr.io/sygmaprotocol/beacon-api-stub
+    container_name: mempool-stub
+    labels:
+      logging: "promtail"
+      logging_jobname: "containerlogs"
     volumes:
       - "./cfg/stubs:/stubs"
     ports:
       - 8882:8882
     environment:
       - STUB_DATA=/stubs/mempool.yml
+    logging: *default-logging
 
   bitcoin:
 #   image: ruimarinho/bitcoin-core:latest
     image: ghcr.io/sygmaprotocol/bitcoin
     container_name: bitcoin
+    labels:
+      logging: "promtail"
+      logging_jobname: "containerlogs"
     ports:
       - "18443:18443"
       - "18332:18332"
@@ -27,40 +43,52 @@ services:
     volumes:
       - ./cfg/entrypoint/entrypoint.sh:/cfg/entrypoint/entrypoint.sh
     entrypoint: /cfg/entrypoint/entrypoint.sh
+    logging: *default-logging
 
   evm1-1:
     image: ghcr.io/sygmaprotocol/sygma-solidity:evm1-v2.10.1
     container_name: evm1-1
     command: --chain.chainId 1337 --db data/ --blockTime 2 --m 'black toward wish jar twin produce remember fluid always confirm bacon slush' > /dev/null
-    logging:
-      driver: none
+    labels:
+      logging: "promtail"
+      logging_jobname: "containerlogs"
+    logging: *default-logging
     ports:
       - "8545:8545"
 
   evm2-1:
     image: ghcr.io/sygmaprotocol/sygma-solidity:evm2-v2.10.1
     command: --chain.chainId 1338 --db data/ --blockTime 2 --m 'black toward wish jar twin produce remember fluid always confirm bacon slush' > /dev/null
     container_name: evm2-1
-    logging:
-      driver: none
+    labels:
+      logging: "promtail"
+      logging_jobname: "containerlogs"
+    logging: *default-logging
     ports:
       - "8547:8545"
 
   substrate-pallet:
-    container_name: substrate-pallet
     image: "ghcr.io/sygmaprotocol/sygma-substrate-pallets:e2e-v0.3.1"
+    container_name: substrate-pallet
+    labels:
+      logging: "promtail"
+      logging_jobname: "containerlogs"
     ports:
       - "9944:9944"
       - "9933:9933"
-
+    logging: *default-logging
+
   ipfs0:
-    container_name: ipfs0
     image: ipfs/kubo:release
+    container_name: ipfs0
+    labels:
+      logging: "promtail"
+      logging_jobname: "containerlogs"
     ports:
-      - "4001:4001" 
-      - "5002:5001" 
-      - "8080:8080" 
-
+      - "4001:4001"
+      - "5002:5001"
+      - "8080:8080"
+    logging: *default-logging
 
   relayer1:
     container_name: example_relayer1
@@ -71,9 +99,13 @@ services:
     depends_on:
       - evm1-1
       - substrate-pallet
+    labels:
+      logging: "promtail"
+      logging_jobname: "containerlogs"
     volumes:
       - ./cfg:/cfg
     restart: always
+    logging: *default-logging
 
   relayer2:
     container_name: example_relayer2
@@ -84,9 +116,13 @@ services:
     depends_on:
       - evm1-1
       - substrate-pallet
+    labels:
+      logging: "promtail"
+      logging_jobname: "containerlogs"
     volumes:
       - ./cfg:/cfg
     restart: always
+    logging: *default-logging
 
   relayer3:
     container_name: example_relayer3
@@ -97,15 +133,61 @@ services:
     depends_on:
       - evm1-1
       - substrate-pallet
+    labels:
+      logging: "promtail"
+      logging_jobname: "containerlogs"
     volumes:
       - ./cfg:/cfg
     restart: always
+    logging: *default-logging
 
   otel-collector:
     container_name: otel-collector
     image: otel/opentelemetry-collector
     command: ["--config=/etc/otel-collector-config.yml"]
+    labels:
+      logging: "promtail"
+      logging_jobname: "containerlogs"
     volumes:
       - ./cfg/otel-collector-config.yml:/etc/otel-collector-config.yml
     ports:
       - "8889:8889"   # Prometheus exporter metrics
+    logging: *default-logging
+
+  grafana:
+    image: grafana/grafana:11.3.0
+    container_name: grafana
+    ports:
+      - 3000:3000
+    volumes:
+      - ./grafana/datasources/grafana.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+      - ./grafana/plugins/app.yaml:/etc/grafana/provisioning/plugins/app.yaml
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+      - GF_FEATURE_TOGGLES_ENABLE=accessControlOnCall lokiLogsDataplane
+      - GF_INSTALL_PLUGINS=grafana-lokiexplore-app
+    logging: *default-logging
+
+  loki:
+    image: grafana/loki:3.0.0
+    container_name: loki
+    ports:
+      - 3100:3100
+    command: -config.file=/etc/loki/loki.yaml
+    volumes:
+      - ./loki/loki.yaml:/etc/loki/loki.yaml
+    logging: *default-logging
+
+  promtail:
+    image:  grafana/promtail:3.0.0
+    container_name: promtail
+    volumes:
+      - ./promtail/promtail.yaml:/etc/promtail/docker-config.yaml
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+    command: -config.file=/etc/promtail/docker-config.yaml
+    depends_on:
+      - loki
+    logging: *default-logging
diff --git a/example/grafana/datasources/grafana.yaml b/example/grafana/datasources/grafana.yaml
@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    version: 1
+    editable: false
+    isDefault: true
diff --git a/example/grafana/plugins/app.yaml b/example/grafana/plugins/app.yaml
@@ -0,0 +1,12 @@
+apiVersion: 1
+
+apps:
+  - type: "grafana-lokiexplore-app"
+    org_id: 1
+    org_name: "Grafana"
+    disabled: false
+    jsonData:
+      apiUrl: http://localhost
+      isApiKeySet: true
+    secureJsonData:
+      apiKey: example-key
diff --git a/example/loki/loki.yaml b/example/loki/loki.yaml
@@ -0,0 +1,49 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+frontend:
+  max_outstanding_per_tenant: 2048
+
+pattern_ingester:
+  enabled: true
+
+limits_config:
+  max_global_streams_per_user: 0
+  ingestion_rate_mb: 50000
+  ingestion_burst_size_mb: 50000
+  volume_enabled: true
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+analytics:
+  reporting_enabled: false
diff --git a/example/promtail/promtail.yaml b/example/promtail/promtail.yaml
@@ -0,0 +1,39 @@
+# https://grafana.com/docs/loki/latest/clients/promtail/configuration/
+# https://docs.docker.com/engine/api/v1.41/#operation/ContainerList
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: flog_scrape 
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 5s
+        filters:
+          - name: label
+            values: ["logging=promtail"] 
+    relabel_configs:
+      - source_labels: ['__meta_docker_container_name']
+        regex: '/(.*)'
+        target_label: 'container'
+      - source_labels: ['__meta_docker_container_log_stream']
+        target_label: 'logstream'
+      - source_labels: ['__meta_docker_container_label_logging_jobname']
+        target_label: 'job'
+    pipeline_stages:
+      - cri: {}
+      - multiline:
+          firstline: ^\d{4}-\d{2}-\d{2} \d{1,2}:\d{2}:\d{2},\d{3}
+          max_wait_time: 3s
+      # https://grafana.com/docs/loki/latest/clients/promtail/stages/json/
+      - json:
+          expressions:
+            #message: message
+            level: level
+            #output: 'message'