Merge pull request #102 from DogCatWorldTeam/feat#101

[feat] ELK 구성

build.gradle

@@ -71,6 +71,8 @@ dependencies {
 	annotationProcessor "com.querydsl:querydsl-apt:5.0.0:jakarta"
 	annotationProcessor "jakarta.annotation:jakarta.annotation-api"
 	annotationProcessor "jakarta.persistence:jakarta.persistence-api"
+
+	implementation 'net.logstash.logback:logstash-logback-encoder:7.4'
 }
 
 def querydslSrcDir = 'src/main/generated'

docker-compose.logging.yml

@@ -0,0 +1,87 @@
+services:
+  elasticsearch:
+    build:
+      context: elk/elasticsearch/
+      args:
+        ELASTIC_VERSION: 8.5.2
+    volumes:
+      - ./elk/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro,Z
+      - elasticsearch:/usr/share/elasticsearch/data:Z
+    ports:
+      - 9200:9200
+      - 9300:9300
+    environment:
+      node.name: elasticsearch
+      ES_JAVA_OPTS: -Xms512m -Xmx512m
+      # Bootstrap password.
+      # Used to initialize the keystore during the initial startup of
+      # Elasticsearch. Ignored on subsequent runs.
+      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
+      # Use single node discovery in order to disable production mode and avoid bootstrap checks.
+      # see: https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html
+      discovery.type: single-node
+    networks:
+      - elk
+    restart: unless-stopped
+
+  logstash:
+    build:
+      context: elk/logstash/
+      args:
+        ELASTIC_VERSION: 8.5.2
+    volumes:
+      - ./elk/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro,Z
+      - ./elk/logstash/pipeline:/usr/share/logstash/pipeline:ro,Z
+    ports:
+      - 5044:5044
+      - 50000:50000/tcp
+      - 50000:50000/udp
+      - 9600:9600
+    environment:
+      LS_JAVA_OPTS: -Xms256m -Xmx256m
+      LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-}
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch
+    restart: unless-stopped
+
+  kibana:
+    build:
+      context: elk/kibana/
+      args:
+        ELASTIC_VERSION: 8.5.2
+    volumes:
+      - ./elk/kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro,Z
+    ports:
+      - 5601:5601
+    environment:
+      KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-}
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch
+    restart: unless-stopped
+
+  filebeat:
+    build:
+      context: elk/filebeat
+      args:
+        ELASTIC_VERSION: 8.5.2
+    entrypoint: "filebeat -e -strict.perms=false"
+    volumes:
+      - ./elk/filebeat/config/filebeat.yml:/usr/share/filebeat/filebeat.yml
+      - ./log/:/var/log/server.log
+    depends_on:
+      - logstash
+      - elasticsearch
+      - kibana
+    networks:
+      - elk
+
+networks:
+  elk:
+    driver: bridge
+
+volumes:
+  elasticsearch:

docker-compose.yml

@@ -51,5 +51,19 @@ services:
     networks:
       - team_network
 
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor
+    container_name: cadvisor
+    ports:
+      - 8080:8080
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+      - /dev/disk/:/dev/disk:ro
+    networks:
+      - team_network
+
 networks:
   team_network:

elk/elasticsearch/.dockerignore

@@ -0,0 +1,6 @@
+# Ignore Docker build files
+Dockerfile
+.dockerignore
+
+# Ignore OS artifacts
+**/.DS_Store

elk/elasticsearch/Dockerfile

@@ -0,0 +1,7 @@
+ARG ELASTIC_VERSION
+
+# https://www.docker.elastic.co/
+FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}
+
+# Add your elasticsearch plugins setup here
+# Example: RUN elasticsearch-plugin install analysis-icu

elk/elasticsearch/config/elasticsearch.yml

@@ -0,0 +1,12 @@
+---
+## Default Elasticsearch configuration from Elasticsearch base image.
+## https://github.com/elastic/elasticsearch/blob/main/distribution/docker/src/docker/config/elasticsearch.yml
+#
+cluster.name: docker-cluster
+network.host: 0.0.0.0
+
+## X-Pack settings
+## see https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html
+#
+xpack.license.self_generated.type: basic
+xpack.security.enabled: false

elk/filebeat/.dockerignore

@@ -0,0 +1,6 @@
+# Ignore Docker build files
+Dockerfile
+.dockerignore
+
+# Ignore OS artifacts
+**/.DS_Store

elk/filebeat/Dockerfile

@@ -0,0 +1,3 @@
+ARG ELASTIC_VERSION
+
+FROM docker.elastic.co/beats/filebeat:${ELASTIC_VERSION}

elk/filebeat/README.md

@@ -0,0 +1,42 @@
+# Filebeat
+
+Filebeat is a lightweight shipper for forwarding and centralizing log data. Installed as an agent on your servers,
+Filebeat monitors the log files or locations that you specify, collects log events, and forwards them either to
+Elasticsearch or Logstash for indexing.
+
+## Usage
+
+**This extension requires the `filebeat_internal` and `beats_system` users to be created and initialized with a
+password.** In case you haven't done that during the initial startup of the stack, please refer to [How to re-execute
+the setup][setup] to run the setup container again and initialize these users.
+
+To include Filebeat in the stack, run Docker Compose from the root of the repository with an additional command line
+argument referencing the `filebeat-compose.yml` file:
+
+```console
+$ docker-compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml up
+```
+
+## Configuring Filebeat
+
+The Filebeat configuration is stored in [`config/filebeat.yml`](config/filebeat.yml). You can modify this file with
+the help of the [Configuration reference][filebeat-config].
+
+Any change to the Filebeat configuration requires a restart of the Filebeat container:
+
+```console
+$ docker-compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml restart filebeat
+```
+
+Please refer to the following documentation page for more details about how to configure Filebeat inside a Docker
+container: [Run Filebeat on Docker][filebeat-docker].
+
+## See also
+
+[Filebeat documentation][filebeat-doc]
+
+[filebeat-config]: https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html
+[filebeat-docker]: https://www.elastic.co/guide/en/beats/filebeat/current/running-on-docker.html
+[filebeat-doc]: https://www.elastic.co/guide/en/beats/filebeat/current/index.html
+
+[setup]: ../../docker-elk/README.md#how-to-re-execute-the-setup

elk/filebeat/config/filebeat.yml

@@ -0,0 +1,14 @@
+filebeat.inputs:
+  - type: log
+    enabled: true
+    paths:
+      - /var/log/*.log  # 수집할 로그 파일 위치
+
+output.logstash:
+  enabled: true
+  hosts: ["logstash:5044"]
+
+setup.kibana:
+  host: "kibana:5601"
+  username: "elastic"
+  password: "changeme"

elk/filebeat/filebeat-compose.yml

@@ -0,0 +1,35 @@
+version: '3.7'
+
+services:
+  filebeat:
+    build:
+      context: extensions/filebeat/
+      args:
+        ELASTIC_VERSION: ${ELASTIC_VERSION}
+    # Run as 'root' instead of 'filebeat' (uid 1000) to allow reading
+    # 'docker.sock' and the host's filesystem.
+    user: root
+    command:
+      # Log to stderr.
+      - -e
+      # Disable config file permissions checks. Allows mounting
+      # 'config/filebeat.yml' even if it's not owned by root.
+      # see: https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html
+      - --strict.perms=false
+    volumes:
+      - ./extensions/filebeat/config/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro,Z
+      - type: bind
+        source: /var/lib/docker/containers
+        target: /var/lib/docker/containers
+        read_only: true
+      - type: bind
+        source: /var/run/docker.sock
+        target: /var/run/docker.sock
+        read_only: true
+    environment:
+      FILEBEAT_INTERNAL_PASSWORD: ${FILEBEAT_INTERNAL_PASSWORD:-}
+      BEATS_SYSTEM_PASSWORD: ${BEATS_SYSTEM_PASSWORD:-}
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch

elk/kibana/.dockerignore

@@ -0,0 +1,6 @@
+# Ignore Docker build files
+Dockerfile
+.dockerignore
+
+# Ignore OS artifacts
+**/.DS_Store

elk/kibana/Dockerfile

@@ -0,0 +1,7 @@
+ARG ELASTIC_VERSION
+
+# https://www.docker.elastic.co/
+FROM docker.elastic.co/kibana/kibana:${ELASTIC_VERSION}
+
+# Add your kibana plugins setup here
+# Example: RUN kibana-plugin install <name|url>

elk/kibana/config/kibana.yml

@@ -0,0 +1,99 @@
+---
+## Default Kibana configuration from Kibana base image.
+## https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.ts
+#
+server.name: kibana
+server.host: 0.0.0.0
+elasticsearch.hosts: [ http://elasticsearch:9200 ]
+
+monitoring.ui.container.elasticsearch.enabled: true
+monitoring.ui.container.logstash.enabled: true
+
+## X-Pack security credentials
+#
+elasticsearch.username: kibana_system
+elasticsearch.password: ${KIBANA_SYSTEM_PASSWORD}
+
+## Encryption keys (optional but highly recommended)
+##
+## Generate with either
+##  $ docker container run --rm docker.elastic.co/kibana/kibana:8.6.2 bin/kibana-encryption-keys generate
+##  $ openssl rand -hex 32
+##
+## https://www.elastic.co/guide/en/kibana/current/using-kibana-with-security.html
+## https://www.elastic.co/guide/en/kibana/current/kibana-encryption-keys.html
+#
+#xpack.security.encryptionKey:
+#xpack.encryptedSavedObjects.encryptionKey:
+#xpack.reporting.encryptionKey:
+
+## Fleet
+## https://www.elastic.co/guide/en/kibana/current/fleet-settings-kb.html
+#
+xpack.fleet.agents.fleet_server.hosts: [ http://fleet-server:8220 ]
+
+xpack.fleet.outputs:
+  - id: fleet-default-output
+    name: default
+    type: elasticsearch
+    hosts: [ http://elasticsearch:9200 ]
+    is_default: true
+    is_default_monitoring: true
+
+xpack.fleet.packages:
+  - name: fleet_server
+    version: latest
+  - name: system
+    version: latest
+  - name: elastic_agent
+    version: latest
+  - name: docker
+    version: latest
+  - name: apm
+    version: latest
+
+xpack.fleet.agentPolicies:
+  - name: Fleet Server Policy
+    id: fleet-server-policy
+    description: Static agent policy for Fleet Server
+    monitoring_enabled:
+      - logs
+      - metrics
+    package_policies:
+      - name: fleet_server-1
+        package:
+          name: fleet_server
+      - name: system-1
+        package:
+          name: system
+      - name: elastic_agent-1
+        package:
+          name: elastic_agent
+      - name: docker-1
+        package:
+          name: docker
+  - name: Agent Policy APM Server
+    id: agent-policy-apm-server
+    description: Static agent policy for the APM Server integration
+    monitoring_enabled:
+      - logs
+      - metrics
+    package_policies:
+      - name: system-1
+        package:
+          name: system
+      - name: elastic_agent-1
+        package:
+          name: elastic_agent
+      - name: apm-1
+        package:
+          name: apm
+        # See the APM package manifest for a list of possible inputs.
+        # https://github.com/elastic/apm-server/blob/v8.5.0/apmpackage/apm/manifest.yml#L41-L168
+        inputs:
+          - type: apm
+            vars:
+              - name: host
+                value: 0.0.0.0:8200
+              - name: url
+                value: http://apm-server:8200

elk/logstash/.dockerignore

@@ -0,0 +1,6 @@
+# Ignore Docker build files
+Dockerfile
+.dockerignore
+
+# Ignore OS artifacts
+**/.DS_Store

elk/logstash/Dockerfile

@@ -0,0 +1,7 @@
+ARG ELASTIC_VERSION
+
+# https://www.docker.elastic.co/
+FROM docker.elastic.co/logstash/logstash:${ELASTIC_VERSION}
+
+# Add your logstash plugins setup here
+# Example: RUN logstash-plugin install logstash-filter-json

elk/logstash/config/logstash.yml

@@ -0,0 +1,7 @@
+---
+## Default Logstash configuration from Logstash base image.
+## https://github.com/elastic/logstash/blob/main/docker/data/logstash/config/logstash-full.yml
+#
+http.host: 0.0.0.0
+
+node.name: logstash