Merge pull request #68 from kikkomep/feat/logs-export

feat/logging improvements
crs4 · Nov 9, 2023 · c6a08e6 · c6a08e6
2 parents 2e79917 + fe7834e
commit c6a08e6
Show file tree

Hide file tree

Showing 7 changed files with 182 additions and 49 deletions.
diff --git a/k8s/lifemonitor-web/templates/deployment.yaml b/k8s/lifemonitor-web/templates/deployment.yaml
@@ -17,6 +17,8 @@ spec:
       helm.sh/chart: {{ include "lifemonitor-web.chart" . }}
   template:
     metadata:
+      annotations:
+        checksum/frontend-config: {{ include (print $.Template.BasePath "/nginx.configmap.yml") . | sha256sum }}
       labels:
         app.kubernetes.io/name: {{ include "lifemonitor-web.name" . }}
         app.kubernetes.io/instance: {{ .Release.Name }}
@@ -31,7 +33,7 @@ spec:
             - name: http
               containerPort: 4200
               protocol: TCP
-          {{ if .Values.monitoring.enabled }}
+          {{- if .Values.monitoring.enabled }}
             - name: metrics
               containerPort: 9090
               protocol: TCP
@@ -46,8 +48,10 @@ spec:
             - name: nginx-config
               mountPath: /etc/nginx/conf.d/webapp.prod.conf
               subPath: app.conf
-            - name: logs-storage
+            - name: nginx-logs
               mountPath: /var/log/nginx
+            - name: nginx-run
+              mountPath: /var/run/nginx
             {{ if .Values.extraVolumeMounts }}
             {{- toYaml .Values.extraVolumeMounts | nindent 12 }}
             {{- end }}
@@ -62,6 +66,24 @@ spec:
           #     port: http
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
+        {{- if .Values.monitoring.enabled }}
+        - name: promtail
+          image: grafana/promtail:master
+          args:
+            - "-config.file=/etc/promtail/promtail.yaml" # Found in the ConfigMap
+          resources:
+            requests:
+              memory: "128Mi"
+              cpu: "100m"
+            limits:
+              memory: "256Mi"
+              cpu: "200m"
+          volumeMounts:
+            - name: promtail-config
+              mountPath: /etc/promtail
+            - name: nginx-logs
+              mountPath: /var/log/nginx
+        {{- end }}
       volumes:
         - name: frontend-config
           configMap:
@@ -71,8 +93,16 @@ spec:
           configMap:
             name: {{ include "lifemonitor-web.fullname" . }}-nginx-config
             defaultMode: 0644
-        - name: logs-storage
+        - name: nginx-logs
           emptyDir: {}
+        - name: nginx-run
+          emptyDir: {}
+        {{- if .Values.monitoring.enabled }}
+        - name: promtail-config
+          configMap:
+            name: {{ include "lifemonitor-web.fullname" . }}-promtail-configmap
+            defaultMode: 0644
+        {{- end }}
         {{ if .Values.extraVolumes }}
           {{- toYaml .Values.extraVolumes | nindent 8 }}
         {{- end }}

diff --git a/k8s/lifemonitor-web/templates/nginx.configmap.yml b/k8s/lifemonitor-web/templates/nginx.configmap.yml
@@ -47,10 +47,6 @@ data:
         server {
             server_name {{ .Values.externalServerName }} {{ include "lifemonitor-web.name" . }}-metrics.default {{ include "lifemonitor-web.name" . }}-metrics.default.svc.cluster.local;
 
-            # save logs here
-            access_log /var/log/nginx/metrics.access.log extended;
-            error_log /var/log/nginx/metrics.error.log;
-
             proxy_read_timeout 600;
             proxy_connect_timeout 600;
             proxy_send_timeout 600;
@@ -69,9 +65,12 @@ data:
             # force HTTP traffic to HTTPS
             # error_page  497 http://$host:4200$request_uri;
 
-            # expose extended metrics
+            # expose standard nginx metrics
             location = /metrics {
                 stub_status on;
+
+                access_log    off;
+                log_not_found off;
             }
         }
         {{- end -}}
@@ -97,8 +96,46 @@ data:
             # force HTTP traffic to HTTPS
             error_page  497 http://$host:4200$request_uri;
 
+            # define error pages
+            error_page 404 /error/404;
+            error_page 405 /error/405;
+            error_page 429 /error/429;
+            error_page 500 /error/500;
+            error_page 502 /error/502;
+
+            # location for error pages
+            location ~ ^/error {
+                # rewrite request uri to point to the api
+                rewrite /error/(.*) /api/error/$1  break;
+
+                # disable redirects
+                proxy_redirect          off;
+
+                # rewrite headers
+                proxy_pass_header       Server;
+                proxy_set_header        X-Real-IP $http_x_forwarded_for;
+                proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+                proxy_set_header        X-Scheme $scheme;
+                proxy_set_header        Host $http_host;
+                proxy_set_header        Cookie $http_cookie;
+                proxy_set_header        X-Requested-With $http_x_requested_with;
+
+                # various proxy settings
+                proxy_connect_timeout   600;
+                proxy_read_timeout      600;
+                proxy_send_timeout      600;
+                #proxy_intercept_errors  on;
+
+                # set uppstream
+                proxy_pass              https://api;
+
+                # log errors using the extended format
+                access_log /var/log/nginx/http.error.log custom_format;
+            }
+
             # disable standard nginx metrics
             location = /nginx-metrics {
+                # stub_status on;
                 deny all;
             }
 
@@ -155,7 +192,6 @@ data:
               proxy_connect_timeout   600;
               proxy_read_timeout      600;
               proxy_send_timeout      600;
-              #proxy_intercept_errors  on;
           }
 
             location ~ ^/(account|oauth2|jobs|github|integrations) {
@@ -177,7 +213,6 @@ data:
                 proxy_connect_timeout   600;
                 proxy_read_timeout      600;
                 proxy_send_timeout      600;
-                #proxy_intercept_errors  on;
             }
 
             location /socket.io/ {
@@ -223,15 +258,15 @@ data:
         pcre_jit on;
 
         # logs
-        pid        /var/run/openresty/nginx.pid;
-        error_log  /var/log/nginx/nginx.error.log crit;
+        pid        /var/log/nginx/nginx.pid;
+        error_log  /var/log/nginx/nginx.error.log warn;
 
         events {
             worker_connections 1024;
         }
 
         http {
-
+            
             include mime.types;
 
             default_type application/octet-stream;
@@ -240,27 +275,35 @@ data:
             # When the use of underscores is disabled, request header fields whose names contain underscores are marked as invalid and become subject to the ignore_invalid_headers directive.
             # underscores_in_headers off;
 
-            server_names_hash_bucket_size  128;
-            server_names_hash_max_size 512;
-        
+            # fix issues with large client headers
             proxy_headers_hash_max_size 512;
             proxy_headers_hash_bucket_size 128;
 
-            # Define a verbose log format
-            log_format extended 
-            '[$http_x_forwarded_for] '
-            '"$request" $status ($body_bytes_sent bytes) - "$http_referer", "$http_user_agent", "$http_x_request_domain"';
-            
+            # Define the log format
+            log_format custom_format '$http_x_client_ip - $remote_user [$time_local] '
+                            '"$request" $status $body_bytes_sent '
+                            '"$http_referer" "$http_user_agent" - '
+                            '$http_x_forwarded_for $request_time $request_length - '
+                            '$connection $connection_requests $pipe - '
+                            '$upstream_addr $upstream_status $upstream_cache_status '
+                            '$upstream_response_time $upstream_response_length';
+
             # Configure Log files
-            access_log /usr/local/openresty/nginx/logs/access.log extended;
-            # error_log /usr/local/openresty/nginx/logs/error.log warn;
+            access_log /var/log/nginx/access.log custom_format;
+            error_log  /var/log/nginx/error.log warn;
+
+            
+            # See Move default writable paths to a dedicated directory (#119)
+            # https://github.com/openresty/docker-openresty/issues/119
+            client_body_temp_path /var/run/nginx/nginx-client-body;
+            proxy_temp_path /var/run/nginx/nginx-proxy;
+            fastcgi_temp_path /var/run/nginx/nginx-fastcgi;
+            uwsgi_temp_path /var/run/nginx/nginx-uwsgi;
+            scgi_temp_path /var/run/nginx/nginx-scgi;
+
 
-            # Extra settings
-            client_body_temp_path /var/run/openresty/nginx-client-body;
-            proxy_temp_path /var/run/openresty/nginx-proxy;
-            fastcgi_temp_path /var/run/openresty/nginx-fastcgi;
-            uwsgi_temp_path /var/run/openresty/nginx-uwsgi;
-            scgi_temp_path /var/run/openresty/nginx-scgi;
+            # Defines a file that will store the process ID of the main process.            
+            server_names_hash_bucket_size  128;
 
             sendfile on;
             #tcp_nopush     on;
@@ -275,4 +318,3 @@ data:
             # Don't reveal OpenResty version to clients.
             # server_tokens off;
         }
-
diff --git a/k8s/lifemonitor-web/templates/promtail.configmap.yaml b/k8s/lifemonitor-web/templates/promtail.configmap.yaml
@@ -0,0 +1,41 @@
+{{- if .Values.monitoring.loki.enabled }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "{{ include "lifemonitor-web.fullname" . }}-promtail-configmap"
+data:
+  promtail.yaml: |
+    server:
+      http_listen_port: 9080
+      grpc_listen_port: 0
+      log_level: "debug"
+    positions:
+      filename: /tmp/positions.yaml
+    clients: # Specify target
+      - url: {{ $.Values.monitoring.loki.url }}/loki/api/v1/push
+    scrape_configs:
+      - job_name:  "lifemonitor-app-proxy-logger" 
+        static_configs:
+          - targets:
+              - localhost 
+            labels:
+              app: "lifemonitor-webapp"
+              component: "{{$.Release.Name}}-proxy"
+              environment: "{{$.Release.Namespace}}"
+              format: "extended"
+              level: "INFO"
+              __path__: /var/log/nginx/access.log
+          - targets: 
+              - localhost 
+            labels:
+              app: "lifemonitor-webapp"
+              component: "{{ $.Release.Name }}-proxy"
+              environment: "{{ $.Release.Namespace }}"
+              format: "extended"
+              level: "ERROR"
+              __path__: /var/log/nginx/*error.log          
+        pipeline_stages:
+        - drop:
+            expression: ".*(DEBUG|health|heartbeat).*"
+{{- end }}
diff --git a/k8s/lifemonitor-web/values.yaml b/k8s/lifemonitor-web/values.yaml
@@ -9,15 +9,15 @@ image:
   tag: latest
   pullPolicy: IfNotPresent
 
-nameOverride: ''
-fullnameOverride: ''
+nameOverride: ""
+fullnameOverride: ""
 
-externalServerName: 'localhost'
+externalServerName: "localhost"
 
 # Setting for the LifeMonitor Backend API
 backend:
-  apiUrl: ''
-  clientId: ''
+  apiUrl: ""
+  clientId: ""
 
 service:
   type: NodePort
@@ -29,6 +29,19 @@ monitoring:
     enabled: false
   prometheus:
     namespace: kube-prometheus-stack
+  loki:
+    enabled: false
+    url: http://loki:3100
+    exporter:
+      image: grafana/promtail:main-60ea954
+      imagePullPolicy: IfNotPresent
+      resources:
+        requests:
+          memory: 128Mi
+          cpu: 0.1
+        limits:
+          memory: 256Mi
+          cpu: 0.2
 
 ingress:
   enabled: false
@@ -39,24 +52,30 @@ ingress:
   hosts:
     - host: localhost
       paths:
-        - '/'
+        - "/"
   # configure TLS for the ingress
   tls:
     - secretName: lifemonitor-web-tls
       hosts:
         - localhost
 
-resources: {}
-# We usually recommend not to specify default resources and to leave this as a conscious
-# choice for the user. This also increases chances charts run on environments with little
-# resources, such as Minikube. If you do want to specify resources, uncomment the following
-# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-# limits:
-#   cpu: 100m
-#   memory: 128Mi
-# requests:
-#   cpu: 100m
-#   memory: 128Mi
+resources:
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+  requests:
+    memory: "4096Mi"
+    cpu: "1000m"
+  limits:
+    memory: "7680Mi"
+    cpu: "1800m"
 
 nodeSelector: {}
 

diff --git a/ngsw-config.json b/ngsw-config.json
@@ -33,6 +33,7 @@
       "name": "api",
       "urls": [
         "/api/**",
+        "/error/**",
         "/account/**",
         "/oauth2/**",
         "/jobs/**",

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "lifemonitor",
-  "version": "0.5.5",
+  "version": "0.5.6",
   "scripts": {
     "start": "ng build --configuration production && http-server -p 4202 --ssl -C ./certs/lm.crt -K ./certs/lm.key --host lm dist/lifemonitor",
     "start:dev": "ng serve --port 4200 --ssl --ssl-key ./certs/lm.key --ssl-cert ./certs/lm.crt --host lm",