From 0633c53928d3b05859965a9d105f42b423daf3ec Mon Sep 17 00:00:00 2001 From: Yuri Smirnov Date: Thu, 12 Dec 2024 14:04:33 +0300 Subject: [PATCH] Limit each log line length (#47) --- .github/workflows/test.yml | 2 - Changelog.md | 9 +- Gemfile | 1 - Gemfile.lock | 256 ++++++++++++++++----------------- lib/lamian/config.rb | 2 + lib/lamian/log_device.rb | 24 +++- spec/lamian/engine_spec.rb | 2 +- spec/lamian/log_device_spec.rb | 22 ++- 8 files changed, 167 insertions(+), 151 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 231cadb..7dcc966 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,8 +24,6 @@ jobs: run: bundle exec ci-helper CheckSpecSuffixes --extra-paths spec/*.rb --ignored-paths spec/*_helper.rb - name: Run specs run: bundle exec ci-helper RunSpecs - - name: Audit - run: bundle exec ci-helper BundlerAudit - name: Documentation coverage run: bundle exec rake doc:coverage - name: Coveralls diff --git a/Changelog.md b/Changelog.md index 878fd91..7e790cf 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,9 @@ +# Changelog + +## 1.11.0 + +* Add `max_log_length` to limit each log line length in order to prevent memory leaks in case of very long logs + ## 1.10.0 * Add an ability to insert Lamian's middleware inside the rails initialization process manually (`Lamian.config.middleware_autoset = true/false`) @@ -14,7 +20,7 @@ ## 1.3.0 -* Add support for the (new sentry gem)[https://github.com/getsentry/sentry-ruby]. +* Add support for the [new sentry gem](https://github.com/getsentry/sentry-ruby). ## 1.2.0 @@ -33,7 +39,6 @@ which ruins concept of single entry point :(. Also tied it to lamian instance * `8136689` fixed crashes when dump used outside lamian context - ## 0.3.2 * `e57e6cec` Changed rails dependency from `~> 4.2` to `>= 4.2` diff --git a/Gemfile b/Gemfile index 46fd231..0973322 100644 --- a/Gemfile +++ b/Gemfile @@ -5,7 +5,6 @@ source "https://rubygems.org" # Specify your gem's dependencies in lamian.gemspec gemspec -gem "bundler-audit" gem "ci-helper" gem "launchy" gem "pry" diff --git a/Gemfile.lock b/Gemfile.lock index 6756be2..bc61cf1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -7,29 +7,29 @@ PATH GEM remote: https://rubygems.org/ specs: - actioncable (7.2.0) - actionpack (= 7.2.0) - activesupport (= 7.2.0) + actioncable (7.2.2.1) + actionpack (= 7.2.2.1) + activesupport (= 7.2.2.1) nio4r (~> 2.0) websocket-driver (>= 0.6.1) zeitwerk (~> 2.6) - actionmailbox (7.2.0) - actionpack (= 7.2.0) - activejob (= 7.2.0) - activerecord (= 7.2.0) - activestorage (= 7.2.0) - activesupport (= 7.2.0) + actionmailbox (7.2.2.1) + actionpack (= 7.2.2.1) + activejob (= 7.2.2.1) + activerecord (= 7.2.2.1) + activestorage (= 7.2.2.1) + activesupport (= 7.2.2.1) mail (>= 2.8.0) - actionmailer (7.2.0) - actionpack (= 7.2.0) - actionview (= 7.2.0) - activejob (= 7.2.0) - activesupport (= 7.2.0) + actionmailer (7.2.2.1) + actionpack (= 7.2.2.1) + actionview (= 7.2.2.1) + activejob (= 7.2.2.1) + activesupport (= 7.2.2.1) mail (>= 2.8.0) rails-dom-testing (~> 2.2) - actionpack (7.2.0) - actionview (= 7.2.0) - activesupport (= 7.2.0) + actionpack (7.2.2.1) + actionview (= 7.2.2.1) + activesupport (= 7.2.2.1) nokogiri (>= 1.8.5) racc rack (>= 2.2.4, < 3.2) @@ -38,36 +38,37 @@ GEM rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) useragent (~> 0.16) - actiontext (7.2.0) - actionpack (= 7.2.0) - activerecord (= 7.2.0) - activestorage (= 7.2.0) - activesupport (= 7.2.0) + actiontext (7.2.2.1) + actionpack (= 7.2.2.1) + activerecord (= 7.2.2.1) + activestorage (= 7.2.2.1) + activesupport (= 7.2.2.1) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (7.2.0) - activesupport (= 7.2.0) + actionview (7.2.2.1) + activesupport (= 7.2.2.1) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activejob (7.2.0) - activesupport (= 7.2.0) + activejob (7.2.2.1) + activesupport (= 7.2.2.1) globalid (>= 0.3.6) - activemodel (7.2.0) - activesupport (= 7.2.0) - activerecord (7.2.0) - activemodel (= 7.2.0) - activesupport (= 7.2.0) + activemodel (7.2.2.1) + activesupport (= 7.2.2.1) + activerecord (7.2.2.1) + activemodel (= 7.2.2.1) + activesupport (= 7.2.2.1) timeout (>= 0.4.0) - activestorage (7.2.0) - actionpack (= 7.2.0) - activejob (= 7.2.0) - activerecord (= 7.2.0) - activesupport (= 7.2.0) + activestorage (7.2.2.1) + actionpack (= 7.2.2.1) + activejob (= 7.2.2.1) + activerecord (= 7.2.2.1) + activesupport (= 7.2.2.1) marcel (~> 1.0) - activesupport (7.2.0) + activesupport (7.2.2.1) base64 + benchmark (>= 0.3) bigdecimal concurrent-ruby (~> 1.0, >= 1.3.1) connection_pool (>= 2.2.5) @@ -81,14 +82,12 @@ GEM public_suffix (>= 2.0.2, < 7.0) ast (2.4.2) base64 (0.2.0) + benchmark (0.4.0) bigdecimal (3.1.8) builder (3.3.0) - bundler-audit (0.9.1) - bundler (>= 1.2.0, < 3) - thor (~> 1.0) childprocess (5.1.0) logger (~> 1.5) - ci-helper (0.6.0) + ci-helper (0.7.0) colorize (~> 1.1) dry-inflector (~> 1.0) umbrellio-sequel-plugins (~> 0.14) @@ -97,32 +96,33 @@ GEM concurrent-ruby (1.3.4) connection_pool (2.4.1) crass (1.0.6) - date (3.3.4) + date (3.4.1) diff-lcs (1.5.1) docile (1.4.1) drb (2.2.1) dry-inflector (1.1.0) erubi (1.13.0) - faraday (2.10.1) - faraday-net_http (>= 2.0, < 3.2) + faraday (2.12.2) + faraday-net_http (>= 2.0, < 3.5) + json logger - faraday-net_http (3.1.1) - net-http + faraday-net_http (3.4.0) + net-http (>= 0.5.0) globalid (1.2.1) activesupport (>= 6.1) - i18n (1.14.5) + i18n (1.14.6) concurrent-ruby (~> 1.0) - io-console (0.7.2) - irb (1.14.0) + io-console (0.8.0) + irb (1.14.1) rdoc (>= 4.0.0) reline (>= 0.4.2) - json (2.7.2) + json (2.9.0) language_server-protocol (3.17.0.3) launchy (3.0.1) addressable (~> 2.8) childprocess (~> 5.0) - logger (1.6.0) - loofah (2.22.0) + logger (1.6.2) + loofah (2.23.1) crass (~> 1.0.2) nokogiri (>= 1.12.0) mail (2.8.1) @@ -133,11 +133,11 @@ GEM marcel (1.0.4) method_source (1.1.0) mini_mime (1.1.5) - mini_portile2 (2.8.7) - minitest (5.25.0) - net-http (0.4.1) + mini_portile2 (2.8.8) + minitest (5.25.4) + net-http (0.6.0) uri - net-imap (0.4.14) + net-imap (0.5.1) date net-protocol net-pop (0.1.2) @@ -146,57 +146,57 @@ GEM timeout net-smtp (0.5.0) net-protocol - nio4r (2.7.3) - nokogiri (1.16.7) + nio4r (2.7.4) + nokogiri (1.17.1) mini_portile2 (~> 2.8.2) racc (~> 1.4) - nokogiri (1.16.7-x86_64-darwin) + nokogiri (1.17.1-x86_64-darwin) racc (~> 1.4) - nokogiri (1.16.7-x86_64-linux) + nokogiri (1.17.1-x86_64-linux) racc (~> 1.4) - parallel (1.26.2) - parser (3.3.4.2) + parallel (1.26.3) + parser (3.3.6.0) ast (~> 2.4.1) racc - pry (0.14.2) + pry (0.15.0) coderay (~> 1.1) method_source (~> 1.0) - psych (5.1.2) + psych (5.2.1) + date stringio public_suffix (6.0.1) racc (1.8.1) - rack (3.1.7) + rack (3.1.8) rack-session (2.0.0) rack (>= 3.0.0) rack-test (2.1.0) rack (>= 1.3) - rackup (2.1.0) + rackup (2.2.1) rack (>= 3) - webrick (~> 1.8) - rails (7.2.0) - actioncable (= 7.2.0) - actionmailbox (= 7.2.0) - actionmailer (= 7.2.0) - actionpack (= 7.2.0) - actiontext (= 7.2.0) - actionview (= 7.2.0) - activejob (= 7.2.0) - activemodel (= 7.2.0) - activerecord (= 7.2.0) - activestorage (= 7.2.0) - activesupport (= 7.2.0) + rails (7.2.2.1) + actioncable (= 7.2.2.1) + actionmailbox (= 7.2.2.1) + actionmailer (= 7.2.2.1) + actionpack (= 7.2.2.1) + actiontext (= 7.2.2.1) + actionview (= 7.2.2.1) + activejob (= 7.2.2.1) + activemodel (= 7.2.2.1) + activerecord (= 7.2.2.1) + activestorage (= 7.2.2.1) + activesupport (= 7.2.2.1) bundler (>= 1.15.0) - railties (= 7.2.0) + railties (= 7.2.2.1) rails-dom-testing (2.2.0) activesupport (>= 5.0.0) minitest nokogiri (>= 1.6) - rails-html-sanitizer (1.6.0) + rails-html-sanitizer (1.6.1) loofah (~> 2.21) - nokogiri (~> 1.14) - railties (7.2.0) - actionpack (= 7.2.0) - activesupport (= 7.2.0) + nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) + railties (7.2.2.1) + actionpack (= 7.2.2.1) + activesupport (= 7.2.2.1) irb (~> 1.13) rackup (>= 1.0.0) rake (>= 12.2) @@ -204,106 +204,93 @@ GEM zeitwerk (~> 2.6) rainbow (3.1.1) rake (13.2.1) - rdoc (6.7.0) + rdoc (6.8.1) psych (>= 4.0.0) - regexp_parser (2.9.2) - reline (0.5.9) + regexp_parser (2.9.3) + reline (0.5.12) io-console (~> 0.5) - rexml (3.3.5) - strscan rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) rspec-mocks (~> 3.13.0) - rspec-core (3.13.0) + rspec-core (3.13.2) rspec-support (~> 3.13.0) - rspec-expectations (3.13.1) + rspec-expectations (3.13.3) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-mocks (3.13.1) + rspec-mocks (3.13.2) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-support (3.13.1) - rubocop (1.63.5) + rspec-support (3.13.2) + rubocop (1.66.1) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) - rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.31.1, < 2.0) + regexp_parser (>= 2.4, < 3.0) + rubocop-ast (>= 1.32.2, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.32.0) + rubocop-ast (1.36.2) parser (>= 3.3.1.0) - rubocop-capybara (2.21.0) - rubocop (~> 1.41) - rubocop-config-umbrellio (1.63.0.93) - rubocop (~> 1.63.0) - rubocop-performance (~> 1.21.0) - rubocop-rails (~> 2.24.0) + rubocop-config-umbrellio (1.66.0.99) + rubocop (~> 1.66.0) + rubocop-factory_bot (~> 2.26.0) + rubocop-performance (~> 1.22.0) + rubocop-rails (~> 2.26.0) rubocop-rake (~> 0.6.0) - rubocop-rspec (~> 2.29.0) + rubocop-rspec (~> 3.0.0) rubocop-sequel (~> 0.3.3) rubocop-factory_bot (2.26.1) rubocop (~> 1.61) - rubocop-performance (1.21.1) + rubocop-performance (1.22.1) rubocop (>= 1.48.1, < 2.0) rubocop-ast (>= 1.31.1, < 2.0) - rubocop-rails (2.24.1) + rubocop-rails (2.26.2) activesupport (>= 4.2.0) rack (>= 1.1) - rubocop (>= 1.33.0, < 2.0) + rubocop (>= 1.52.0, < 2.0) rubocop-ast (>= 1.31.1, < 2.0) rubocop-rake (0.6.0) rubocop (~> 1.0) - rubocop-rspec (2.29.2) - rubocop (~> 1.40) - rubocop-capybara (~> 2.17) - rubocop-factory_bot (~> 2.22) - rubocop-rspec_rails (~> 2.28) - rubocop-rspec_rails (2.29.1) + rubocop-rspec (3.0.5) rubocop (~> 1.61) - rubocop-sequel (0.3.4) + rubocop-sequel (0.3.7) rubocop (~> 1.0) ruby-progressbar (1.13.0) - securerandom (0.3.1) - semantic_logger (4.16.0) + securerandom (0.4.0) + semantic_logger (4.16.1) concurrent-ruby (~> 1.0) sentry-raven (3.1.2) faraday (>= 1.0) - sentry-ruby (5.19.0) + sentry-ruby (5.22.0) bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) - sequel (5.83.1) + sequel (5.87.0) bigdecimal simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) - simplecov-html (0.12.3) + simplecov-html (0.13.1) simplecov-lcov (0.8.0) simplecov_json_formatter (0.1.4) - stringio (3.1.1) - strscan (3.1.0) - symbiont-ruby (0.7.0) - thor (1.3.1) - timeout (0.4.1) + stringio (3.1.2) + thor (1.3.2) + timeout (0.4.2) tzinfo (2.0.6) concurrent-ruby (~> 1.0) - umbrellio-sequel-plugins (0.15.0.198) + umbrellio-sequel-plugins (0.17.0) sequel - symbiont-ruby - unicode-display_width (2.5.0) - uri (0.13.0) - useragent (0.16.10) - webrick (1.8.1) + unicode-display_width (2.6.0) + uri (1.0.2) + useragent (0.16.11) websocket-driver (0.7.6) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) - yard (0.9.36) - zeitwerk (2.6.17) + yard (0.9.37) + zeitwerk (2.6.18) PLATFORMS ruby @@ -312,7 +299,6 @@ PLATFORMS x86_64-linux DEPENDENCIES - bundler-audit ci-helper lamian! launchy diff --git a/lib/lamian/config.rb b/lib/lamian/config.rb index f1e4618..1be04ff 100644 --- a/lib/lamian/config.rb +++ b/lib/lamian/config.rb @@ -13,12 +13,14 @@ module Lamian Config = Struct.new( :formatter, :max_log_lines, + :max_log_length, :raven_log_size_limit, :middleware_autoset, ) do def initialize self.formatter = ::Logger::Formatter.new self.max_log_lines = 5000 + self.max_log_length = 10_000 self.raven_log_size_limit = 500_000 self.middleware_autoset = true end diff --git a/lib/lamian/log_device.rb b/lib/lamian/log_device.rb index b3ba958..339c107 100644 --- a/lib/lamian/log_device.rb +++ b/lib/lamian/log_device.rb @@ -2,14 +2,16 @@ module Lamian class LogDevice # :nodoc: - def initialize(size = Lamian.config.max_log_lines) - self.size = size + def initialize(max_log_lines: Lamian.config.max_log_lines, + max_log_length: Lamian.config.max_log_length) + self.max_log_lines = max_log_lines + self.max_log_length = max_log_length self.lines = [] end - def write(string) # :nodoc: - lines << string - lines.shift if lines.size > size + def write(msg) # :nodoc: + lines << truncate(msg) + lines.shift if lines.size > max_log_lines true end @@ -19,6 +21,16 @@ def string # :nodoc: private - attr_accessor :size, :lines + attr_accessor :lines, :max_log_lines, :max_log_length + + def truncate(msg) # :nodoc: + return msg unless msg.size > max_log_length + + suffix = +"..." + suffix << "\n" if msg.end_with?("\n") + msg = msg[0, max_log_length - suffix.size] + + "#{msg}#{suffix}" + end end end diff --git a/spec/lamian/engine_spec.rb b/spec/lamian/engine_spec.rb index f448921..f12826d 100644 --- a/spec/lamian/engine_spec.rb +++ b/spec/lamian/engine_spec.rb @@ -9,7 +9,7 @@ allow(event).to receive(:extra).and_return(extra_hash) end end - let(:extra_hash) { Hash[] } + let(:extra_hash) { {} } let(:callback) { instance.rebuild_before_send } diff --git a/spec/lamian/log_device_spec.rb b/spec/lamian/log_device_spec.rb index 467b013..5ba7de8 100644 --- a/spec/lamian/log_device_spec.rb +++ b/spec/lamian/log_device_spec.rb @@ -1,16 +1,30 @@ # frozen_string_literal: true describe Lamian::LogDevice do - subject(:logdev) { described_class.new(5) } + subject(:logdev) { described_class.new(max_log_lines: 5, max_log_length: 20) } it "saves log" do - logdev.write("Hello ") - logdev.write("world!") - expect(logdev.string).to eq("Hello world!") + logdev.write("Hello\n") + logdev.write("world!\n") + expect(logdev.string).to eq("Hello\nworld!\n") end it "only stores 5 latest log lines" do 8.times { |x| logdev.write(x + 1) } expect(logdev.string).to eq("45678") end + + context "long log lines" do + it "truncates log" do + logdev.write("Hello world, this is me!") + expect(logdev.string).to eq("Hello world, this...") + end + + context "line with newline in the end" do + it "truncates log and adds newline" do + logdev.write("Hello world, this is me!\n") + expect(logdev.string).to eq("Hello world, thi...\n") + end + end + end end