From 751f7f7b66479d289cdab318693cd73762498e51 Mon Sep 17 00:00:00 2001 From: Anton Fomenko Date: Tue, 30 Jun 2020 19:41:40 +0300 Subject: [PATCH 1/5] allow module to work with puppet 6.16 and higher remove url encoding on new puppet agent versions as it's done internally --- lib/puppetdb/connection.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/puppetdb/connection.rb b/lib/puppetdb/connection.rb index 426b265..ee0c502 100644 --- a/lib/puppetdb/connection.rb +++ b/lib/puppetdb/connection.rb @@ -60,7 +60,11 @@ def query(endpoint, query = nil, options = {}, version = :v4) end uri = "/pdb/query/#{version}/#{endpoint}" - uri += URI.escape "?query=#{query.to_json}" unless query.nil? || query.empty? + if Gem::Version.new(Puppet.version) < Gem::Version.new('6.16.0') + uri += URI.escape "?query=#{query.to_json}" unless query.nil? || query.empty? + else + uri += "?query=#{query.to_json}" unless query.nil? || query.empty? + end debug("PuppetDB query: #{query.to_json}") From e36ef766fb6fa67d233ce70bbe353784024051fc Mon Sep 17 00:00:00 2001 From: Anton Fomenko Date: Wed, 1 Jul 2020 21:08:34 +0300 Subject: [PATCH 2/5] fix puppet 6 compatibility it resolved that face and function have different internal processing face does noe require URL encoding and funtions do --- lib/hiera/backend/puppetdb_backend.rb | 4 ++-- lib/puppet/functions/query_facts.rb | 2 +- lib/puppet/functions/query_nodes.rb | 4 ++-- lib/puppet/parser/functions/query_facts.rb | 2 +- lib/puppet/parser/functions/query_nodes.rb | 4 ++-- lib/puppet/parser/functions/query_resources.rb | 2 +- lib/puppetdb/connection.rb | 7 +++++-- 7 files changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/hiera/backend/puppetdb_backend.rb b/lib/hiera/backend/puppetdb_backend.rb index b870c07..8acc334 100644 --- a/lib/hiera/backend/puppetdb_backend.rb +++ b/lib/hiera/backend/puppetdb_backend.rb @@ -45,10 +45,10 @@ def lookup(key, scope, order_override, _resolution_type) if fact query = @parser.facts_query query, [fact] - @puppetdb.query(:facts, query).collect { |f| f['value'] }.sort + @puppetdb.query(:facts, query, { :source => 'function' }).collect { |f| f['value'] }.sort else query = @parser.parse query, :nodes if query.is_a? String - @puppetdb.query(:nodes, query).collect { |n| n['name'] } + @puppetdb.query(:nodes, query, { :source => 'function' }).collect { |n| n['name'] } end end end diff --git a/lib/puppet/functions/query_facts.rb b/lib/puppet/functions/query_facts.rb index eceb1db..25c596f 100644 --- a/lib/puppet/functions/query_facts.rb +++ b/lib/puppet/functions/query_facts.rb @@ -33,6 +33,6 @@ def query_facts(query, facts) puppetdb = PuppetDB::Connection.new(uri.host, uri.port, uri.scheme == 'https') parser = PuppetDB::Parser.new query = parser.facts_query query, facts_for_query if query.is_a? String - parser.facts_hash(puppetdb.query(:facts, query, :extract => [:certname, :name, :value]), facts) + parser.facts_hash(puppetdb.query(:facts, query, { :extract => [:certname, :name, :value], :source => 'function' }), facts) end end diff --git a/lib/puppet/functions/query_nodes.rb b/lib/puppet/functions/query_nodes.rb index 412355e..7eafb0c 100644 --- a/lib/puppet/functions/query_nodes.rb +++ b/lib/puppet/functions/query_nodes.rb @@ -42,7 +42,7 @@ def puppetdb def query_nodes(query) query = parser.parse(query, :nodes) if query.is_a? String - puppetdb.query(:nodes, query, :extract => :certname).collect do |n| + puppetdb.query(:nodes, query, { :extract => :certname, :source => 'function' }).collect do |n| n['certname'] end end @@ -51,7 +51,7 @@ def query_nodes_fact(query, fact) fact_for_query = fact.split('.').first query = parser.facts_query(query, [fact_for_query]) - response = puppetdb.query(:facts, query, :extract => :value) + response = puppetdb.query(:facts, query, { :extract => :value, :source => 'function' }) if fact.split('.').size > 1 parser.extract_nested_fact(response, fact.split('.')[1..-1]) diff --git a/lib/puppet/parser/functions/query_facts.rb b/lib/puppet/parser/functions/query_facts.rb index db73817..4487838 100644 --- a/lib/puppet/parser/functions/query_facts.rb +++ b/lib/puppet/parser/functions/query_facts.rb @@ -34,5 +34,5 @@ puppetdb = PuppetDB::Connection.new(uri.host, uri.port, uri.scheme == 'https') parser = PuppetDB::Parser.new query = parser.facts_query query, facts_for_query if query.is_a? String - parser.facts_hash(puppetdb.query(:facts, query, :extract => [:certname, :name, :value]), facts) + parser.facts_hash(puppetdb.query(:facts, query, { :extract => [:certname, :name, :value], :source => 'function' }), facts) end diff --git a/lib/puppet/parser/functions/query_nodes.rb b/lib/puppet/parser/functions/query_nodes.rb index 83898bc..056fe50 100644 --- a/lib/puppet/parser/functions/query_nodes.rb +++ b/lib/puppet/parser/functions/query_nodes.rb @@ -36,7 +36,7 @@ parser = PuppetDB::Parser.new if fact_for_query query = parser.facts_query(query, [fact_for_query]) - response = puppetdb.query(:facts, query, :extract => :value) + response = puppetdb.query(:facts, query, { :extract => :value, :source => 'function' }) if fact.split('.').size > 1 parser.extract_nested_fact(response, fact.split('.')[1..-1]) @@ -45,6 +45,6 @@ end else query = parser.parse(query, :nodes) if query.is_a? String - puppetdb.query(:nodes, query, :extract => :certname).collect { |n| n['certname'] } + puppetdb.query(:nodes, query, { :extract => :certname, :source => 'function' }).collect { |n| n['certname'] } end end diff --git a/lib/puppet/parser/functions/query_resources.rb b/lib/puppet/parser/functions/query_resources.rb index 5f86359..16d6f41 100644 --- a/lib/puppet/parser/functions/query_resources.rb +++ b/lib/puppet/parser/functions/query_resources.rb @@ -61,7 +61,7 @@ end # Fetch the results - results = puppetdb.query(:resources, q) + results = puppetdb.query(:resources, q, { :source => 'function' }) # If grouphosts is true create a nested hash with nodes and resources if grouphosts diff --git a/lib/puppetdb/connection.rb b/lib/puppetdb/connection.rb index ee0c502..76e3b4e 100644 --- a/lib/puppetdb/connection.rb +++ b/lib/puppetdb/connection.rb @@ -38,7 +38,8 @@ def query(endpoint, query = nil, options = {}, version = :v4) default_options = { :http => nil, # A HTTP object to be used for the connection - :extract => nil # An array of fields to extract + :extract => nil, # An array of fields to extract + :source => 'face' # Source of request (face or function) } if options.is_a? Hash @@ -49,6 +50,8 @@ def query(endpoint, query = nil, options = {}, version = :v4) end http = options[:http] + source = options[:source] + unless http require 'puppet/network/http_pool' http = Puppet::Network::HttpPool.http_instance(@host, @port, @use_ssl) @@ -60,7 +63,7 @@ def query(endpoint, query = nil, options = {}, version = :v4) end uri = "/pdb/query/#{version}/#{endpoint}" - if Gem::Version.new(Puppet.version) < Gem::Version.new('6.16.0') + if source == 'function' uri += URI.escape "?query=#{query.to_json}" unless query.nil? || query.empty? else uri += "?query=#{query.to_json}" unless query.nil? || query.empty? From 993273cfb61d9285260eecc5e835f91ac4bad26b Mon Sep 17 00:00:00 2001 From: Anton Samets Date: Thu, 20 Jun 2024 13:57:43 +0300 Subject: [PATCH 3/5] Fix warning about using 'Puppet::Network::HttpPool.http_instance' Fix deprecation warning: Puppet The method 'Puppet::Network::HttpPool.http_instance' is deprecated. Use Puppet.runtime[:http] instead --- lib/puppetdb/connection.rb | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/puppetdb/connection.rb b/lib/puppetdb/connection.rb index 76e3b4e..6245c19 100644 --- a/lib/puppetdb/connection.rb +++ b/lib/puppetdb/connection.rb @@ -1,12 +1,14 @@ +# frozen_string_literal: true + require 'puppetdb' +require 'rubygems' +require 'puppetdb/parser' +require 'uri' +require 'puppet' +require 'puppet/util/logging' +require 'json' class PuppetDB::Connection - require 'rubygems' - require 'puppetdb/parser' - require 'uri' - require 'puppet' - require 'puppet/util/logging' - include Puppet::Util::Logging def initialize(host = 'puppetdb', port = 443, use_ssl = true) @@ -34,7 +36,6 @@ def self.check_version # @param options [Hash] specify extract values or http connection # @return [Array] the results of the query def query(endpoint, query = nil, options = {}, version = :v4) - require 'json' default_options = { :http => nil, # A HTTP object to be used for the connection @@ -53,8 +54,7 @@ def query(endpoint, query = nil, options = {}, version = :v4) source = options[:source] unless http - require 'puppet/network/http_pool' - http = Puppet::Network::HttpPool.http_instance(@host, @port, @use_ssl) + http = Puppet.runtime[:http].create_session.server(@host, @port).connection(use_ssl: @use_ssl) end headers = { 'Accept' => 'application/json' } @@ -64,15 +64,15 @@ def query(endpoint, query = nil, options = {}, version = :v4) uri = "/pdb/query/#{version}/#{endpoint}" if source == 'function' - uri += URI.escape "?query=#{query.to_json}" unless query.nil? || query.empty? + uri = "#{uri}#{URI.escape("?query=#{query.to_json}")}" unless query.nil? || query.empty? else - uri += "?query=#{query.to_json}" unless query.nil? || query.empty? + uri = "#{uri}?query=#{query.to_json}" unless query.nil? || query.empty? end debug("PuppetDB query: #{query.to_json}") - resp = http.get(uri, headers) - fail "PuppetDB query error: [#{resp.code}] #{resp.msg}, query: #{query.to_json}" unless resp.is_a?(Net::HTTPSuccess) + resp = http.get(uri, headers: headers) + raise "PuppetDB query error: [#{resp.code}] #{resp.msg}, query: #{query.to_json}" unless resp.success? JSON.parse(resp.body) end end From 81d7256154c977e8cf0cebb1c49e369b7acf28c9 Mon Sep 17 00:00:00 2001 From: Andrey Burmenko Date: Thu, 27 Jun 2024 14:10:12 +0300 Subject: [PATCH 4/5] correct using Puppet::runtime[:http] with URI --- lib/puppetdb/connection.rb | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/lib/puppetdb/connection.rb b/lib/puppetdb/connection.rb index 6245c19..467651a 100644 --- a/lib/puppetdb/connection.rb +++ b/lib/puppetdb/connection.rb @@ -49,26 +49,18 @@ def query(endpoint, query = nil, options = {}, version = :v4) Puppet.deprecation_warning 'Specify http object with :http key instead' options = default_options.merge(:http => options) end - http = options[:http] source = options[:source] - - unless http - http = Puppet.runtime[:http].create_session.server(@host, @port).connection(use_ssl: @use_ssl) - end + http = options[:http] || Puppet.runtime[:http] headers = { 'Accept' => 'application/json' } - if options[:extract] - query = PuppetDB::ParserHelper.extract(*Array(options[:extract]), query) - end + query = PuppetDB::ParserHelper.extract(*Array(options[:extract]), query) if options[:extract] - uri = "/pdb/query/#{version}/#{endpoint}" - if source == 'function' - uri = "#{uri}#{URI.escape("?query=#{query.to_json}")}" unless query.nil? || query.empty? - else - uri = "#{uri}?query=#{query.to_json}" unless query.nil? || query.empty? - end + uri = URI("#{@use_ssl ? 'https' : 'http'}://#{@host}:#{@port}/pdb/query/#{version}/#{endpoint}") + _query = "query=#{query.to_json}" unless query.nil? || query.empty? + uri.query = (source == 'function' ? URI.encode_www_form(query: _query) : _query) unless _query.empty? + debug("PuppetDB uri: #{uri.to_s}") debug("PuppetDB query: #{query.to_json}") resp = http.get(uri, headers: headers) From 37eda4b4a3d763bb10fd5a62dd2d36951da5e7e3 Mon Sep 17 00:00:00 2001 From: Andrey Burmenko Date: Thu, 27 Jun 2024 15:25:15 +0300 Subject: [PATCH 5/5] correct using Puppet::runtime[:http] with URI #2 --- lib/puppetdb/connection.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/puppetdb/connection.rb b/lib/puppetdb/connection.rb index 467651a..e6810b5 100644 --- a/lib/puppetdb/connection.rb +++ b/lib/puppetdb/connection.rb @@ -57,14 +57,14 @@ def query(endpoint, query = nil, options = {}, version = :v4) query = PuppetDB::ParserHelper.extract(*Array(options[:extract]), query) if options[:extract] uri = URI("#{@use_ssl ? 'https' : 'http'}://#{@host}:#{@port}/pdb/query/#{version}/#{endpoint}") - _query = "query=#{query.to_json}" unless query.nil? || query.empty? - uri.query = (source == 'function' ? URI.encode_www_form(query: _query) : _query) unless _query.empty? + _query = "#{query.to_json}" unless query.nil? || query.empty? + uri.query = (source == 'function' ? URI.encode_www_form(query: _query) : "query=#{_query}") unless _query.empty? debug("PuppetDB uri: #{uri.to_s}") debug("PuppetDB query: #{query.to_json}") resp = http.get(uri, headers: headers) - raise "PuppetDB query error: [#{resp.code}] #{resp.msg}, query: #{query.to_json}" unless resp.success? + raise "PuppetDB query error: [#{resp.code}] #{resp.reason}, url: #{uri.to_s}, query: #{query.to_json}" unless resp.success? JSON.parse(resp.body) end end