From 35963ca8eb836c02f70d0acf4581026d20c553a5 Mon Sep 17 00:00:00 2001 From: Michael Hashizume Date: Thu, 11 Apr 2024 13:59:35 -0700 Subject: [PATCH] Allow cloud metadata to ignore proxy settings Prior to this commit, if the http_proxy environment variable was set for Net::HTTP, cloud resolvers (ec2, GCE, Azure) would fail to resolve metadata. This commit adds a new positional argument to Facter::Util::Resolvers::Http.get_request and #put_request to allow those resolvers to disable proxy settings. --- .rubocop_todo.yml | 8 + lib/facter/resolvers/az.rb | 2 +- lib/facter/resolvers/ec2.rb | 2 +- lib/facter/resolvers/gce.rb | 2 +- lib/facter/util/resolvers/http.rb | 26 ++- spec/facter/resolvers/az_spec.rb | 14 +- spec/facter/resolvers/ec2_spec.rb | 12 ++ spec/facter/resolvers/gce_spec.rb | 258 ++++++++++++------------ spec/facter/util/resolvers/http_spec.rb | 34 +++- 9 files changed, 215 insertions(+), 143 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 7bdfc55fc4..11e618abb2 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -42,3 +42,11 @@ Style/Documentation: - 'spec_integration/**/*' - 'scripts/*' - 'install.rb' + +# While it would be preferable to use a keyword argument for the proxy setting in #get_request and #put_request, if we +# add keyword arguments to those methods Ruby < 3 misinterprets earlier positional arguments as a keyword arguments. +# This is because those positional arguments are hashes that use symbols as keys. +# TODO: revisit this after we drop Ruby < 3 support. +Style/OptionalBooleanParameter: + Exclude: + - 'lib/facter/util/resolvers/http.rb' \ No newline at end of file diff --git a/lib/facter/resolvers/az.rb b/lib/facter/resolvers/az.rb index 0f558ccb2c..3935156422 100644 --- a/lib/facter/resolvers/az.rb +++ b/lib/facter/resolvers/az.rb @@ -26,7 +26,7 @@ def read_facts(fact_name) def get_data_from(url) headers = { Metadata: 'true' } - Facter::Util::Resolvers::Http.get_request(url, headers, { session: determine_session_timeout }) + Facter::Util::Resolvers::Http.get_request(url, headers, { session: determine_session_timeout }, false) end def determine_session_timeout diff --git a/lib/facter/resolvers/ec2.rb b/lib/facter/resolvers/ec2.rb index b6c9f36dbc..adaeb56db7 100644 --- a/lib/facter/resolvers/ec2.rb +++ b/lib/facter/resolvers/ec2.rb @@ -52,7 +52,7 @@ def build_path_component(line) def get_data_from(url) headers = {} headers['X-aws-ec2-metadata-token'] = v2_token if v2_token - Facter::Util::Resolvers::Http.get_request(url, headers, { session: determine_session_timeout }) + Facter::Util::Resolvers::Http.get_request(url, headers, { session: determine_session_timeout }, false) end def determine_session_timeout diff --git a/lib/facter/resolvers/gce.rb b/lib/facter/resolvers/gce.rb index d5ae616a91..ba612ddd2a 100644 --- a/lib/facter/resolvers/gce.rb +++ b/lib/facter/resolvers/gce.rb @@ -22,7 +22,7 @@ def read_facts(fact_name) end def query_for_metadata - gce_data = extract_to_hash(Facter::Util::Resolvers::Http.get_request(METADATA_URL, HEADERS)) + gce_data = extract_to_hash(Facter::Util::Resolvers::Http.get_request(METADATA_URL, HEADERS, false)) parse_instance(gce_data) gce_data.empty? ? nil : gce_data diff --git a/lib/facter/util/resolvers/http.rb b/lib/facter/util/resolvers/http.rb index 163321945f..0709faf07e 100644 --- a/lib/facter/util/resolvers/http.rb +++ b/lib/facter/util/resolvers/http.rb @@ -17,28 +17,30 @@ class << self # Defaults to an empty hash. # @param timeouts [Hash] Values for the session and connection # timeouts. + # @param proxy [Boolean] Whether to use proxy settings when calling + # Net::HTTP.new. Defaults to true. # @returns [String] the response body if the response code is 200. # If the response code is not 200, an empty string is returned. # @example # get_request('https://example.com', { "Accept": 'application/json' }, { session: 2.4, connection: 5 }) - def get_request(url, headers = {}, timeouts = {}) - make_request(url, headers, timeouts, 'GET') + def get_request(url, headers = {}, timeouts = {}, proxy = true) + make_request(url, headers, timeouts, 'GET', proxy) end # Makes a PUT HTTP request and returns its response # @param (see #get_request) # @return (see #get_request) - def put_request(url, headers = {}, timeouts = {}) - make_request(url, headers, timeouts, 'PUT') + def put_request(url, headers = {}, timeouts = {}, proxy = true) + make_request(url, headers, timeouts, 'PUT', proxy) end private - def make_request(url, headers, timeouts, request_type) + def make_request(url, headers, timeouts, request_type, proxy) require 'net/http' uri = URI.parse(url) - http = http_obj(uri, timeouts) + http = http_obj(uri, timeouts, proxy) request = request_obj(headers, uri, request_type) # The Windows implementation of sockets does not respect net/http @@ -57,8 +59,16 @@ def make_request(url, headers, timeouts, request_type) '' end - def http_obj(parsed_url, timeouts) - http = Net::HTTP.new(parsed_url.host) + def http_obj(parsed_url, timeouts, proxy) + # If get_request or put_request are called and set proxy to false, + # manually set Net::HTTP.new's p_addr (proxy address) positional + # argument to nil to override anywhere else a proxy may be set + # (e.g. the http_proxy environment variable). + http = if proxy + Net::HTTP.new(parsed_url.host) + else + Net::HTTP.new(parsed_url.host, 80, nil) + end http.read_timeout = timeouts[:session] || SESSION_TIMEOUT http.open_timeout = timeouts[:connection] || CONNECTION_TIMEOUT diff --git a/spec/facter/resolvers/az_spec.rb b/spec/facter/resolvers/az_spec.rb index 72606c7546..8e0416976b 100644 --- a/spec/facter/resolvers/az_spec.rb +++ b/spec/facter/resolvers/az_spec.rb @@ -8,7 +8,7 @@ before do allow(Facter::Util::Resolvers::Http).to receive(:get_request) - .with(uri, { Metadata: 'true' }, { session: 5 }).and_return(output) + .with(uri, { Metadata: 'true' }, { session: 5 }, false).and_return(output) az.instance_variable_set(:@log, log_spy) end @@ -24,6 +24,18 @@ end end + context "when a proxy is set with ENV['http_proxy']" do + before do + stub_const('ENV', { 'http_proxy' => 'http://example.com' }) + end + + let(:output) { '{"azEnvironment":"AzurePublicCloud"}' } + + it 'returns az metadata' do + expect(az.resolve(:metadata)).to eq({ 'azEnvironment' => 'AzurePublicCloud' }) + end + end + context 'when an exception is thrown' do let(:output) { '' } diff --git a/spec/facter/resolvers/ec2_spec.rb b/spec/facter/resolvers/ec2_spec.rb index 18d17f63da..31305b1276 100644 --- a/spec/facter/resolvers/ec2_spec.rb +++ b/spec/facter/resolvers/ec2_spec.rb @@ -158,4 +158,16 @@ expect(ec2.resolve(:userdata)).to eql(expected_str) end end + + context "when a proxy is set with ENV['http_proxy']" do + before do + stub_const('ENV', { 'http_proxy' => 'http://example.com' }) + stub_request(:put, token_uri).to_return(status: 200, body: token) + end + + let(:headers) { { 'Accept' => '*/*' } } + let(:token) { 'v2_token' } + + it_behaves_like 'ec2' + end end diff --git a/spec/facter/resolvers/gce_spec.rb b/spec/facter/resolvers/gce_spec.rb index b5f3824a28..8584f0d43e 100644 --- a/spec/facter/resolvers/gce_spec.rb +++ b/spec/facter/resolvers/gce_spec.rb @@ -6,7 +6,7 @@ before do allow(Facter::Util::Resolvers::Http).to receive(:get_request) - .with(gce_metadata_url, gce_url_headers) + .with(gce_metadata_url, gce_url_headers, false) .and_return(http_response_body) end @@ -14,139 +14,141 @@ Facter::Resolvers::Gce.invalidate_cache end - context 'when http request is successful' do - let(:http_response_body) { load_fixture('gce').read } - let(:value) do - { - 'instance' => { - 'attributes' => { - # resolver transforms key1\nkey2 into array of keys - 'ssh-keys' => [ - 'john_doe:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA9D8Op48TtEiDmb+Gtna3Bs9B google-ssh {"userName":"john.doe@puppet.com","expireOn":"2020-08-13T12:17:19+0000"}', - 'jane_doe:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA9D8Op48TtEiDmb+Gtna3Bs9B google-ssh {"userName":"jane.doe@puppet.com","expireOn":"2020-08-13T12:17:19+0000"}' + shared_examples 'check GCE resolver called with metadata' do + context 'when http request is successful' do + let(:http_response_body) { load_fixture('gce').read } + let(:value) do + { + 'instance' => { + 'attributes' => { + # resolver transforms key1\nkey2 into array of keys + 'ssh-keys' => [ + 'john_doe:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA9D8Op48TtEiDmb+Gtna3Bs9B google-ssh {"userName":"john.doe@puppet.com","expireOn":"2020-08-13T12:17:19+0000"}', + 'jane_doe:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA9D8Op48TtEiDmb+Gtna3Bs9B google-ssh {"userName":"jane.doe@puppet.com","expireOn":"2020-08-13T12:17:19+0000"}' + ], + 'sshKeys' => [ + 'jill_doe:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA9D8Op48TtEiDmb+Gtna3Bs9B google-ssh {"userName":"jill.doe@puppet.com","expireOn":"2020-08-13T12:17:19+0000"}', + 'jacob_doe:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA9D8Op48TtEiDmb+Gtna3Bs9B google-ssh {"userName":"jacob.doe@puppet.com","expireOn":"2020-08-13T12:17:19+0000"}' + ] + }, + 'cpuPlatform' => 'Intel Broadwell', + 'description' => '', + 'disks' => [ + { + 'deviceName' => 'instance-3', + 'index' => 0, + 'interface' => 'SCSI', + 'mode' => 'READ_WRITE', + 'type' => 'PERSISTENT' + } ], - 'sshKeys' => [ - 'jill_doe:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA9D8Op48TtEiDmb+Gtna3Bs9B google-ssh {"userName":"jill.doe@puppet.com","expireOn":"2020-08-13T12:17:19+0000"}', - 'jacob_doe:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA9D8Op48TtEiDmb+Gtna3Bs9B google-ssh {"userName":"jacob.doe@puppet.com","expireOn":"2020-08-13T12:17:19+0000"}' - ] - }, - 'cpuPlatform' => 'Intel Broadwell', - 'description' => '', - 'disks' => [ - { - 'deviceName' => 'instance-3', - 'index' => 0, - 'interface' => 'SCSI', - 'mode' => 'READ_WRITE', - 'type' => 'PERSISTENT' - } - ], - 'guestAttributes' => {}, - 'hostname' => 'instance-3.c.facter-performance-history.internal', - 'id' => 2_206_944_706_428_651_580, - 'image' => 'ubuntu-2004-focal-v20200810', - 'legacyEndpointAccess' => { - '0.1' => 0, - 'v1beta1' => 0 + 'guestAttributes' => {}, + 'hostname' => 'instance-3.c.facter-performance-history.internal', + 'id' => 2_206_944_706_428_651_580, + 'image' => 'ubuntu-2004-focal-v20200810', + 'legacyEndpointAccess' => { + '0.1' => 0, + 'v1beta1' => 0 + }, + 'licenses' => [ + { + 'id' => '2211838267635035815' + } + ], + 'machineType' => 'n1-standard-2', + 'maintenanceEvent' => 'NONE', + 'name' => 'instance-3', + 'networkInterfaces' => [ + { + 'accessConfigs' => [ + { + 'externalIp' => '34.89.230.102', + 'type' => 'ONE_TO_ONE_NAT' + } + ], + 'dnsServers' => [ + '169.254.169.254' + ], + 'forwardedIps' => [], + 'gateway' => '10.156.0.1', + 'ip' => '10.156.0.4', + 'ipAliases' => [], + 'mac' => '42:01:0a:9c:00:04', + 'mtu' => 1460, + 'network' => 'default', + 'subnetmask' => '255.255.240.0', + 'targetInstanceIps' => [] + } + ], + 'preempted' => 'FALSE', + 'remainingCpuTime' => -1, + 'scheduling' => { + 'automaticRestart' => 'TRUE', + 'onHostMaintenance' => 'MIGRATE', + 'preemptible' => 'FALSE' + }, + 'serviceAccounts' => { + '728618928092-compute@developer.gserviceaccount.com' => { + 'aliases' => [ + 'default' + ], + 'email' => '728618928092-compute@developer.gserviceaccount.com', + 'scopes' => [ + 'https://www.googleapis.com/auth/devstorage.read_only', + 'https://www.googleapis.com/auth/logging.write', + 'https://www.googleapis.com/auth/monitoring.write', + 'https://www.googleapis.com/auth/servicecontrol', + 'https://www.googleapis.com/auth/service.management.readonly', + 'https://www.googleapis.com/auth/trace.append' + ] + }, + 'default' => { + 'aliases' => [ + 'default' + ], + 'email' => '728618928092-compute@developer.gserviceaccount.com', + 'scopes' => [ + 'https://www.googleapis.com/auth/devstorage.read_only', + 'https://www.googleapis.com/auth/logging.write', + 'https://www.googleapis.com/auth/monitoring.write', + 'https://www.googleapis.com/auth/servicecontrol', + 'https://www.googleapis.com/auth/service.management.readonly', + 'https://www.googleapis.com/auth/trace.append' + ] + } + }, + 'tags' => [], + 'virtualClock' => { + 'driftToken' => '0' + }, + 'zone' => 'europe-west3-c' }, - 'licenses' => [ - { - 'id' => '2211838267635035815' - } - ], - 'machineType' => 'n1-standard-2', - 'maintenanceEvent' => 'NONE', - 'name' => 'instance-3', - 'networkInterfaces' => [ - { - 'accessConfigs' => [ - { - 'externalIp' => '34.89.230.102', - 'type' => 'ONE_TO_ONE_NAT' - } - ], - 'dnsServers' => [ - '169.254.169.254' - ], - 'forwardedIps' => [], - 'gateway' => '10.156.0.1', - 'ip' => '10.156.0.4', - 'ipAliases' => [], - 'mac' => '42:01:0a:9c:00:04', - 'mtu' => 1460, - 'network' => 'default', - 'subnetmask' => '255.255.240.0', - 'targetInstanceIps' => [] + 'oslogin' => { + 'authenticate' => { + 'sessions' => {} } - ], - 'preempted' => 'FALSE', - 'remainingCpuTime' => -1, - 'scheduling' => { - 'automaticRestart' => 'TRUE', - 'onHostMaintenance' => 'MIGRATE', - 'preemptible' => 'FALSE' }, - 'serviceAccounts' => { - '728618928092-compute@developer.gserviceaccount.com' => { - 'aliases' => [ - 'default' - ], - 'email' => '728618928092-compute@developer.gserviceaccount.com', - 'scopes' => [ - 'https://www.googleapis.com/auth/devstorage.read_only', - 'https://www.googleapis.com/auth/logging.write', - 'https://www.googleapis.com/auth/monitoring.write', - 'https://www.googleapis.com/auth/servicecontrol', - 'https://www.googleapis.com/auth/service.management.readonly', - 'https://www.googleapis.com/auth/trace.append' + 'project' => { + 'attributes' => { + # resolver transforms key1\nkey2 into array of keys + 'ssh-keys' => ['john_doe:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA9D8Op48TtEiDmb+Gtna3Bs9B google-ssh {"userName":"john.doe@puppet.com","expireOn":"2020-08-13T12:17:19+0000"}', + 'jane_doe:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA9D8Op48TtEiDmb+Gtna3Bs9B google-ssh {"userName":"jane.doe@puppet.com","expireOn":"2020-08-13T12:17:19+0000"}'], + 'sshKeys' => [ + 'jill_doe:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA9D8Op48TtEiDmb+Gtna3Bs9B google-ssh {"userName":"jill.doe@puppet.com","expireOn":"2020-08-13T12:17:19+0000"}', + 'jacob_doe:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA9D8Op48TtEiDmb+Gtna3Bs9B google-ssh {"userName":"jacob.doe@puppet.com","expireOn":"2020-08-13T12:17:19+0000"}' ] }, - 'default' => { - 'aliases' => [ - 'default' - ], - 'email' => '728618928092-compute@developer.gserviceaccount.com', - 'scopes' => [ - 'https://www.googleapis.com/auth/devstorage.read_only', - 'https://www.googleapis.com/auth/logging.write', - 'https://www.googleapis.com/auth/monitoring.write', - 'https://www.googleapis.com/auth/servicecontrol', - 'https://www.googleapis.com/auth/service.management.readonly', - 'https://www.googleapis.com/auth/trace.append' - ] - } - }, - 'tags' => [], - 'virtualClock' => { - 'driftToken' => '0' - }, - 'zone' => 'europe-west3-c' - }, - 'oslogin' => { - 'authenticate' => { - 'sessions' => {} + 'numericProjectId' => 728_618_928_092, + 'projectId' => 'facter-performance-history' } - }, - 'project' => { - 'attributes' => { - # resolver transforms key1\nkey2 into array of keys - 'ssh-keys' => ['john_doe:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA9D8Op48TtEiDmb+Gtna3Bs9B google-ssh {"userName":"john.doe@puppet.com","expireOn":"2020-08-13T12:17:19+0000"}', - 'jane_doe:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA9D8Op48TtEiDmb+Gtna3Bs9B google-ssh {"userName":"jane.doe@puppet.com","expireOn":"2020-08-13T12:17:19+0000"}'], - 'sshKeys' => [ - 'jill_doe:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA9D8Op48TtEiDmb+Gtna3Bs9B google-ssh {"userName":"jill.doe@puppet.com","expireOn":"2020-08-13T12:17:19+0000"}', - 'jacob_doe:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA9D8Op48TtEiDmb+Gtna3Bs9B google-ssh {"userName":"jacob.doe@puppet.com","expireOn":"2020-08-13T12:17:19+0000"}' - ] - }, - 'numericProjectId' => 728_618_928_092, - 'projectId' => 'facter-performance-history' } - } - end + end - it 'returns gce data' do - result = Facter::Resolvers::Gce.resolve(:metadata) + it 'returns gce data' do + result = Facter::Resolvers::Gce.resolve(:metadata) - expect(result).to eq(value) + expect(result).to eq(value) + end end end @@ -159,4 +161,12 @@ expect(result).to be(nil) end end + + describe 'with a proxy set' do + before do + stub_const('ENV', { 'http_proxy' => 'http://example.com' }) + end + + it_behaves_like 'check GCE resolver called with metadata' + end end diff --git a/spec/facter/util/resolvers/http_spec.rb b/spec/facter/util/resolvers/http_spec.rb index 490785965a..48984c4da8 100644 --- a/spec/facter/util/resolvers/http_spec.rb +++ b/spec/facter/util/resolvers/http_spec.rb @@ -11,7 +11,7 @@ allow(Gem).to receive(:win_platform?).and_return(false) end - RSpec.shared_examples 'a http request' do + RSpec.shared_examples 'an http request' do context 'when success' do before do stub_request(http_verb, url).to_return(status: 200, body: 'success') @@ -97,8 +97,8 @@ end end - RSpec.shared_examples 'a http request on windows' do - it_behaves_like 'a http request' + RSpec.shared_examples 'an http request on windows' do + it_behaves_like 'an http request' context 'when host is unreachable ' do before do @@ -138,14 +138,14 @@ let(:http_verb) { :get } let(:client_method) { :get_request } - it_behaves_like 'a http request' + it_behaves_like 'an http request' end describe '#put_request' do let(:http_verb) { :put } let(:client_method) { :put_request } - it_behaves_like 'a http request' + it_behaves_like 'an http request' end context 'when windows' do @@ -160,14 +160,34 @@ let(:http_verb) { :get } let(:client_method) { :get_request } - it_behaves_like 'a http request on windows' + it_behaves_like 'an http request on windows' end describe '#put_request' do let(:http_verb) { :put } let(:client_method) { :put_request } - it_behaves_like 'a http request on windows' + it_behaves_like 'an http request on windows' + end + end + + context 'when using a proxy' do + before do + stub_const('ENV', { 'http_proxy' => 'http://example.com' }) + end + + describe '#get_request' do + let(:http_verb) { :get } + let(:client_method) { :get_request } + + it_behaves_like 'an http request' + end + + describe '#put_request' do + let(:http_verb) { :put } + let(:client_method) { :put_request } + + it_behaves_like 'an http request' end end end