Skip to content

Commit

Permalink
Frozen string literal, docs
Browse files Browse the repository at this point in the history
* Add `frozen_string_literal: true`

* Add some YARD docs

* Update CHANGELOG
  • Loading branch information
nbulaj committed Dec 29, 2017
1 parent 35d5c17 commit 30c6d40
Show file tree
Hide file tree
Showing 43 changed files with 148 additions and 1 deletion.
6 changes: 5 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,14 @@

Reverse Chronological Order:

## `0.6.2` (2017-12-27)

* Fix ProxyDocker provider.

## `0.6.1` (2017-12-11)

* Fix gem executable to check dependencies for adapters
* Code clean
* Code cleanup
* Some new specs

## `0.6.0` (2017-12-08)
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

require 'uri'
require 'net/https'

Expand Down
12 changes: 12 additions & 0 deletions lib/proxy_fetcher/client/client.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
# ProxyFetcher HTTP client that encapsulates all the logic for sending
# HTTP(S) requests using proxies, automatically fetched and validated by the gem.
Expand Down Expand Up @@ -122,6 +124,8 @@ def patch(url, payload, headers: {}, options: {})

private

# Executes HTTP request with user payload.
#
def request_with_payload(method, url, payload, headers, options)
safe_request_to(url, options.fetch(:max_retries, 1000)) do |proxy|
opts = options.merge(url: url, payload: payload, proxy: proxy, headers: default_headers.merge(headers))
Expand All @@ -130,6 +134,8 @@ def request_with_payload(method, url, payload, headers, options)
end
end

# Executes HTTP request without user payload.
#
def request_without_payload(method, url, headers, options)
safe_request_to(url, options.fetch(:max_retries, 1000)) do |proxy|
opts = options.merge(url: url, proxy: proxy, headers: default_headers.merge(headers))
Expand All @@ -138,6 +144,12 @@ def request_without_payload(method, url, headers, options)
end
end

# Default ProxyFetcher::Client http headers. Uses some options
# from the configuration object, such as User-Agent string.
#
# @return [Hash]
# headers
#
def default_headers
{
'User-Agent' => ProxyFetcher.config.user_agent
Expand Down
24 changes: 24 additions & 0 deletions lib/proxy_fetcher/client/proxies_registry.rb
Original file line number Diff line number Diff line change
@@ -1,12 +1,30 @@
# frozen_string_literal: true

module ProxyFetcher
module Client
class ProxiesRegistry
class << self
# Removes proxy from the list of the current proxy manager
# instance. If no more proxy available, refreshes the list.
#
# @param proxy [ProxyFetcher::Proxy]
# proxy object to remove
#
def invalidate_proxy!(proxy)
manager.proxies.delete(proxy)
manager.refresh_list! if manager.proxies.empty?
end

# Searches for valid proxy or required type (HTTP or secure)
# for requested URL. If no proxy found, than it refreshes proxy list
# and tries again.
#
# @param url [String]
# URL to process with proxy
#
# @return [ProxyFetcher::Proxy]
# gems proxy object
#
def find_proxy_for(url)
proxy = if URI.parse(url).is_a?(URI::HTTPS)
manager.proxies.detect(&:ssl?)
Expand All @@ -20,6 +38,12 @@ def find_proxy_for(url)
find_proxy_for(url)
end

# Instantiate or returns <code>ProxyFetcher::Manager</code> instance
# for current <code>Thread</code>.
#
# @return [ProxyFetcher::Manager]
# ProxyFetcher manager class
#
def manager
manager = Thread.current[:proxy_fetcher_manager]
return manager unless manager.nil?
Expand Down
14 changes: 14 additions & 0 deletions lib/proxy_fetcher/client/request.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
module Client
# ProxyFetcher::Client HTTP request abstraction.
Expand Down Expand Up @@ -37,6 +39,8 @@ def initialize(args)
build_http_client
end

# Executes HTTP request with defined options.
#
def execute
request = request_class_for(method).new(uri, headers)

Expand All @@ -47,6 +51,9 @@ def execute

private

# Converts payload to the required format, so <code>Hash</code>
# must be a WWW-Form encoded for example.
#
def preprocess_payload(payload)
return if payload.nil?

Expand All @@ -58,6 +65,8 @@ def preprocess_payload(payload)
end
end

# Builds HTTP client based on stdlib Net::HTTP.
#
def build_http_client
@http = Net::HTTP.new(uri.host, uri.port, proxy.addr, proxy.port)

Expand All @@ -76,6 +85,8 @@ def process_response!(http_response)
end
end

# Follows redirection for response.
#
def follow_redirection(http_response)
raise ProxyFetcher::Exceptions::MaximumRedirectsReached if max_redirects <= 0

Expand All @@ -85,6 +96,9 @@ def follow_redirection(http_response)
Request.execute(method: :get, url: url, proxy: proxy, headers: headers, timeout: timeout, max_redirects: max_redirects - 1)
end

# Returns particular Net::HTTP method object
# for processing required request.
#
def request_class_for(method)
Net::HTTP.const_get(method, false)
end
Expand Down
17 changes: 17 additions & 0 deletions lib/proxy_fetcher/configuration.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
# ProxyFetcher configuration. Stores all the options for dealing
# with HTTP requests, adapters, custom classes.
Expand All @@ -21,6 +23,11 @@ class Configuration
DEFAULT_ADAPTER = :nokogiri

class << self
# Registry for handling proxy providers.
#
# @return [ProxyFetcher::ProvidersRegistry]
# providers registry
#
def providers_registry
@registry ||= ProvidersRegistry.new
end
Expand All @@ -38,11 +45,21 @@ def register_provider(name, klass)
providers_registry.register(name, klass)
end

# Returns registered providers names.
#
# @return [Array<String>, Array<Symbol>]
# registered providers names
#
def registered_providers
providers_registry.providers.keys
end
end

# Initialize ProxyFetcher configuration with default options.
#
# @return [ProxyFetcher::Configuration]
# ProxyFetcher gem configuration object
#
def initialize
reset!
end
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/configuration/providers_registry.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
# ProxyFetcher providers registry that stores all registered proxy providers.
class ProvidersRegistry
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/document.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
# HTML document abstraction class. Used to work with different HTML parser adapters
# such as Nokogiri, Oga or a custom one. Stores <i>backend</i< that will handle all
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/document/adapters.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
class Document
# ProxyFetcher HTML parser adapters.
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/document/adapters/abstract_adapter.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
class Document
# Abstract HTML parser adapter class.
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/document/adapters/nokogiri_adapter.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
class Document
# HTML parser adapter that uses Nokogiri as a backend.
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/document/adapters/oga_adapter.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
class Document
# HTML parser adapter that uses Oga as a backend.
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/document/node.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
class Document
# Abstract class for storing HTML elements that was parsed by
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/exceptions.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
# Base exception class for all the ProxyFetcher exceptions.
Error = Class.new(StandardError)
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/manager.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
# ProxyFetcher Manager class for interacting with proxy lists from various providers.
class Manager
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/providers/base.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
module Providers
# Base class for all the ProxyFetcher providers.
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/providers/free_proxy_list.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
module Providers
# FreeProxyList provider class.
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/providers/free_proxy_list_ssl.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
module Providers
# FreeProxyListSSL provider class.
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/providers/gather_proxy.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

require 'json'

module ProxyFetcher
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/providers/http_tunnel.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
module Providers
# HTTPTunnel provider class.
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/providers/proxy_docker.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
module Providers
# ProxyDocker provider class.
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/providers/proxy_list.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

require 'base64'

module ProxyFetcher
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/providers/xroxy.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
module Providers
# XRoxy provider class.
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/proxy.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
# Proxy object
class Proxy
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/utils/http_client.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
# Default ProxyFetcher HTTP client used to fetch proxy lists from
# the different providers. Uses ProxyFetcher configuration options
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/utils/proxy_validator.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
# Default ProxyFetcher proxy validator that checks either proxy
# connectable or not. It tries to send HEAD request to default
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy_fetcher/version.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

module ProxyFetcher
##
# ProxyFetcher gem version.
Expand Down
2 changes: 2 additions & 0 deletions spec/proxy_fetcher/client_spec.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

require 'spec_helper'
require 'json'

Expand Down
2 changes: 2 additions & 0 deletions spec/proxy_fetcher/configuration_spec.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

require 'spec_helper'

describe ProxyFetcher::Configuration do
Expand Down
2 changes: 2 additions & 0 deletions spec/proxy_fetcher/document/adapters_spec.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

require 'spec_helper'

describe ProxyFetcher::Document::Adapters do
Expand Down
2 changes: 2 additions & 0 deletions spec/proxy_fetcher/document/node_spec.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

require 'spec_helper'

describe ProxyFetcher::Document::Node do
Expand Down
2 changes: 2 additions & 0 deletions spec/proxy_fetcher/providers/base_spec.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

require 'spec_helper'

describe ProxyFetcher::Providers::Base do
Expand Down
2 changes: 2 additions & 0 deletions spec/proxy_fetcher/providers/free_proxy_list_spec.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

require 'spec_helper'

describe ProxyFetcher::Providers::FreeProxyList do
Expand Down
2 changes: 2 additions & 0 deletions spec/proxy_fetcher/providers/free_proxy_list_ssl_spec.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

require 'spec_helper'

describe ProxyFetcher::Providers::FreeProxyListSSL do
Expand Down
2 changes: 2 additions & 0 deletions spec/proxy_fetcher/providers/gather_proxy_spec.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

require 'spec_helper'

describe ProxyFetcher::Providers::GatherProxy do
Expand Down
2 changes: 2 additions & 0 deletions spec/proxy_fetcher/providers/http_tunnel_spec.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

require 'spec_helper'

describe ProxyFetcher::Providers::HTTPTunnel do
Expand Down
2 changes: 2 additions & 0 deletions spec/proxy_fetcher/providers/multiple_providers_spec.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

require 'spec_helper'

describe 'Multiple proxy providers' do
Expand Down
2 changes: 2 additions & 0 deletions spec/proxy_fetcher/providers/proxy_docker_spec.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

require 'spec_helper'

describe ProxyFetcher::Providers::ProxyDocker do
Expand Down
2 changes: 2 additions & 0 deletions spec/proxy_fetcher/providers/proxy_list_spec.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

require 'spec_helper'

describe ProxyFetcher::Providers::ProxyList do
Expand Down
2 changes: 2 additions & 0 deletions spec/proxy_fetcher/providers/xroxy_spec.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# frozen_string_literal: true

require 'spec_helper'

describe ProxyFetcher::Providers::XRoxy do
Expand Down
Loading

0 comments on commit 30c6d40

Please sign in to comment.