diff --git a/CHANGELOG.md b/CHANGELOG.md index dc62176..7c56200 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * [#18](https://github.com/dblock/ruby-link-checker/pull/18): Added Ruby 4.0 to the CI test matrix - [@dblock](https://github.com/dblock), [@Copilot](https://github.com/Copilot). * [#19](https://github.com/dblock/ruby-link-checker/pull/19): Added `LinkChecker::Async::HTTP`, a link checker based on [async-http](https://github.com/socketry/async-http) - [@dblock](https://github.com/dblock), [@Copilot](https://github.com/Copilot). * [#19](https://github.com/dblock/ruby-link-checker/pull/19): Dropped support for Ruby < 3.3 - [@dblock](https://github.com/dblock), [@Copilot](https://github.com/Copilot). +* [#20](https://github.com/dblock/ruby-link-checker/pull/20): Added support for custom headers - [@dblock](https://github.com/dblock), [@Copilot](https://github.com/Copilot). * Your contribution here. ### 0.3.0 (2025-07-05) diff --git a/README.md b/README.md index ed0df99..7d93dd8 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ A fast Ruby link checker with support for multiple HTTP libraries. Does not pars - [Methods](#methods) - [Logger](#logger) - [User-Agent](#user-agent) + - [Headers](#headers) - [Global Configuration](#global-configuration) - [Callbacks and Events](#callbacks-and-events) - [Benchmarks](#benchmarks) @@ -271,6 +272,20 @@ Pass your own user-agent. Default is `Ruby Link Checker/x.y.z`. checker = LinkChecker::Net::HTTP::Checker.new(user_agent: 'Custom Agent/1.0') ``` +#### Headers + +Some sites reject or return errors (403, 404) to requests that look like bots. Pass additional headers to mimic a browser. + +```ruby +checker = LinkChecker::Net::HTTP::Checker.new( + headers: { + 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language' => 'en-US,en;q=0.5', + 'DNT' => '1' + } +) +``` + ### Global Configuration All options can also be configured globally. @@ -278,6 +293,7 @@ All options can also be configured globally. ```ruby LinkChecker.configure do |config| config.user_agent = 'Custom Agent/1.0' + config.headers = { 'DNT' => '1' } config.methods = ['HEAD', 'GET'] config.logger = ::Logger.new(STDOUT) end diff --git a/lib/ruby-link-checker/async/http/checker.rb b/lib/ruby-link-checker/async/http/checker.rb index c661dfe..70ee9dd 100644 --- a/lib/ruby-link-checker/async/http/checker.rb +++ b/lib/ruby-link-checker/async/http/checker.rb @@ -10,7 +10,9 @@ def run! def _run! client = checker._client_for(uri) - headers = ::Protocol::HTTP::Headers[[['user-agent', checker.user_agent]]] + headers = ::Protocol::HTTP::Headers[ + [['user-agent', checker.user_agent]] + checker.headers.map { |key, value| [key, value] } + ] request = ::Protocol::HTTP::Request[method, uri.request_uri, headers] response = _call(client, request) result! Result.new(uri, method, original_uri, request, response, options) diff --git a/lib/ruby-link-checker/async/http/result.rb b/lib/ruby-link-checker/async/http/result.rb index 250bf55..4a686a4 100644 --- a/lib/ruby-link-checker/async/http/result.rb +++ b/lib/ruby-link-checker/async/http/result.rb @@ -9,7 +9,8 @@ def initialize(headers) end def [](key) - @headers[key.to_s.downcase] + value = @headers[key.to_s.downcase] + value.is_a?(Array) ? value.join(', ') : value end end diff --git a/lib/ruby-link-checker/config.rb b/lib/ruby-link-checker/config.rb index 0b9e50d..4bb7d9a 100644 --- a/lib/ruby-link-checker/config.rb +++ b/lib/ruby-link-checker/config.rb @@ -7,6 +7,7 @@ module Config ATTRIBUTES = %i[ methods user_agent + headers logger retries ].freeze @@ -16,6 +17,7 @@ module Config def reset self.methods = %w[HEAD GET] self.user_agent = "Ruby Link Checker/#{LinkChecker::VERSION}" + self.headers = {} self.logger = nil self.retries = 0 end diff --git a/lib/ruby-link-checker/net/http/checker.rb b/lib/ruby-link-checker/net/http/checker.rb index ad35b6d..6f16ebe 100644 --- a/lib/ruby-link-checker/net/http/checker.rb +++ b/lib/ruby-link-checker/net/http/checker.rb @@ -8,6 +8,7 @@ def run! http.open_timeout = checker.open_timeout if checker.open_timeout request = ::Net::HTTPGenericRequest.new(method, false, true, uri) request['User-Agent'] = checker.user_agent + checker.headers.each { |key, value| request[key] = value } response = http.request(request) result! Result.new(uri, method, original_uri, request, response, options) end diff --git a/lib/ruby-link-checker/typhoeus/hydra/checker.rb b/lib/ruby-link-checker/typhoeus/hydra/checker.rb index 7d47dac..d41afd5 100644 --- a/lib/ruby-link-checker/typhoeus/hydra/checker.rb +++ b/lib/ruby-link-checker/typhoeus/hydra/checker.rb @@ -11,7 +11,7 @@ def run! connecttimeout: checker.connecttimeout, headers: { 'User-Agent' => checker.user_agent - } + }.merge(checker.headers) } ) request.on_complete do |response| diff --git a/spec/support/link_checker.rb b/spec/support/link_checker.rb index d1cd9af..783f545 100644 --- a/spec/support/link_checker.rb +++ b/spec/support/link_checker.rb @@ -7,6 +7,10 @@ it 'has a default user-agent' do expect(subject.user_agent).to eq "Ruby Link Checker/#{LinkChecker::VERSION}" end + + it 'has no default headers' do + expect(subject.headers).to eq({}) + end end context 'user-agent' do @@ -19,6 +23,16 @@ end end + context 'headers' do + subject do + described_class.new(headers: { 'DNT' => '1' }) + end + + it 'updates headers' do + expect(subject.headers).to eq('DNT' => '1') + end + end + context 'check' do let(:url) { 'https://www.example.org' } @@ -104,6 +118,16 @@ expect(result.options).to eq({}) end + context 'with custom headers' do + subject do + described_class.new(methods: ['GET'], headers: { 'DNT' => '1' }) + end + + it 'sends custom headers' do + expect(result.request_headers['DNT']).to eq '1' + end + end + it 'returns results' do expect(subject.results).to eq( error: [],