Add support for custom headers

CHANGELOG.md

@@ -7,6 +7,7 @@
 * [#18](https://github.com/dblock/ruby-link-checker/pull/18): Added Ruby 4.0 to the CI test matrix - [@dblock](https://github.com/dblock), [@Copilot](https://github.com/Copilot).
 * [#19](https://github.com/dblock/ruby-link-checker/pull/19): Added `LinkChecker::Async::HTTP`, a link checker based on [async-http](https://github.com/socketry/async-http) - [@dblock](https://github.com/dblock), [@Copilot](https://github.com/Copilot).
 * [#19](https://github.com/dblock/ruby-link-checker/pull/19): Dropped support for Ruby < 3.3 - [@dblock](https://github.com/dblock), [@Copilot](https://github.com/Copilot).
+* [#20](https://github.com/dblock/ruby-link-checker/pull/20): Added support for custom headers - [@dblock](https://github.com/dblock), [@Copilot](https://github.com/Copilot).
 * Your contribution here.
 
 ### 0.3.0 (2025-07-05)

README.md

@@ -24,6 +24,7 @@ A fast Ruby link checker with support for multiple HTTP libraries. Does not pars
     - [Methods](#methods)
     - [Logger](#logger)
     - [User-Agent](#user-agent)
+    - [Headers](#headers)
   - [Global Configuration](#global-configuration)
   - [Callbacks and Events](#callbacks-and-events)
 - [Benchmarks](#benchmarks)
@@ -271,13 +272,28 @@ Pass your own user-agent. Default is `Ruby Link Checker/x.y.z`.
 checker = LinkChecker::Net::HTTP::Checker.new(user_agent: 'Custom Agent/1.0')
 ```
 
+#### Headers
+
+Some sites reject or return errors (403, 404) to requests that look like bots. Pass additional headers to mimic a browser.
+
+```ruby
+checker = LinkChecker::Net::HTTP::Checker.new(
+  headers: {
+    'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Language' => 'en-US,en;q=0.5',
+    'DNT' => '1'
+  }
+)
+```
+
 ### Global Configuration
 
 All options can also be configured globally.
 
 ```ruby
 LinkChecker.configure do |config|
   config.user_agent = 'Custom Agent/1.0'
+  config.headers = { 'DNT' => '1' }
   config.methods = ['HEAD', 'GET']
   config.logger = ::Logger.new(STDOUT)
 end

lib/ruby-link-checker/async/http/checker.rb

@@ -10,7 +10,9 @@ def run!
 
         def _run!
           client = checker._client_for(uri)
-          headers = ::Protocol::HTTP::Headers[[['user-agent', checker.user_agent]]]
+          headers = ::Protocol::HTTP::Headers[
+            [['user-agent', checker.user_agent]] + checker.headers.map { |key, value| [key, value] }
+          ]
           request = ::Protocol::HTTP::Request[method, uri.request_uri, headers]
           response = _call(client, request)
           result! Result.new(uri, method, original_uri, request, response, options)

lib/ruby-link-checker/async/http/result.rb

@@ -9,7 +9,8 @@ def initialize(headers)
         end
 
         def [](key)
-          @headers[key.to_s.downcase]
+          value = @headers[key.to_s.downcase]
+          value.is_a?(Array) ? value.join(', ') : value
         end
       end
 

lib/ruby-link-checker/config.rb

@@ -7,6 +7,7 @@ module Config
     ATTRIBUTES = %i[
       methods
       user_agent
+      headers
       logger
       retries
     ].freeze
@@ -16,6 +17,7 @@ module Config
     def reset
       self.methods = %w[HEAD GET]
       self.user_agent = "Ruby Link Checker/#{LinkChecker::VERSION}"
+      self.headers = {}
       self.logger = nil
       self.retries = 0
     end

lib/ruby-link-checker/net/http/checker.rb

@@ -8,6 +8,7 @@ def run!
             http.open_timeout = checker.open_timeout if checker.open_timeout
             request = ::Net::HTTPGenericRequest.new(method, false, true, uri)
             request['User-Agent'] = checker.user_agent
+            checker.headers.each { |key, value| request[key] = value }
             response = http.request(request)
             result! Result.new(uri, method, original_uri, request, response, options)
           end

lib/ruby-link-checker/typhoeus/hydra/checker.rb

@@ -11,7 +11,7 @@ def run!
               connecttimeout: checker.connecttimeout,
               headers: {
                 'User-Agent' => checker.user_agent
-              }
+              }.merge(checker.headers)
             }
           )
           request.on_complete do |response|

spec/support/link_checker.rb

@@ -7,6 +7,10 @@
     it 'has a default user-agent' do
       expect(subject.user_agent).to eq "Ruby Link Checker/#{LinkChecker::VERSION}"
     end
+
+    it 'has no default headers' do
+      expect(subject.headers).to eq({})
+    end
   end
 
   context 'user-agent' do
@@ -19,6 +23,16 @@
     end
   end
 
+  context 'headers' do
+    subject do
+      described_class.new(headers: { 'DNT' => '1' })
+    end
+
+    it 'updates headers' do
+      expect(subject.headers).to eq('DNT' => '1')
+    end
+  end
+
   context 'check' do
     let(:url) { 'https://www.example.org' }
 
@@ -104,6 +118,16 @@
               expect(result.options).to eq({})
             end
 
+            context 'with custom headers' do
+              subject do
+                described_class.new(methods: ['GET'], headers: { 'DNT' => '1' })
+              end
+
+              it 'sends custom headers' do
+                expect(result.request_headers['DNT']).to eq '1'
+              end
+            end
+
             it 'returns results' do
               expect(subject.results).to eq(
                 error: [],