From 028b0284d6f5f46001b6abc7448a2a3c4884a295 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 25 Nov 2015 17:51:26 +0100 Subject: [PATCH 1/2] Add use_firehose ,configuration conflit --- lib/logstash/inputs/twitter.rb | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/logstash/inputs/twitter.rb b/lib/logstash/inputs/twitter.rb index 313250d..ab2e050 100644 --- a/lib/logstash/inputs/twitter.rb +++ b/lib/logstash/inputs/twitter.rb @@ -86,6 +86,9 @@ class LogStash::Inputs::Twitter < LogStash::Inputs::Base # follows, locations, and languages options will be ignored. Default => false config :use_samples, :validate => :boolean, :default => false + # Returns all tweet from firehose + config :use_firehose, :validate => :boolean, :default => false + # Lets you ingore the retweets coming out of the Twitter API. Default => false config :ignore_retweets, :validate => :boolean, :default => false @@ -101,10 +104,14 @@ class LogStash::Inputs::Twitter < LogStash::Inputs::Base def register require "twitter" - if !@use_samples && ( @keywords.nil? && @follows.nil? && @locations.nil? ) + if !@use_samples && !@use_firehose && ( @keywords.nil? && @follows.nil? && @locations.nil? ) raise LogStash::ConfigurationError.new("At least one parameter (follows, locations or keywords) must be specified.") end + if @use_samples && @use_firehose + raise LogStash::ConfigurationError.new("Only use_firehose or use_samples can be choose") + end + # monkey patch twitter gem to ignore json parsing error. # at the same time, use our own json parser # this has been tested with a specific gem version, raise if not the same @@ -124,6 +131,11 @@ def run(queue) return if stop? tweet_processor(queue, tweet) end + elsif @use_firehose + @stream_client.firehose do |tweet| + return if stop? + tweet_processor(queue, tweet) + end else @stream_client.filter(twitter_options) do |tweet| return if stop? From d8d9a8e93d7348a3f455af4a57da5c1ae1a46d59 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 10 Dec 2015 10:29:34 +0100 Subject: [PATCH 2/2] Edit lib/logstash/inputs/twitter.rb ConfigurationError reply (jordansissel) --- lib/logstash/inputs/twitter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logstash/inputs/twitter.rb b/lib/logstash/inputs/twitter.rb index ab2e050..de2ddbe 100644 --- a/lib/logstash/inputs/twitter.rb +++ b/lib/logstash/inputs/twitter.rb @@ -109,7 +109,7 @@ def register end if @use_samples && @use_firehose - raise LogStash::ConfigurationError.new("Only use_firehose or use_samples can be choose") + raise LogStash::ConfigurationError.new("use_firehose and use_samples cannot both be true") end # monkey patch twitter gem to ignore json parsing error.