From 84b89212c30d247bfd92b84b81800781bb00f09a Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Tue, 6 Oct 2020 14:38:16 +0200 Subject: [PATCH 01/16] Refactor: only require plugin not its deps --- spec/spec_helper.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index e65ae3e..0a51c31 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,8 +1,8 @@ require 'logstash/devutils/rspec/spec_helper' -require 'logstash/inputs/twitter' -require 'twitter' require 'rspec_sequencing' +require 'logstash/inputs/twitter' + creds_file_path = File.expand_path('spec/integration_credentials.rb') if File.exist?(creds_file_path) load creds_file_path From 238fb4b5675b22a584efc701990c8d088c08719f Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Tue, 6 Oct 2020 14:38:42 +0200 Subject: [PATCH 02/16] Test: swap out integration specs to run --- spec/integration/twitter_pubsub_spec.rb | 13 +++++++++++-- spec/integration/twitter_spec.rb | 3 --- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/spec/integration/twitter_pubsub_spec.rb b/spec/integration/twitter_pubsub_spec.rb index fb72b7b..4eff083 100644 --- a/spec/integration/twitter_pubsub_spec.rb +++ b/spec/integration/twitter_pubsub_spec.rb @@ -1,9 +1,18 @@ +__END__ require_relative "../spec_helper" -Thread.abort_on_exception = true - describe LogStash::Inputs::Twitter do describe "full integration", :integration => true do + + before(:all) do + @_abort_on_exception = Thread.abort_on_exception + Thread.abort_on_exception = true + end + + after(:all) do + Thread.abort_on_exception = @_abort_on_exception + end + let(:pub_config) do { :consumer_key => ENV['PUB_TWITTER_CONSUMER_KEY'], diff --git a/spec/integration/twitter_spec.rb b/spec/integration/twitter_spec.rb index d503406..a6d10a7 100644 --- a/spec/integration/twitter_spec.rb +++ b/spec/integration/twitter_spec.rb @@ -1,6 +1,3 @@ -__END__ -# this test is quite unreliable - commenting it out - require_relative "../spec_helper" describe LogStash::Inputs::Twitter do From 9126812af4d16a60a1295429839e1911d205d8ff Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Tue, 6 Oct 2020 14:39:10 +0200 Subject: [PATCH 03/16] CI: let's see how it breaks ... --- .travis.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a50fc73..060bbf4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,2 +1,11 @@ import: -- logstash-plugins/.ci:travis/travis.yml@1.x \ No newline at end of file + - source: logstash-plugins/.ci:travis/travis.yml@1.x + mode: deep_merge_append + +before_install: + - sudo apt install -y squid + - netstat -plunt | grep 3128 + +env: + jobs: + - SPEC_OPTS="--tag integration" ELASTIC_STACK_VERSION=7.x From 8b3fa4c7056dd6022ac97ebfb74df150412ebea7 Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Tue, 6 Oct 2020 16:04:37 +0200 Subject: [PATCH 04/16] CI: use an explicit squid.conf --- .ci/squid.conf | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++ .travis.yml | 3 ++ 2 files changed, 85 insertions(+) create mode 100644 .ci/squid.conf diff --git a/.ci/squid.conf b/.ci/squid.conf new file mode 100644 index 0000000..acf48fc --- /dev/null +++ b/.ci/squid.conf @@ -0,0 +1,82 @@ +# +# Recommended minimum configuration: +# +# ACLs all, manager, localhost, and to_localhost are predefined. +#acl localhost src 127.0.0.1/32 ::1 +#acl to_localhost dst 127.0.0.0/8 0.0.0.0/32 ::1 + +# Example rule allowing access from your local networks. +# Adapt to list your (internal) IP networks from where browsing should be allowed + +acl localnet src 0.0.0.1-0.255.255.255 # RFC 1122 "this" network (LAN) +acl localnet src 10.0.0.0/8 # RFC 1918 local private network (LAN) +#acl localnet src 100.64.0.0/10 # RFC 6598 shared address space (CGN) +#acl localnet src 169.254.0.0/16 # RFC 3927 link-local (directly plugged) machines +acl localnet src 172.16.0.0/12 # RFC 1918 local private network (LAN) +acl localnet src 192.168.0.0/16 # RFC 1918 local private network (LAN) +acl localnet src fc00::/7 # RFC 4193 local private network range +acl localnet src fe80::/10 # RFC 4291 link-local (directly plugged) machines + +acl SSL_ports port 443 +acl Safe_ports port 80 # http +#acl Safe_ports port 21 # ftp +acl Safe_ports port 443 # https +#acl Safe_ports port 70 # gopher +#acl Safe_ports port 210 # wais +acl Safe_ports port 1025-65535 # unregistered ports +acl Safe_ports port 280 # http-mgmt +#acl Safe_ports port 488 # gss-http +#acl Safe_ports port 591 # filemaker +#acl Safe_ports port 777 # multiling http +acl CONNECT method CONNECT + +# +# Recommended minimum Access Permission configuration: +# +# Only allow cachemgr access from localhost +http_access allow manager localhost +http_access deny manager + +# Deny requests to certain unsafe ports +http_access deny !Safe_ports + +# Deny CONNECT to other than secure SSL ports +#http_access deny CONNECT !SSL_ports + +# We strongly recommend the following be uncommented to protect innocent +# web applications running on the proxy server who think the only +# one who can access services on "localhost" is a local user +#http_access deny to_localhost + +# Example rule allowing access from your local networks. +# Adapt localnet in the ACL section to list your (internal) IP networks +# from where browsing should be allowed +http_access allow localnet +http_access allow localhost + +# And finally deny all other access to this proxy +http_access deny all + +# Squid normally listens to port 3128 +http_port 0.0.0.0:3128 + +#cache deny all + +#debug_options ALL,1 11,3 + +# Leave coredumps in the first cache dir +coredump_dir /var/spool/squid + +# +# Add any of your own refresh_pattern entries above these. +# +#refresh_pattern ^ftp: 1440 20% 10080 +#refresh_pattern ^gopher: 1440 0% 1440 +#refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 +#refresh_pattern \/(Packages|Sources)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims +#refresh_pattern \/Release(|\.gpg)$ 0 0% 0 refresh-ims +#refresh_pattern \/InRelease$ 0 0% 0 refresh-ims +#refresh_pattern \/(Translation-.*)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims +## example pattern for deb packages +##refresh_pattern (\.deb|\.udeb)$ 129600 100% 129600 +#refresh_pattern . 0 20% 4320 diff --git a/.travis.yml b/.travis.yml index 060bbf4..a7485fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,9 @@ import: before_install: - sudo apt install -y squid - netstat -plunt | grep 3128 + - sudo cp -f .ci/squid.conf /etc/squid/squid.conf + - sudo service squid restart + - netstat -plunt | grep 3128 env: jobs: From efec1167a1f87c8cf5dba997a26d65b45a831a27 Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Tue, 6 Oct 2020 17:13:27 +0200 Subject: [PATCH 05/16] CI: debug what's going on --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a7485fe..fd9f659 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ before_install: - netstat -plunt | grep 3128 - sudo cp -f .ci/squid.conf /etc/squid/squid.conf - sudo service squid restart - - netstat -plunt | grep 3128 + - sudo netstat -plunt env: jobs: From 08d0d51ffa9f30ff697c56377cd722c01d634684 Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Tue, 6 Oct 2020 18:32:14 +0200 Subject: [PATCH 06/16] CI: override to have network_mode: host --- .ci/docker-compose.override.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .ci/docker-compose.override.yml diff --git a/.ci/docker-compose.override.yml b/.ci/docker-compose.override.yml new file mode 100644 index 0000000..ea7c493 --- /dev/null +++ b/.ci/docker-compose.override.yml @@ -0,0 +1,5 @@ +version: '3' + +services: + logstash: + network_mode: host From 58e7b392d57409f299570eb8fdc921f15f8d4a3c Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Wed, 7 Oct 2020 12:25:21 +0200 Subject: [PATCH 07/16] CI: use (but hide) TWITTER env variables --- .ci/docker-compose.override.yml | 7 +++++++ .ci/run.sh | 9 +++++++++ 2 files changed, 16 insertions(+) create mode 100755 .ci/run.sh diff --git a/.ci/docker-compose.override.yml b/.ci/docker-compose.override.yml index ea7c493..a5f92da 100644 --- a/.ci/docker-compose.override.yml +++ b/.ci/docker-compose.override.yml @@ -3,3 +3,10 @@ version: '3' services: logstash: network_mode: host + environment: + - TWITTER_CONSUMER_KEY + - TWITTER_CONSUMER_SECRET + - TWITTER_OAUTH_TOKEN + - TWITTER_OAUTH_TOKEN_SECRET + - SPEC_OPTS + - LOGGER_LEVEL diff --git a/.ci/run.sh b/.ci/run.sh new file mode 100755 index 0000000..e71a8ff --- /dev/null +++ b/.ci/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# This is intended to be run inside the docker container as the command of the docker-compose. + +# Do not print TWITTER_ (auth) secrets +env | grep -v 'TWITTER_' + +set -ex + +jruby -rbundler/setup -S rspec -fd From 8b38870cf12ec60b1068130f8977d92b20088da1 Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Wed, 7 Oct 2020 17:12:57 +0200 Subject: [PATCH 08/16] Temp: debug logging for the twitter plugins --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fd9f659..5a283bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,4 +11,4 @@ before_install: env: jobs: - - SPEC_OPTS="--tag integration" ELASTIC_STACK_VERSION=7.x + - SPEC_OPTS="--tag integration" LOGGER_LEVEL="logstash.inputs.twitter=debug" ELASTIC_STACK_VERSION=7.x From 78d781e49a630d72d8e01e9c3ad215b32971a466 Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Wed, 7 Oct 2020 21:20:25 +0200 Subject: [PATCH 09/16] do we need any of the refresh patterns? --- .ci/squid.conf | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.ci/squid.conf b/.ci/squid.conf index acf48fc..8a9b2fd 100644 --- a/.ci/squid.conf +++ b/.ci/squid.conf @@ -70,13 +70,13 @@ coredump_dir /var/spool/squid # # Add any of your own refresh_pattern entries above these. # -#refresh_pattern ^ftp: 1440 20% 10080 -#refresh_pattern ^gopher: 1440 0% 1440 -#refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 -#refresh_pattern \/(Packages|Sources)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims -#refresh_pattern \/Release(|\.gpg)$ 0 0% 0 refresh-ims -#refresh_pattern \/InRelease$ 0 0% 0 refresh-ims -#refresh_pattern \/(Translation-.*)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims -## example pattern for deb packages -##refresh_pattern (\.deb|\.udeb)$ 129600 100% 129600 -#refresh_pattern . 0 20% 4320 +refresh_pattern ^ftp: 1440 20% 10080 +refresh_pattern ^gopher: 1440 0% 1440 +refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 +refresh_pattern \/(Packages|Sources)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims +refresh_pattern \/Release(|\.gpg)$ 0 0% 0 refresh-ims +refresh_pattern \/InRelease$ 0 0% 0 refresh-ims +refresh_pattern \/(Translation-.*)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims +# example pattern for deb packages +#refresh_pattern (\.deb|\.udeb)$ 129600 100% 129600 +refresh_pattern . 0 20% 4320 From 38051b4f977716f47483d68a37f58c51a822299b Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Wed, 7 Oct 2020 21:37:17 +0200 Subject: [PATCH 10/16] bring back squid includes + debug logging --- .ci/squid.conf | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.ci/squid.conf b/.ci/squid.conf index 8a9b2fd..79f59ca 100644 --- a/.ci/squid.conf +++ b/.ci/squid.conf @@ -30,11 +30,14 @@ acl Safe_ports port 280 # http-mgmt #acl Safe_ports port 777 # multiling http acl CONNECT method CONNECT +# +include /etc/squid/conf.d/* + # # Recommended minimum Access Permission configuration: # # Only allow cachemgr access from localhost -http_access allow manager localhost +http_access allow localhost manager http_access deny manager # Deny requests to certain unsafe ports @@ -55,14 +58,14 @@ http_access allow localnet http_access allow localhost # And finally deny all other access to this proxy -http_access deny all +#http_access deny all # Squid normally listens to port 3128 http_port 0.0.0.0:3128 #cache deny all -#debug_options ALL,1 11,3 +debug_options ALL,1 11,3 # Leave coredumps in the first cache dir coredump_dir /var/spool/squid From 9be27eea5bc1ea061db6570d6ce7449e8e4dc6e3 Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Wed, 7 Oct 2020 21:37:33 +0200 Subject: [PATCH 11/16] CI: roll on Ubuntu 20.04 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5a283bb..f3399e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,5 @@ +dist: focal # 20.04 LTS + import: - source: logstash-plugins/.ci:travis/travis.yml@1.x mode: deep_merge_append From ed719bbbddb4a232e5583497a82104b024b3b29f Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Wed, 7 Oct 2020 21:47:40 +0200 Subject: [PATCH 12/16] CI: deal with dist: focal --- .travis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f3399e9..a7b21dc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,12 @@ dist: focal # 20.04 LTS +language: minimal import: - - source: logstash-plugins/.ci:travis/travis.yml@1.x - mode: deep_merge_append + # NOTE: can not be used with dist: focal due docker-ce addon + #- source: logstash-plugins/.ci:travis/travis.yml@1.x + # mode: deep_merge_append + - logstash-plugins/.ci:travis/matrix.yml@1.x + - logstash-plugins/.ci:travis/exec.yml@1.x before_install: - sudo apt install -y squid From cb702414f0f60de4d5f601ec889520f0cdb91a7b Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Wed, 7 Oct 2020 21:47:56 +0200 Subject: [PATCH 13/16] Temp: test the proxy on restart --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index a7b21dc..b35c9d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ before_install: - sudo cp -f .ci/squid.conf /etc/squid/squid.conf - sudo service squid restart - sudo netstat -plunt + - curl "https://devdocs.io" --proxy http://127.0.0.1:3128 env: jobs: From f246ee6d46d22075884c75e0ba16474c515885de Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Wed, 7 Oct 2020 21:59:03 +0200 Subject: [PATCH 14/16] more squid-ing! --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b35c9d6..eec5086 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,13 @@ import: - logstash-plugins/.ci:travis/matrix.yml@1.x - logstash-plugins/.ci:travis/exec.yml@1.x +addons: + apt: + packages: + - squid + before_install: - - sudo apt install -y squid + #- sudo apt install -y squid - netstat -plunt | grep 3128 - sudo cp -f .ci/squid.conf /etc/squid/squid.conf - sudo service squid restart From c9ab0323fdd39b988d035054e9d0909147eccf0e Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Wed, 7 Oct 2020 22:02:19 +0200 Subject: [PATCH 15/16] just roll with the squid package AS IS --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index eec5086..7853e29 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,10 +16,10 @@ addons: before_install: #- sudo apt install -y squid - netstat -plunt | grep 3128 - - sudo cp -f .ci/squid.conf /etc/squid/squid.conf - - sudo service squid restart - - sudo netstat -plunt - - curl "https://devdocs.io" --proxy http://127.0.0.1:3128 + #- sudo cp -f .ci/squid.conf /etc/squid/squid.conf + #- sudo service squid restart + #- sudo netstat -plunt + #- curl "https://devdocs.io" --proxy http://127.0.0.1:3128 env: jobs: From c44257f80469ca6efa4cf896d02816042b94e0d4 Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Wed, 21 Oct 2020 12:27:38 +0200 Subject: [PATCH 16/16] cleanup --- .ci/squid.conf | 85 -------------------------------------------------- .travis.yml | 7 +---- 2 files changed, 1 insertion(+), 91 deletions(-) delete mode 100644 .ci/squid.conf diff --git a/.ci/squid.conf b/.ci/squid.conf deleted file mode 100644 index 79f59ca..0000000 --- a/.ci/squid.conf +++ /dev/null @@ -1,85 +0,0 @@ -# -# Recommended minimum configuration: -# -# ACLs all, manager, localhost, and to_localhost are predefined. -#acl localhost src 127.0.0.1/32 ::1 -#acl to_localhost dst 127.0.0.0/8 0.0.0.0/32 ::1 - -# Example rule allowing access from your local networks. -# Adapt to list your (internal) IP networks from where browsing should be allowed - -acl localnet src 0.0.0.1-0.255.255.255 # RFC 1122 "this" network (LAN) -acl localnet src 10.0.0.0/8 # RFC 1918 local private network (LAN) -#acl localnet src 100.64.0.0/10 # RFC 6598 shared address space (CGN) -#acl localnet src 169.254.0.0/16 # RFC 3927 link-local (directly plugged) machines -acl localnet src 172.16.0.0/12 # RFC 1918 local private network (LAN) -acl localnet src 192.168.0.0/16 # RFC 1918 local private network (LAN) -acl localnet src fc00::/7 # RFC 4193 local private network range -acl localnet src fe80::/10 # RFC 4291 link-local (directly plugged) machines - -acl SSL_ports port 443 -acl Safe_ports port 80 # http -#acl Safe_ports port 21 # ftp -acl Safe_ports port 443 # https -#acl Safe_ports port 70 # gopher -#acl Safe_ports port 210 # wais -acl Safe_ports port 1025-65535 # unregistered ports -acl Safe_ports port 280 # http-mgmt -#acl Safe_ports port 488 # gss-http -#acl Safe_ports port 591 # filemaker -#acl Safe_ports port 777 # multiling http -acl CONNECT method CONNECT - -# -include /etc/squid/conf.d/* - -# -# Recommended minimum Access Permission configuration: -# -# Only allow cachemgr access from localhost -http_access allow localhost manager -http_access deny manager - -# Deny requests to certain unsafe ports -http_access deny !Safe_ports - -# Deny CONNECT to other than secure SSL ports -#http_access deny CONNECT !SSL_ports - -# We strongly recommend the following be uncommented to protect innocent -# web applications running on the proxy server who think the only -# one who can access services on "localhost" is a local user -#http_access deny to_localhost - -# Example rule allowing access from your local networks. -# Adapt localnet in the ACL section to list your (internal) IP networks -# from where browsing should be allowed -http_access allow localnet -http_access allow localhost - -# And finally deny all other access to this proxy -#http_access deny all - -# Squid normally listens to port 3128 -http_port 0.0.0.0:3128 - -#cache deny all - -debug_options ALL,1 11,3 - -# Leave coredumps in the first cache dir -coredump_dir /var/spool/squid - -# -# Add any of your own refresh_pattern entries above these. -# -refresh_pattern ^ftp: 1440 20% 10080 -refresh_pattern ^gopher: 1440 0% 1440 -refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 -refresh_pattern \/(Packages|Sources)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims -refresh_pattern \/Release(|\.gpg)$ 0 0% 0 refresh-ims -refresh_pattern \/InRelease$ 0 0% 0 refresh-ims -refresh_pattern \/(Translation-.*)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims -# example pattern for deb packages -#refresh_pattern (\.deb|\.udeb)$ 129600 100% 129600 -refresh_pattern . 0 20% 4320 diff --git a/.travis.yml b/.travis.yml index 7853e29..f294def 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,13 +14,8 @@ addons: - squid before_install: - #- sudo apt install -y squid - netstat -plunt | grep 3128 - #- sudo cp -f .ci/squid.conf /etc/squid/squid.conf - #- sudo service squid restart - #- sudo netstat -plunt - #- curl "https://devdocs.io" --proxy http://127.0.0.1:3128 env: jobs: - - SPEC_OPTS="--tag integration" LOGGER_LEVEL="logstash.inputs.twitter=debug" ELASTIC_STACK_VERSION=7.x + - SPEC_OPTS="--tag integration" ELASTIC_STACK_VERSION=7.x