Seams like there needs to be added a try catch around fetching the web content.
data-scraper | 2022-03-19T17:10:20.156173 => {'ConnectionError': 200, 'register': 164, 'ReadTimeout': 22481, 'unfollow': 45902, 'follow': 103508, 'tweet': 27042, 'latest': 988179}
data-scraper | 2022-03-19T17:15:22.084095 => {'ConnectionError': 200, 'register': 164, 'ReadTimeout': 22481, 'unfollow': 45902, 'follow': 103508, 'tweet': 27042, 'latest': 988179}
data-scraper | Traceback (most recent call last):
data-scraper | File "/usr/local/lib/python3.9/site-packages/urllib3/response.py", line 441, in _error_catcher
data-scraper | yield
data-scraper | File "/usr/local/lib/python3.9/site-packages/urllib3/response.py", line 522, in read
data-scraper | data = self._fp.read(amt) if not fp_closed else b""
data-scraper | File "/usr/local/lib/python3.9/http/client.py", line 463, in read
data-scraper | n = self.readinto(b)
data-scraper | File "/usr/local/lib/python3.9/http/client.py", line 507, in readinto
data-scraper | n = self.fp.readinto(b)
data-scraper | File "/usr/local/lib/python3.9/socket.py", line 704, in readinto
data-scraper | return self._sock.recv_into(b)
data-scraper | ConnectionResetError: [Errno 104] Connection reset by peer
data-scraper |
data-scraper | During handling of the above exception, another exception occurred:
data-scraper |
data-scraper | Traceback (most recent call last):
data-scraper | File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 760, in generate
data-scraper | for chunk in self.raw.stream(chunk_size, decode_content=True):
data-scraper | File "/usr/local/lib/python3.9/site-packages/urllib3/response.py", line 579, in stream
data-scraper | data = self.read(amt=amt, decode_content=decode_content)
data-scraper | File "/usr/local/lib/python3.9/site-packages/urllib3/response.py", line 544, in read
data-scraper | raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
data-scraper | File "/usr/local/lib/python3.9/contextlib.py", line 137, in __exit__
data-scraper | self.gen.throw(typ, value, traceback)
data-scraper | File "/usr/local/lib/python3.9/site-packages/urllib3/response.py", line 458, in _error_catcher
data-scraper | raise ProtocolError("Connection broken: %r" % e, e)
data-scraper | urllib3.exceptions.ProtocolError: ("Connection broken: ConnectionResetError(104, 'Connection reset by peer')", ConnectionResetError(104, 'Connection reset by peer'))
data-scraper |
data-scraper | During handling of the above exception, another exception occurred:
data-scraper |
data-scraper | Traceback (most recent call last):
data-scraper | File "/app/./main.py", line 28, in <module>
data-scraper | data = scraper.getData()
data-scraper | File "/app/scraper/scraper.py", line 6, in getData
data-scraper | data.setdefault("latest", getLatest())
data-scraper | File "/app/scraper/scraper.py", line 14, in getLatest
data-scraper | result = requests.get(latestUrl)
data-scraper | File "/usr/local/lib/python3.9/site-packages/requests/api.py", line 75, in get
data-scraper | return request('get', url, params=params, **kwargs)
data-scraper | File "/usr/local/lib/python3.9/site-packages/requests/api.py", line 61, in request
data-scraper | return session.request(method=method, url=url, **kwargs)
data-scraper | File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 529, in request
data-scraper | resp = self.send(prep, **send_kwargs)
data-scraper | File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 687, in send
data-scraper | r.content
data-scraper | File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 838, in content
data-scraper | self._content = b''.join(self.iter_content(CONTENT_CHUNK_SIZE)) or b''
data-scraper | File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 763, in generate
data-scraper | raise ChunkedEncodingError(e)
```
Seams like there needs to be added a try catch around fetching the web content.
stack trace: