mop-546f67dd66-cpsv7 mop 127.0.0.1 - - [29/Mar/2026:03:22:49 +0000] "GET /favicon.ico HTTP/1.1" 404 5231 "https://mop.lco.global/observations/LCO/create/?target_id=56066" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"
mop-546f67dd66-2fbfk mop Instruments not cached, getting them again!!!
mop-546f67dd66-2fbfk mop Internal Server Error: /observations/LCO/create/
mop-546f67dd66-2fbfk mop Traceback (most recent call last):
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/django/core/handlers/exception.py", line 55, in inner
mop-546f67dd66-2fbfk mop response = get_response(request)
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/django/core/handlers/base.py", line 199, in _get_response
mop-546f67dd66-2fbfk mop response = self.process_exception_by_middleware(e, request)
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/django/core/handlers/base.py", line 362, in process_exception_by_middleware
mop-546f67dd66-2fbfk mop response = middleware_method(request, exception)
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/tom_common/middleware.py", line 30, in process_exception
mop-546f67dd66-2fbfk mop raise exception
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/django/core/handlers/base.py", line 197, in _get_response
mop-546f67dd66-2fbfk mop response = wrapped_callback(request, *callback_args, **callback_kwargs)
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/django/views/generic/base.py", line 105, in view
mop-546f67dd66-2fbfk mop return self.dispatch(request, *args, **kwargs)
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/tom_observations/views.py", line 162, in dispatch
mop-546f67dd66-2fbfk mop return super().dispatch(request, *args, **kwargs)
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/django/contrib/auth/mixins.py", line 73, in dispatch
mop-546f67dd66-2fbfk mop return super().dispatch(request, *args, **kwargs)
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/django/views/generic/base.py", line 144, in dispatch
mop-546f67dd66-2fbfk mop return handler(request, *args, **kwargs)
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/tom_observations/views.py", line 367, in post
mop-546f67dd66-2fbfk mop if form.is_valid():
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/tom_observations/facilities/ocs.py", line 796, in is_valid
mop-546f67dd66-2fbfk mop self.validate_at_facility()
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/tom_observations/facilities/ocs.py", line 787, in validate_at_facility
mop-546f67dd66-2fbfk mop response = obs_module().validate_observation(self.observation_payload())
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/tom_observations/facilities/ocs.py", line 1350, in observation_payload
mop-546f67dd66-2fbfk mop payload = self._expand_cadence_request(payload)
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/tom_observations/facilities/ocs.py", line 913, in _expand_cadence_request
mop-546f67dd66-2fbfk mop response = make_request(
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/tom_observations/facilities/ocs.py", line 186, in make_request
mop-546f67dd66-2fbfk mop response.raise_for_status()
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/requests/models.py", line 1026, in raise_for_status
mop-546f67dd66-2fbfk mop raise HTTPError(http_error_msg, response=self)
mop-546f67dd66-2fbfk mop requests.exceptions.HTTPError: 502 Server Error: Bad Gateway for url: https://observe.lco.global/api/requestgroups/cadence/
mo
mop-546f67dd66-cpsv7 mop 127.0.0.1 - - [29/Mar/2026:03:22:49 +0000] "GET /favicon.ico HTTP/1.1" 404 5231 "https://mop.lco.global/observations/LCO/create/?target_id=56066" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"
mop-546f67dd66-2fbfk mop Instruments not cached, getting them again!!!
mop-546f67dd66-2fbfk mop Internal Server Error: /observations/LCO/create/
mop-546f67dd66-2fbfk mop Traceback (most recent call last):
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/django/core/handlers/exception.py", line 55, in inner
mop-546f67dd66-2fbfk mop response = get_response(request)
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/django/core/handlers/base.py", line 199, in _get_response
mop-546f67dd66-2fbfk mop response = self.process_exception_by_middleware(e, request)
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/django/core/handlers/base.py", line 362, in process_exception_by_middleware
mop-546f67dd66-2fbfk mop response = middleware_method(request, exception)
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/tom_common/middleware.py", line 30, in process_exception
mop-546f67dd66-2fbfk mop raise exception
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/django/core/handlers/base.py", line 197, in _get_response
mop-546f67dd66-2fbfk mop response = wrapped_callback(request, *callback_args, **callback_kwargs)
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/django/views/generic/base.py", line 105, in view
mop-546f67dd66-2fbfk mop return self.dispatch(request, *args, **kwargs)
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/tom_observations/views.py", line 162, in dispatch
mop-546f67dd66-2fbfk mop return super().dispatch(request, *args, **kwargs)
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/django/contrib/auth/mixins.py", line 73, in dispatch
mop-546f67dd66-2fbfk mop return super().dispatch(request, *args, **kwargs)
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/django/views/generic/base.py", line 144, in dispatch
mop-546f67dd66-2fbfk mop return handler(request, *args, **kwargs)
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/tom_observations/views.py", line 367, in post
mop-546f67dd66-2fbfk mop if form.is_valid():
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/tom_observations/facilities/ocs.py", line 796, in is_valid
mop-546f67dd66-2fbfk mop self.validate_at_facility()
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/tom_observations/facilities/ocs.py", line 787, in validate_at_facility
mop-546f67dd66-2fbfk mop response = obs_module().validate_observation(self.observation_payload())
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/tom_observations/facilities/ocs.py", line 1350, in observation_payload
mop-546f67dd66-2fbfk mop payload = self._expand_cadence_request(payload)
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/tom_observations/facilities/ocs.py", line 913, in _expand_cadence_request
mop-546f67dd66-2fbfk mop response = make_request(
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/tom_observations/facilities/ocs.py", line 186, in make_request
mop-546f67dd66-2fbfk mop response.raise_for_status()
mop-546f67dd66-2fbfk mop File "/mop/.venv/lib/python3.10/site-packages/requests/models.py", line 1026, in raise_for_status
mop-546f67dd66-2fbfk mop raise HTTPError(http_error_msg, response=self)
mop-546f67dd66-2fbfk mop requests.exceptions.HTTPError: 502 Server Error: Bad Gateway for url: https://observe.lco.global/api/requestgroups/cadence/
mo