From 5df7bbd10afaf2abeff25ecbda943c71226e53c8 Mon Sep 17 00:00:00 2001 From: Moira Andrews Date: Wed, 18 Feb 2026 13:51:48 -0800 Subject: [PATCH 1/9] add a check and update for observation status --- tom_observations/cadences/retry_failed_observations.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tom_observations/cadences/retry_failed_observations.py b/tom_observations/cadences/retry_failed_observations.py index fc15217bb..157069494 100644 --- a/tom_observations/cadences/retry_failed_observations.py +++ b/tom_observations/cadences/retry_failed_observations.py @@ -2,7 +2,7 @@ from dateutil.parser import parse from tom_observations.cadence import BaseCadenceForm, CadenceStrategy -from tom_observations.models import ObservationRecord +from tom_observations.models import ObservationRecord, DynamicCadence from tom_observations.facility import get_service_class @@ -23,6 +23,11 @@ class RetryFailedObservationsStrategy(CadenceStrategy): form = RetryFailedObservationsForm def run(self): + last_obs = self.dynamic_cadence.observation_group.observation_records.order_by('-created').first() + facility = get_service_class(last_obs.facility)() + facility.update_observation_status(last_obs.observation_id) # Updates the DB record + last_obs.refresh_from_db() + failed_observations = [obsr for obsr in self.dynamic_cadence.observation_group.observation_records.all() if obsr.failed] From 0c2c80dd1753ee242a20b27b5727d4411f926c60 Mon Sep 17 00:00:00 2001 From: Moira Andrews Date: Wed, 18 Feb 2026 13:52:12 -0800 Subject: [PATCH 2/9] update dynamic cadence on obs complete --- tom_observations/cadences/retry_failed_observations.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tom_observations/cadences/retry_failed_observations.py b/tom_observations/cadences/retry_failed_observations.py index 157069494..413ff1c33 100644 --- a/tom_observations/cadences/retry_failed_observations.py +++ b/tom_observations/cadences/retry_failed_observations.py @@ -28,6 +28,12 @@ def run(self): facility.update_observation_status(last_obs.observation_id) # Updates the DB record last_obs.refresh_from_db() + if last_obs.status == 'COMPLETED': + obs_group = last_obs.observationgroup_set.first() + dynamic_cadence = DynamicCadence.objects.get(observation_group=obs_group) + dynamic_cadence.active = False + dynamic_cadence.save() + failed_observations = [obsr for obsr in self.dynamic_cadence.observation_group.observation_records.all() if obsr.failed] From 25298c3f7dc08e595b9db26f3e4b6a5571890027 Mon Sep 17 00:00:00 2001 From: Moira Andrews Date: Wed, 18 Feb 2026 13:52:54 -0800 Subject: [PATCH 3/9] remove excess whitespace --- tom_observations/cadences/retry_failed_observations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_observations/cadences/retry_failed_observations.py b/tom_observations/cadences/retry_failed_observations.py index 413ff1c33..e10be522e 100644 --- a/tom_observations/cadences/retry_failed_observations.py +++ b/tom_observations/cadences/retry_failed_observations.py @@ -27,7 +27,7 @@ def run(self): facility = get_service_class(last_obs.facility)() facility.update_observation_status(last_obs.observation_id) # Updates the DB record last_obs.refresh_from_db() - + if last_obs.status == 'COMPLETED': obs_group = last_obs.observationgroup_set.first() dynamic_cadence = DynamicCadence.objects.get(observation_group=obs_group) From 982fe86b19f3a68e3fc5180a54e62ad906df1205 Mon Sep 17 00:00:00 2001 From: Joey Chatelain Date: Wed, 4 Mar 2026 15:29:53 -0700 Subject: [PATCH 4/9] mock status check --- tom_observations/tests/test_cadence.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tom_observations/tests/test_cadence.py b/tom_observations/tests/test_cadence.py index 66cf49511..a106168ae 100644 --- a/tom_observations/tests/test_cadence.py +++ b/tom_observations/tests/test_cadence.py @@ -59,7 +59,9 @@ def setUp(self): cadence_strategy='Test Strategy', cadence_parameters={'cadence_frequency': 72}, active=True, observation_group=self.group) - def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4): + @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', return_value={'state': 'CANCELED', + 'scheduled_start': None, 'scheduled_end': None}) + def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4, mock_get_obs_status): num_records = self.group.observation_records.count() observing_record = self.group.observation_records.first() observing_record.status = 'CANCELED' From aed35bf7a9dd9cad0edd9aea4184aa320ca0a328 Mon Sep 17 00:00:00 2001 From: moira-andrews <85570657+moira-andrews@users.noreply.github.com> Date: Wed, 4 Mar 2026 15:07:02 -0800 Subject: [PATCH 5/9] Cleaning up logic and pulling changes from snex2 --- .../cadences/retry_failed_observations.py | 79 +++++++++++-------- 1 file changed, 46 insertions(+), 33 deletions(-) diff --git a/tom_observations/cadences/retry_failed_observations.py b/tom_observations/cadences/retry_failed_observations.py index e10be522e..0f6f25ec1 100644 --- a/tom_observations/cadences/retry_failed_observations.py +++ b/tom_observations/cadences/retry_failed_observations.py @@ -23,44 +23,57 @@ class RetryFailedObservationsStrategy(CadenceStrategy): form = RetryFailedObservationsForm def run(self): - last_obs = self.dynamic_cadence.observation_group.observation_records.order_by('-created').first() - facility = get_service_class(last_obs.facility)() - facility.update_observation_status(last_obs.observation_id) # Updates the DB record + records = self.dynamic_cadence.observation_group.observation_records.all().order_by('-created') + last_obs = records.first() + + if not last_obs: + return + + facility_class = get_service_class(last_obs.facility) + facility = facility_class() + facility.update_observation_status(last_obs.observation_id) last_obs.refresh_from_db() - if last_obs.status == 'COMPLETED': - obs_group = last_obs.observationgroup_set.first() - dynamic_cadence = DynamicCadence.objects.get(observation_group=obs_group) - dynamic_cadence.active = False - dynamic_cadence.save() + if not last_obs.terminal: + return + elif last_obs.status == 'COMPLETED': + self.dynamic_cadence.active = False + self.dynamic_cadence.save() + return + + if not last_obs.failed: + return + + observation_payload = last_obs.parameters.copy() - failed_observations = [obsr for obsr - in self.dynamic_cadence.observation_group.observation_records.all() - if obsr.failed] + start_keyword, end_keyword = facility.get_start_end_keywords() + observation_payload = self.advance_window( + observation_payload, start_keyword=start_keyword, end_keyword=end_keyword + ) + + obs_type = observation_payload.get('observation_type') + form = facility.get_form(obs_type)(observation_payload) + + if not form.is_valid(): + return + + observation_ids = facility.submit_observation(form.observation_payload()) new_observations = [] - for obs in failed_observations: - observation_payload = obs.parameters - facility = get_service_class(obs.facility)() - start_keyword, end_keyword = facility.get_start_end_keywords() - observation_payload = self.advance_window( - observation_payload, start_keyword=start_keyword, end_keyword=end_keyword + + for observation_id in observation_ids: + record = ObservationRecord.objects.create( + target=last_obs.target, + facility=facility.name, + parameters=observation_payload, + observation_id=observation_id ) - obs_type = obs.parameters.get('observation_type', None) - form = facility.get_form(obs_type)(data=observation_payload) - form.is_valid() - observation_ids = facility.submit_observation(form.observation_payload()) - - for observation_id in observation_ids: - # Create Observation record - record = ObservationRecord.objects.create( - target=obs.target, - facility=facility.name, - parameters=observation_payload, - observation_id=observation_id - ) - self.dynamic_cadence.observation_group.observation_records.add(record) - self.dynamic_cadence.observation_group.save() - new_observations.append(record) + self.dynamic_cadence.observation_group.observation_records.add(record) + new_observations.append(record) + + self.dynamic_cadence.observation_group.save() + + for obsr in new_observations: + facility.update_observation_status(obsr.observation_id) return new_observations From 834339d60dd62710d628df3c85b7881fa11632f2 Mon Sep 17 00:00:00 2001 From: Joey Chatelain Date: Wed, 4 Mar 2026 16:20:45 -0700 Subject: [PATCH 6/9] fix linting --- tom_observations/cadences/retry_failed_observations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tom_observations/cadences/retry_failed_observations.py b/tom_observations/cadences/retry_failed_observations.py index 0f6f25ec1..ecbf6ef82 100644 --- a/tom_observations/cadences/retry_failed_observations.py +++ b/tom_observations/cadences/retry_failed_observations.py @@ -2,7 +2,7 @@ from dateutil.parser import parse from tom_observations.cadence import BaseCadenceForm, CadenceStrategy -from tom_observations.models import ObservationRecord, DynamicCadence +from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class @@ -50,7 +50,7 @@ def run(self): observation_payload = self.advance_window( observation_payload, start_keyword=start_keyword, end_keyword=end_keyword ) - + obs_type = observation_payload.get('observation_type') form = facility.get_form(obs_type)(observation_payload) @@ -59,7 +59,7 @@ def run(self): observation_ids = facility.submit_observation(form.observation_payload()) new_observations = [] - + for observation_id in observation_ids: record = ObservationRecord.objects.create( target=last_obs.target, From c55dece2a16e9e106351aeeb7fd33cf164b1443c Mon Sep 17 00:00:00 2001 From: Joey Chatelain Date: Wed, 4 Mar 2026 16:22:44 -0700 Subject: [PATCH 7/9] missed one --- tom_observations/cadences/retry_failed_observations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_observations/cadences/retry_failed_observations.py b/tom_observations/cadences/retry_failed_observations.py index ecbf6ef82..264ad736b 100644 --- a/tom_observations/cadences/retry_failed_observations.py +++ b/tom_observations/cadences/retry_failed_observations.py @@ -53,7 +53,7 @@ def run(self): obs_type = observation_payload.get('observation_type') form = facility.get_form(obs_type)(observation_payload) - + if not form.is_valid(): return From 666c48d6185e3c547e94ad9147d4b9d082d93a89 Mon Sep 17 00:00:00 2001 From: Moira Andrews Date: Wed, 1 Apr 2026 17:23:10 -0400 Subject: [PATCH 8/9] updated logic and handling to be more direct and have runcadencestrategies display expected response --- .../cadences/retry_failed_observations.py | 78 ++++++++++--------- .../commands/runcadencestrategies.py | 4 + 2 files changed, 45 insertions(+), 37 deletions(-) diff --git a/tom_observations/cadences/retry_failed_observations.py b/tom_observations/cadences/retry_failed_observations.py index 264ad736b..a65640bf4 100644 --- a/tom_observations/cadences/retry_failed_observations.py +++ b/tom_observations/cadences/retry_failed_observations.py @@ -1,10 +1,13 @@ from datetime import timedelta from dateutil.parser import parse +import logging from tom_observations.cadence import BaseCadenceForm, CadenceStrategy -from tom_observations.models import ObservationRecord +from tom_observations.models import ObservationRecord, DynamicCadence from tom_observations.facility import get_service_class +logger = logging.getLogger(__name__) + class RetryFailedObservationsForm(BaseCadenceForm): pass @@ -31,51 +34,52 @@ def run(self): facility_class = get_service_class(last_obs.facility) facility = facility_class() + start_keyword, end_keyword = facility.get_start_end_keywords() facility.update_observation_status(last_obs.observation_id) last_obs.refresh_from_db() - if not last_obs.terminal: + if not last_obs.terminal: #observation is still pending, do nothing return - elif last_obs.status == 'COMPLETED': + + elif not last_obs.failed: #observation succeeded self.dynamic_cadence.active = False self.dynamic_cadence.save() - return - - if not last_obs.failed: - return - - observation_payload = last_obs.parameters.copy() - - start_keyword, end_keyword = facility.get_start_end_keywords() - observation_payload = self.advance_window( - observation_payload, start_keyword=start_keyword, end_keyword=end_keyword - ) - - obs_type = observation_payload.get('observation_type') - form = facility.get_form(obs_type)(observation_payload) - - if not form.is_valid(): - return + return 'COMPLETED' - observation_ids = facility.submit_observation(form.observation_payload()) - new_observations = [] + else: #observation failed, submit a new one + observation_payload = last_obs.parameters.copy() - for observation_id in observation_ids: - record = ObservationRecord.objects.create( - target=last_obs.target, - facility=facility.name, - parameters=observation_payload, - observation_id=observation_id + observation_payload = self.advance_window( + observation_payload, start_keyword=start_keyword, end_keyword=end_keyword ) - self.dynamic_cadence.observation_group.observation_records.add(record) - new_observations.append(record) - - self.dynamic_cadence.observation_group.save() - - for obsr in new_observations: - facility.update_observation_status(obsr.observation_id) - - return new_observations + + obs_type = observation_payload.get('observation_type') + form = facility.get_form(obs_type)(observation_payload) + + if not form.is_valid(): + logger.error(msg=f'Unable to submit next cadenced observation: {form.errors}') + raise Exception(f'Unable to submit next cadenced observation: {form.errors}') + + observation_ids = facility.submit_observation(form.observation_payload()) + new_observations = [] + + for observation_id in observation_ids: + record = ObservationRecord.objects.create( + target=last_obs.target, + facility=facility.name, + parameters=observation_payload, + observation_id=observation_id + ) + self.dynamic_cadence.observation_group.observation_records.add(record) + new_observations.append(record) + + self.dynamic_cadence.observation_group.save() + + for obsr in new_observations: + facility.update_observation_status(obsr.observation_id) + obsr.refresh_from_db() + + return new_observations def advance_window(self, observation_payload, start_keyword='start', end_keyword='end'): cadence_frequency = self.dynamic_cadence.cadence_parameters.get('cadence_frequency') diff --git a/tom_observations/management/commands/runcadencestrategies.py b/tom_observations/management/commands/runcadencestrategies.py index 73adbbec8..2ce50529e 100644 --- a/tom_observations/management/commands/runcadencestrategies.py +++ b/tom_observations/management/commands/runcadencestrategies.py @@ -36,6 +36,10 @@ def handle(self, *args, **kwargs): continue if not new_observations: logger.log(msg=f'No changes from dynamic cadence {cg}', level=logging.INFO) + elif new_observations == 'COMPLETED': + logger.log(msg=f'''Single observation obtained for {cg}, + no new observation submitted.''', + level=logging.INFO) else: logger.log(msg=f'''Cadence update completed for dynamic cadence {cg}, {len(new_observations)} new observations created.''', From 52743dc1ac8bf756657d9796e396a93619a6238f Mon Sep 17 00:00:00 2001 From: Moira Andrews Date: Wed, 1 Apr 2026 17:33:24 -0400 Subject: [PATCH 9/9] updated testing to follow the resume cadence strategy of setting no validation errors, added a test for a completed observation to turn off the cadence --- tom_observations/tests/test_cadence.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tom_observations/tests/test_cadence.py b/tom_observations/tests/test_cadence.py index a106168ae..e6e6ee207 100644 --- a/tom_observations/tests/test_cadence.py +++ b/tom_observations/tests/test_cadence.py @@ -61,7 +61,8 @@ def setUp(self): @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', return_value={'state': 'CANCELED', 'scheduled_start': None, 'scheduled_end': None}) - def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4, mock_get_obs_status): + def test_retry_when_failed_cadence_failed_obs(self, patch1, patch2, patch3, patch4, mock_get_obs_status, mock_validate_obs): + mock_validate_obs.return_value = {} num_records = self.group.observation_records.count() observing_record = self.group.observation_records.first() observing_record.status = 'CANCELED' @@ -78,6 +79,23 @@ def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4, mock_ge parse(observing_record.parameters['start']), parse(new_records[0].parameters['start']) - timedelta(days=3) ) + + @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', return_value={'state': 'CANCELED', + 'scheduled_start': None, 'scheduled_end': None}) + def test_retry_when_failed_cadence_successful_obs(self, patch1, patch2, patch3, patch4, mock_get_obs_status, mock_validate_obs): + mock_validate_obs.return_value = {} + observing_record = self.group.observation_records.first() + observing_record.status = 'COMPLETE' + observing_record.save() + + strategy = RetryFailedObservationsStrategy(self.dynamic_cadence) + new_records = strategy.run() + self.group.refresh_from_db() + # Make sure the candence returned 'COMPLETED' + self.assertEqual(new_records, 'COMPLETED') + # Make sure the dynamic cadence was turned off + self.assertEqual(self.dynamic_cadence.active, False) + @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', return_value={'state': 'CANCELED', 'scheduled_start': None, 'scheduled_end': None})