diff --git a/tom_observations/cadences/retry_failed_observations.py b/tom_observations/cadences/retry_failed_observations.py index fc15217bb..a65640bf4 100644 --- a/tom_observations/cadences/retry_failed_observations.py +++ b/tom_observations/cadences/retry_failed_observations.py @@ -1,10 +1,13 @@ from datetime import timedelta from dateutil.parser import parse +import logging from tom_observations.cadence import BaseCadenceForm, CadenceStrategy -from tom_observations.models import ObservationRecord +from tom_observations.models import ObservationRecord, DynamicCadence from tom_observations.facility import get_service_class +logger = logging.getLogger(__name__) + class RetryFailedObservationsForm(BaseCadenceForm): pass @@ -23,35 +26,60 @@ class RetryFailedObservationsStrategy(CadenceStrategy): form = RetryFailedObservationsForm def run(self): - failed_observations = [obsr for obsr - in self.dynamic_cadence.observation_group.observation_records.all() - if obsr.failed] - new_observations = [] - for obs in failed_observations: - observation_payload = obs.parameters - facility = get_service_class(obs.facility)() - start_keyword, end_keyword = facility.get_start_end_keywords() + records = self.dynamic_cadence.observation_group.observation_records.all().order_by('-created') + last_obs = records.first() + + if not last_obs: + return + + facility_class = get_service_class(last_obs.facility) + facility = facility_class() + start_keyword, end_keyword = facility.get_start_end_keywords() + facility.update_observation_status(last_obs.observation_id) + last_obs.refresh_from_db() + + if not last_obs.terminal: #observation is still pending, do nothing + return + + elif not last_obs.failed: #observation succeeded + self.dynamic_cadence.active = False + self.dynamic_cadence.save() + return 'COMPLETED' + + else: #observation failed, submit a new one + observation_payload = last_obs.parameters.copy() + observation_payload = self.advance_window( observation_payload, start_keyword=start_keyword, end_keyword=end_keyword ) - obs_type = obs.parameters.get('observation_type', None) - form = facility.get_form(obs_type)(data=observation_payload) - form.is_valid() - observation_ids = facility.submit_observation(form.observation_payload()) + + obs_type = observation_payload.get('observation_type') + form = facility.get_form(obs_type)(observation_payload) + + if not form.is_valid(): + logger.error(msg=f'Unable to submit next cadenced observation: {form.errors}') + raise Exception(f'Unable to submit next cadenced observation: {form.errors}') + observation_ids = facility.submit_observation(form.observation_payload()) + new_observations = [] + for observation_id in observation_ids: - # Create Observation record record = ObservationRecord.objects.create( - target=obs.target, + target=last_obs.target, facility=facility.name, parameters=observation_payload, observation_id=observation_id ) self.dynamic_cadence.observation_group.observation_records.add(record) - self.dynamic_cadence.observation_group.save() new_observations.append(record) - return new_observations + self.dynamic_cadence.observation_group.save() + + for obsr in new_observations: + facility.update_observation_status(obsr.observation_id) + obsr.refresh_from_db() + + return new_observations def advance_window(self, observation_payload, start_keyword='start', end_keyword='end'): cadence_frequency = self.dynamic_cadence.cadence_parameters.get('cadence_frequency') diff --git a/tom_observations/management/commands/runcadencestrategies.py b/tom_observations/management/commands/runcadencestrategies.py index 73adbbec8..2ce50529e 100644 --- a/tom_observations/management/commands/runcadencestrategies.py +++ b/tom_observations/management/commands/runcadencestrategies.py @@ -36,6 +36,10 @@ def handle(self, *args, **kwargs): continue if not new_observations: logger.log(msg=f'No changes from dynamic cadence {cg}', level=logging.INFO) + elif new_observations == 'COMPLETED': + logger.log(msg=f'''Single observation obtained for {cg}, + no new observation submitted.''', + level=logging.INFO) else: logger.log(msg=f'''Cadence update completed for dynamic cadence {cg}, {len(new_observations)} new observations created.''', diff --git a/tom_observations/tests/test_cadence.py b/tom_observations/tests/test_cadence.py index 66cf49511..e6e6ee207 100644 --- a/tom_observations/tests/test_cadence.py +++ b/tom_observations/tests/test_cadence.py @@ -59,7 +59,10 @@ def setUp(self): cadence_strategy='Test Strategy', cadence_parameters={'cadence_frequency': 72}, active=True, observation_group=self.group) - def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4): + @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', return_value={'state': 'CANCELED', + 'scheduled_start': None, 'scheduled_end': None}) + def test_retry_when_failed_cadence_failed_obs(self, patch1, patch2, patch3, patch4, mock_get_obs_status, mock_validate_obs): + mock_validate_obs.return_value = {} num_records = self.group.observation_records.count() observing_record = self.group.observation_records.first() observing_record.status = 'CANCELED' @@ -76,6 +79,23 @@ def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4): parse(observing_record.parameters['start']), parse(new_records[0].parameters['start']) - timedelta(days=3) ) + + @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', return_value={'state': 'CANCELED', + 'scheduled_start': None, 'scheduled_end': None}) + def test_retry_when_failed_cadence_successful_obs(self, patch1, patch2, patch3, patch4, mock_get_obs_status, mock_validate_obs): + mock_validate_obs.return_value = {} + observing_record = self.group.observation_records.first() + observing_record.status = 'COMPLETE' + observing_record.save() + + strategy = RetryFailedObservationsStrategy(self.dynamic_cadence) + new_records = strategy.run() + self.group.refresh_from_db() + # Make sure the candence returned 'COMPLETED' + self.assertEqual(new_records, 'COMPLETED') + # Make sure the dynamic cadence was turned off + self.assertEqual(self.dynamic_cadence.active, False) + @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', return_value={'state': 'CANCELED', 'scheduled_start': None, 'scheduled_end': None})