From cb4293c6e0ac67bba47fbe8d92878477c77af26d Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Thu, 2 Jan 2020 16:10:36 -0800 Subject: [PATCH 001/424] Basic drf setup with target viewset --- setup.py | 1 + tom_base/settings.py | 1 + tom_common/urls.py | 6 ++++++ tom_setup/templates/tom_setup/settings.tmpl | 1 + tom_targets/api_urls.py | 8 ++++++++ tom_targets/api_views.py | 9 +++++++++ tom_targets/serializers.py | 8 ++++++++ tom_targets/urls.py | 6 ++++++ 8 files changed, 40 insertions(+) create mode 100644 tom_targets/api_urls.py create mode 100644 tom_targets/api_views.py create mode 100644 tom_targets/serializers.py diff --git a/setup.py b/setup.py index 4ff599b95..aabb6acff 100644 --- a/setup.py +++ b/setup.py @@ -46,6 +46,7 @@ 'pillow', 'fits2image', 'specutils', + 'djangorestframework', 'dataclasses; python_version < "3.7"', ], extras_require={ diff --git a/tom_base/settings.py b/tom_base/settings.py index 8429129ed..e0ea21693 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -45,6 +45,7 @@ 'django_comments', 'bootstrap4', 'crispy_forms', + 'rest_framework', 'django_filters', 'django_gravatar', 'tom_targets', diff --git a/tom_common/urls.py b/tom_common/urls.py index 66f0957ed..f2132ad3a 100644 --- a/tom_common/urls.py +++ b/tom_common/urls.py @@ -24,6 +24,10 @@ from tom_common.views import UserListView, UserPasswordChangeView, UserCreateView, UserDeleteView, UserUpdateView from tom_common.views import CommentDeleteView, GroupCreateView, GroupUpdateView, GroupDeleteView +api_urlpatterns = [ + path('', include('tom_targets.api_urls')) +] + urlpatterns = [ path('', TemplateView.as_view(template_name='tom_common/index.html'), name='home'), path('targets/', include('tom_targets.urls', namespace='targets')), @@ -44,6 +48,8 @@ path('accounts/logout/', LogoutView.as_view(), name='logout'), path('comment//delete', CommentDeleteView.as_view(), name='comment-delete'), path('admin/', admin.site.urls), + path('api-auth/', include('rest_framework.urls')), + path('api/', include(api_urlpatterns)) # The static helper below only works in development see # https://docs.djangoproject.com/en/2.1/howto/static-files/#serving-files-uploaded-by-a-user-during-development ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/tom_setup/templates/tom_setup/settings.tmpl b/tom_setup/templates/tom_setup/settings.tmpl index 981d1e973..4615078a6 100644 --- a/tom_setup/templates/tom_setup/settings.tmpl +++ b/tom_setup/templates/tom_setup/settings.tmpl @@ -46,6 +46,7 @@ INSTALLED_APPS = [ 'django_comments', 'bootstrap4', 'crispy_forms', + 'rest_framework', 'django_filters', 'django_gravatar', 'tom_targets', diff --git a/tom_targets/api_urls.py b/tom_targets/api_urls.py new file mode 100644 index 000000000..5138640b3 --- /dev/null +++ b/tom_targets/api_urls.py @@ -0,0 +1,8 @@ +from rest_framework.routers import DefaultRouter + +from .api_views import TargetViewSet + +router = DefaultRouter() +router.register(r'targets', TargetViewSet, 'targets') + +urlpatterns = router.urls diff --git a/tom_targets/api_views.py b/tom_targets/api_views.py new file mode 100644 index 000000000..ac2a0f60a --- /dev/null +++ b/tom_targets/api_views.py @@ -0,0 +1,9 @@ +from rest_framework import viewsets + +from .serializers import TargetSerializer +from .models import Target + + +class TargetViewSet(viewsets.ModelViewSet): + queryset = Target.objects.all() + serializer_class = TargetSerializer diff --git a/tom_targets/serializers.py b/tom_targets/serializers.py new file mode 100644 index 000000000..09060067e --- /dev/null +++ b/tom_targets/serializers.py @@ -0,0 +1,8 @@ +from rest_framework import serializers +from .models import Target + + +class TargetSerializer(serializers.ModelSerializer): + class Meta: + model = Target + fields = '__all__' diff --git a/tom_targets/urls.py b/tom_targets/urls.py index 7c52d41dc..f261ca31f 100644 --- a/tom_targets/urls.py +++ b/tom_targets/urls.py @@ -1,8 +1,10 @@ from django.urls import path +from rest_framework.routers import DefaultRouter from .views import TargetCreateView, TargetUpdateView, TargetDetailView from .views import TargetDeleteView, TargetListView, TargetImportView, TargetExportView from .views import TargetGroupingView, TargetGroupingDeleteView, TargetGroupingCreateView, TargetAddRemoveGroupingView +from .api_views import TargetViewSet app_name = 'tom_targets' @@ -19,3 +21,7 @@ path('targetgrouping//delete/', TargetGroupingDeleteView.as_view(), name='delete-group'), path('targetgrouping/create/', TargetGroupingCreateView.as_view(), name='create-group') ] + +router = DefaultRouter() +router.register(r'targets', TargetViewSet, 'targets') +apiurlpatterns = router.urls From b46a0932b5738696a7624d256c8e8ca7d3b3a0c8 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Mon, 6 Jan 2020 16:40:08 -0800 Subject: [PATCH 002/424] Initial target api views --- tom_base/settings.py | 6 +++++ tom_targets/api_urls.py | 3 +++ tom_targets/api_views.py | 6 ++++- tom_targets/serializers.py | 38 ++++++++++++++++++++++++++++++- tom_targets/tests/factories.py | 26 ++++++++++++++++----- tom_targets/tests/test_api.py | 41 ++++++++++++++++++++++++++++++++++ 6 files changed, 113 insertions(+), 7 deletions(-) create mode 100644 tom_targets/tests/test_api.py diff --git a/tom_base/settings.py b/tom_base/settings.py index e0ea21693..5ee9d9223 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -245,6 +245,12 @@ HINTS_ENABLED = False HINT_LEVEL = 20 +REST_FRAMEWORK = { + 'TEST_REQUEST_DEFAULT_FORMAT': 'json', + 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', + 'PAGE_SIZE': 100 +} + try: from local_settings import * # noqa except ImportError: diff --git a/tom_targets/api_urls.py b/tom_targets/api_urls.py index 5138640b3..def9740bd 100644 --- a/tom_targets/api_urls.py +++ b/tom_targets/api_urls.py @@ -2,6 +2,9 @@ from .api_views import TargetViewSet + +app_name = 'api' + router = DefaultRouter() router.register(r'targets', TargetViewSet, 'targets') diff --git a/tom_targets/api_views.py b/tom_targets/api_views.py index ac2a0f60a..0c715fc0c 100644 --- a/tom_targets/api_views.py +++ b/tom_targets/api_views.py @@ -1,9 +1,13 @@ from rest_framework import viewsets +from guardian.mixins import PermissionListMixin from .serializers import TargetSerializer from .models import Target +from .filters import TargetFilter -class TargetViewSet(viewsets.ModelViewSet): +class TargetViewSet(PermissionListMixin, viewsets.ModelViewSet): queryset = Target.objects.all() serializer_class = TargetSerializer + filterset_class = TargetFilter + permission_required = 'tom_targets.view_target' diff --git a/tom_targets/serializers.py b/tom_targets/serializers.py index 09060067e..a611fe00e 100644 --- a/tom_targets/serializers.py +++ b/tom_targets/serializers.py @@ -1,8 +1,44 @@ from rest_framework import serializers -from .models import Target +from .models import Target, TargetExtra, TargetName + + +class TargetNameSerializer(serializers.ModelSerializer): + class Meta: + model = TargetName + fields = ('name',) + + +class TargetExtraSerializer(serializers.ModelSerializer): + class Meta: + model = TargetExtra + fields = ('key', 'value') class TargetSerializer(serializers.ModelSerializer): + targetextra_set = TargetExtraSerializer(many=True) + aliases = TargetNameSerializer(many=True) + class Meta: model = Target fields = '__all__' + + def create(self, validated_data): + """ + DRF requires explicitly handling writeable nested serializers, + here we pop the alias/tag data and save it using thier respective + serializers + """ + aliases = validated_data.pop('aliases', []) + targetextras = validated_data.pop('targetextra_set', []) + + target = Target.objects.create(**validated_data) + + tns = TargetNameSerializer(data=aliases, many=True) + if tns.is_valid(): + tns.save(target=target) + + tes = TargetExtraSerializer(data=targetextras, many=True) + if tes.is_valid(): + tes.save(target=target) + + return target diff --git a/tom_targets/tests/factories.py b/tom_targets/tests/factories.py index 808d49fcd..9ae657455 100644 --- a/tom_targets/tests/factories.py +++ b/tom_targets/tests/factories.py @@ -1,6 +1,21 @@ import factory -from tom_targets.models import Target, TargetName, TargetList +from tom_targets.models import Target, TargetName, TargetList, TargetExtra + + +class TargetExtraFactory(factory.django.DjangoModelFactory): + class Meta: + model = TargetExtra + + key = factory.Faker('pystr') + value = factory.Faker('pystr') + + +class TargetNameFactory(factory.django.DjangoModelFactory): + class Meta: + model = TargetName + + name = factory.Faker('pystr') class SiderealTargetFactory(factory.django.DjangoModelFactory): @@ -15,6 +30,9 @@ class Meta: pm_ra = factory.Faker('pyfloat') pm_dec = factory.Faker('pyfloat') + targetextra_set = factory.RelatedFactoryList(TargetExtraFactory, factory_related_name='target', size=3) + aliases = factory.RelatedFactoryList(TargetNameFactory, factory_related_name='target', size=2) + class NonSiderealTargetFactory(factory.django.DjangoModelFactory): class Meta: @@ -33,10 +51,8 @@ class Meta: ephemeris_epoch = factory.Faker('pyfloat') ephemeris_epoch_err = factory.Faker('pyfloat') - -class TargetNameFactory(factory.django.DjangoModelFactory): - class Meta: - model = TargetName + targetextra_set = factory.RelatedFactoryList(TargetExtraFactory, factory_related_name='target', size=3) + aliases = factory.RelatedFactoryList(TargetNameFactory, factory_related_name='target', size=2) class TargetGroupingFactory(factory.django.DjangoModelFactory): diff --git a/tom_targets/tests/test_api.py b/tom_targets/tests/test_api.py new file mode 100644 index 000000000..fb71e23da --- /dev/null +++ b/tom_targets/tests/test_api.py @@ -0,0 +1,41 @@ +from rest_framework.test import APITestCase +from rest_framework import status +from guardian.shortcuts import assign_perm +from django.urls import reverse +from django.contrib.auth.models import User + +from tom_targets.models import Target +from .factories import SiderealTargetFactory, NonSiderealTargetFactory + + +class TestTargetViewset(APITestCase): + def setUp(self): + user = User.objects.create(username='testuser') + self.client.force_login(user) + self.st = SiderealTargetFactory.create() + self.nst = NonSiderealTargetFactory.create() + assign_perm('tom_targets.view_target', user, self.st) + assign_perm('tom_targets.view_target', user, self.nst) + + def test_target_detail(self): + response = self.client.get(reverse('api:targets-detail', args=(self.st.id,))) + self.assertEqual(response.json()['name'], self.st.name) + + def test_target_create(self): + target_data = { + 'name': 'test_target_name_wtf', + 'type': Target.SIDEREAL, + 'ra': 123.456, + 'dec': -32.1, + 'targetextra_set': [ + {'key': 'foo', 'value': 5} + ], + 'aliases': [ + {'name': 'alternative name'} + ] + } + response = self.client.post(reverse('api:targets-list'), data=target_data) + print(str(response.content)) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(response.json()['name'], target_data['name']) + self.assertEqual(response.json()['aliases'][0]['name'], target_data['aliases'][0]['name']) From 73110e3375935ad485acca2f376734873e75ad57 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Tue, 7 Jan 2020 12:35:08 -0800 Subject: [PATCH 003/424] Docstrings --- .gitignore | 3 +++ tom_targets/api_urls.py | 3 +++ tom_targets/api_views.py | 3 +++ tom_targets/serializers.py | 7 +++++-- tom_targets/tests/test_api.py | 8 +++++++- 5 files changed, 21 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 8f1636a1c..a5e7ba9ed 100644 --- a/.gitignore +++ b/.gitignore @@ -111,3 +111,6 @@ ENV/ # Rope project settings .ropeproject + +# OSX Files +.DS_Store diff --git a/tom_targets/api_urls.py b/tom_targets/api_urls.py index def9740bd..0225feb0b 100644 --- a/tom_targets/api_urls.py +++ b/tom_targets/api_urls.py @@ -2,6 +2,9 @@ from .api_views import TargetViewSet +"""A url module specifically for api paths, seperate from +the rest so it can be included modularly. +""" app_name = 'api' diff --git a/tom_targets/api_views.py b/tom_targets/api_views.py index 0c715fc0c..d8f1a8891 100644 --- a/tom_targets/api_views.py +++ b/tom_targets/api_views.py @@ -7,6 +7,9 @@ class TargetViewSet(PermissionListMixin, viewsets.ModelViewSet): + """Viewset for Target objects. By default supports CRUD operations. + See the docs on viewsets: https://www.django-rest-framework.org/api-guide/viewsets/ + """ queryset = Target.objects.all() serializer_class = TargetSerializer filterset_class = TargetFilter diff --git a/tom_targets/serializers.py b/tom_targets/serializers.py index a611fe00e..c418b71c0 100644 --- a/tom_targets/serializers.py +++ b/tom_targets/serializers.py @@ -15,6 +15,10 @@ class Meta: class TargetSerializer(serializers.ModelSerializer): + """Target serializer responsbile for transforming models to/from + json (or other representations). See + https://www.django-rest-framework.org/api-guide/serializers/#modelserializer + """ targetextra_set = TargetExtraSerializer(many=True) aliases = TargetNameSerializer(many=True) @@ -23,8 +27,7 @@ class Meta: fields = '__all__' def create(self, validated_data): - """ - DRF requires explicitly handling writeable nested serializers, + """DRF requires explicitly handling writeable nested serializers, here we pop the alias/tag data and save it using thier respective serializers """ diff --git a/tom_targets/tests/test_api.py b/tom_targets/tests/test_api.py index fb71e23da..effba0fef 100644 --- a/tom_targets/tests/test_api.py +++ b/tom_targets/tests/test_api.py @@ -35,7 +35,13 @@ def test_target_create(self): ] } response = self.client.post(reverse('api:targets-list'), data=target_data) - print(str(response.content)) self.assertEqual(response.status_code, status.HTTP_201_CREATED) self.assertEqual(response.json()['name'], target_data['name']) self.assertEqual(response.json()['aliases'][0]['name'], target_data['aliases'][0]['name']) + + def test_target_detail_bad_permissions(self): + other_user = User.objects.create(username='otheruser') + self.client.force_login(other_user) + response = self.client.get(reverse('api:targets-detail', kwargs={'pk': self.st.id}), follow=True) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + self.assertEqual(response.json()['detail'], 'Not found.') From 0d747daf70191609ac195d844d60b3785c134a4f Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Tue, 7 Jan 2020 12:56:29 -0800 Subject: [PATCH 004/424] Add some more tests to illustrate how the api works --- tom_targets/tests/test_api.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/tom_targets/tests/test_api.py b/tom_targets/tests/test_api.py index effba0fef..0304f08ae 100644 --- a/tom_targets/tests/test_api.py +++ b/tom_targets/tests/test_api.py @@ -17,10 +17,21 @@ def setUp(self): assign_perm('tom_targets.view_target', user, self.st) assign_perm('tom_targets.view_target', user, self.nst) + def test_target_list(self): + response = self.client.get(reverse('api:targets-list')) + self.assertEqual(response.json()['count'], 2) + def test_target_detail(self): response = self.client.get(reverse('api:targets-detail', args=(self.st.id,))) self.assertEqual(response.json()['name'], self.st.name) + def test_target_detail_bad_permissions(self): + other_user = User.objects.create(username='otheruser') + self.client.force_login(other_user) + response = self.client.get(reverse('api:targets-detail', args=(self.st.id,)), follow=True) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + self.assertEqual(response.json()['detail'], 'Not found.') + def test_target_create(self): target_data = { 'name': 'test_target_name_wtf', @@ -39,9 +50,14 @@ def test_target_create(self): self.assertEqual(response.json()['name'], target_data['name']) self.assertEqual(response.json()['aliases'][0]['name'], target_data['aliases'][0]['name']) - def test_target_detail_bad_permissions(self): - other_user = User.objects.create(username='otheruser') - self.client.force_login(other_user) - response = self.client.get(reverse('api:targets-detail', kwargs={'pk': self.st.id}), follow=True) - self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) - self.assertEqual(response.json()['detail'], 'Not found.') + def test_target_update(self): + updates = {'ra': 123.456} + response = self.client.patch(reverse('api:targets-detail', args=(self.st.id,)), data=updates) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.st.refresh_from_db() + self.assertEqual(self.st.ra, updates['ra']) + + def test_target_delete(self): + response = self.client.delete(reverse('api:targets-detail', args=(self.st.id,))) + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + self.assertFalse(Target.objects.filter(pk=self.st.id).exists()) From 35b62593cb3e8953654115561226f3eb39ca7613 Mon Sep 17 00:00:00 2001 From: Austin Riba Date: Tue, 7 Jan 2020 14:36:31 -0800 Subject: [PATCH 005/424] Remove some dead code --- tom_targets/urls.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tom_targets/urls.py b/tom_targets/urls.py index f261ca31f..7c52d41dc 100644 --- a/tom_targets/urls.py +++ b/tom_targets/urls.py @@ -1,10 +1,8 @@ from django.urls import path -from rest_framework.routers import DefaultRouter from .views import TargetCreateView, TargetUpdateView, TargetDetailView from .views import TargetDeleteView, TargetListView, TargetImportView, TargetExportView from .views import TargetGroupingView, TargetGroupingDeleteView, TargetGroupingCreateView, TargetAddRemoveGroupingView -from .api_views import TargetViewSet app_name = 'tom_targets' @@ -21,7 +19,3 @@ path('targetgrouping//delete/', TargetGroupingDeleteView.as_view(), name='delete-group'), path('targetgrouping/create/', TargetGroupingCreateView.as_view(), name='create-group') ] - -router = DefaultRouter() -router.register(r'targets', TargetViewSet, 'targets') -apiurlpatterns = router.urls From b79acbb94e7ebe99d1d622adc8e4462e298ee927 Mon Sep 17 00:00:00 2001 From: fraserw Date: Wed, 12 Feb 2020 12:43:48 -0800 Subject: [PATCH 006/424] Adding missed files --- tom_targets/models.py | 23 +++- .../tom_targets/partials/target_data.html | 13 +- .../templates/tom_targets/target_list.html | 1 + tom_targets/templatetags/targets_extras.py | 35 +++++ tom_targets/urls.py | 3 +- tom_targets/utils.py | 130 +++++++++++++++++- tom_targets/views.py | 27 +++- 7 files changed, 221 insertions(+), 11 deletions(-) diff --git a/tom_targets/models.py b/tom_targets/models.py index 892a2efa0..3839929f3 100644 --- a/tom_targets/models.py +++ b/tom_targets/models.py @@ -17,19 +17,20 @@ NON_SIDEREAL_FIELDS = GLOBAL_TARGET_FIELDS + [ 'scheme', 'mean_anomaly', 'arg_of_perihelion', 'lng_asc_node', 'inclination', 'mean_daily_motion', 'semimajor_axis', 'eccentricity', 'epoch_of_elements', 'epoch_of_perihelion', 'ephemeris_period', 'ephemeris_period_err', - 'ephemeris_epoch', 'ephemeris_epoch_err', 'perihdist' + 'ephemeris_epoch', 'ephemeris_epoch_err', 'perihdist', 'centsite', 'eph_json' ] REQUIRED_SIDEREAL_FIELDS = ['ra', 'dec'] REQUIRED_NON_SIDEREAL_FIELDS = [ - 'scheme', 'epoch_of_elements', 'inclination', 'lng_asc_node', 'arg_of_perihelion', 'eccentricity', + 'scheme', 'epoch_of_elements', ] # Additional non-sidereal fields that are required for specific orbital element # schemes REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME = { - 'MPC_COMET': ['perihdist', 'epoch_of_perihelion'], - 'MPC_MINOR_PLANET': ['mean_anomaly', 'semimajor_axis'], - 'JPL_MAJOR_PLANET': ['mean_daily_motion', 'mean_anomaly', 'semimajor_axis'] + 'MPC_COMET': ['perihdist', 'epoch_of_perihelion', 'inclination', 'lng_asc_node', 'arg_of_perihelion', 'eccentricity'], + 'MPC_MINOR_PLANET': ['mean_anomaly', 'semimajor_axis', 'inclination', 'lng_asc_node', 'arg_of_perihelion', 'eccentricity'], + 'JPL_MAJOR_PLANET': ['mean_daily_motion', 'mean_anomaly', 'semimajor_axis', 'inclination', 'lng_asc_node', 'arg_of_perihelion', 'eccentricity'], + 'EPHEMERIS':['eph_json'] } @@ -120,6 +121,9 @@ class Target(models.Model): :param ephemeris_epoch_err: Days :type ephemeris_epoch_err: float + + :param eph_json: + : type eph_json: str """ SIDEREAL = 'SIDEREAL' @@ -129,7 +133,8 @@ class Target(models.Model): TARGET_SCHEMES = ( ('MPC_MINOR_PLANET', 'MPC Minor Planet'), ('MPC_COMET', 'MPC Comet'), - ('JPL_MAJOR_PLANET', 'JPL Major Planet') + ('JPL_MAJOR_PLANET', 'JPL Major Planet'), + ('EPHEMERIS', 'Custom Ephemeris') ) name = models.CharField( @@ -226,6 +231,12 @@ class Target(models.Model): perihdist = models.FloatField( null=True, blank=True, verbose_name='Perihelion Distance', help_text='AU' ) + centsite = models.CharField( + max_length=50, null=True, blank=True, verbose_name='Centre-Site Name', help_text='Observatory Site Code' + ) + eph_json = models.TextField( + null=True, blank=True, verbose_name='Ephemeris JSON', help_text='MJD in days, RA and Dec in degress' + ) class Meta: ordering = ('id',) diff --git a/tom_targets/templates/tom_targets/partials/target_data.html b/tom_targets/templates/tom_targets/partials/target_data.html index 4a2ef0aeb..0f558e14a 100644 --- a/tom_targets/templates/tom_targets/partials/target_data.html +++ b/tom_targets/templates/tom_targets/partials/target_data.html @@ -12,8 +12,17 @@ {% endfor %} {% for key, value in target.as_dict.items %} {% if value and key != 'name' %} -
{% verbose_name target key %}
-
{{ value|truncate_number }}
+ {% if key == 'eph_json' %} +
Typical RA
+
{{ value|eph_json_to_value_ra }}
+
Typical Dec
+
{{ value|eph_json_to_value_dec }}
+
At MJD
+
{{ value|eph_json_to_value_mjd }}
+ {% else %} +
{% verbose_name target key %}
+
{{ value|truncate_number }}
+ {% endif %} {% endif %} {% if key == 'ra' %}
 
diff --git a/tom_targets/templates/tom_targets/target_list.html b/tom_targets/templates/tom_targets/target_list.html index f958874b7..78e5e9b1e 100644 --- a/tom_targets/templates/tom_targets/target_list.html +++ b/tom_targets/templates/tom_targets/target_list.html @@ -14,6 +14,7 @@
Create a Target Import Targets + Import Ephemeris Catalog Search
Update Targets diff --git a/tom_targets/templatetags/targets_extras.py b/tom_targets/templatetags/targets_extras.py index a301c9a40..af53c11b4 100644 --- a/tom_targets/templatetags/targets_extras.py +++ b/tom_targets/templatetags/targets_extras.py @@ -10,6 +10,8 @@ from tom_targets.forms import TargetVisibilityForm from tom_observations.utils import get_sidereal_visibility +import json + register = template.Library() @@ -176,3 +178,36 @@ def aladin(target): Displays Aladin skyview of the given target. """ return {'target': target} + +@register.filter +def eph_json_to_value_ra(value): + """ + Returns the middle RA and Dec of the json_ephemeris + """ + eph_json = json.loads(value) + keys = list(eph_json.keys()) + k = keys[0] + l = len(eph_json[k][0]) + return( deg_to_sexigesimal(float(eph_json[k][int(l/2)]['R']),'hms')) + +@register.filter +def eph_json_to_value_dec(value): + """ + Returns the middle RA and Dec of the json_ephemeris + """ + eph_json = json.loads(value) + keys = list(eph_json.keys()) + k = keys[0] + l = len(eph_json[k][0]) + return( deg_to_sexigesimal(float(eph_json[k][int(l/2)]['D']),'dms')) + +@register.filter +def eph_json_to_value_mjd(value): + """ + Returns the middle RA and Dec of the json_ephemeris + """ + eph_json = json.loads(value) + keys = list(eph_json.keys()) + k = keys[0] + l = len(eph_json[k][0]) + return( float(eph_json[k][int(l/2)]['t'])) diff --git a/tom_targets/urls.py b/tom_targets/urls.py index 7c52d41dc..0bc88ee2c 100644 --- a/tom_targets/urls.py +++ b/tom_targets/urls.py @@ -1,7 +1,7 @@ from django.urls import path from .views import TargetCreateView, TargetUpdateView, TargetDetailView -from .views import TargetDeleteView, TargetListView, TargetImportView, TargetExportView +from .views import TargetDeleteView, TargetListView, TargetImportView, TargetImportEphemerisView, TargetExportView from .views import TargetGroupingView, TargetGroupingDeleteView, TargetGroupingCreateView, TargetAddRemoveGroupingView app_name = 'tom_targets' @@ -11,6 +11,7 @@ path('targetgrouping/', TargetGroupingView.as_view(), name='targetgrouping'), path('create/', TargetCreateView.as_view(), name='create'), path('import/', TargetImportView.as_view(), name='import'), + path('ephemeris-import/', TargetImportEphemerisView.as_view(), name='ephemeris-import'), path('export/', TargetExportView.as_view(), name='export'), path('add-remove-grouping/', TargetAddRemoveGroupingView.as_view(), name='add-remove-grouping'), path('/update/', TargetUpdateView.as_view(), name='update'), diff --git a/tom_targets/utils.py b/tom_targets/utils.py index 9408b1316..905f93318 100644 --- a/tom_targets/utils.py +++ b/tom_targets/utils.py @@ -3,7 +3,7 @@ import csv from .models import Target, TargetExtra, TargetName from io import StringIO - +import json # NOTE: This saves locally. To avoid this, create file buffer. # referenced https://www.codingforentrepreneurs.com/blog/django-queryset-to-csv-files-datasets/ @@ -93,3 +93,131 @@ def import_targets(targets): errors.append(error) return {'targets': targets, 'errors': errors} + +def import_ephemeris_target(stream): + """ + Reads in a custom ephemeris from provided file stream. + + Currently only reads in the first site-code ephemeris. + """ + + + #need to make robust to input date type + #need to make robuest to input coordinate type + + errors = [] + targets = [] + + + jpl_ra_key = 'R.A._____(ICRF)_____DEC' + jpl_jd_key = 'Date_________JDUT' + + eph = stream.getvalue().split('\n') + + num_sites = 0 + for i in range(len(eph)): + if 'Center-site name' in eph[i]: + num_sites+=1 + + if num_sites!=8: + errors.append(Warning('WARNING: Provided file does not have ephemerides for all 7 LCO sites.')) + + + eph_json = {} + end_ind = 0 + for ns in range(num_sites): + + centre_site_name = '' + name = 'custom' + jd_inds = None + ra_inds = None + loop_inds = [-1,-1] + for i in range(end_ind,len(eph)): + if 'Center-site name' in eph[i]: + s = eph[i].split(': ')[-1] + if 'Mauna Kea' in s: + centre_site_name = '568' + elif 'Haleakala' in s: + centre_site_name = 'T04' + elif 'McDonald' in s: + centre_site_name = '711' + elif 'Tololo' in s: + centre_site_name = 'W85' + elif 'Teide' in s: + centre_site_name = '954' + elif 'Sunderland' in s: + centre_site_name = 'K91' + elif 'Wise' in s: + centre_site_name = '097' + elif 'Siding Spring' in s: + centre_site_name = 'Q63' + else: + centre_site_name = s + + if 'Target body name' in eph[i]: + name = "-".join(eph[i].split(': ')[1].split('{source')[0].split()) + + if jpl_ra_key in eph[i] and jpl_jd_key in eph[i]: + ra_inds = [eph[i].index(jpl_ra_key),eph[i].index(jpl_ra_key)+len(jpl_ra_key)] + jd_inds = [eph[i].index(jpl_jd_key),eph[i].index(jpl_jd_key)+len(jpl_jd_key)] + + if '$$SOE' in eph[i]: + if ra_inds is not None and loop_inds[0]==-1: + loop_inds[0] = i+1 + if '$$EOE' in eph[i]: + if ra_inds is not None and loop_inds[0]!=-1: + loop_inds[1] = i + break + + end_ind = loop_inds[1]+1 + + #throw HTML screen of warning if I cannot find the coordinates or ephemerides + #here we will put a better error check and correctly thrown warning + #for now being lazy + if loop_inds == [-1,-1] or ra_inds is None or jd_inds is None: + errors.append(Exception('We were not able to understand that ephemeris file.')) + + mjds = [] + ras = [] + decs = [] + R = 0.0 + D = 0.0 + n = 0.0 + for i in range(loop_inds[0],loop_inds[1]): + mjds.append(str(float(eph[i][jd_inds[0]:jd_inds[1]])-2400000.5)) + s = eph[i][ra_inds[0]:ra_inds[1]].split() + r = 15.0*float(s[0])+float(s[1])/60.0+float(s[2])/3600.0 + ras.append("{:11.7f}".format(r)) + d = abs(float(s[3]))+float(s[4])/60.0+float(s[5])/3600.0 + if '-' in s[3]: + d*=-1.0 + decs.append("{:10.6f}".format(d)) + + R+=r + D+=d + n+=1.0 + + eph_json[centre_site_name] = [] + for i in range(len(ras)): + entry = {} + entry['t'] = mjds[i] + entry['R'] = ras[i] + entry['D'] = decs[i] + entry['dR'] = 0.0 + entry['dD'] = 0.0 + + eph_json[centre_site_name].append(entry) + + try: + target_fields = {} + target_fields['type'] = 'NON_SIDEREAL' + target_fields['scheme'] = 'EPHEMERIS' + target_fields['name'] = name + target_fields['eph_json'] = json.dumps(eph_json) + + target = Target.objects.create(**target_fields) + targets.append(target) + except Exception as e: + errors.append(str(e)) + + return {'targets': targets, 'errors': errors} diff --git a/tom_targets/views.py b/tom_targets/views.py index 776a936f8..52919dab0 100644 --- a/tom_targets/views.py +++ b/tom_targets/views.py @@ -29,7 +29,7 @@ from tom_targets.forms import ( SiderealTargetCreateForm, NonSiderealTargetCreateForm, TargetExtraFormset, TargetNamesFormset ) -from tom_targets.utils import import_targets, export_targets +from tom_targets.utils import import_targets, import_ephemeris_target, export_targets from tom_targets.filters import TargetFilter from tom_targets.groups import add_all_to_grouping, add_selected_to_grouping from tom_targets.groups import remove_all_from_grouping, remove_selected_from_grouping @@ -376,6 +376,31 @@ def post(self, request): return redirect(reverse('tom_targets:list')) +class TargetImportEphemerisView(LoginRequiredMixin, TemplateView): + """ + View that handles the import of targets from a .eph . Requires authentication. + """ + template_name = 'tom_targets/target_ephemeris_import.html' + + def post(self, request): + """ + Handles the POST requests to this view. Creates a StringIO object and passes it to ``import_ephemeris_targets``. + + :param request: the request object passed to this view + :type request: HTTPRequest + """ + eph_file = request.FILES['target_eph'] + eph_stream = StringIO(eph_file.read().decode('utf-8'), newline=None) + result = import_ephemeris_target(eph_stream) + messages.success( + request, + 'Targets created: {}'.format(len(result['targets'])) + ) + for error in result['errors']: + messages.warning(request, error) + return redirect(reverse('tom_targets:list')) + + class TargetExportView(TargetListView): """ View that handles the export of targets to a CSV. Only exports selected targets. From d3110556cb5e25a73b44219f873958367217a132 Mon Sep 17 00:00:00 2001 From: fraserw Date: Wed, 12 Feb 2020 14:13:53 -0800 Subject: [PATCH 007/424] adding missing epehemeris template --- .../tom_targets/target_ephemeris_import.eph | 1904 +++++++++++++++++ 1 file changed, 1904 insertions(+) create mode 100644 tom_targets/static/tom_targets/target_ephemeris_import.eph diff --git a/tom_targets/static/tom_targets/target_ephemeris_import.eph b/tom_targets/static/tom_targets/target_ephemeris_import.eph new file mode 100644 index 000000000..1bd033fc3 --- /dev/null +++ b/tom_targets/static/tom_targets/target_ephemeris_import.eph @@ -0,0 +1,1904 @@ +******************************************************************************* +JPL/HORIZONS 136472 Makemake (2005 FY9) 2020-Feb-11 14:59:30 +Rec #: 136472 (+COV) Soln.date: 2019-May-30_07:13:05 # obs: 1949 (1955-2019) + +IAU76/J2000 helio. ecliptic osc. elements (au, days, deg., period=Julian yrs): + + EPOCH= 2456685.5 ! 2014-Jan-28.00 (TDB) Residual RMS= .29893 + EC= .1578018175114501 QR= 38.44081962777469 TP= 2408107.5732444809 + OM= 79.3100646538354 W= 297.2844380342058 IN= 29.01129634081993 + A= 45.64343693332217 MA= 155.2657054431733 ADIST= 52.84605423886966 + PER= 308.37256 N= .003196219 ANGMOM= .114761145 + DAN= 41.50451 DDN= 47.97743 L= 19.8421916 + B= -25.532549 MOID= 37.54140091 TP= 1881-Jan-27.0732444809 + +Asteroid physical parameters (km, seconds, rotational period in hours): + GM= n.a. RAD= n.a. ROTPER= 22.48 + H= -.100 G= .150 B-V= n.a. + ALBEDO= n.a. STYP= n.a. + +ASTEROID comments: +1: soln ref.= JPL#91, OCC=2 +2: source=ORB +******************************************************************************* + + +******************************************************************************* +Ephemeris / WWW_USER Tue Feb 11 14:59:30 2020 Pasadena, USA / Horizons +******************************************************************************* +Target body name: 136472 Makemake (2005 FY9) {source: JPL#91} +Center body name: Earth (399) {source: DE431} +Center-site name: Mauna Kea +******************************************************************************* +Start time : A.D. 2020-Jan-30 00:00:00.0000 UT +Stop time : A.D. 2020-Feb-29 00:00:00.0000 UT +Step-size : 1440 minutes +******************************************************************************* +Target pole/equ : No model available +Target radii : (unavailable) +Center geodetic : 204.527800,19.8260847,4.2102393 {E-lon(deg),Lat(deg),Alt(km)} +Center cylindric: 204.527800,6006.35451,2151.0229 {E-lon(deg),Dxy(km),Dz(km)} +Center pole/equ : High-precision EOP model {East-longitude positive} +Center radii : 6378.1 x 6378.1 x 6356.8 km {Equator, meridian, pole} +Target primary : Sun +Vis. interferer : MOON (R_eq= 1737.400) km {source: DE431} +Rel. light bend : Sun, EARTH {source: DE431} +Rel. lght bnd GM: 1.3271E+11, 3.9860E+05 km^3/s^2 +Small-body perts: Yes {source: SB431-N16} +Atmos refraction: NO (AIRLESS) +RA format : HMS +Time format : JD +EOP file : eop.200210.p200503 +EOP coverage : DATA-BASED 1962-JAN-20 TO 2020-FEB-10. PREDICTS-> 2020-MAY-02 +Units conversion: 1 au= 149597870.700 km, c= 299792.458 km/s, 1 day= 86400.0 s +Table cut-offs 1: Elevation (-90.0deg=NO ),Airmass (>38.000=NO), Daylight (NO ) +Table cut-offs 2: Solar elongation ( 0.0,180.0=NO ),Local Hour Angle( 0.0=NO ) +Table cut-offs 3: RA/DEC angular rate ( 0.0=NO ) +******************************************************************************* +Initial IAU76/J2000 heliocentric ecliptic osculating elements (au, days, deg.): + EPOCH= 2456685.5 ! 2014-Jan-28.00 (TDB) Residual RMS= .29893 + EC= .1578018175114501 QR= 38.44081962777469 TP= 2408107.5732444809 + OM= 79.3100646538354 W= 297.2844380342058 IN= 29.01129634081993 + Equivalent ICRF heliocentric equatorial cartesian coordinates (au, au/d): + X=-4.594801534867052E+01 Y=-9.620475793810611E+00 Z= 2.316346263166737E+01 + VX=-2.210217587691426E-04 VY=-1.960899352961049E-03 VZ=-9.635666653763773E-04 +Asteroid physical parameters (km, seconds, rotational period in hours): + GM= n.a. RAD= n.a. ROTPER= 22.48 + H= -.100 G= .150 B-V= n.a. + ALBEDO= n.a. STYP= n.a. +*************************************************************************************************************************** +Date_________JDUT R.A._____(ICRF)_____DEC APmag delta deldot S-O-T /r S-T-O RA_3sigma DEC_3sigma +*************************************************************************************************************************** +$$SOE +2458878.500000000 *m 13 10 55.49 +23 20 00.8 17.24 52.0964446099905 -21.9976367 118.4783 /L 0.9464 0.184 0.134 +2458879.500000000 *m 13 10 54.41 +23 20 37.0 17.24 52.0836517068990 -21.7418719 119.3373 /L 0.9387 0.184 0.133 +2458880.500000000 *m 13 10 53.25 +23 21 13.5 17.23 52.0710113839675 -21.4793565 120.1932 /L 0.9310 0.183 0.133 +2458881.500000000 *m 13 10 52.02 +23 21 50.1 17.23 52.0585275377521 -21.2102653 121.0456 /L 0.9230 0.183 0.132 +2458882.500000000 *m 13 10 50.71 +23 22 26.8 17.23 52.0462039613305 -20.9347792 121.8945 /L 0.9148 0.183 0.132 +2458883.500000000 *m 13 10 49.33 +23 23 03.6 17.23 52.0340443403728 -20.6530863 122.7396 /L 0.9065 0.182 0.131 +2458884.500000000 *m 13 10 47.88 +23 23 40.6 17.23 52.0220522489986 -20.3653812 123.5808 /L 0.8981 0.182 0.131 +2458885.500000000 * 13 10 46.35 +23 24 17.6 17.23 52.0102311475133 -20.0718589 124.4178 /L 0.8895 0.181 0.130 +2458886.500000000 * 13 10 44.74 +23 24 54.7 17.23 51.9985843852066 -19.7727031 125.2505 /L 0.8807 0.181 0.130 +2458887.500000000 * 13 10 43.07 +23 25 31.9 17.22 51.9871152119603 -19.4680675 126.0785 /L 0.8719 0.180 0.129 +2458888.500000000 * 13 10 41.32 +23 26 09.1 17.22 51.9758268014836 -19.1580555 126.9018 /L 0.8628 0.180 0.129 +2458889.500000000 * 13 10 39.51 +23 26 46.4 17.22 51.9647222858040 -18.8427030 127.7200 /L 0.8537 0.179 0.128 +2458890.500000000 * 13 10 37.62 +23 27 23.7 17.22 51.9538047956317 -18.5219763 128.5330 /L 0.8445 0.179 0.128 +2458891.500000000 * 13 10 35.66 +23 28 01.0 17.22 51.9430774969079 -18.1957879 129.3404 /L 0.8351 0.178 0.127 +2458892.500000000 * 13 10 33.64 +23 28 38.3 17.22 51.9325436135004 -17.8640269 130.1420 /L 0.8257 0.177 0.127 +2458893.500000000 * 13 10 31.55 +23 29 15.6 17.22 51.9222064306642 -17.5265934 130.9375 /L 0.8161 0.177 0.126 +2458894.500000000 * 13 10 29.39 +23 29 52.9 17.21 51.9120692807985 -17.1834239 131.7265 /L 0.8065 0.176 0.126 +2458895.500000000 * 13 10 27.16 +23 30 30.1 17.21 51.9021355180121 -16.8345039 132.5088 /L 0.7968 0.175 0.126 +2458896.500000000 * 13 10 24.87 +23 31 07.3 17.21 51.8924084889643 -16.4798694 133.2839 /L 0.7871 0.175 0.125 +2458897.500000000 * 13 10 22.51 +23 31 44.5 17.21 51.8828915053709 -16.1196006 134.0515 /L 0.7773 0.174 0.125 +2458898.500000000 *m 13 10 20.09 +23 32 21.5 17.21 51.8735878206606 -15.7538141 134.8111 /L 0.7675 0.173 0.124 +2458899.500000000 *m 13 10 17.60 +23 32 58.5 17.21 51.8645006111004 -15.3826555 135.5624 /L 0.7576 0.173 0.124 +2458900.500000000 *m 13 10 15.05 +23 33 35.3 17.21 51.8556329607036 -15.0062941 136.3048 /L 0.7477 0.172 0.124 +2458901.500000000 *m 13 10 12.44 +23 34 12.1 17.20 51.8469878491023 -14.6249175 137.0379 /L 0.7379 0.171 0.123 +2458902.500000000 *m 13 10 09.77 +23 34 48.7 17.20 51.8385681418577 -14.2387286 137.7612 /L 0.7280 0.170 0.123 +2458903.500000000 *m 13 10 07.04 +23 35 25.1 17.20 51.8303765830087 -13.8479408 138.4741 /L 0.7181 0.169 0.123 +2458904.500000000 *m 13 10 04.26 +23 36 01.4 17.20 51.8224157898311 -13.4527750 139.1762 /L 0.7083 0.169 0.122 +2458905.500000000 *m 13 10 01.41 +23 36 37.5 17.20 51.8146882497194 -13.0534556 139.8667 /L 0.6986 0.168 0.122 +2458906.500000000 *m 13 09 58.51 +23 37 13.5 17.20 51.8071963188999 -12.6502081 140.5451 /L 0.6889 0.167 0.122 +2458907.500000000 *m 13 09 55.56 +23 37 49.2 17.20 51.7999422224342 -12.2432570 141.2108 /L 0.6793 0.166 0.121 +2458908.500000000 *m 13 09 52.55 +23 38 24.7 17.19 51.7929280548173 -11.8328258 141.8630 /L 0.6698 0.165 0.121 +$$EOE +*************************************************************************************************************************** +Column meaning: + +TIME + + Times PRIOR to 1962 are UT1, a mean-solar time closely related to the +prior but now-deprecated GMT. Times AFTER 1962 are in UTC, the current +civil or "wall-clock" time-scale. UTC is kept within 0.9 seconds of UT1 +using integer leap-seconds for 1972 and later years. + + Conversion from the internal Barycentric Dynamical Time (TDB) of solar +system dynamics to the non-uniform civil UT time-scale requested for output +has not been determined for UTC times after the next July or January 1st. +Therefore, the last known leap-second is used as a constant over future +intervals. + + Time tags refer to the UT time-scale conversion on Earth regardless of +observer location within the solar system, where clock rates may differ +due to the local gravity field and there is no precisely defined or adopted +"UT" analog timescale. + + Any 'b' symbol in the 1st-column denotes a B.C. date. First-column blank +(" ") denotes an A.D. date. Calendar dates prior to 1582-Oct-15 are in the +Julian calendar system. Later calendar dates are in the Gregorian system. + + NOTE: "n.a." in output means quantity "not available" at the print-time. + +SOLAR PRESENCE (OBSERVING SITE) + Time tag is followed by a blank, then a solar-presence symbol: + + '*' Daylight (refracted solar upper-limb on or above apparent horizon) + 'C' Civil twilight/dawn + 'N' Nautical twilight/dawn + 'A' Astronomical twilight/dawn + ' ' Night OR geocentric ephemeris + +LUNAR PRESENCE (OBSERVING SITE) + The solar-presence symbol is immediately followed by a lunar-presence symbol: + + 'm' Refracted upper-limb of Moon on or above apparent horizon + ' ' Refracted upper-limb of Moon below apparent horizon OR geocentric + ephemeris + +STATISTICAL UNCERTAINTIES + + Output includes formal +/- 3 standard-deviation statistical orbit uncertainty +quantities. There is a 99.7% chance the actual value is within given bounds. +These statistical calculations assume observational data errors are random. If +there are systematic biases (such as timing, reduction or star-catalog errors), +results can be optimistic. Because the epoch covariance is mapped using +linearized variational partial derivatives, results can also be optimistic for +times far from the solution epoch, particularly for objects having close +planetary encounters. + + R.A._____(ICRF)_____DEC = + Astrometric right ascension and declination of the target center with +respect to the observing site (coordinate origin) in the reference frame of +the planetary ephemeris (ICRF). Compensated for down-leg light-time delay +aberration. + + Units: RA in hours-minutes-seconds of time (HH MM SS.ff) + DEC in degrees-minutes-seconds of arc (sDD MN SC.f) + + APmag = + Asteroid's approximate apparent visual magnitude from the standard +IAU H-G magnitude relationship: + APmag = H + 5*log10(delta) + 5*log10(r) - 2.5*log10((1-G)*phi1 + G*phi2). +For solar phase angles > 90 deg, the error could exceed 1 magnitude. For phase +angles > 120 degrees, output values are rounded to the nearest integer to +indicate error could be large and unknown. + Units: MAGNITUDE + + delta deldot = + Range ("delta") and range-rate ("delta-dot") of target center with respect +to the observer at the instant light seen by the observer at print-time would +have left the target center (print-time minus down-leg light-time); the +distance traveled by a light ray emanating from the center of the target and +recorded by the observer at print-time. "deldot" is a projection of the +velocity vector along this ray, the light-time-corrected line-of-sight from +the coordinate center, and indicates relative motion. A positive "deldot" +means the target center is moving away from the observer (coordinate center). +A negative "deldot" means the target center is moving toward the observer. +Units: AU and KM/S + + S-O-T /r = + Sun-Observer-Target angle; target's apparent SOLAR ELONGATION seen from +the observer location at print-time. Angular units: DEGREES + + The '/r' column indicates the target's apparent position relative to +the Sun in the observer's sky, as described below: + + For an observing location on the surface of a rotating body +(considering its rotational sense): + + /T indicates target TRAILS Sun (evening sky; rises and sets AFTER Sun) + /L indicates target LEADS Sun (morning sky; rises and sets BEFORE Sun) + +For an observing point NOT on a rotating body (such as a spacecraft), the +"leading" and "trailing" condition is defined by the observer's +heliocentric orbital motion: if continuing in the observer's current +direction of heliocentric motion would encounter the target's apparent +longitude first, followed by the Sun's, the target LEADS the Sun as seen by +the observer. If the Sun's apparent longitude would be encountered first, +followed by the target's, the target TRAILS the Sun. + +NOTE: The S-O-T solar elongation angle is numerically the minimum +separation angle of the Sun and target in the sky in any direction. It +does NOT indicate the amount of separation in the leading or trailing +directions, which are defined in the equator of a spherical coordinate +system. + + S-T-O = + "S-T-O" is the Sun->Target->Observer angle; the interior vertex angle at +target center formed by a vector to the apparent center of the Sun at +reflection time on the target and the apparent vector to the observer at +print-time. Slightly different from true PHASE ANGLE (requestable separately) +at the few arcsecond level in that it includes stellar aberration on the +down-leg from target to observer. Units: DEGREES + + RA_3sigma DEC_3sigma = + Uncertainty in Right-Ascension and Declination. Output values are the formal ++/- 3 standard-deviations (sigmas) around nominal position. Units: ARCSECONDS + + + Computations by ... + Solar System Dynamics Group, Horizons On-Line Ephemeris System + 4800 Oak Grove Drive, Jet Propulsion Laboratory + Pasadena, CA 91109 USA + Information: http://ssd.jpl.nasa.gov/ + Connect : telnet://ssd.jpl.nasa.gov:6775 (via browser) + telnet ssd.jpl.nasa.gov 6775 (via command-line) + Author : Jon.D.Giorgini@jpl.nasa.gov + +********************************************************************************************************* +******************************************************************************* +JPL/HORIZONS 136472 Makemake (2005 FY9) 2020-Feb-11 13:53:47 +Rec #: 136472 (+COV) Soln.date: 2019-May-30_07:13:05 # obs: 1949 (1955-2019) + +IAU76/J2000 helio. ecliptic osc. elements (au, days, deg., period=Julian yrs): + + EPOCH= 2456685.5 ! 2014-Jan-28.00 (TDB) Residual RMS= .29893 + EC= .1578018175114501 QR= 38.44081962777469 TP= 2408107.5732444809 + OM= 79.3100646538354 W= 297.2844380342058 IN= 29.01129634081993 + A= 45.64343693332217 MA= 155.2657054431733 ADIST= 52.84605423886966 + PER= 308.37256 N= .003196219 ANGMOM= .114761145 + DAN= 41.50451 DDN= 47.97743 L= 19.8421916 + B= -25.532549 MOID= 37.54140091 TP= 1881-Jan-27.0732444809 + +Asteroid physical parameters (km, seconds, rotational period in hours): + GM= n.a. RAD= n.a. ROTPER= 22.48 + H= -.100 G= .150 B-V= n.a. + ALBEDO= n.a. STYP= n.a. + +ASTEROID comments: +1: soln ref.= JPL#91, OCC=2 +2: source=ORB +******************************************************************************* + + +******************************************************************************* +Ephemeris / WWW_USER Tue Feb 11 13:53:47 2020 Pasadena, USA / Horizons +******************************************************************************* +Target body name: 136472 Makemake (2005 FY9) {source: JPL#91} +Center body name: Earth (399) {source: DE431} +Center-site name: Haleakala-LCOGT OGG B #2 +******************************************************************************* +Start time : A.D. 2020-Jan-30 00:00:00.0000 UT +Stop time : A.D. 2020-Feb-29 00:00:00.0000 UT +Step-size : 1440 minutes +******************************************************************************* +Target pole/equ : No model available +Target radii : (unavailable) +Center geodetic : 203.742500,20.7069361,3.0658029 {E-lon(deg),Lat(deg),Alt(km)} +Center cylindric: 203.742500,5971.48324,2242.1579 {E-lon(deg),Dxy(km),Dz(km)} +Center pole/equ : High-precision EOP model {East-longitude positive} +Center radii : 6378.1 x 6378.1 x 6356.8 km {Equator, meridian, pole} +Target primary : Sun +Vis. interferer : MOON (R_eq= 1737.400) km {source: DE431} +Rel. light bend : Sun, EARTH {source: DE431} +Rel. lght bnd GM: 1.3271E+11, 3.9860E+05 km^3/s^2 +Small-body perts: Yes {source: SB431-N16} +Atmos refraction: NO (AIRLESS) +RA format : HMS +Time format : JD +EOP file : eop.200210.p200503 +EOP coverage : DATA-BASED 1962-JAN-20 TO 2020-FEB-10. PREDICTS-> 2020-MAY-02 +Units conversion: 1 au= 149597870.700 km, c= 299792.458 km/s, 1 day= 86400.0 s +Table cut-offs 1: Elevation (-90.0deg=NO ),Airmass (>38.000=NO), Daylight (NO ) +Table cut-offs 2: Solar elongation ( 0.0,180.0=NO ),Local Hour Angle( 0.0=NO ) +Table cut-offs 3: RA/DEC angular rate ( 0.0=NO ) +******************************************************************************* +Initial IAU76/J2000 heliocentric ecliptic osculating elements (au, days, deg.): + EPOCH= 2456685.5 ! 2014-Jan-28.00 (TDB) Residual RMS= .29893 + EC= .1578018175114501 QR= 38.44081962777469 TP= 2408107.5732444809 + OM= 79.3100646538354 W= 297.2844380342058 IN= 29.01129634081993 + Equivalent ICRF heliocentric equatorial cartesian coordinates (au, au/d): + X=-4.594801534867052E+01 Y=-9.620475793810611E+00 Z= 2.316346263166737E+01 + VX=-2.210217587691426E-04 VY=-1.960899352961049E-03 VZ=-9.635666653763773E-04 +Asteroid physical parameters (km, seconds, rotational period in hours): + GM= n.a. RAD= n.a. ROTPER= 22.48 + H= -.100 G= .150 B-V= n.a. + ALBEDO= n.a. STYP= n.a. +*************************************************************************************************************************** +Date_________JDUT R.A._____(ICRF)_____DEC APmag delta deldot S-O-T /r S-T-O RA_3sigma DEC_3sigma +*************************************************************************************************************************** +$$SOE +2458878.500000000 *m 13 10 55.49 +23 20 00.8 17.24 52.0964438612435 -21.9954126 118.4783 /L 0.9464 0.184 0.134 +2458879.500000000 *m 13 10 54.41 +23 20 37.0 17.24 52.0836509616908 -21.7395525 119.3374 /L 0.9387 0.184 0.133 +2458880.500000000 *m 13 10 53.25 +23 21 13.5 17.23 52.0710106424492 -21.4769424 120.1932 /L 0.9310 0.183 0.133 +2458881.500000000 *m 13 10 52.02 +23 21 50.1 17.23 52.0585268000740 -21.2077572 121.0456 /L 0.9230 0.183 0.132 +2458882.500000000 *m 13 10 50.71 +23 22 26.8 17.23 52.0462032276418 -20.9321779 121.8945 /L 0.9148 0.183 0.132 +2458883.500000000 *m 13 10 49.33 +23 23 03.6 17.23 52.0340436108214 -20.6503926 122.7396 /L 0.9065 0.182 0.131 +2458884.500000000 *m 13 10 47.88 +23 23 40.6 17.23 52.0220515237316 -20.3625958 123.5808 /L 0.8981 0.182 0.131 +2458885.500000000 * 13 10 46.35 +23 24 17.6 17.23 52.0102304266763 -20.0689827 124.4178 /L 0.8895 0.181 0.130 +2458886.500000000 * 13 10 44.74 +23 24 54.7 17.23 51.9985836689442 -19.7697368 125.2505 /L 0.8807 0.181 0.130 +2458887.500000000 * 13 10 43.07 +23 25 31.9 17.22 51.9871145004155 -19.4650121 126.0786 /L 0.8719 0.180 0.129 +2458888.500000000 * 13 10 41.32 +23 26 09.1 17.22 51.9758260947983 -19.1549119 126.9018 /L 0.8628 0.180 0.129 +2458889.500000000 * 13 10 39.51 +23 26 46.4 17.22 51.9647215841187 -18.8394721 127.7201 /L 0.8537 0.179 0.128 +2458890.500000000 * 13 10 37.62 +23 27 23.7 17.22 51.9538040990853 -18.5186591 128.5330 /L 0.8445 0.179 0.128 +2458891.500000000 * 13 10 35.66 +23 28 01.0 17.22 51.9430768056378 -18.1923853 129.3405 /L 0.8351 0.178 0.127 +2458892.500000000 * 13 10 33.64 +23 28 38.3 17.22 51.9325429276428 -17.8605401 130.1421 /L 0.8257 0.177 0.127 +2458893.500000000 * 13 10 31.55 +23 29 15.6 17.22 51.9222057503535 -17.5230233 130.9375 /L 0.8161 0.177 0.126 +2458894.500000000 * 13 10 29.39 +23 29 52.9 17.21 51.9120686061678 -17.1797715 131.7266 /L 0.8065 0.176 0.126 +2458895.500000000 * 13 10 27.16 +23 30 30.1 17.21 51.9021348491925 -16.8307704 132.5088 /L 0.7968 0.175 0.126 +2458896.500000000 * 13 10 24.87 +23 31 07.3 17.21 51.8924078260855 -16.4760559 133.2839 /L 0.7871 0.175 0.125 +2458897.500000000 * 13 10 22.51 +23 31 44.5 17.21 51.8828908485607 -16.1157082 134.0515 /L 0.7773 0.174 0.125 +2458898.500000000 *m 13 10 20.09 +23 32 21.5 17.21 51.8735871700451 -15.7498440 134.8111 /L 0.7675 0.173 0.124 +2458899.500000000 *m 13 10 17.60 +23 32 58.5 17.21 51.8644999668039 -15.3786089 135.5624 /L 0.7576 0.173 0.124 +2458900.500000000 *m 13 10 15.05 +23 33 35.3 17.21 51.8556323228483 -15.0021721 136.3048 /L 0.7477 0.172 0.124 +2458901.500000000 *m 13 10 12.44 +23 34 12.1 17.20 51.8469872178088 -14.6207215 137.0379 /L 0.7379 0.171 0.123 +2458902.500000000 *m 13 10 09.77 +23 34 48.7 17.20 51.8385675172443 -14.2344597 137.7612 /L 0.7280 0.170 0.123 +2458903.500000000 *m 13 10 07.04 +23 35 25.1 17.20 51.8303759651921 -13.8436004 138.4742 /L 0.7181 0.169 0.123 +2458904.500000000 *m 13 10 04.26 +23 36 01.4 17.20 51.8224151789256 -13.4483644 139.1762 /L 0.7083 0.169 0.122 +2458905.500000000 *m 13 10 01.41 +23 36 37.5 17.20 51.8146876458376 -13.0489762 139.8667 /L 0.6986 0.168 0.122 +2458906.500000000 *m 13 09 58.51 +23 37 13.5 17.20 51.8071957221522 -12.6456611 140.5451 /L 0.6889 0.167 0.122 +2458907.500000000 *m 13 09 55.56 +23 37 49.2 17.20 51.7999416329288 -12.2386438 141.2108 /L 0.6793 0.166 0.121 +2458908.500000000 *m 13 09 52.55 +23 38 24.7 17.19 51.7929274726603 -11.8281478 141.8631 /L 0.6698 0.165 0.121 +$$EOE +*************************************************************************************************************************** +Column meaning: + +TIME + + Times PRIOR to 1962 are UT1, a mean-solar time closely related to the +prior but now-deprecated GMT. Times AFTER 1962 are in UTC, the current +civil or "wall-clock" time-scale. UTC is kept within 0.9 seconds of UT1 +using integer leap-seconds for 1972 and later years. + + Conversion from the internal Barycentric Dynamical Time (TDB) of solar +system dynamics to the non-uniform civil UT time-scale requested for output +has not been determined for UTC times after the next July or January 1st. +Therefore, the last known leap-second is used as a constant over future +intervals. + + Time tags refer to the UT time-scale conversion on Earth regardless of +observer location within the solar system, where clock rates may differ +due to the local gravity field and there is no precisely defined or adopted +"UT" analog timescale. + + Any 'b' symbol in the 1st-column denotes a B.C. date. First-column blank +(" ") denotes an A.D. date. Calendar dates prior to 1582-Oct-15 are in the +Julian calendar system. Later calendar dates are in the Gregorian system. + + NOTE: "n.a." in output means quantity "not available" at the print-time. + +SOLAR PRESENCE (OBSERVING SITE) + Time tag is followed by a blank, then a solar-presence symbol: + + '*' Daylight (refracted solar upper-limb on or above apparent horizon) + 'C' Civil twilight/dawn + 'N' Nautical twilight/dawn + 'A' Astronomical twilight/dawn + ' ' Night OR geocentric ephemeris + +LUNAR PRESENCE (OBSERVING SITE) + The solar-presence symbol is immediately followed by a lunar-presence symbol: + + 'm' Refracted upper-limb of Moon on or above apparent horizon + ' ' Refracted upper-limb of Moon below apparent horizon OR geocentric + ephemeris + +STATISTICAL UNCERTAINTIES + + Output includes formal +/- 3 standard-deviation statistical orbit uncertainty +quantities. There is a 99.7% chance the actual value is within given bounds. +These statistical calculations assume observational data errors are random. If +there are systematic biases (such as timing, reduction or star-catalog errors), +results can be optimistic. Because the epoch covariance is mapped using +linearized variational partial derivatives, results can also be optimistic for +times far from the solution epoch, particularly for objects having close +planetary encounters. + + R.A._____(ICRF)_____DEC = + Astrometric right ascension and declination of the target center with +respect to the observing site (coordinate origin) in the reference frame of +the planetary ephemeris (ICRF). Compensated for down-leg light-time delay +aberration. + + Units: RA in hours-minutes-seconds of time (HH MM SS.ff) + DEC in degrees-minutes-seconds of arc (sDD MN SC.f) + + APmag = + Asteroid's approximate apparent visual magnitude from the standard +IAU H-G magnitude relationship: + APmag = H + 5*log10(delta) + 5*log10(r) - 2.5*log10((1-G)*phi1 + G*phi2). +For solar phase angles > 90 deg, the error could exceed 1 magnitude. For phase +angles > 120 degrees, output values are rounded to the nearest integer to +indicate error could be large and unknown. + Units: MAGNITUDE + + delta deldot = + Range ("delta") and range-rate ("delta-dot") of target center with respect +to the observer at the instant light seen by the observer at print-time would +have left the target center (print-time minus down-leg light-time); the +distance traveled by a light ray emanating from the center of the target and +recorded by the observer at print-time. "deldot" is a projection of the +velocity vector along this ray, the light-time-corrected line-of-sight from +the coordinate center, and indicates relative motion. A positive "deldot" +means the target center is moving away from the observer (coordinate center). +A negative "deldot" means the target center is moving toward the observer. +Units: AU and KM/S + + S-O-T /r = + Sun-Observer-Target angle; target's apparent SOLAR ELONGATION seen from +the observer location at print-time. Angular units: DEGREES + + The '/r' column indicates the target's apparent position relative to +the Sun in the observer's sky, as described below: + + For an observing location on the surface of a rotating body +(considering its rotational sense): + + /T indicates target TRAILS Sun (evening sky; rises and sets AFTER Sun) + /L indicates target LEADS Sun (morning sky; rises and sets BEFORE Sun) + +For an observing point NOT on a rotating body (such as a spacecraft), the +"leading" and "trailing" condition is defined by the observer's +heliocentric orbital motion: if continuing in the observer's current +direction of heliocentric motion would encounter the target's apparent +longitude first, followed by the Sun's, the target LEADS the Sun as seen by +the observer. If the Sun's apparent longitude would be encountered first, +followed by the target's, the target TRAILS the Sun. + +NOTE: The S-O-T solar elongation angle is numerically the minimum +separation angle of the Sun and target in the sky in any direction. It +does NOT indicate the amount of separation in the leading or trailing +directions, which are defined in the equator of a spherical coordinate +system. + + S-T-O = + "S-T-O" is the Sun->Target->Observer angle; the interior vertex angle at +target center formed by a vector to the apparent center of the Sun at +reflection time on the target and the apparent vector to the observer at +print-time. Slightly different from true PHASE ANGLE (requestable separately) +at the few arcsecond level in that it includes stellar aberration on the +down-leg from target to observer. Units: DEGREES + + RA_3sigma DEC_3sigma = + Uncertainty in Right-Ascension and Declination. Output values are the formal ++/- 3 standard-deviations (sigmas) around nominal position. Units: ARCSECONDS + + + Computations by ... + Solar System Dynamics Group, Horizons On-Line Ephemeris System + 4800 Oak Grove Drive, Jet Propulsion Laboratory + Pasadena, CA 91109 USA + Information: http://ssd.jpl.nasa.gov/ + Connect : telnet://ssd.jpl.nasa.gov:6775 (via browser) + telnet ssd.jpl.nasa.gov 6775 (via command-line) + Author : Jon.D.Giorgini@jpl.nasa.gov + +************************************************************************************************ +******************************************************************************* +JPL/HORIZONS 136472 Makemake (2005 FY9) 2020-Feb-11 13:53:47 +Rec #: 136472 (+COV) Soln.date: 2019-May-30_07:13:05 # obs: 1949 (1955-2019) + +IAU76/J2000 helio. ecliptic osc. elements (au, days, deg., period=Julian yrs): + + EPOCH= 2456685.5 ! 2014-Jan-28.00 (TDB) Residual RMS= .29893 + EC= .1578018175114501 QR= 38.44081962777469 TP= 2408107.5732444809 + OM= 79.3100646538354 W= 297.2844380342058 IN= 29.01129634081993 + A= 45.64343693332217 MA= 155.2657054431733 ADIST= 52.84605423886966 + PER= 308.37256 N= .003196219 ANGMOM= .114761145 + DAN= 41.50451 DDN= 47.97743 L= 19.8421916 + B= -25.532549 MOID= 37.54140091 TP= 1881-Jan-27.0732444809 + +Asteroid physical parameters (km, seconds, rotational period in hours): + GM= n.a. RAD= n.a. ROTPER= 22.48 + H= -.100 G= .150 B-V= n.a. + ALBEDO= n.a. STYP= n.a. + +ASTEROID comments: +1: soln ref.= JPL#91, OCC=2 +2: source=ORB +******************************************************************************* + + +******************************************************************************* +Ephemeris / WWW_USER Tue Feb 11 13:53:47 2020 Pasadena, USA / Horizons +******************************************************************************* +Target body name: 136472 Makemake (2005 FY9) {source: JPL#91} +Center body name: Earth (399) {source: DE431} +Center-site name: Haleakala-LCOGT OGG B #2 +******************************************************************************* +Start time : A.D. 2020-Jan-30 00:00:00.0000 UT +Stop time : A.D. 2020-Feb-29 00:00:00.0000 UT +Step-size : 1440 minutes +******************************************************************************* +Target pole/equ : No model available +Target radii : (unavailable) +Center geodetic : 203.742500,20.7069361,3.0658029 {E-lon(deg),Lat(deg),Alt(km)} +Center cylindric: 203.742500,5971.48324,2242.1579 {E-lon(deg),Dxy(km),Dz(km)} +Center pole/equ : High-precision EOP model {East-longitude positive} +Center radii : 6378.1 x 6378.1 x 6356.8 km {Equator, meridian, pole} +Target primary : Sun +Vis. interferer : MOON (R_eq= 1737.400) km {source: DE431} +Rel. light bend : Sun, EARTH {source: DE431} +Rel. lght bnd GM: 1.3271E+11, 3.9860E+05 km^3/s^2 +Small-body perts: Yes {source: SB431-N16} +Atmos refraction: NO (AIRLESS) +RA format : HMS +Time format : JD +EOP file : eop.200210.p200503 +EOP coverage : DATA-BASED 1962-JAN-20 TO 2020-FEB-10. PREDICTS-> 2020-MAY-02 +Units conversion: 1 au= 149597870.700 km, c= 299792.458 km/s, 1 day= 86400.0 s +Table cut-offs 1: Elevation (-90.0deg=NO ),Airmass (>38.000=NO), Daylight (NO ) +Table cut-offs 2: Solar elongation ( 0.0,180.0=NO ),Local Hour Angle( 0.0=NO ) +Table cut-offs 3: RA/DEC angular rate ( 0.0=NO ) +******************************************************************************* +Initial IAU76/J2000 heliocentric ecliptic osculating elements (au, days, deg.): + EPOCH= 2456685.5 ! 2014-Jan-28.00 (TDB) Residual RMS= .29893 + EC= .1578018175114501 QR= 38.44081962777469 TP= 2408107.5732444809 + OM= 79.3100646538354 W= 297.2844380342058 IN= 29.01129634081993 + Equivalent ICRF heliocentric equatorial cartesian coordinates (au, au/d): + X=-4.594801534867052E+01 Y=-9.620475793810611E+00 Z= 2.316346263166737E+01 + VX=-2.210217587691426E-04 VY=-1.960899352961049E-03 VZ=-9.635666653763773E-04 +Asteroid physical parameters (km, seconds, rotational period in hours): + GM= n.a. RAD= n.a. ROTPER= 22.48 + H= -.100 G= .150 B-V= n.a. + ALBEDO= n.a. STYP= n.a. +*************************************************************************************************************************** +Date_________JDUT R.A._____(ICRF)_____DEC APmag delta deldot S-O-T /r S-T-O RA_3sigma DEC_3sigma +*************************************************************************************************************************** +$$SOE +2458878.500000000 *m 13 10 55.49 +23 20 00.8 17.24 52.0964438612435 -21.9954126 118.4783 /L 0.9464 0.184 0.134 +2458879.500000000 *m 13 10 54.41 +23 20 37.0 17.24 52.0836509616908 -21.7395525 119.3374 /L 0.9387 0.184 0.133 +2458880.500000000 *m 13 10 53.25 +23 21 13.5 17.23 52.0710106424492 -21.4769424 120.1932 /L 0.9310 0.183 0.133 +2458881.500000000 *m 13 10 52.02 +23 21 50.1 17.23 52.0585268000740 -21.2077572 121.0456 /L 0.9230 0.183 0.132 +2458882.500000000 *m 13 10 50.71 +23 22 26.8 17.23 52.0462032276418 -20.9321779 121.8945 /L 0.9148 0.183 0.132 +2458883.500000000 *m 13 10 49.33 +23 23 03.6 17.23 52.0340436108214 -20.6503926 122.7396 /L 0.9065 0.182 0.131 +2458884.500000000 *m 13 10 47.88 +23 23 40.6 17.23 52.0220515237316 -20.3625958 123.5808 /L 0.8981 0.182 0.131 +2458885.500000000 * 13 10 46.35 +23 24 17.6 17.23 52.0102304266763 -20.0689827 124.4178 /L 0.8895 0.181 0.130 +2458886.500000000 * 13 10 44.74 +23 24 54.7 17.23 51.9985836689442 -19.7697368 125.2505 /L 0.8807 0.181 0.130 +2458887.500000000 * 13 10 43.07 +23 25 31.9 17.22 51.9871145004155 -19.4650121 126.0786 /L 0.8719 0.180 0.129 +2458888.500000000 * 13 10 41.32 +23 26 09.1 17.22 51.9758260947983 -19.1549119 126.9018 /L 0.8628 0.180 0.129 +2458889.500000000 * 13 10 39.51 +23 26 46.4 17.22 51.9647215841187 -18.8394721 127.7201 /L 0.8537 0.179 0.128 +2458890.500000000 * 13 10 37.62 +23 27 23.7 17.22 51.9538040990853 -18.5186591 128.5330 /L 0.8445 0.179 0.128 +2458891.500000000 * 13 10 35.66 +23 28 01.0 17.22 51.9430768056378 -18.1923853 129.3405 /L 0.8351 0.178 0.127 +2458892.500000000 * 13 10 33.64 +23 28 38.3 17.22 51.9325429276428 -17.8605401 130.1421 /L 0.8257 0.177 0.127 +2458893.500000000 * 13 10 31.55 +23 29 15.6 17.22 51.9222057503535 -17.5230233 130.9375 /L 0.8161 0.177 0.126 +2458894.500000000 * 13 10 29.39 +23 29 52.9 17.21 51.9120686061678 -17.1797715 131.7266 /L 0.8065 0.176 0.126 +2458895.500000000 * 13 10 27.16 +23 30 30.1 17.21 51.9021348491925 -16.8307704 132.5088 /L 0.7968 0.175 0.126 +2458896.500000000 * 13 10 24.87 +23 31 07.3 17.21 51.8924078260855 -16.4760559 133.2839 /L 0.7871 0.175 0.125 +2458897.500000000 * 13 10 22.51 +23 31 44.5 17.21 51.8828908485607 -16.1157082 134.0515 /L 0.7773 0.174 0.125 +2458898.500000000 *m 13 10 20.09 +23 32 21.5 17.21 51.8735871700451 -15.7498440 134.8111 /L 0.7675 0.173 0.124 +2458899.500000000 *m 13 10 17.60 +23 32 58.5 17.21 51.8644999668039 -15.3786089 135.5624 /L 0.7576 0.173 0.124 +2458900.500000000 *m 13 10 15.05 +23 33 35.3 17.21 51.8556323228483 -15.0021721 136.3048 /L 0.7477 0.172 0.124 +2458901.500000000 *m 13 10 12.44 +23 34 12.1 17.20 51.8469872178088 -14.6207215 137.0379 /L 0.7379 0.171 0.123 +2458902.500000000 *m 13 10 09.77 +23 34 48.7 17.20 51.8385675172443 -14.2344597 137.7612 /L 0.7280 0.170 0.123 +2458903.500000000 *m 13 10 07.04 +23 35 25.1 17.20 51.8303759651921 -13.8436004 138.4742 /L 0.7181 0.169 0.123 +2458904.500000000 *m 13 10 04.26 +23 36 01.4 17.20 51.8224151789256 -13.4483644 139.1762 /L 0.7083 0.169 0.122 +2458905.500000000 *m 13 10 01.41 +23 36 37.5 17.20 51.8146876458376 -13.0489762 139.8667 /L 0.6986 0.168 0.122 +2458906.500000000 *m 13 09 58.51 +23 37 13.5 17.20 51.8071957221522 -12.6456611 140.5451 /L 0.6889 0.167 0.122 +2458907.500000000 *m 13 09 55.56 +23 37 49.2 17.20 51.7999416329288 -12.2386438 141.2108 /L 0.6793 0.166 0.121 +2458908.500000000 *m 13 09 52.55 +23 38 24.7 17.19 51.7929274726603 -11.8281478 141.8631 /L 0.6698 0.165 0.121 +$$EOE +*************************************************************************************************************************** +Column meaning: + +TIME + + Times PRIOR to 1962 are UT1, a mean-solar time closely related to the +prior but now-deprecated GMT. Times AFTER 1962 are in UTC, the current +civil or "wall-clock" time-scale. UTC is kept within 0.9 seconds of UT1 +using integer leap-seconds for 1972 and later years. + + Conversion from the internal Barycentric Dynamical Time (TDB) of solar +system dynamics to the non-uniform civil UT time-scale requested for output +has not been determined for UTC times after the next July or January 1st. +Therefore, the last known leap-second is used as a constant over future +intervals. + + Time tags refer to the UT time-scale conversion on Earth regardless of +observer location within the solar system, where clock rates may differ +due to the local gravity field and there is no precisely defined or adopted +"UT" analog timescale. + + Any 'b' symbol in the 1st-column denotes a B.C. date. First-column blank +(" ") denotes an A.D. date. Calendar dates prior to 1582-Oct-15 are in the +Julian calendar system. Later calendar dates are in the Gregorian system. + + NOTE: "n.a." in output means quantity "not available" at the print-time. + +SOLAR PRESENCE (OBSERVING SITE) + Time tag is followed by a blank, then a solar-presence symbol: + + '*' Daylight (refracted solar upper-limb on or above apparent horizon) + 'C' Civil twilight/dawn + 'N' Nautical twilight/dawn + 'A' Astronomical twilight/dawn + ' ' Night OR geocentric ephemeris + +LUNAR PRESENCE (OBSERVING SITE) + The solar-presence symbol is immediately followed by a lunar-presence symbol: + + 'm' Refracted upper-limb of Moon on or above apparent horizon + ' ' Refracted upper-limb of Moon below apparent horizon OR geocentric + ephemeris + +STATISTICAL UNCERTAINTIES + + Output includes formal +/- 3 standard-deviation statistical orbit uncertainty +quantities. There is a 99.7% chance the actual value is within given bounds. +These statistical calculations assume observational data errors are random. If +there are systematic biases (such as timing, reduction or star-catalog errors), +results can be optimistic. Because the epoch covariance is mapped using +linearized variational partial derivatives, results can also be optimistic for +times far from the solution epoch, particularly for objects having close +planetary encounters. + + R.A._____(ICRF)_____DEC = + Astrometric right ascension and declination of the target center with +respect to the observing site (coordinate origin) in the reference frame of +the planetary ephemeris (ICRF). Compensated for down-leg light-time delay +aberration. + + Units: RA in hours-minutes-seconds of time (HH MM SS.ff) + DEC in degrees-minutes-seconds of arc (sDD MN SC.f) + + APmag = + Asteroid's approximate apparent visual magnitude from the standard +IAU H-G magnitude relationship: + APmag = H + 5*log10(delta) + 5*log10(r) - 2.5*log10((1-G)*phi1 + G*phi2). +For solar phase angles > 90 deg, the error could exceed 1 magnitude. For phase +angles > 120 degrees, output values are rounded to the nearest integer to +indicate error could be large and unknown. + Units: MAGNITUDE + + delta deldot = + Range ("delta") and range-rate ("delta-dot") of target center with respect +to the observer at the instant light seen by the observer at print-time would +have left the target center (print-time minus down-leg light-time); the +distance traveled by a light ray emanating from the center of the target and +recorded by the observer at print-time. "deldot" is a projection of the +velocity vector along this ray, the light-time-corrected line-of-sight from +the coordinate center, and indicates relative motion. A positive "deldot" +means the target center is moving away from the observer (coordinate center). +A negative "deldot" means the target center is moving toward the observer. +Units: AU and KM/S + + S-O-T /r = + Sun-Observer-Target angle; target's apparent SOLAR ELONGATION seen from +the observer location at print-time. Angular units: DEGREES + + The '/r' column indicates the target's apparent position relative to +the Sun in the observer's sky, as described below: + + For an observing location on the surface of a rotating body +(considering its rotational sense): + + /T indicates target TRAILS Sun (evening sky; rises and sets AFTER Sun) + /L indicates target LEADS Sun (morning sky; rises and sets BEFORE Sun) + +For an observing point NOT on a rotating body (such as a spacecraft), the +"leading" and "trailing" condition is defined by the observer's +heliocentric orbital motion: if continuing in the observer's current +direction of heliocentric motion would encounter the target's apparent +longitude first, followed by the Sun's, the target LEADS the Sun as seen by +the observer. If the Sun's apparent longitude would be encountered first, +followed by the target's, the target TRAILS the Sun. + +NOTE: The S-O-T solar elongation angle is numerically the minimum +separation angle of the Sun and target in the sky in any direction. It +does NOT indicate the amount of separation in the leading or trailing +directions, which are defined in the equator of a spherical coordinate +system. + + S-T-O = + "S-T-O" is the Sun->Target->Observer angle; the interior vertex angle at +target center formed by a vector to the apparent center of the Sun at +reflection time on the target and the apparent vector to the observer at +print-time. Slightly different from true PHASE ANGLE (requestable separately) +at the few arcsecond level in that it includes stellar aberration on the +down-leg from target to observer. Units: DEGREES + + RA_3sigma DEC_3sigma = + Uncertainty in Right-Ascension and Declination. Output values are the formal ++/- 3 standard-deviations (sigmas) around nominal position. Units: ARCSECONDS + + + Computations by ... + Solar System Dynamics Group, Horizons On-Line Ephemeris System + 4800 Oak Grove Drive, Jet Propulsion Laboratory + Pasadena, CA 91109 USA + Information: http://ssd.jpl.nasa.gov/ + Connect : telnet://ssd.jpl.nasa.gov:6775 (via browser) + telnet ssd.jpl.nasa.gov 6775 (via command-line) + Author : Jon.D.Giorgini@jpl.nasa.gov + +************************************************************************************************ +******************************************************************************* +JPL/HORIZONS 136472 Makemake (2005 FY9) 2020-Feb-11 13:57:56 +Rec #: 136472 (+COV) Soln.date: 2019-May-30_07:13:05 # obs: 1949 (1955-2019) + +IAU76/J2000 helio. ecliptic osc. elements (au, days, deg., period=Julian yrs): + + EPOCH= 2456685.5 ! 2014-Jan-28.00 (TDB) Residual RMS= .29893 + EC= .1578018175114501 QR= 38.44081962777469 TP= 2408107.5732444809 + OM= 79.3100646538354 W= 297.2844380342058 IN= 29.01129634081993 + A= 45.64343693332217 MA= 155.2657054431733 ADIST= 52.84605423886966 + PER= 308.37256 N= .003196219 ANGMOM= .114761145 + DAN= 41.50451 DDN= 47.97743 L= 19.8421916 + B= -25.532549 MOID= 37.54140091 TP= 1881-Jan-27.0732444809 + +Asteroid physical parameters (km, seconds, rotational period in hours): + GM= n.a. RAD= n.a. ROTPER= 22.48 + H= -.100 G= .150 B-V= n.a. + ALBEDO= n.a. STYP= n.a. + +ASTEROID comments: +1: soln ref.= JPL#91, OCC=2 +2: source=ORB +******************************************************************************* + + +******************************************************************************* +Ephemeris / WWW_USER Tue Feb 11 13:57:56 2020 Pasadena, USA / Horizons +******************************************************************************* +Target body name: 136472 Makemake (2005 FY9) {source: JPL#91} +Center body name: Earth (399) {source: DE431} +Center-site name: Cerro Tololo-LCOGT A +******************************************************************************* +Start time : A.D. 2020-Jan-30 00:00:00.0000 UT +Stop time : A.D. 2020-Feb-29 00:00:00.0000 UT +Step-size : 1440 minutes +******************************************************************************* +Target pole/equ : No model available +Target radii : (unavailable) +Center geodetic : 289.195200,-30.167405,2.2093405 {E-lon(deg),Lat(deg),Alt(km)} +Center cylindric: 289.195200,5520.86454,-3187.542 {E-lon(deg),Dxy(km),Dz(km)} +Center pole/equ : High-precision EOP model {East-longitude positive} +Center radii : 6378.1 x 6378.1 x 6356.8 km {Equator, meridian, pole} +Target primary : Sun +Vis. interferer : MOON (R_eq= 1737.400) km {source: DE431} +Rel. light bend : Sun, EARTH {source: DE431} +Rel. lght bnd GM: 1.3271E+11, 3.9860E+05 km^3/s^2 +Small-body perts: Yes {source: SB431-N16} +Atmos refraction: NO (AIRLESS) +RA format : HMS +Time format : JD +EOP file : eop.200210.p200503 +EOP coverage : DATA-BASED 1962-JAN-20 TO 2020-FEB-10. PREDICTS-> 2020-MAY-02 +Units conversion: 1 au= 149597870.700 km, c= 299792.458 km/s, 1 day= 86400.0 s +Table cut-offs 1: Elevation (-90.0deg=NO ),Airmass (>38.000=NO), Daylight (NO ) +Table cut-offs 2: Solar elongation ( 0.0,180.0=NO ),Local Hour Angle( 0.0=NO ) +Table cut-offs 3: RA/DEC angular rate ( 0.0=NO ) +******************************************************************************* +Initial IAU76/J2000 heliocentric ecliptic osculating elements (au, days, deg.): + EPOCH= 2456685.5 ! 2014-Jan-28.00 (TDB) Residual RMS= .29893 + EC= .1578018175114501 QR= 38.44081962777469 TP= 2408107.5732444809 + OM= 79.3100646538354 W= 297.2844380342058 IN= 29.01129634081993 + Equivalent ICRF heliocentric equatorial cartesian coordinates (au, au/d): + X=-4.594801534867052E+01 Y=-9.620475793810611E+00 Z= 2.316346263166737E+01 + VX=-2.210217587691426E-04 VY=-1.960899352961049E-03 VZ=-9.635666653763773E-04 +Asteroid physical parameters (km, seconds, rotational period in hours): + GM= n.a. RAD= n.a. ROTPER= 22.48 + H= -.100 G= .150 B-V= n.a. + ALBEDO= n.a. STYP= n.a. +*************************************************************************************************************************** +Date_________JDUT R.A._____(ICRF)_____DEC APmag delta deldot S-O-T /r S-T-O RA_3sigma DEC_3sigma +*************************************************************************************************************************** +$$SOE +2458878.500000000 Cm 13 10 55.51 +23 20 00.9 17.24 52.0964584853686 -22.5183802 118.4762 /L 0.9464 0.184 0.134 +2458879.500000000 Cm 13 10 54.43 +23 20 37.2 17.24 52.0836647632730 -22.2624611 119.3352 /L 0.9388 0.184 0.133 +2458880.500000000 Cm 13 10 53.27 +23 21 13.6 17.23 52.0710236215262 -21.9996357 120.1910 /L 0.9310 0.183 0.133 +2458881.500000000 Cm 13 10 52.04 +23 21 50.2 17.23 52.0585389569307 -21.7300787 121.0434 /L 0.9230 0.183 0.132 +2458882.500000000 Cm 13 10 50.73 +23 22 26.9 17.23 52.0462145628112 -21.4539711 121.8923 /L 0.9149 0.183 0.132 +2458883.500000000 Cm 13 10 49.35 +23 23 03.8 17.23 52.0340541250853 -21.1715011 122.7374 /L 0.9066 0.182 0.131 +2458884.500000000 Cm 13 10 47.89 +23 23 40.7 17.23 52.0220612181212 -20.8828634 123.5786 /L 0.8982 0.182 0.131 +2458885.500000000 Cm 13 10 46.36 +23 24 17.7 17.23 52.0102393024728 -20.5882532 124.4156 /L 0.8896 0.181 0.130 +2458886.500000000 Cm 13 10 44.76 +23 24 54.8 17.23 51.9985917276797 -20.2878543 125.2483 /L 0.8808 0.181 0.130 +2458887.500000000 Cm 13 10 43.08 +23 25 32.0 17.22 51.9871217438731 -19.9818209 126.0764 /L 0.8719 0.180 0.129 +2458888.500000000 Cm 13 10 41.34 +23 26 09.3 17.22 51.9758325250120 -19.6702565 126.8996 /L 0.8629 0.180 0.129 +2458889.500000000 C 13 10 39.52 +23 26 46.5 17.22 51.9647272033719 -19.3531975 127.7179 /L 0.8538 0.179 0.128 +2458890.500000000 C 13 10 37.63 +23 27 23.8 17.22 51.9538089099089 -19.0306105 128.5308 /L 0.8445 0.179 0.128 +2458891.500000000 N 13 10 35.68 +23 28 01.1 17.22 51.9430808108077 -18.7024085 129.3382 /L 0.8352 0.178 0.127 +2458892.500000000 N 13 10 33.65 +23 28 38.5 17.22 51.9325461301780 -18.3684812 130.1398 /L 0.8258 0.177 0.127 +2458893.500000000 N 13 10 31.56 +23 29 15.8 17.22 51.9222081535159 -18.0287292 130.9353 /L 0.8162 0.177 0.126 +2458894.500000000 N 13 10 29.40 +23 29 53.0 17.21 51.9120702134623 -17.6830896 131.7243 /L 0.8066 0.176 0.126 +2458895.500000000 N 13 10 27.17 +23 30 30.3 17.21 51.9021356643679 -17.3315486 132.5065 /L 0.7969 0.175 0.126 +2458896.500000000 N 13 10 24.88 +23 31 07.5 17.21 51.8924078531342 -16.9741430 133.2816 /L 0.7872 0.175 0.125 +2458897.500000000 N 13 10 22.52 +23 31 44.6 17.21 51.8828900917194 -16.6109537 134.0492 /L 0.7774 0.174 0.125 +2458898.500000000 N 13 10 20.10 +23 32 21.7 17.21 51.8735856337934 -16.2420980 134.8088 /L 0.7676 0.173 0.124 +2458899.500000000 N 13 10 17.61 +23 32 58.6 17.21 51.8644976558637 -15.8677225 135.5601 /L 0.7577 0.173 0.124 +2458900.500000000 N 13 10 15.07 +23 33 35.5 17.21 51.8556292421821 -15.4879973 136.3025 /L 0.7478 0.172 0.124 +2458901.500000000 N 13 10 12.46 +23 34 12.2 17.20 51.8469833726175 -15.1031110 137.0356 /L 0.7380 0.171 0.123 +2458902.500000000 N 13 10 09.79 +23 34 48.8 17.20 51.8385629129657 -14.7132674 137.7589 /L 0.7281 0.170 0.123 +2458903.500000000 N 13 10 07.06 +23 35 25.3 17.20 51.8303706074984 -14.3186811 138.4718 /L 0.7182 0.169 0.123 +2458904.500000000 Nm 13 10 04.27 +23 36 01.6 17.20 51.8224090737217 -13.9195740 139.1738 /L 0.7084 0.169 0.122 +2458905.500000000 Nm 13 10 01.43 +23 36 37.7 17.20 51.8146807992588 -13.5161717 139.8643 /L 0.6987 0.168 0.122 +2458906.500000000 Nm 13 09 58.53 +23 37 13.6 17.20 51.8071881405623 -13.1087008 140.5427 /L 0.6890 0.167 0.122 +2458907.500000000 Nm 13 09 55.57 +23 37 49.4 17.20 51.7999333229185 -12.6973871 141.2084 /L 0.6794 0.166 0.121 +2458908.500000000 Nm 13 09 52.56 +23 38 24.9 17.19 51.7929184410454 -12.2824554 141.8606 /L 0.6699 0.165 0.121 +$$EOE +*************************************************************************************************************************** +Column meaning: + +TIME + + Times PRIOR to 1962 are UT1, a mean-solar time closely related to the +prior but now-deprecated GMT. Times AFTER 1962 are in UTC, the current +civil or "wall-clock" time-scale. UTC is kept within 0.9 seconds of UT1 +using integer leap-seconds for 1972 and later years. + + Conversion from the internal Barycentric Dynamical Time (TDB) of solar +system dynamics to the non-uniform civil UT time-scale requested for output +has not been determined for UTC times after the next July or January 1st. +Therefore, the last known leap-second is used as a constant over future +intervals. + + Time tags refer to the UT time-scale conversion on Earth regardless of +observer location within the solar system, where clock rates may differ +due to the local gravity field and there is no precisely defined or adopted +"UT" analog timescale. + + Any 'b' symbol in the 1st-column denotes a B.C. date. First-column blank +(" ") denotes an A.D. date. Calendar dates prior to 1582-Oct-15 are in the +Julian calendar system. Later calendar dates are in the Gregorian system. + + NOTE: "n.a." in output means quantity "not available" at the print-time. + +SOLAR PRESENCE (OBSERVING SITE) + Time tag is followed by a blank, then a solar-presence symbol: + + '*' Daylight (refracted solar upper-limb on or above apparent horizon) + 'C' Civil twilight/dawn + 'N' Nautical twilight/dawn + 'A' Astronomical twilight/dawn + ' ' Night OR geocentric ephemeris + +LUNAR PRESENCE (OBSERVING SITE) + The solar-presence symbol is immediately followed by a lunar-presence symbol: + + 'm' Refracted upper-limb of Moon on or above apparent horizon + ' ' Refracted upper-limb of Moon below apparent horizon OR geocentric + ephemeris + +STATISTICAL UNCERTAINTIES + + Output includes formal +/- 3 standard-deviation statistical orbit uncertainty +quantities. There is a 99.7% chance the actual value is within given bounds. +These statistical calculations assume observational data errors are random. If +there are systematic biases (such as timing, reduction or star-catalog errors), +results can be optimistic. Because the epoch covariance is mapped using +linearized variational partial derivatives, results can also be optimistic for +times far from the solution epoch, particularly for objects having close +planetary encounters. + + R.A._____(ICRF)_____DEC = + Astrometric right ascension and declination of the target center with +respect to the observing site (coordinate origin) in the reference frame of +the planetary ephemeris (ICRF). Compensated for down-leg light-time delay +aberration. + + Units: RA in hours-minutes-seconds of time (HH MM SS.ff) + DEC in degrees-minutes-seconds of arc (sDD MN SC.f) + + APmag = + Asteroid's approximate apparent visual magnitude from the standard +IAU H-G magnitude relationship: + APmag = H + 5*log10(delta) + 5*log10(r) - 2.5*log10((1-G)*phi1 + G*phi2). +For solar phase angles > 90 deg, the error could exceed 1 magnitude. For phase +angles > 120 degrees, output values are rounded to the nearest integer to +indicate error could be large and unknown. + Units: MAGNITUDE + + delta deldot = + Range ("delta") and range-rate ("delta-dot") of target center with respect +to the observer at the instant light seen by the observer at print-time would +have left the target center (print-time minus down-leg light-time); the +distance traveled by a light ray emanating from the center of the target and +recorded by the observer at print-time. "deldot" is a projection of the +velocity vector along this ray, the light-time-corrected line-of-sight from +the coordinate center, and indicates relative motion. A positive "deldot" +means the target center is moving away from the observer (coordinate center). +A negative "deldot" means the target center is moving toward the observer. +Units: AU and KM/S + + S-O-T /r = + Sun-Observer-Target angle; target's apparent SOLAR ELONGATION seen from +the observer location at print-time. Angular units: DEGREES + + The '/r' column indicates the target's apparent position relative to +the Sun in the observer's sky, as described below: + + For an observing location on the surface of a rotating body +(considering its rotational sense): + + /T indicates target TRAILS Sun (evening sky; rises and sets AFTER Sun) + /L indicates target LEADS Sun (morning sky; rises and sets BEFORE Sun) + +For an observing point NOT on a rotating body (such as a spacecraft), the +"leading" and "trailing" condition is defined by the observer's +heliocentric orbital motion: if continuing in the observer's current +direction of heliocentric motion would encounter the target's apparent +longitude first, followed by the Sun's, the target LEADS the Sun as seen by +the observer. If the Sun's apparent longitude would be encountered first, +followed by the target's, the target TRAILS the Sun. + +NOTE: The S-O-T solar elongation angle is numerically the minimum +separation angle of the Sun and target in the sky in any direction. It +does NOT indicate the amount of separation in the leading or trailing +directions, which are defined in the equator of a spherical coordinate +system. + + S-T-O = + "S-T-O" is the Sun->Target->Observer angle; the interior vertex angle at +target center formed by a vector to the apparent center of the Sun at +reflection time on the target and the apparent vector to the observer at +print-time. Slightly different from true PHASE ANGLE (requestable separately) +at the few arcsecond level in that it includes stellar aberration on the +down-leg from target to observer. Units: DEGREES + + RA_3sigma DEC_3sigma = + Uncertainty in Right-Ascension and Declination. Output values are the formal ++/- 3 standard-deviations (sigmas) around nominal position. Units: ARCSECONDS + + + Computations by ... + Solar System Dynamics Group, Horizons On-Line Ephemeris System + 4800 Oak Grove Drive, Jet Propulsion Laboratory + Pasadena, CA 91109 USA + Information: http://ssd.jpl.nasa.gov/ + Connect : telnet://ssd.jpl.nasa.gov:6775 (via browser) + telnet ssd.jpl.nasa.gov 6775 (via command-line) + Author : Jon.D.Giorgini@jpl.nasa.gov + +*********************************************************************************************** +******************************************************************************* +JPL/HORIZONS 136472 Makemake (2005 FY9) 2020-Feb-11 13:58:56 +Rec #: 136472 (+COV) Soln.date: 2019-May-30_07:13:05 # obs: 1949 (1955-2019) + +IAU76/J2000 helio. ecliptic osc. elements (au, days, deg., period=Julian yrs): + + EPOCH= 2456685.5 ! 2014-Jan-28.00 (TDB) Residual RMS= .29893 + EC= .1578018175114501 QR= 38.44081962777469 TP= 2408107.5732444809 + OM= 79.3100646538354 W= 297.2844380342058 IN= 29.01129634081993 + A= 45.64343693332217 MA= 155.2657054431733 ADIST= 52.84605423886966 + PER= 308.37256 N= .003196219 ANGMOM= .114761145 + DAN= 41.50451 DDN= 47.97743 L= 19.8421916 + B= -25.532549 MOID= 37.54140091 TP= 1881-Jan-27.0732444809 + +Asteroid physical parameters (km, seconds, rotational period in hours): + GM= n.a. RAD= n.a. ROTPER= 22.48 + H= -.100 G= .150 B-V= n.a. + ALBEDO= n.a. STYP= n.a. + +ASTEROID comments: +1: soln ref.= JPL#91, OCC=2 +2: source=ORB +******************************************************************************* + + +******************************************************************************* +Ephemeris / WWW_USER Tue Feb 11 13:58:56 2020 Pasadena, USA / Horizons +******************************************************************************* +Target body name: 136472 Makemake (2005 FY9) {source: JPL#91} +Center body name: Earth (399) {source: DE431} +Center-site name: Teide Observatory +******************************************************************************* +Start time : A.D. 2020-Jan-30 00:00:00.0000 UT +Stop time : A.D. 2020-Feb-29 00:00:00.0000 UT +Step-size : 1440 minutes +******************************************************************************* +Target pole/equ : No model available +Target radii : (unavailable) +Center geodetic : 343.490600,28.2983824,2.3649836 {E-lon(deg),Lat(deg),Alt(km)} +Center cylindric: 343.490600,5622.20214,3006.7826 {E-lon(deg),Dxy(km),Dz(km)} +Center pole/equ : High-precision EOP model {East-longitude positive} +Center radii : 6378.1 x 6378.1 x 6356.8 km {Equator, meridian, pole} +Target primary : Sun +Vis. interferer : MOON (R_eq= 1737.400) km {source: DE431} +Rel. light bend : Sun, EARTH {source: DE431} +Rel. lght bnd GM: 1.3271E+11, 3.9860E+05 km^3/s^2 +Small-body perts: Yes {source: SB431-N16} +Atmos refraction: NO (AIRLESS) +RA format : HMS +Time format : JD +EOP file : eop.200210.p200503 +EOP coverage : DATA-BASED 1962-JAN-20 TO 2020-FEB-10. PREDICTS-> 2020-MAY-02 +Units conversion: 1 au= 149597870.700 km, c= 299792.458 km/s, 1 day= 86400.0 s +Table cut-offs 1: Elevation (-90.0deg=NO ),Airmass (>38.000=NO), Daylight (NO ) +Table cut-offs 2: Solar elongation ( 0.0,180.0=NO ),Local Hour Angle( 0.0=NO ) +Table cut-offs 3: RA/DEC angular rate ( 0.0=NO ) +******************************************************************************* +Initial IAU76/J2000 heliocentric ecliptic osculating elements (au, days, deg.): + EPOCH= 2456685.5 ! 2014-Jan-28.00 (TDB) Residual RMS= .29893 + EC= .1578018175114501 QR= 38.44081962777469 TP= 2408107.5732444809 + OM= 79.3100646538354 W= 297.2844380342058 IN= 29.01129634081993 + Equivalent ICRF heliocentric equatorial cartesian coordinates (au, au/d): + X=-4.594801534867052E+01 Y=-9.620475793810611E+00 Z= 2.316346263166737E+01 + VX=-2.210217587691426E-04 VY=-1.960899352961049E-03 VZ=-9.635666653763773E-04 +Asteroid physical parameters (km, seconds, rotational period in hours): + GM= n.a. RAD= n.a. ROTPER= 22.48 + H= -.100 G= .150 B-V= n.a. + ALBEDO= n.a. STYP= n.a. +*************************************************************************************************************************** +Date_________JDUT R.A._____(ICRF)_____DEC APmag delta deldot S-O-T /r S-T-O RA_3sigma DEC_3sigma +*************************************************************************************************************************** +$$SOE +2458878.500000000 13 10 55.51 +23 20 00.8 17.24 52.0964136051386 -22.6566754 118.4778 /L 0.9465 0.184 0.134 +2458879.500000000 13 10 54.43 +23 20 37.1 17.24 52.0836196637411 -22.3953441 119.3369 /L 0.9389 0.184 0.133 +2458880.500000000 m 13 10 53.27 +23 21 13.5 17.23 52.0709783112294 -22.1270656 120.1927 /L 0.9311 0.183 0.133 +2458881.500000000 m 13 10 52.04 +23 21 50.1 17.23 52.0584934444773 -21.8520167 121.0451 /L 0.9231 0.183 0.132 +2458882.500000000 m 13 10 50.73 +23 22 26.8 17.23 52.0461688568781 -21.5703797 121.8940 /L 0.9150 0.183 0.132 +2458883.500000000 m 13 10 49.35 +23 23 03.7 17.23 52.0340082344152 -21.2823445 122.7391 /L 0.9067 0.182 0.131 +2458884.500000000 m 13 10 47.89 +23 23 40.6 17.23 52.0220151515194 -20.9881075 123.5803 /L 0.8982 0.182 0.131 +2458885.500000000 m 13 10 46.36 +23 24 17.6 17.23 52.0101930688048 -20.6878655 124.4173 /L 0.8896 0.181 0.130 +2458886.500000000 m 13 10 44.76 +23 24 54.7 17.23 51.9985453358673 -20.3818042 125.2500 /L 0.8809 0.181 0.130 +2458887.500000000 m 13 10 43.09 +23 25 31.9 17.22 51.9870752028913 -20.0700793 126.0781 /L 0.8720 0.180 0.129 +2458888.500000000 m 13 10 41.34 +23 26 09.2 17.22 51.9757858438856 -19.7527961 126.9014 /L 0.8630 0.180 0.129 +2458889.500000000 m 13 10 39.52 +23 26 46.4 17.22 51.9646803911733 -19.4299926 127.7196 /L 0.8539 0.179 0.128 +2458890.500000000 m 13 10 37.64 +23 27 23.7 17.22 51.9537619757551 -19.1016373 128.5326 /L 0.8446 0.179 0.128 +2458891.500000000 m 13 10 35.68 +23 28 01.0 17.22 51.9430337638591 -18.7676448 129.3400 /L 0.8353 0.178 0.127 +2458892.500000000 m 13 10 33.66 +23 28 38.4 17.22 51.9324989796363 -18.4279067 130.1416 /L 0.8258 0.177 0.127 +2458893.500000000 m 13 10 31.56 +23 29 15.7 17.22 51.9221609086218 -18.0823251 130.9370 /L 0.8163 0.177 0.126 +2458894.500000000 13 10 29.40 +23 29 52.9 17.21 51.9120228834925 -17.7308390 131.7261 /L 0.8067 0.176 0.126 +2458895.500000000 13 10 27.17 +23 30 30.2 17.21 51.9020882586319 -17.3734365 132.5083 /L 0.7970 0.175 0.126 +2458896.500000000 13 10 24.88 +23 31 07.4 17.21 51.8923603809708 -17.0101559 133.2835 /L 0.7873 0.175 0.125 +2458897.500000000 13 10 22.52 +23 31 44.5 17.21 51.8828425624928 -16.6410801 134.0510 /L 0.7775 0.174 0.125 +2458898.500000000 13 10 20.10 +23 32 21.6 17.21 51.8735380568906 -16.2663281 134.8107 /L 0.7676 0.173 0.124 +2458899.500000000 13 10 17.61 +23 32 58.5 17.21 51.8644500406906 -15.8860484 135.5620 /L 0.7578 0.173 0.124 +2458900.500000000 13 10 15.07 +23 33 35.4 17.21 51.8555815981610 -15.5004128 136.3044 /L 0.7479 0.172 0.124 +2458901.500000000 13 10 12.46 +23 34 12.1 17.20 51.8469357091844 -15.1096118 137.0375 /L 0.7380 0.171 0.123 +2458902.500000000 13 10 09.79 +23 34 48.7 17.20 51.8385152395672 -14.7138509 137.7608 /L 0.7282 0.170 0.123 +2458903.500000000 13 10 07.06 +23 35 25.2 17.20 51.8303229335892 -14.3133466 138.4738 /L 0.7183 0.169 0.123 +2458904.500000000 13 10 04.27 +23 36 01.5 17.20 51.8223614087620 -13.9083226 139.1758 /L 0.7085 0.169 0.122 +2458905.500000000 13 10 01.43 +23 36 37.6 17.20 51.8146331527114 -13.4990062 139.8663 /L 0.6987 0.168 0.122 +2458906.500000000 13 09 58.52 +23 37 13.6 17.20 51.8071405218899 -13.0856260 140.5448 /L 0.6890 0.167 0.122 +2458907.500000000 13 09 55.57 +23 37 49.3 17.20 51.7998857415806 -12.6684094 141.2105 /L 0.6794 0.166 0.121 +2458908.500000000 13 09 52.56 +23 38 24.8 17.19 51.7928709064952 -12.2475832 141.8627 /L 0.6699 0.165 0.121 +$$EOE +*************************************************************************************************************************** +Column meaning: + +TIME + + Times PRIOR to 1962 are UT1, a mean-solar time closely related to the +prior but now-deprecated GMT. Times AFTER 1962 are in UTC, the current +civil or "wall-clock" time-scale. UTC is kept within 0.9 seconds of UT1 +using integer leap-seconds for 1972 and later years. + + Conversion from the internal Barycentric Dynamical Time (TDB) of solar +system dynamics to the non-uniform civil UT time-scale requested for output +has not been determined for UTC times after the next July or January 1st. +Therefore, the last known leap-second is used as a constant over future +intervals. + + Time tags refer to the UT time-scale conversion on Earth regardless of +observer location within the solar system, where clock rates may differ +due to the local gravity field and there is no precisely defined or adopted +"UT" analog timescale. + + Any 'b' symbol in the 1st-column denotes a B.C. date. First-column blank +(" ") denotes an A.D. date. Calendar dates prior to 1582-Oct-15 are in the +Julian calendar system. Later calendar dates are in the Gregorian system. + + NOTE: "n.a." in output means quantity "not available" at the print-time. + +SOLAR PRESENCE (OBSERVING SITE) + Time tag is followed by a blank, then a solar-presence symbol: + + '*' Daylight (refracted solar upper-limb on or above apparent horizon) + 'C' Civil twilight/dawn + 'N' Nautical twilight/dawn + 'A' Astronomical twilight/dawn + ' ' Night OR geocentric ephemeris + +LUNAR PRESENCE (OBSERVING SITE) + The solar-presence symbol is immediately followed by a lunar-presence symbol: + + 'm' Refracted upper-limb of Moon on or above apparent horizon + ' ' Refracted upper-limb of Moon below apparent horizon OR geocentric + ephemeris + +STATISTICAL UNCERTAINTIES + + Output includes formal +/- 3 standard-deviation statistical orbit uncertainty +quantities. There is a 99.7% chance the actual value is within given bounds. +These statistical calculations assume observational data errors are random. If +there are systematic biases (such as timing, reduction or star-catalog errors), +results can be optimistic. Because the epoch covariance is mapped using +linearized variational partial derivatives, results can also be optimistic for +times far from the solution epoch, particularly for objects having close +planetary encounters. + + R.A._____(ICRF)_____DEC = + Astrometric right ascension and declination of the target center with +respect to the observing site (coordinate origin) in the reference frame of +the planetary ephemeris (ICRF). Compensated for down-leg light-time delay +aberration. + + Units: RA in hours-minutes-seconds of time (HH MM SS.ff) + DEC in degrees-minutes-seconds of arc (sDD MN SC.f) + + APmag = + Asteroid's approximate apparent visual magnitude from the standard +IAU H-G magnitude relationship: + APmag = H + 5*log10(delta) + 5*log10(r) - 2.5*log10((1-G)*phi1 + G*phi2). +For solar phase angles > 90 deg, the error could exceed 1 magnitude. For phase +angles > 120 degrees, output values are rounded to the nearest integer to +indicate error could be large and unknown. + Units: MAGNITUDE + + delta deldot = + Range ("delta") and range-rate ("delta-dot") of target center with respect +to the observer at the instant light seen by the observer at print-time would +have left the target center (print-time minus down-leg light-time); the +distance traveled by a light ray emanating from the center of the target and +recorded by the observer at print-time. "deldot" is a projection of the +velocity vector along this ray, the light-time-corrected line-of-sight from +the coordinate center, and indicates relative motion. A positive "deldot" +means the target center is moving away from the observer (coordinate center). +A negative "deldot" means the target center is moving toward the observer. +Units: AU and KM/S + + S-O-T /r = + Sun-Observer-Target angle; target's apparent SOLAR ELONGATION seen from +the observer location at print-time. Angular units: DEGREES + + The '/r' column indicates the target's apparent position relative to +the Sun in the observer's sky, as described below: + + For an observing location on the surface of a rotating body +(considering its rotational sense): + + /T indicates target TRAILS Sun (evening sky; rises and sets AFTER Sun) + /L indicates target LEADS Sun (morning sky; rises and sets BEFORE Sun) + +For an observing point NOT on a rotating body (such as a spacecraft), the +"leading" and "trailing" condition is defined by the observer's +heliocentric orbital motion: if continuing in the observer's current +direction of heliocentric motion would encounter the target's apparent +longitude first, followed by the Sun's, the target LEADS the Sun as seen by +the observer. If the Sun's apparent longitude would be encountered first, +followed by the target's, the target TRAILS the Sun. + +NOTE: The S-O-T solar elongation angle is numerically the minimum +separation angle of the Sun and target in the sky in any direction. It +does NOT indicate the amount of separation in the leading or trailing +directions, which are defined in the equator of a spherical coordinate +system. + + S-T-O = + "S-T-O" is the Sun->Target->Observer angle; the interior vertex angle at +target center formed by a vector to the apparent center of the Sun at +reflection time on the target and the apparent vector to the observer at +print-time. Slightly different from true PHASE ANGLE (requestable separately) +at the few arcsecond level in that it includes stellar aberration on the +down-leg from target to observer. Units: DEGREES + + RA_3sigma DEC_3sigma = + Uncertainty in Right-Ascension and Declination. Output values are the formal ++/- 3 standard-deviations (sigmas) around nominal position. Units: ARCSECONDS + + + Computations by ... + Solar System Dynamics Group, Horizons On-Line Ephemeris System + 4800 Oak Grove Drive, Jet Propulsion Laboratory + Pasadena, CA 91109 USA + Information: http://ssd.jpl.nasa.gov/ + Connect : telnet://ssd.jpl.nasa.gov:6775 (via browser) + telnet ssd.jpl.nasa.gov 6775 (via command-line) + Author : Jon.D.Giorgini@jpl.nasa.gov + +*************************************************************************************************************************** +******************************************************************************* +JPL/HORIZONS 136472 Makemake (2005 FY9) 2020-Feb-11 14:00:10 +Rec #: 136472 (+COV) Soln.date: 2019-May-30_07:13:05 # obs: 1949 (1955-2019) + +IAU76/J2000 helio. ecliptic osc. elements (au, days, deg., period=Julian yrs): + + EPOCH= 2456685.5 ! 2014-Jan-28.00 (TDB) Residual RMS= .29893 + EC= .1578018175114501 QR= 38.44081962777469 TP= 2408107.5732444809 + OM= 79.3100646538354 W= 297.2844380342058 IN= 29.01129634081993 + A= 45.64343693332217 MA= 155.2657054431733 ADIST= 52.84605423886966 + PER= 308.37256 N= .003196219 ANGMOM= .114761145 + DAN= 41.50451 DDN= 47.97743 L= 19.8421916 + B= -25.532549 MOID= 37.54140091 TP= 1881-Jan-27.0732444809 + +Asteroid physical parameters (km, seconds, rotational period in hours): + GM= n.a. RAD= n.a. ROTPER= 22.48 + H= -.100 G= .150 B-V= n.a. + ALBEDO= n.a. STYP= n.a. + +ASTEROID comments: +1: soln ref.= JPL#91, OCC=2 +2: source=ORB +******************************************************************************* + + +******************************************************************************* +Ephemeris / WWW_USER Tue Feb 11 14:00:10 2020 Pasadena, USA / Horizons +******************************************************************************* +Target body name: 136472 Makemake (2005 FY9) {source: JPL#91} +Center body name: Earth (399) {source: DE431} +Center-site name: Sunderland-LCOGT A +******************************************************************************* +Start time : A.D. 2020-Jan-30 00:00:00.0000 UT +Stop time : A.D. 2020-Feb-29 00:00:00.0000 UT +Step-size : 1440 minutes +******************************************************************************* +Target pole/equ : No model available +Target radii : (unavailable) +Center geodetic : 20.8102000,-32.380541,1.8117864 {E-lon(deg),Lat(deg),Alt(km)} +Center cylindric: 20.8102000,5393.10796,-3397.113 {E-lon(deg),Dxy(km),Dz(km)} +Center pole/equ : High-precision EOP model {East-longitude positive} +Center radii : 6378.1 x 6378.1 x 6356.8 km {Equator, meridian, pole} +Target primary : Sun +Vis. interferer : MOON (R_eq= 1737.400) km {source: DE431} +Rel. light bend : Sun, EARTH {source: DE431} +Rel. lght bnd GM: 1.3271E+11, 3.9860E+05 km^3/s^2 +Small-body perts: Yes {source: SB431-N16} +Atmos refraction: NO (AIRLESS) +RA format : HMS +Time format : JD +EOP file : eop.200210.p200503 +EOP coverage : DATA-BASED 1962-JAN-20 TO 2020-FEB-10. PREDICTS-> 2020-MAY-02 +Units conversion: 1 au= 149597870.700 km, c= 299792.458 km/s, 1 day= 86400.0 s +Table cut-offs 1: Elevation (-90.0deg=NO ),Airmass (>38.000=NO), Daylight (NO ) +Table cut-offs 2: Solar elongation ( 0.0,180.0=NO ),Local Hour Angle( 0.0=NO ) +Table cut-offs 3: RA/DEC angular rate ( 0.0=NO ) +******************************************************************************* +Initial IAU76/J2000 heliocentric ecliptic osculating elements (au, days, deg.): + EPOCH= 2456685.5 ! 2014-Jan-28.00 (TDB) Residual RMS= .29893 + EC= .1578018175114501 QR= 38.44081962777469 TP= 2408107.5732444809 + OM= 79.3100646538354 W= 297.2844380342058 IN= 29.01129634081993 + Equivalent ICRF heliocentric equatorial cartesian coordinates (au, au/d): + X=-4.594801534867052E+01 Y=-9.620475793810611E+00 Z= 2.316346263166737E+01 + VX=-2.210217587691426E-04 VY=-1.960899352961049E-03 VZ=-9.635666653763773E-04 +Asteroid physical parameters (km, seconds, rotational period in hours): + GM= n.a. RAD= n.a. ROTPER= 22.48 + H= -.100 G= .150 B-V= n.a. + ALBEDO= n.a. STYP= n.a. +*************************************************************************************************************************** +Date_________JDUT R.A._____(ICRF)_____DEC APmag delta deldot S-O-T /r S-T-O RA_3sigma DEC_3sigma +*************************************************************************************************************************** +$$SOE +2458878.500000000 13 10 55.51 +23 20 01.0 17.24 52.0964110616823 -22.5514333 118.4784 /L 0.9465 0.184 0.134 +2458879.500000000 13 10 54.43 +23 20 37.3 17.24 52.0836172982328 -22.2864637 119.3374 /L 0.9389 0.184 0.133 +2458880.500000000 13 10 53.27 +23 21 13.7 17.23 52.0709761294870 -22.0145789 120.1933 /L 0.9311 0.183 0.133 +2458881.500000000 13 10 52.04 +23 21 50.3 17.23 52.0584914522655 -21.7359566 121.0457 /L 0.9232 0.183 0.132 +2458882.500000000 13 10 50.73 +23 22 27.0 17.23 52.0461670599064 -21.4507804 121.8946 /L 0.9150 0.183 0.132 +2458883.500000000 13 10 49.35 +23 23 03.8 17.23 52.0340066383367 -21.1592413 122.7397 /L 0.9067 0.182 0.131 +2458884.500000000 m 13 10 47.89 +23 23 40.8 17.23 52.0220137619291 -20.8615367 123.5809 /L 0.8983 0.182 0.131 +2458885.500000000 m 13 10 46.36 +23 24 17.8 17.23 52.0101918912385 -20.5578646 124.4179 /L 0.8897 0.181 0.130 +2458886.500000000 m 13 10 44.76 +23 24 54.9 17.23 51.9985443758007 -20.2484115 125.2505 /L 0.8809 0.181 0.130 +2458887.500000000 m 13 10 43.08 +23 25 32.1 17.22 51.9870744657397 -19.9333345 126.0786 /L 0.8720 0.180 0.129 +2458888.500000000 m 13 10 41.34 +23 26 09.3 17.22 51.9757853350024 -19.6127398 126.9019 /L 0.8630 0.180 0.129 +2458889.500000000 m 13 10 39.52 +23 26 46.6 17.22 51.9646801158482 -19.2866664 127.7201 /L 0.8539 0.179 0.128 +2458890.500000000 m 13 10 37.63 +23 27 23.9 17.22 51.9537619392119 -18.9550839 128.5331 /L 0.8446 0.179 0.128 +2458891.500000000 m 13 10 35.68 +23 28 01.2 17.22 51.9430339712517 -18.6179078 129.3405 /L 0.8353 0.178 0.127 +2458892.500000000 m 13 10 33.65 +23 28 38.5 17.22 51.9324994360456 -18.2750307 130.1420 /L 0.8259 0.177 0.127 +2458893.500000000 m 13 10 31.56 +23 29 15.8 17.22 51.9221616190533 -17.9263557 130.9375 /L 0.8163 0.177 0.126 +2458894.500000000 m 13 10 29.40 +23 29 53.1 17.21 51.9120238528744 -17.5718228 131.7265 /L 0.8067 0.176 0.126 +2458895.500000000 m 13 10 27.17 +23 30 30.4 17.21 51.9020894918144 -17.2114211 132.5088 /L 0.7970 0.175 0.126 +2458896.500000000 m 13 10 24.88 +23 31 07.6 17.21 51.8923618827255 -16.8451898 133.2839 /L 0.7873 0.175 0.125 +2458897.500000000 m 13 10 22.52 +23 31 44.7 17.21 51.8828443375124 -16.4732126 134.0514 /L 0.7775 0.174 0.125 +2458898.500000000 13 10 20.10 +23 32 21.8 17.21 51.8735401097874 -16.0956096 134.8110 /L 0.7677 0.173 0.124 +2458899.500000000 13 10 17.61 +23 32 58.7 17.21 51.8644523759956 -15.7125302 135.5623 /L 0.7578 0.173 0.124 +2458900.500000000 13 10 15.06 +23 33 35.6 17.21 51.8555842203225 -15.3241469 136.3047 /L 0.7479 0.172 0.124 +2458901.500000000 13 10 12.45 +23 34 12.3 17.20 51.8469386225658 -14.9306513 137.0378 /L 0.7380 0.171 0.123 +2458902.500000000 13 10 09.78 +23 34 48.9 17.20 51.8385184484454 -14.5322496 137.7611 /L 0.7282 0.170 0.123 +2458903.500000000 13 10 07.05 +23 35 25.4 17.20 51.8303264421528 -14.1291592 138.4740 /L 0.7183 0.169 0.123 +2458904.500000000 13 10 04.26 +23 36 01.7 17.20 51.8223652211090 -13.7216045 139.1760 /L 0.7085 0.169 0.122 +2458905.500000000 13 10 01.42 +23 36 37.8 17.20 51.8146372728473 -13.3098138 139.8665 /L 0.6988 0.168 0.122 +2458906.500000000 13 09 58.52 +23 37 13.7 17.20 51.8071449537264 -12.8940163 140.5449 /L 0.6891 0.167 0.122 +2458907.500000000 13 09 55.56 +23 37 49.5 17.20 51.7998904889337 -12.4744403 141.2105 /L 0.6795 0.166 0.121 +2458908.500000000 13 09 52.55 +23 38 25.0 17.19 51.7928759730841 -12.0513133 141.8627 /L 0.6700 0.165 0.121 +$$EOE +*************************************************************************************************************************** +Column meaning: + +TIME + + Times PRIOR to 1962 are UT1, a mean-solar time closely related to the +prior but now-deprecated GMT. Times AFTER 1962 are in UTC, the current +civil or "wall-clock" time-scale. UTC is kept within 0.9 seconds of UT1 +using integer leap-seconds for 1972 and later years. + + Conversion from the internal Barycentric Dynamical Time (TDB) of solar +system dynamics to the non-uniform civil UT time-scale requested for output +has not been determined for UTC times after the next July or January 1st. +Therefore, the last known leap-second is used as a constant over future +intervals. + + Time tags refer to the UT time-scale conversion on Earth regardless of +observer location within the solar system, where clock rates may differ +due to the local gravity field and there is no precisely defined or adopted +"UT" analog timescale. + + Any 'b' symbol in the 1st-column denotes a B.C. date. First-column blank +(" ") denotes an A.D. date. Calendar dates prior to 1582-Oct-15 are in the +Julian calendar system. Later calendar dates are in the Gregorian system. + + NOTE: "n.a." in output means quantity "not available" at the print-time. + +SOLAR PRESENCE (OBSERVING SITE) + Time tag is followed by a blank, then a solar-presence symbol: + + '*' Daylight (refracted solar upper-limb on or above apparent horizon) + 'C' Civil twilight/dawn + 'N' Nautical twilight/dawn + 'A' Astronomical twilight/dawn + ' ' Night OR geocentric ephemeris + +LUNAR PRESENCE (OBSERVING SITE) + The solar-presence symbol is immediately followed by a lunar-presence symbol: + + 'm' Refracted upper-limb of Moon on or above apparent horizon + ' ' Refracted upper-limb of Moon below apparent horizon OR geocentric + ephemeris + +STATISTICAL UNCERTAINTIES + + Output includes formal +/- 3 standard-deviation statistical orbit uncertainty +quantities. There is a 99.7% chance the actual value is within given bounds. +These statistical calculations assume observational data errors are random. If +there are systematic biases (such as timing, reduction or star-catalog errors), +results can be optimistic. Because the epoch covariance is mapped using +linearized variational partial derivatives, results can also be optimistic for +times far from the solution epoch, particularly for objects having close +planetary encounters. + + R.A._____(ICRF)_____DEC = + Astrometric right ascension and declination of the target center with +respect to the observing site (coordinate origin) in the reference frame of +the planetary ephemeris (ICRF). Compensated for down-leg light-time delay +aberration. + + Units: RA in hours-minutes-seconds of time (HH MM SS.ff) + DEC in degrees-minutes-seconds of arc (sDD MN SC.f) + + APmag = + Asteroid's approximate apparent visual magnitude from the standard +IAU H-G magnitude relationship: + APmag = H + 5*log10(delta) + 5*log10(r) - 2.5*log10((1-G)*phi1 + G*phi2). +For solar phase angles > 90 deg, the error could exceed 1 magnitude. For phase +angles > 120 degrees, output values are rounded to the nearest integer to +indicate error could be large and unknown. + Units: MAGNITUDE + + delta deldot = + Range ("delta") and range-rate ("delta-dot") of target center with respect +to the observer at the instant light seen by the observer at print-time would +have left the target center (print-time minus down-leg light-time); the +distance traveled by a light ray emanating from the center of the target and +recorded by the observer at print-time. "deldot" is a projection of the +velocity vector along this ray, the light-time-corrected line-of-sight from +the coordinate center, and indicates relative motion. A positive "deldot" +means the target center is moving away from the observer (coordinate center). +A negative "deldot" means the target center is moving toward the observer. +Units: AU and KM/S + + S-O-T /r = + Sun-Observer-Target angle; target's apparent SOLAR ELONGATION seen from +the observer location at print-time. Angular units: DEGREES + + The '/r' column indicates the target's apparent position relative to +the Sun in the observer's sky, as described below: + + For an observing location on the surface of a rotating body +(considering its rotational sense): + + /T indicates target TRAILS Sun (evening sky; rises and sets AFTER Sun) + /L indicates target LEADS Sun (morning sky; rises and sets BEFORE Sun) + +For an observing point NOT on a rotating body (such as a spacecraft), the +"leading" and "trailing" condition is defined by the observer's +heliocentric orbital motion: if continuing in the observer's current +direction of heliocentric motion would encounter the target's apparent +longitude first, followed by the Sun's, the target LEADS the Sun as seen by +the observer. If the Sun's apparent longitude would be encountered first, +followed by the target's, the target TRAILS the Sun. + +NOTE: The S-O-T solar elongation angle is numerically the minimum +separation angle of the Sun and target in the sky in any direction. It +does NOT indicate the amount of separation in the leading or trailing +directions, which are defined in the equator of a spherical coordinate +system. + + S-T-O = + "S-T-O" is the Sun->Target->Observer angle; the interior vertex angle at +target center formed by a vector to the apparent center of the Sun at +reflection time on the target and the apparent vector to the observer at +print-time. Slightly different from true PHASE ANGLE (requestable separately) +at the few arcsecond level in that it includes stellar aberration on the +down-leg from target to observer. Units: DEGREES + + RA_3sigma DEC_3sigma = + Uncertainty in Right-Ascension and Declination. Output values are the formal ++/- 3 standard-deviations (sigmas) around nominal position. Units: ARCSECONDS + + + Computations by ... + Solar System Dynamics Group, Horizons On-Line Ephemeris System + 4800 Oak Grove Drive, Jet Propulsion Laboratory + Pasadena, CA 91109 USA + Information: http://ssd.jpl.nasa.gov/ + Connect : telnet://ssd.jpl.nasa.gov:6775 (via browser) + telnet ssd.jpl.nasa.gov 6775 (via command-line) + Author : Jon.D.Giorgini@jpl.nasa.gov + +******************************************************************************************************************** +******************************************************************************* +JPL/HORIZONS 136472 Makemake (2005 FY9) 2020-Feb-11 14:01:12 +Rec #: 136472 (+COV) Soln.date: 2019-May-30_07:13:05 # obs: 1949 (1955-2019) + +IAU76/J2000 helio. ecliptic osc. elements (au, days, deg., period=Julian yrs): + + EPOCH= 2456685.5 ! 2014-Jan-28.00 (TDB) Residual RMS= .29893 + EC= .1578018175114501 QR= 38.44081962777469 TP= 2408107.5732444809 + OM= 79.3100646538354 W= 297.2844380342058 IN= 29.01129634081993 + A= 45.64343693332217 MA= 155.2657054431733 ADIST= 52.84605423886966 + PER= 308.37256 N= .003196219 ANGMOM= .114761145 + DAN= 41.50451 DDN= 47.97743 L= 19.8421916 + B= -25.532549 MOID= 37.54140091 TP= 1881-Jan-27.0732444809 + +Asteroid physical parameters (km, seconds, rotational period in hours): + GM= n.a. RAD= n.a. ROTPER= 22.48 + H= -.100 G= .150 B-V= n.a. + ALBEDO= n.a. STYP= n.a. + +ASTEROID comments: +1: soln ref.= JPL#91, OCC=2 +2: source=ORB +******************************************************************************* + + +******************************************************************************* +Ephemeris / WWW_USER Tue Feb 11 14:01:12 2020 Pasadena, USA / Horizons +******************************************************************************* +Target body name: 136472 Makemake (2005 FY9) {source: JPL#91} +Center body name: Earth (399) {source: DE431} +Center-site name: Wise Observatory, Mitzpeh Ramon +******************************************************************************* +Start time : A.D. 2020-Jan-30 00:00:00.0000 UT +Stop time : A.D. 2020-Feb-29 00:00:00.0000 UT +Step-size : 1440 minutes +******************************************************************************* +Target pole/equ : No model available +Target radii : (unavailable) +Center geodetic : 34.7625000,30.5956453,0.8980474 {E-lon(deg),Lat(deg),Alt(km)} +Center cylindric: 34.7625000,5495.71714,3227.8433 {E-lon(deg),Dxy(km),Dz(km)} +Center pole/equ : High-precision EOP model {East-longitude positive} +Center radii : 6378.1 x 6378.1 x 6356.8 km {Equator, meridian, pole} +Target primary : Sun +Vis. interferer : MOON (R_eq= 1737.400) km {source: DE431} +Rel. light bend : Sun, EARTH {source: DE431} +Rel. lght bnd GM: 1.3271E+11, 3.9860E+05 km^3/s^2 +Small-body perts: Yes {source: SB431-N16} +Atmos refraction: NO (AIRLESS) +RA format : HMS +Time format : JD +EOP file : eop.200210.p200503 +EOP coverage : DATA-BASED 1962-JAN-20 TO 2020-FEB-10. PREDICTS-> 2020-MAY-02 +Units conversion: 1 au= 149597870.700 km, c= 299792.458 km/s, 1 day= 86400.0 s +Table cut-offs 1: Elevation (-90.0deg=NO ),Airmass (>38.000=NO), Daylight (NO ) +Table cut-offs 2: Solar elongation ( 0.0,180.0=NO ),Local Hour Angle( 0.0=NO ) +Table cut-offs 3: RA/DEC angular rate ( 0.0=NO ) +******************************************************************************* +Initial IAU76/J2000 heliocentric ecliptic osculating elements (au, days, deg.): + EPOCH= 2456685.5 ! 2014-Jan-28.00 (TDB) Residual RMS= .29893 + EC= .1578018175114501 QR= 38.44081962777469 TP= 2408107.5732444809 + OM= 79.3100646538354 W= 297.2844380342058 IN= 29.01129634081993 + Equivalent ICRF heliocentric equatorial cartesian coordinates (au, au/d): + X=-4.594801534867052E+01 Y=-9.620475793810611E+00 Z= 2.316346263166737E+01 + VX=-2.210217587691426E-04 VY=-1.960899352961049E-03 VZ=-9.635666653763773E-04 +Asteroid physical parameters (km, seconds, rotational period in hours): + GM= n.a. RAD= n.a. ROTPER= 22.48 + H= -.100 G= .150 B-V= n.a. + ALBEDO= n.a. STYP= n.a. +*************************************************************************************************************************** +Date_________JDUT R.A._____(ICRF)_____DEC APmag delta deldot S-O-T /r S-T-O RA_3sigma DEC_3sigma +*************************************************************************************************************************** +$$SOE +2458878.500000000 13 10 55.51 +23 20 00.8 17.24 52.0963877454596 -22.4895734 118.4796 /L 0.9465 0.184 0.134 +2458879.500000000 13 10 54.43 +23 20 37.1 17.24 52.0835940741697 -22.2235150 119.3386 /L 0.9389 0.184 0.133 +2458880.500000000 13 10 53.27 +23 21 13.6 17.23 52.0709529992902 -21.9505601 120.1945 /L 0.9311 0.183 0.133 +2458881.500000000 13 10 52.04 +23 21 50.1 17.23 52.0584684176161 -21.6708866 121.0469 /L 0.9232 0.183 0.132 +2458882.500000000 13 10 50.73 +23 22 26.9 17.23 52.0461441224593 -21.3846786 121.8958 /L 0.9150 0.183 0.132 +2458883.500000000 13 10 49.34 +23 23 03.7 17.23 52.0339837997196 -21.0921273 122.7409 /L 0.9067 0.182 0.131 +2458884.500000000 m 13 10 47.89 +23 23 40.6 17.23 52.0219910237416 -20.7934305 123.5821 /L 0.8983 0.182 0.131 +2458885.500000000 m 13 10 46.36 +23 24 17.7 17.23 52.0101692550509 -20.4887863 124.4191 /L 0.8897 0.181 0.130 +2458886.500000000 m 13 10 44.76 +23 24 54.8 17.23 51.9985218431529 -20.1783819 125.2518 /L 0.8809 0.181 0.130 +2458887.500000000 m 13 10 43.08 +23 25 32.0 17.22 51.9870520381392 -19.8623744 126.0799 /L 0.8720 0.180 0.129 +2458888.500000000 m 13 10 41.33 +23 26 09.2 17.22 51.9757630139242 -19.5408703 126.9031 /L 0.8630 0.180 0.129 +2458889.500000000 m 13 10 39.52 +23 26 46.5 17.22 51.9646579027340 -19.2139091 127.7214 /L 0.8539 0.179 0.128 +2458890.500000000 m 13 10 37.63 +23 27 23.8 17.22 51.9537398354708 -18.8814604 128.5343 /L 0.8446 0.179 0.128 +2458891.500000000 m 13 10 35.67 +23 28 01.1 17.22 51.9430119782617 -18.5434403 129.3418 /L 0.8353 0.178 0.127 +2458892.500000000 m 13 10 33.65 +23 28 38.4 17.22 51.9324775551541 -18.1997414 130.1433 /L 0.8259 0.177 0.127 +2458893.500000000 m 13 10 31.56 +23 29 15.7 17.22 51.9221398515773 -17.8502672 130.9388 /L 0.8163 0.177 0.126 +2458894.500000000 m 13 10 29.40 +23 29 53.0 17.21 51.9120022000996 -17.4949579 131.7279 /L 0.8067 0.176 0.126 +2458895.500000000 m 13 10 27.17 +23 30 30.2 17.21 51.9020679549938 -17.1338028 132.5101 /L 0.7970 0.175 0.126 +2458896.500000000 m 13 10 24.87 +23 31 07.4 17.21 51.8923404630782 -16.7668415 133.2852 /L 0.7873 0.175 0.125 +2458897.500000000 13 10 22.52 +23 31 44.5 17.21 51.8828230362219 -16.3941578 134.0528 /L 0.7775 0.174 0.125 +2458898.500000000 13 10 20.09 +23 32 21.6 17.21 51.8735189280010 -16.0158721 134.8124 /L 0.7677 0.173 0.124 +2458899.500000000 13 10 17.61 +23 32 58.6 17.21 51.8644313148237 -15.6321340 135.5637 /L 0.7578 0.173 0.124 +2458900.500000000 13 10 15.06 +23 33 35.4 17.21 51.8555632808383 -15.2431162 136.3061 /L 0.7479 0.172 0.124 +2458901.500000000 13 10 12.45 +23 34 12.2 17.20 51.8469178058053 -14.8490105 137.0393 /L 0.7380 0.171 0.123 +2458902.500000000 13 10 09.78 +23 34 48.8 17.20 51.8384977554075 -14.4500234 137.7625 /L 0.7282 0.170 0.123 +2458903.500000000 13 10 07.05 +23 35 25.2 17.20 51.8303058737993 -14.0463723 138.4755 /L 0.7183 0.169 0.123 +2458904.500000000 13 10 04.26 +23 36 01.5 17.20 51.8223447783649 -13.6382820 139.1775 /L 0.7085 0.169 0.122 +2458905.500000000 13 10 01.42 +23 36 37.6 17.20 51.8146169566007 -13.2259809 139.8680 /L 0.6988 0.168 0.122 +2458906.500000000 13 09 58.52 +23 37 13.6 17.20 51.8071247648286 -12.8096982 140.5465 /L 0.6891 0.167 0.122 +2458907.500000000 13 09 55.56 +23 37 49.3 17.20 51.7998704281988 -12.3896626 141.2121 /L 0.6794 0.166 0.121 +2458908.500000000 13 09 52.55 +23 38 24.8 17.19 51.7928560412889 -11.9661016 141.8644 /L 0.6699 0.165 0.121 +$$EOE +*************************************************************************************************************************** +Column meaning: + +TIME + + Times PRIOR to 1962 are UT1, a mean-solar time closely related to the +prior but now-deprecated GMT. Times AFTER 1962 are in UTC, the current +civil or "wall-clock" time-scale. UTC is kept within 0.9 seconds of UT1 +using integer leap-seconds for 1972 and later years. + + Conversion from the internal Barycentric Dynamical Time (TDB) of solar +system dynamics to the non-uniform civil UT time-scale requested for output +has not been determined for UTC times after the next July or January 1st. +Therefore, the last known leap-second is used as a constant over future +intervals. + + Time tags refer to the UT time-scale conversion on Earth regardless of +observer location within the solar system, where clock rates may differ +due to the local gravity field and there is no precisely defined or adopted +"UT" analog timescale. + + Any 'b' symbol in the 1st-column denotes a B.C. date. First-column blank +(" ") denotes an A.D. date. Calendar dates prior to 1582-Oct-15 are in the +Julian calendar system. Later calendar dates are in the Gregorian system. + + NOTE: "n.a." in output means quantity "not available" at the print-time. + +SOLAR PRESENCE (OBSERVING SITE) + Time tag is followed by a blank, then a solar-presence symbol: + + '*' Daylight (refracted solar upper-limb on or above apparent horizon) + 'C' Civil twilight/dawn + 'N' Nautical twilight/dawn + 'A' Astronomical twilight/dawn + ' ' Night OR geocentric ephemeris + +LUNAR PRESENCE (OBSERVING SITE) + The solar-presence symbol is immediately followed by a lunar-presence symbol: + + 'm' Refracted upper-limb of Moon on or above apparent horizon + ' ' Refracted upper-limb of Moon below apparent horizon OR geocentric + ephemeris + +STATISTICAL UNCERTAINTIES + + Output includes formal +/- 3 standard-deviation statistical orbit uncertainty +quantities. There is a 99.7% chance the actual value is within given bounds. +These statistical calculations assume observational data errors are random. If +there are systematic biases (such as timing, reduction or star-catalog errors), +results can be optimistic. Because the epoch covariance is mapped using +linearized variational partial derivatives, results can also be optimistic for +times far from the solution epoch, particularly for objects having close +planetary encounters. + + R.A._____(ICRF)_____DEC = + Astrometric right ascension and declination of the target center with +respect to the observing site (coordinate origin) in the reference frame of +the planetary ephemeris (ICRF). Compensated for down-leg light-time delay +aberration. + + Units: RA in hours-minutes-seconds of time (HH MM SS.ff) + DEC in degrees-minutes-seconds of arc (sDD MN SC.f) + + APmag = + Asteroid's approximate apparent visual magnitude from the standard +IAU H-G magnitude relationship: + APmag = H + 5*log10(delta) + 5*log10(r) - 2.5*log10((1-G)*phi1 + G*phi2). +For solar phase angles > 90 deg, the error could exceed 1 magnitude. For phase +angles > 120 degrees, output values are rounded to the nearest integer to +indicate error could be large and unknown. + Units: MAGNITUDE + + delta deldot = + Range ("delta") and range-rate ("delta-dot") of target center with respect +to the observer at the instant light seen by the observer at print-time would +have left the target center (print-time minus down-leg light-time); the +distance traveled by a light ray emanating from the center of the target and +recorded by the observer at print-time. "deldot" is a projection of the +velocity vector along this ray, the light-time-corrected line-of-sight from +the coordinate center, and indicates relative motion. A positive "deldot" +means the target center is moving away from the observer (coordinate center). +A negative "deldot" means the target center is moving toward the observer. +Units: AU and KM/S + + S-O-T /r = + Sun-Observer-Target angle; target's apparent SOLAR ELONGATION seen from +the observer location at print-time. Angular units: DEGREES + + The '/r' column indicates the target's apparent position relative to +the Sun in the observer's sky, as described below: + + For an observing location on the surface of a rotating body +(considering its rotational sense): + + /T indicates target TRAILS Sun (evening sky; rises and sets AFTER Sun) + /L indicates target LEADS Sun (morning sky; rises and sets BEFORE Sun) + +For an observing point NOT on a rotating body (such as a spacecraft), the +"leading" and "trailing" condition is defined by the observer's +heliocentric orbital motion: if continuing in the observer's current +direction of heliocentric motion would encounter the target's apparent +longitude first, followed by the Sun's, the target LEADS the Sun as seen by +the observer. If the Sun's apparent longitude would be encountered first, +followed by the target's, the target TRAILS the Sun. + +NOTE: The S-O-T solar elongation angle is numerically the minimum +separation angle of the Sun and target in the sky in any direction. It +does NOT indicate the amount of separation in the leading or trailing +directions, which are defined in the equator of a spherical coordinate +system. + + S-T-O = + "S-T-O" is the Sun->Target->Observer angle; the interior vertex angle at +target center formed by a vector to the apparent center of the Sun at +reflection time on the target and the apparent vector to the observer at +print-time. Slightly different from true PHASE ANGLE (requestable separately) +at the few arcsecond level in that it includes stellar aberration on the +down-leg from target to observer. Units: DEGREES + + RA_3sigma DEC_3sigma = + Uncertainty in Right-Ascension and Declination. Output values are the formal ++/- 3 standard-deviations (sigmas) around nominal position. Units: ARCSECONDS + + + Computations by ... + Solar System Dynamics Group, Horizons On-Line Ephemeris System + 4800 Oak Grove Drive, Jet Propulsion Laboratory + Pasadena, CA 91109 USA + Information: http://ssd.jpl.nasa.gov/ + Connect : telnet://ssd.jpl.nasa.gov:6775 (via browser) + telnet ssd.jpl.nasa.gov 6775 (via command-line) + Author : Jon.D.Giorgini@jpl.nasa.gov + +*************************************************************************************************************************** +******************************************************************************* +JPL/HORIZONS 136472 Makemake (2005 FY9) 2020-Feb-11 14:02:10 +Rec #: 136472 (+COV) Soln.date: 2019-May-30_07:13:05 # obs: 1949 (1955-2019) + +IAU76/J2000 helio. ecliptic osc. elements (au, days, deg., period=Julian yrs): + + EPOCH= 2456685.5 ! 2014-Jan-28.00 (TDB) Residual RMS= .29893 + EC= .1578018175114501 QR= 38.44081962777469 TP= 2408107.5732444809 + OM= 79.3100646538354 W= 297.2844380342058 IN= 29.01129634081993 + A= 45.64343693332217 MA= 155.2657054431733 ADIST= 52.84605423886966 + PER= 308.37256 N= .003196219 ANGMOM= .114761145 + DAN= 41.50451 DDN= 47.97743 L= 19.8421916 + B= -25.532549 MOID= 37.54140091 TP= 1881-Jan-27.0732444809 + +Asteroid physical parameters (km, seconds, rotational period in hours): + GM= n.a. RAD= n.a. ROTPER= 22.48 + H= -.100 G= .150 B-V= n.a. + ALBEDO= n.a. STYP= n.a. + +ASTEROID comments: +1: soln ref.= JPL#91, OCC=2 +2: source=ORB +******************************************************************************* + + +******************************************************************************* +Ephemeris / WWW_USER Tue Feb 11 14:02:10 2020 Pasadena, USA / Horizons +******************************************************************************* +Target body name: 136472 Makemake (2005 FY9) {source: JPL#91} +Center body name: Earth (399) {source: DE431} +Center-site name: Siding Spring-LCOGT A +******************************************************************************* +Start time : A.D. 2020-Jan-30 00:00:00.0000 UT +Stop time : A.D. 2020-Feb-29 00:00:00.0000 UT +Step-size : 1440 minutes +******************************************************************************* +Target pole/equ : No model available +Target radii : (unavailable) +Center geodetic : 149.070600,-31.272987,1.1750341 {E-lon(deg),Lat(deg),Alt(km)} +Center cylindric: 149.070600,5457.34528,-3292.410 {E-lon(deg),Dxy(km),Dz(km)} +Center pole/equ : High-precision EOP model {East-longitude positive} +Center radii : 6378.1 x 6378.1 x 6356.8 km {Equator, meridian, pole} +Target primary : Sun +Vis. interferer : MOON (R_eq= 1737.400) km {source: DE431} +Rel. light bend : Sun, EARTH {source: DE431} +Rel. lght bnd GM: 1.3271E+11, 3.9860E+05 km^3/s^2 +Small-body perts: Yes {source: SB431-N16} +Atmos refraction: NO (AIRLESS) +RA format : HMS +Time format : JD +EOP file : eop.200210.p200503 +EOP coverage : DATA-BASED 1962-JAN-20 TO 2020-FEB-10. PREDICTS-> 2020-MAY-02 +Units conversion: 1 au= 149597870.700 km, c= 299792.458 km/s, 1 day= 86400.0 s +Table cut-offs 1: Elevation (-90.0deg=NO ),Airmass (>38.000=NO), Daylight (NO ) +Table cut-offs 2: Solar elongation ( 0.0,180.0=NO ),Local Hour Angle( 0.0=NO ) +Table cut-offs 3: RA/DEC angular rate ( 0.0=NO ) +******************************************************************************* +Initial IAU76/J2000 heliocentric ecliptic osculating elements (au, days, deg.): + EPOCH= 2456685.5 ! 2014-Jan-28.00 (TDB) Residual RMS= .29893 + EC= .1578018175114501 QR= 38.44081962777469 TP= 2408107.5732444809 + OM= 79.3100646538354 W= 297.2844380342058 IN= 29.01129634081993 + Equivalent ICRF heliocentric equatorial cartesian coordinates (au, au/d): + X=-4.594801534867052E+01 Y=-9.620475793810611E+00 Z= 2.316346263166737E+01 + VX=-2.210217587691426E-04 VY=-1.960899352961049E-03 VZ=-9.635666653763773E-04 +Asteroid physical parameters (km, seconds, rotational period in hours): + GM= n.a. RAD= n.a. ROTPER= 22.48 + H= -.100 G= .150 B-V= n.a. + ALBEDO= n.a. STYP= n.a. +*************************************************************************************************************************** +Date_________JDUT R.A._____(ICRF)_____DEC APmag delta deldot S-O-T /r S-T-O RA_3sigma DEC_3sigma +*************************************************************************************************************************** +$$SOE +2458878.500000000 *m 13 10 55.49 +23 20 00.9 17.24 52.0964268193931 -21.9210153 118.4795 /L 0.9464 0.184 0.134 +2458879.500000000 * 13 10 54.41 +23 20 37.2 17.24 52.0836340507385 -21.6592092 119.3386 /L 0.9388 0.184 0.133 +2458880.500000000 * 13 10 53.25 +23 21 13.7 17.23 52.0709938718657 -21.3906760 120.1944 /L 0.9310 0.183 0.133 +2458881.500000000 * 13 10 52.02 +23 21 50.3 17.23 52.0585101792924 -21.1155927 121.0468 /L 0.9230 0.183 0.132 +2458882.500000000 * 13 10 50.71 +23 22 27.0 17.23 52.0461867660547 -20.8341417 121.8957 /L 0.9149 0.183 0.132 +2458883.500000000 * 13 10 49.33 +23 23 03.8 17.23 52.0340273177788 -20.5465132 122.7408 /L 0.9066 0.182 0.131 +2458884.500000000 * 13 10 47.87 +23 23 40.8 17.23 52.0220354085375 -20.2529035 123.5820 /L 0.8981 0.182 0.131 +2458885.500000000 * 13 10 46.34 +23 24 17.8 17.23 52.0102144985870 -19.9535093 124.4190 /L 0.8895 0.181 0.130 +2458886.500000000 * 13 10 44.74 +23 24 54.9 17.23 51.9985679371663 -19.6485162 125.2517 /L 0.8808 0.181 0.130 +2458887.500000000 * 13 10 43.07 +23 25 32.1 17.22 51.9870989741038 -19.3380799 126.0797 /L 0.8719 0.180 0.129 +2458888.500000000 * 13 10 41.32 +23 26 09.3 17.22 51.9758107830535 -19.0223053 126.9030 /L 0.8629 0.180 0.129 +2458889.500000000 * 13 10 39.50 +23 26 46.6 17.22 51.9647064959845 -18.7012302 127.7212 /L 0.8538 0.179 0.128 +2458890.500000000 * 13 10 37.62 +23 27 23.9 17.22 51.9537892435448 -18.3748226 128.5342 /L 0.8445 0.179 0.128 +2458891.500000000 * 13 10 35.66 +23 28 01.2 17.22 51.9430621916089 -18.0429969 129.3416 /L 0.8352 0.178 0.127 +2458892.500000000 * 13 10 33.63 +23 28 38.5 17.22 51.9325285639731 -17.7056438 130.1431 /L 0.8257 0.177 0.127 +2458893.500000000 *m 13 10 31.54 +23 29 15.8 17.22 51.9221916458173 -17.3626651 130.9386 /L 0.8162 0.177 0.126 +2458894.500000000 *m 13 10 29.38 +23 29 53.1 17.21 51.9120547694629 -17.0139990 131.7276 /L 0.8066 0.176 0.126 +2458895.500000000 *m 13 10 27.15 +23 30 30.3 17.21 51.9021212889384 -16.6596329 132.5099 /L 0.7969 0.175 0.126 +2458896.500000000 *m 13 10 24.86 +23 31 07.5 17.21 51.8923945508215 -16.2996044 133.2850 /L 0.7872 0.175 0.125 +2458897.500000000 *m 13 10 22.50 +23 31 44.6 17.21 51.8828778667442 -15.9339952 134.0525 /L 0.7774 0.174 0.125 +2458898.500000000 *m 13 10 20.08 +23 32 21.7 17.21 51.8735744900492 -15.5629237 134.8121 /L 0.7675 0.173 0.124 +2458899.500000000 *m 13 10 17.59 +23 32 58.7 17.21 51.8644875969152 -15.1865373 135.5634 /L 0.7577 0.173 0.124 +2458900.500000000 *m 13 10 15.05 +23 33 35.5 17.21 51.8556202712643 -14.8050067 136.3058 /L 0.7478 0.172 0.124 +2458901.500000000 *m 13 10 12.44 +23 34 12.2 17.20 51.8469754926344 -14.4185213 137.0388 /L 0.7379 0.171 0.123 +2458902.500000000 *m 13 10 09.77 +23 34 48.9 17.20 51.8385561264894 -14.0272853 137.7621 /L 0.7280 0.170 0.123 +2458903.500000000 *m 13 10 07.04 +23 35 25.3 17.20 51.8303649167678 -13.6315140 138.4750 /L 0.7182 0.169 0.123 +2458904.500000000 *m 13 10 04.25 +23 36 01.6 17.20 51.8224044806417 -13.2314298 139.1770 /L 0.7084 0.169 0.122 +2458905.500000000 *m 13 10 01.41 +23 36 37.7 17.20 51.8146773053988 -12.8272585 139.8675 /L 0.6986 0.168 0.122 +2458906.500000000 *m 13 09 58.51 +23 37 13.7 17.20 51.8071857471559 -12.4192271 140.5459 /L 0.6889 0.167 0.122 +2458907.500000000 *m 13 09 55.55 +23 37 49.4 17.20 51.7999320308620 -12.0075618 141.2115 /L 0.6793 0.166 0.121 +2458908.500000000 * 13 09 52.54 +23 38 24.9 17.19 51.7929182508972 -11.5924874 141.8637 /L 0.6698 0.165 0.121 +$$EOE +*************************************************************************************************************************** +Column meaning: + +TIME + + Times PRIOR to 1962 are UT1, a mean-solar time closely related to the +prior but now-deprecated GMT. Times AFTER 1962 are in UTC, the current +civil or "wall-clock" time-scale. UTC is kept within 0.9 seconds of UT1 +using integer leap-seconds for 1972 and later years. + + Conversion from the internal Barycentric Dynamical Time (TDB) of solar +system dynamics to the non-uniform civil UT time-scale requested for output +has not been determined for UTC times after the next July or January 1st. +Therefore, the last known leap-second is used as a constant over future +intervals. + + Time tags refer to the UT time-scale conversion on Earth regardless of +observer location within the solar system, where clock rates may differ +due to the local gravity field and there is no precisely defined or adopted +"UT" analog timescale. + + Any 'b' symbol in the 1st-column denotes a B.C. date. First-column blank +(" ") denotes an A.D. date. Calendar dates prior to 1582-Oct-15 are in the +Julian calendar system. Later calendar dates are in the Gregorian system. + + NOTE: "n.a." in output means quantity "not available" at the print-time. + +SOLAR PRESENCE (OBSERVING SITE) + Time tag is followed by a blank, then a solar-presence symbol: + + '*' Daylight (refracted solar upper-limb on or above apparent horizon) + 'C' Civil twilight/dawn + 'N' Nautical twilight/dawn + 'A' Astronomical twilight/dawn + ' ' Night OR geocentric ephemeris + +LUNAR PRESENCE (OBSERVING SITE) + The solar-presence symbol is immediately followed by a lunar-presence symbol: + + 'm' Refracted upper-limb of Moon on or above apparent horizon + ' ' Refracted upper-limb of Moon below apparent horizon OR geocentric + ephemeris + +STATISTICAL UNCERTAINTIES + + Output includes formal +/- 3 standard-deviation statistical orbit uncertainty +quantities. There is a 99.7% chance the actual value is within given bounds. +These statistical calculations assume observational data errors are random. If +there are systematic biases (such as timing, reduction or star-catalog errors), +results can be optimistic. Because the epoch covariance is mapped using +linearized variational partial derivatives, results can also be optimistic for +times far from the solution epoch, particularly for objects having close +planetary encounters. + + R.A._____(ICRF)_____DEC = + Astrometric right ascension and declination of the target center with +respect to the observing site (coordinate origin) in the reference frame of +the planetary ephemeris (ICRF). Compensated for down-leg light-time delay +aberration. + + Units: RA in hours-minutes-seconds of time (HH MM SS.ff) + DEC in degrees-minutes-seconds of arc (sDD MN SC.f) + + APmag = + Asteroid's approximate apparent visual magnitude from the standard +IAU H-G magnitude relationship: + APmag = H + 5*log10(delta) + 5*log10(r) - 2.5*log10((1-G)*phi1 + G*phi2). +For solar phase angles > 90 deg, the error could exceed 1 magnitude. For phase +angles > 120 degrees, output values are rounded to the nearest integer to +indicate error could be large and unknown. + Units: MAGNITUDE + + delta deldot = + Range ("delta") and range-rate ("delta-dot") of target center with respect +to the observer at the instant light seen by the observer at print-time would +have left the target center (print-time minus down-leg light-time); the +distance traveled by a light ray emanating from the center of the target and +recorded by the observer at print-time. "deldot" is a projection of the +velocity vector along this ray, the light-time-corrected line-of-sight from +the coordinate center, and indicates relative motion. A positive "deldot" +means the target center is moving away from the observer (coordinate center). +A negative "deldot" means the target center is moving toward the observer. +Units: AU and KM/S + + S-O-T /r = + Sun-Observer-Target angle; target's apparent SOLAR ELONGATION seen from +the observer location at print-time. Angular units: DEGREES + + The '/r' column indicates the target's apparent position relative to +the Sun in the observer's sky, as described below: + + For an observing location on the surface of a rotating body +(considering its rotational sense): + + /T indicates target TRAILS Sun (evening sky; rises and sets AFTER Sun) + /L indicates target LEADS Sun (morning sky; rises and sets BEFORE Sun) + +For an observing point NOT on a rotating body (such as a spacecraft), the +"leading" and "trailing" condition is defined by the observer's +heliocentric orbital motion: if continuing in the observer's current +direction of heliocentric motion would encounter the target's apparent +longitude first, followed by the Sun's, the target LEADS the Sun as seen by +the observer. If the Sun's apparent longitude would be encountered first, +followed by the target's, the target TRAILS the Sun. + +NOTE: The S-O-T solar elongation angle is numerically the minimum +separation angle of the Sun and target in the sky in any direction. It +does NOT indicate the amount of separation in the leading or trailing +directions, which are defined in the equator of a spherical coordinate +system. + + S-T-O = + "S-T-O" is the Sun->Target->Observer angle; the interior vertex angle at +target center formed by a vector to the apparent center of the Sun at +reflection time on the target and the apparent vector to the observer at +print-time. Slightly different from true PHASE ANGLE (requestable separately) +at the few arcsecond level in that it includes stellar aberration on the +down-leg from target to observer. Units: DEGREES + + RA_3sigma DEC_3sigma = + Uncertainty in Right-Ascension and Declination. Output values are the formal ++/- 3 standard-deviations (sigmas) around nominal position. Units: ARCSECONDS + + + Computations by ... + Solar System Dynamics Group, Horizons On-Line Ephemeris System + 4800 Oak Grove Drive, Jet Propulsion Laboratory + Pasadena, CA 91109 USA + Information: http://ssd.jpl.nasa.gov/ + Connect : telnet://ssd.jpl.nasa.gov:6775 (via browser) + telnet ssd.jpl.nasa.gov 6775 (via command-line) + Author : Jon.D.Giorgini@jpl.nasa.gov + +*************************************************************************************************************************** From 4b1bc52f3da256417c9ba15d89ceab44f5ce3f8b Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 14 Apr 2020 13:01:26 -0700 Subject: [PATCH 008/424] Pinned all versions to latest available, removed matplotlib --- setup.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/setup.py b/setup.py index 0de499e21..4eb32d0d8 100644 --- a/setup.py +++ b/setup.py @@ -28,24 +28,24 @@ packages=find_packages(), install_requires=[ 'django>=2.2', # TOM Toolkit requires db math functions - 'django-bootstrap4', - 'django-extensions', - 'django-filter', + 'django-bootstrap4==1.1.1', + 'django-extensions==2.2.9', + 'django-filter==2.2.0', 'django-contrib-comments>=1.9.2', # Earlier version are incompatible with Django >= 3.0 - 'django-gravatar2', - 'django-crispy-forms', - 'django-guardian', - 'numpy', - 'python-dateutil', - 'requests', - 'astroquery', + 'django-gravatar2==1.4.3', + 'django-crispy-forms==1.9.0', + 'django-guardian==2.2.0', + 'numpy==1.18.2', + 'python-dateutil==2.8.1', + 'requests==2.23.0', + 'astroquery==0.4', 'astropy==4.0', - 'astroplan', - 'plotly', - 'matplotlib', - 'pillow', - 'fits2image', - 'specutils==0.7', + 'astroplan==0.6', + 'plotly==4.6.0', + # 'matplotlib', + 'pillow==7.1.0', + 'fits2image==0.4.3', + 'specutils==1.0', 'dataclasses; python_version < "3.7"', ], extras_require={ From dda5c11f9190da23d216da2eac0a5dceb5986da7 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 14 Apr 2020 14:49:19 -0700 Subject: [PATCH 009/424] Added documentation on writing a custom template tag --- docs/customization/common_customizations.rst | 1 + docs/customization/customize_template_tags.md | 266 ++++++++++++++++++ docs/customization/index.rst | 4 + docs/index.rst | 3 + docs/introduction/resources.rst | 4 +- 5 files changed, 277 insertions(+), 1 deletion(-) create mode 100644 docs/customization/customize_template_tags.md diff --git a/docs/customization/common_customizations.rst b/docs/customization/common_customizations.rst index 526b00b10..83cfd3ef3 100644 --- a/docs/customization/common_customizations.rst +++ b/docs/customization/common_customizations.rst @@ -7,3 +7,4 @@ When starting a new TOM, we're sure there are a few things a user might want to * Not happy with the appearance? Jump straight in with :doc:`customizing TOM Templates `. You may want to take a look at the available Template Tags in each modules' respective `API Documentation `_ to see what you can do. * Need another view? Take a peek at :doc:`Adding Pages `. * Want to automate something? Look at the :doc:`Automation Guide `. Feeling bold? Set up :doc:`background tasks <../advanced/backgroundtasks>`. +* SQLite not meeting your needs? Check out Django's documentation on `databases `_. \ No newline at end of file diff --git a/docs/customization/customize_template_tags.md b/docs/customization/customize_template_tags.md new file mode 100644 index 000000000..cb81c5e47 --- /dev/null +++ b/docs/customization/customize_template_tags.md @@ -0,0 +1,266 @@ +# Customizing Template Tags + +The TOM Toolkit is designed to be as customizable as possible. A number of UI objects are rendered as Django templatetags. +Django has quite a few [built-in template tags](https://docs.djangoproject.com/en/3.0/ref/templates/builtins/), but also +allows the creation of [custom template tags](https://docs.djangoproject.com/en/3.0/howto/custom-template-tags/), which +the TOM Toolkit leverages heavily. + +However, it's possible that a TOM Toolkit template tag doesn't quite meet your needs. Maybe the axis labels for photometry +plotting aren't quite what you're looking for, or the target data isn't formatted the way you'd like. This tutorial will +show you how to write your own template tag to suit your own program better. + + +## Preparing your project for custom template tags + +The first thing your project will need is a custom app. You can read about custom apps in the Django tutorial +[here](https://docs.djangoproject.com/en/dev/intro/tutorial01/), but to quickly get started, the command to create a new +app is as follows: + +```python +./manage.py startapp custom_code +``` + +Where `custom_code` is the name of your app. You will also need to ensure that `custom_code` is in your `settings.py`. +Append it to the end of `INSTALLED_APPS`: + +```python +... +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + ... + 'tom_dataproducts', + 'custom_code', +] +... +``` + +You should now have a directory within your TOM called `custom_code`, which looks like this: + + ├── custom_code + | ├── __init__.py + │ ├── admin.py + │ ├── apps.py + │ ├── models.py + │ ├── tests.py + │ └── views.py + ├── data + ├── db.sqlite3 + ├── manage.py + ├── mytom + │ ├── __init__.py + │ ├── settings.py + │ ├── urls.py + │ └── wsgi.py + ├── static + ├── templates + └── tmp + +Next, you'll need to add a `templatetags` directory within `custom_code`. Create an empty file called `__init__.py` within +that directory. Finally, we need a file to put the code for our custom template tags. Add a file in `custom_code` called +`custom_extras`. It's convention to use `_extras` within your template tag module name. + +Your `custom_code` directory should look like this: + + └── custom_code + ├── __init__.py + ├── admin.py + ├── apps.py + ├── models.py + ├── templatetags + | ├── __init__.py + | └── custom_extras.py + ├── tests.py + └── views.py + + +## Writing a custom template tag + +For our template tag, we're going to write a tag that displays the timestamp and magnitude for the most recent photometry +point available for a target. There are three aspects to a template tag: + +* The code in `custom_extras` to run the logic to get the data we'll be displaying +* The partial template to render the data +* Putting the custom tag somewhere we'd like it displayed + +### The Python code + +We're going to write a `recent_photometry` function in our `custom_extras` first. Step one is the necessary import and +initialization of the template library: + +```python +from django import template + + +register = template.Library() +``` + +Now, to the `recent_photometry` function. A couple notes about the approach here: + +* The function will have the decorator `@register.inclusion_tag()`. There are a couple of different types of template tags, +but we're using the `inclusion_tag` because it renders a template, allowing us to customize how it looks. The `simple_tag` +is a different type of template tag that simply modifies data, so that won't work for us. +* Within the decorator is a path to the partial template that will render the data--this doesn't exist yet, but remember the file name we're using! +* We'd like to get the latest photometry values for a specific target, so we'll need to pass a `Target` as a parameter. +* We'd also like to be able to specify how many photometry points we care about, so let's also include a keyword argument that defaults to just 1. + +```python +from django import template + + +register = template.Library() + + +@register.inclusion_tag('custom_code/partials/recent_photometry.html') +def recent_photometry(target, num_points=1): + return {} +``` + +You can see that we'll eventually be returning a dictionary, but first we need to add our logic. We'll need to use the +`Target` passed in to get all `ReducedDatum` objects for that `Target` with a `data_type` of `photometry`. Then we'll +need to order by `timestamp` descending, and slice just the first few. Make sure to take note of the imports in this step! + +```python +import json + +from django import template + +from tom_dataproducts.models import ReducedDatum + + +register = template.Library() + + +@register.inclusion_tag('custom_code/partials/recent_photometry.html') +def recent_photometry(target, num_points=1): + photometry = ReducedDatum.objects.filter(data_type='photometry').order_by('-timestamp')[:num_points] + return {'recent_photometry': [(datum.timestamp, json.loads(datum.value)['magnitude']) for datum in photometry]} +``` + +It's only a couple of lines, but there's a lot going on here. The first line does the aforemention database query and +slices the first point of the `QuerySet`. The second line constructs a dictionary--the only key is `recent_photometry`, +and the corresponding value is a list of tuples. Each tuple has the timestamp as the first item, and the magnitude as +the second item. + +Ultimately, this template tag will, when included, return the most recent photometry points for a `Target`. But it +can't display anything! + +### The partial template + +So now we need to create `custom_code/templates/custom_code/partials/recent_photometry.html`. We'll need to add yet +another series of directories and files. Your directory structure should now look like this: + +Let's start with the partial template. We'll need to add yet another series of directories and files. Add the following +to your directory structure: + + └── custom_code + └── templates + └── custom_code + └── partials + └── recent_photometry.html + +Your complete directory structure should look like this: + + └── custom_code + ├── __init__.py + ├── admin.py + ├── apps.py + ├── models.py + ├── templates + | └── custom_code + | └── partials + | └── recent_photometry.html + ├── templatetags + | ├── __init__.py + | └── custom_extras.py + ├── tests.py + └── views.py + +And let's open up `recent_photometry.html` and get to work. + + +```html +
+
+ Recent Photometry +
+ + + + {% for datum in recent_photometry %} + + + + + {% empty %} + + + + {% endfor %} + +
TimestampMagnitude
{{ datum.0 }}{{ datum.1 }}
No recent photometry.
+
+``` + +This template looks suspiciously like a few others in the TOM Toolkit, but that's okay! It will just render a two-column +table with columns for timestamp and magnitude. The dictionary we returned is accessible to the template, which is why +this line works: + +```html +{% for datum in recent_photometry %} +``` +It iterates over the value referred to by `recent_photometry`, which, if you recall, is a list of tuples. Then it +renders each element of the tuple in a `` element. + +So we have a partial template and a template tag that can be used anywhere, but we have to put it somewhere! + +### Using the template tag + +The target detail page seems like a logical place for this, so let's go there. First, we need to override our `target_detail.html` +template. If you haven't read the tutorial on template overriding, you can do so [here](customize_templates)-- +in the meantime, you'll need to add `target_detail.html` to `templates/tom_targets/` in the top level of your project. +Your project directory should look like this: + + ├── custom_code + ├── data + ├── db.sqlite3 + ├── manage.py + ├── mytom + ├── static + ├── templates + │ └── tom_targets + │ └── target_detail.html + └── tmp + +Then, you'll need to copy the contents of `target_detail.html` in the base TOM Toolkit to your `target_detail.html`. You +can find that file on [Github](https://github.com/TOMToolkit/tom_base/blob/master/tom_targets/templates/tom_targets/target_detail.html). + +Near the top of the file, there's a series of template tags that are loaded in. Add `custom_extras` to that list: + +```html +{% extends 'tom_common/base.html' %} +{% load comments bootstrap4 tom_common_extras targets_extras observation_extras dataproduct_extras publication_extras custom_extras static cache %} +... +``` + +Then, put your templatetag in the HTML somewhere, passing in `object` (which refers to the object value of the current +template context) and the desired number of photometry points: + +```html +... +{% endif %} +{% target_buttons object %} +{% target_data object %} +{% if object.type == 'SIDEREAL' %} +{% aladin object %} +{% recent_photometry object num_points=3 %} +{% endif %} +... +``` + +The new table should be displayed on your target detail page! Not only that, but you'll now be able to include that +template tag on other pages, too. And if it doesn't quite meet your needs--perhaps you want the most recent photometry +points for all targets, for example--it can be easily modified. + +As far as this template tag goes, as of this tutorial, it's now a part of the base TOM Toolkit, but all of the information +here should provide you with the ability to write your own. diff --git a/docs/customization/index.rst b/docs/customization/index.rst index b64aa7c83..87eb76722 100644 --- a/docs/customization/index.rst +++ b/docs/customization/index.rst @@ -8,6 +8,7 @@ Customizing customsettings customize_templates + customize_template_tags adding_pages target_fields customizing_data_processing @@ -25,6 +26,9 @@ configure. :doc:`Customizing TOM Templates ` - Learn how to override built in TOM templates to change the look and feel of your TOM. +:doc:`Customizing Template Tag ` - Learn how to write your own template tags to display +the data you need. + :doc:`Adding new Pages to your TOM ` - Learn how to add entirely new pages to your TOM, displaying static html pages or dynamic database-driven content. diff --git a/docs/index.rst b/docs/index.rst index 8a07b681e..9cf1110b6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -47,6 +47,9 @@ configure. :doc:`Customizing TOM Templates ` - Learn how to override built in TOM templates to change the look and feel of your TOM. +:doc:`Customizing Template Tag ` - Learn how to write your own template tags to display +the data you need. + :doc:`Adding new Pages to your TOM ` - Learn how to add entirely new pages to your TOM, displaying static html pages or dynamic database-driven content. diff --git a/docs/introduction/resources.rst b/docs/introduction/resources.rst index 54e0bb163..ee4eb64d5 100644 --- a/docs/introduction/resources.rst +++ b/docs/introduction/resources.rst @@ -31,7 +31,9 @@ Django `Classy Class-Based Views `_ - Inheritance can be confusing. The CCBV reference page shows the class hierarchy of Django's built-in class-based views, along with the available attributes and methods, and the class they each come from. -`Security in Django `_ - Take special note of +`Databases `_ - Information on setting up different database backends in Django. + +`Security in Django `_ - Make sure your Django application is secure. Python From 3a1c5b321e2f89606a8a5a8a9640af41af0ca451 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 14 Apr 2020 15:01:38 -0700 Subject: [PATCH 010/424] Added templatetags for recent photometry and fixed typo in docs --- docs/customization/customize_template_tags.md | 2 +- .../partials/recent_photometry.html | 21 +++++++++++++++++++ .../templatetags/dataproduct_extras.py | 6 ++++++ .../templates/tom_targets/target_detail.html | 2 ++ 4 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 tom_dataproducts/templates/tom_dataproducts/partials/recent_photometry.html diff --git a/docs/customization/customize_template_tags.md b/docs/customization/customize_template_tags.md index cb81c5e47..e6a2fd7b0 100644 --- a/docs/customization/customize_template_tags.md +++ b/docs/customization/customize_template_tags.md @@ -253,8 +253,8 @@ template context) and the desired number of photometry points: {% target_data object %} {% if object.type == 'SIDEREAL' %} {% aladin object %} -{% recent_photometry object num_points=3 %} {% endif %} +{% recent_photometry object num_points=3 %} ... ``` diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/recent_photometry.html b/tom_dataproducts/templates/tom_dataproducts/partials/recent_photometry.html new file mode 100644 index 000000000..034d14906 --- /dev/null +++ b/tom_dataproducts/templates/tom_dataproducts/partials/recent_photometry.html @@ -0,0 +1,21 @@ +{% load tom_common_extras %} +
+
+ Recent Photometry +
+ + + + {% for datum in recent_photometry %} + + + + + {% empty %} + + + + {% endfor %} + +
TimestampMagnitude
{{ datum.0 }}{{ datum.1|truncate_number }}
No recent photometry.
+
\ No newline at end of file diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index 666071b13..5d29fecf7 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -94,6 +94,12 @@ def upload_dataproduct(context, obj): return {'data_product_form': form} +@register.inclusion_tag('tom_dataproducts/partials/recent_photometry.html') +def recent_photometry(target, num_points=1): + photometry = ReducedDatum.objects.filter(data_type='photometry').order_by('-timestamp')[:num_points] + return {'recent_photometry': [(datum.timestamp, json.loads(datum.value)['magnitude']) for datum in photometry]} + + @register.inclusion_tag('tom_dataproducts/partials/photometry_for_target.html', takes_context=True) def photometry_for_target(context, target): """ diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index 885627a3b..f3c3aaa76 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -17,9 +17,11 @@ {% endif %} {% target_buttons object %} {% target_data object %} + {% recent_photometry object num_points=3 %} {% if object.type == 'SIDEREAL' %} {% aladin object %} {% endif %} +
From d8479a79150af5ff3ab815cd0bd438e7982bcf1c Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 14 Apr 2020 15:24:20 -0700 Subject: [PATCH 011/424] Updated recent_photometry tag to be more elegant --- .../tom_dataproducts/partials/recent_photometry.html | 6 +++--- tom_dataproducts/templatetags/dataproduct_extras.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/recent_photometry.html b/tom_dataproducts/templates/tom_dataproducts/partials/recent_photometry.html index 034d14906..80baa0c56 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/recent_photometry.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/recent_photometry.html @@ -6,10 +6,10 @@ - {% for datum in recent_photometry %} + {% for datum in data %} - - + + {% empty %} diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index 5d29fecf7..4de4f5dac 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -97,7 +97,7 @@ def upload_dataproduct(context, obj): @register.inclusion_tag('tom_dataproducts/partials/recent_photometry.html') def recent_photometry(target, num_points=1): photometry = ReducedDatum.objects.filter(data_type='photometry').order_by('-timestamp')[:num_points] - return {'recent_photometry': [(datum.timestamp, json.loads(datum.value)['magnitude']) for datum in photometry]} + return {'data': [{'timestamp': rd.timestamp, 'magnitude': json.loads(rd.value)['magnitude']} for rd in photometry]} @register.inclusion_tag('tom_dataproducts/partials/photometry_for_target.html', takes_context=True) From 673ddcf78f3b3db042d99f0d260feec0b6550952 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 14 Apr 2020 15:35:06 -0700 Subject: [PATCH 012/424] Added message for if filter doesn't match any targets --- tom_targets/templates/tom_targets/target_list.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_targets/templates/tom_targets/target_list.html b/tom_targets/templates/tom_targets/target_list.html index f958874b7..445c172cb 100644 --- a/tom_targets/templates/tom_targets/target_list.html +++ b/tom_targets/templates/tom_targets/target_list.html @@ -73,7 +73,7 @@ {% empty %} + {% empty %} + + + {% endfor %}
TimestampMagnitude
{{ datum.0 }}{{ datum.1|truncate_number }}{{ datum.timestamp }}{{ datum.magnitude|truncate_number }}
- {% if target_count == 0 %} + {% if target_count == 0 and not query_string %} No targets yet. You might want to create a target manually or import one from an alert broker. {% else %} From b355600aec203fc7222733758ad2faf5dbc554ae Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 14 Apr 2020 15:37:50 -0700 Subject: [PATCH 013/424] changed name of kwarg for recent_photometry --- tom_dataproducts/templatetags/dataproduct_extras.py | 5 ++++- tom_targets/templates/tom_targets/target_detail.html | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index 4de4f5dac..d760a1914 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -95,7 +95,10 @@ def upload_dataproduct(context, obj): @register.inclusion_tag('tom_dataproducts/partials/recent_photometry.html') -def recent_photometry(target, num_points=1): +def recent_photometry(target, limit=1): + """ + Displays a table of the most recent photometric points for a target. + """ photometry = ReducedDatum.objects.filter(data_type='photometry').order_by('-timestamp')[:num_points] return {'data': [{'timestamp': rd.timestamp, 'magnitude': json.loads(rd.value)['magnitude']} for rd in photometry]} diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index f3c3aaa76..d596a3b8d 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -17,7 +17,7 @@ {% endif %} {% target_buttons object %} {% target_data object %} - {% recent_photometry object num_points=3 %} + {% recent_photometry object limit=3 %} {% if object.type == 'SIDEREAL' %} {% aladin object %} {% endif %} From 48ffaa12c57472a4408b92662001be76d5f81c6e Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 14 Apr 2020 15:42:40 -0700 Subject: [PATCH 014/424] Added templatetag for recently updated targets --- .../templatetags/dataproduct_extras.py | 2 +- .../partials/recently_updated_targets.html | 23 +++++++++++++++++++ tom_targets/templatetags/targets_extras.py | 9 ++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 tom_targets/templates/tom_targets/partials/recently_updated_targets.html diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index d760a1914..98621fac9 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -99,7 +99,7 @@ def recent_photometry(target, limit=1): """ Displays a table of the most recent photometric points for a target. """ - photometry = ReducedDatum.objects.filter(data_type='photometry').order_by('-timestamp')[:num_points] + photometry = ReducedDatum.objects.filter(data_type='photometry').order_by('-timestamp')[:limit] return {'data': [{'timestamp': rd.timestamp, 'magnitude': json.loads(rd.value)['magnitude']} for rd in photometry]} diff --git a/tom_targets/templates/tom_targets/partials/recently_updated_targets.html b/tom_targets/templates/tom_targets/partials/recently_updated_targets.html new file mode 100644 index 000000000..3be8ce2ad --- /dev/null +++ b/tom_targets/templates/tom_targets/partials/recently_updated_targets.html @@ -0,0 +1,23 @@ + + + + {% for target in targets %} + + + + + {% empty %} + + + + {% endfor %} + +
IDModified
+ + {{ target.name }} + + + {{ target.modified|date }} +
+ No targets. Create a target. +
diff --git a/tom_targets/templatetags/targets_extras.py b/tom_targets/templatetags/targets_extras.py index 9eba25300..f6ec0e90f 100644 --- a/tom_targets/templatetags/targets_extras.py +++ b/tom_targets/templatetags/targets_extras.py @@ -28,6 +28,15 @@ def recent_targets(context, limit=10): return {'targets': get_objects_for_user(user, 'tom_targets.view_target').order_by('-created')[:limit]} +@register.inclusion_tag('tom_targets/partials/recently_updated_targets.html', takes_context=True) +def recently_updated_targets(context, limit=10): + """ + Displays a list of the most recently updated targets in the TOM up to the given limit, or 10 if not specified. + """ + user = context['request'].user + return {'targets': get_objects_for_user(user, 'tom_targets.view_target').order_by('-modified')[:limit]} + + @register.inclusion_tag('tom_targets/partials/target_feature.html') def target_feature(target): """ From af37126a56eb8ffd14d2fd566c46b02b9ca0a96d Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 15 Apr 2020 08:50:51 -0700 Subject: [PATCH 015/424] Added documentation on querying TargetExtras --- docs/advanced/index.rst | 6 +++- docs/advanced/querying.md | 70 +++++++++++++++++++++++++++++++++++++++ docs/index.rst | 2 ++ 3 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 docs/advanced/querying.md diff --git a/docs/advanced/index.rst b/docs/advanced/index.rst index a3e7a032e..3b487b8e1 100644 --- a/docs/advanced/index.rst +++ b/docs/advanced/index.rst @@ -12,6 +12,8 @@ Advanced Topics scripts strategies latex_generation + querying + :doc:`Background Tasks ` - Learn how to set up an asynchronous task library to handle long running and/or concurrent functions. @@ -28,4 +30,6 @@ console/scripts) to interact directly with your TOM. :doc:`Observing and cadence strategies ` - Learn about observing and cadence strategies and how to write a custom cadence strategy to automate a series of observations -:doc:`LaTeX table generation ` - Learn how to generate LaTeX for certain models and add LaTeX generators for other models \ No newline at end of file +:doc:`LaTeX table generation ` - Learn how to generate LaTeX for certain models and add LaTeX generators for other models + +:doc: `Advanced Querying ` - Get a couple of tips on programmatic querying with Django's QuerySet API \ No newline at end of file diff --git a/docs/advanced/querying.md b/docs/advanced/querying.md new file mode 100644 index 000000000..26585e309 --- /dev/null +++ b/docs/advanced/querying.md @@ -0,0 +1,70 @@ +# Querying on related objects + +An aspect of programmatic TOM Toolkit access that is often desired is filtering by related objects. While this is +extensively documented in the Django documentation, it's certainly helpful to see a couple of examples in action. + +## Identifying Targets by TargetExtra values + +There may be times that you want to find Targets by specific parameters. It's fairly trivial to, for example, find a set +of Targets by RA: + +```python +>>> from tom_targets.models import Target +>>> Target.objects.filter(ra=356.58) +``` + +However, this isn't terribly helpful, as you need to know the exact value of RA that you're looking for to find your +Target. Fortunately, the Django QuerySet API offers a number of additional functions, including +[Field lookups](https://docs.djangoproject.com/en/3.0/ref/models/querysets/#field-lookups): + +```python +>>> Target.objects.filter(ra__lte=357, ra__gte=356) +``` + +The above query will look for Targets with RAs between 356 and 357. Field lookups can be used for more granular queries, +and it's encouraged to reference the Django Queryset API Docs to familiarize yourself. + +While the previous query is very useful for searching in a range, what about when you aren't filtering on base Target +fields? A common use case of the TOM `TargetExtra` model is for fields that aren't on Targets by default. Let's take the +example of supernovae. Let's say that you have a TOM for tracking supernovae, and you've added redshift as a TargetExtra. +How does one find Targets with the appropriate redshift? + +```python +>>> Target.objects.filter(targetextra__key='redshift', targetextra__value__gt=0.5) +``` + +That query will first find Targets with a TargetExtra of `redshift`, and will filter those particular TargetExtras for a +value of greater than 0.5. + + +## Adding Targets to Groups programmatically + +Another operation that one might desire to do programmatically is adding Targets to Groups. This can be done in a +relatively straightforward manner as well: + +```python +from tom_targets.models import TargetList +>>> Target.objects.all() +, ]> +>>> TargetList.objects.all() + +>>> tl = TargetList(name='My Target List') +>>> tl.save() +>>> tl.refresh_from_db() +>>> tl.id +1 +>>> tl.targets.all() + +>>> tl.targets.add(Target.objects.first()) +>>> tl.targets.all() +]> +``` + +Related objects can be obtained in either direction: + +```python +>>> t.targetlist_set.all() +]> +>>> tl.targets.all() +]> +``` diff --git a/docs/index.rst b/docs/index.rst index 9cf1110b6..1fde41757 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -95,6 +95,8 @@ custom cadence strategy to automate a series of observations. :doc:`LaTeX table generation ` - Learn how to generate LaTeX for certain models and add LaTeX generators for other models. +:doc: `Advanced Querying ` - Get a couple of tips on programmatic querying with Django's QuerySet API + Deployment ---------- From 8abd9b8033e04d31e5f144f7c63cd122b5847ecb Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 15 Apr 2020 09:46:22 -0700 Subject: [PATCH 016/424] Fixed tns harvester exception handling, improved api key acquisition --- tom_catalogs/harvesters/tns.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tom_catalogs/harvesters/tns.py b/tom_catalogs/harvesters/tns.py index 2ec540e74..51b98cbd1 100644 --- a/tom_catalogs/harvesters/tns.py +++ b/tom_catalogs/harvesters/tns.py @@ -1,5 +1,4 @@ import json -import os import requests from astropy import units as u From 55f1e7ca41e65cb5c679672e1d9e7079a0b4f5dc Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 15 Apr 2020 09:47:36 -0700 Subject: [PATCH 017/424] Actually did the things I said I did this time --- tom_catalogs/harvesters/tns.py | 40 +++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/tom_catalogs/harvesters/tns.py b/tom_catalogs/harvesters/tns.py index 51b98cbd1..43c25b35b 100644 --- a/tom_catalogs/harvesters/tns.py +++ b/tom_catalogs/harvesters/tns.py @@ -7,29 +7,39 @@ from django.conf import settings from tom_catalogs.harvester import AbstractHarvester +from tom_common.exceptions import ImproperCredentialsException + +TNS_URL = 'https://wis-tns.weizmann.ac.il' + +try: + TNS_CREDENTIALS = settings.BROKER_CREDENTIALS['TNS'] +except (AttributeError, KeyError): + TNS_CREDENTIALS = { + 'api_key': '' + } def get(term): - api_key = settings.BROKER_CREDENTIALS['TNS_APIKEY'] - url = "https://wis-tns.weizmann.ac.il/api/get" + # url = "https://wis-tns.weizmann.ac.il/api/get" + + get_url = TNS_URL + '/api/get/object' - try: - get_url = url + '/object' + # change term to json format + json_list = [("objname", term)] + json_file = OrderedDict(json_list) - # change term to json format - json_list = [("objname", term)] - json_file = OrderedDict(json_list) + # construct the list of (key,value) pairs + get_data = [('api_key', (None, TNS_CREDENTIALS['api_key'])), + ('data', (None, json.dumps(json_file)))] - # construct the list of (key,value) pairs - get_data = [('api_key', (None, api_key)), - ('data', (None, json.dumps(json_file)))] + response = requests.post(get_url, files=get_data) + response_data = json.loads(response.text) + print(response_data) - response = requests.post(get_url, files=get_data) - response = json.loads(response.text)['data']['reply'] - return response + if 400 <= response_data.get('id_code') <= 403: + raise ImproperCredentialsException('TNS: ' + str(response_data.get('id_message'))) - except Exception as e: - return [None, 'Error message : \n' + str(e)] + return response_data['data']['reply'] class TNSHarvester(AbstractHarvester): From 9988f9778791432cb293ea3449f4597f3cbf3582 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 15 Apr 2020 09:51:00 -0700 Subject: [PATCH 018/424] removed print statement --- tom_catalogs/harvesters/tns.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tom_catalogs/harvesters/tns.py b/tom_catalogs/harvesters/tns.py index 43c25b35b..b0c2ef577 100644 --- a/tom_catalogs/harvesters/tns.py +++ b/tom_catalogs/harvesters/tns.py @@ -34,7 +34,6 @@ def get(term): response = requests.post(get_url, files=get_data) response_data = json.loads(response.text) - print(response_data) if 400 <= response_data.get('id_code') <= 403: raise ImproperCredentialsException('TNS: ' + str(response_data.get('id_message'))) From 917b25d4db5ceeabd4c499a0b3ac198541e38198 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 15 Apr 2020 10:58:51 -0700 Subject: [PATCH 019/424] added troubleshooting doc --- docs/index.rst | 2 ++ docs/introduction/index.rst | 5 ++++- docs/introduction/troubleshooting.md | 24 ++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 docs/introduction/troubleshooting.md diff --git a/docs/index.rst b/docs/index.rst index 1fde41757..c389a0414 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -36,6 +36,8 @@ HTML, CSS, Python, and Django :doc:`Frequently Asked Questions ` - Look here for a potential quick answer to a common question. +:doc:`Troubleshooting ` - Find solutions to common problems or information on how to debug an issue. + Extending and Customizing ------------------------- diff --git a/docs/introduction/index.rst b/docs/introduction/index.rst index 79a1bdfac..7abb22e78 100644 --- a/docs/introduction/index.rst +++ b/docs/introduction/index.rst @@ -10,6 +10,7 @@ Introduction workflow resources faqs + troubleshooting :doc:`Architecture ` - This document describes the architecture of the TOM Toolkit at a high level. Read this first if you're interested in how the TOM Toolkit works. @@ -21,4 +22,6 @@ high level. Read this first if you're interested in how the TOM Toolkit works. :doc:`Programming Resources ` - Resources for learning the elements of programming used in the TOM Toolkit: HTML, CSS, Python, and Django -:doc:`Frequently Asked Questions ` - Look here for a potential quick answer to a common question. \ No newline at end of file +:doc:`Frequently Asked Questions ` - Look here for a potential quick answer to a common question. + +:doc:`Troubleshooting ` - Find solutions to common problems or information on how to debug an issue. \ No newline at end of file diff --git a/docs/introduction/troubleshooting.md b/docs/introduction/troubleshooting.md new file mode 100644 index 000000000..17a0a11f5 --- /dev/null +++ b/docs/introduction/troubleshooting.md @@ -0,0 +1,24 @@ +# Troubleshooting your TOM Toolkit + +When first installing or later updating your TOM, you may run into a few common issues. Fortunately, you can stand on our +shoulders and hopefully find a solution here! + + +## Check that you've migrated + +Oftentimes, updating the TOM Toolkit requires running migrations. Usually, a directive to do so will be included in the +release notes, or Django will remind you that `You have unapplied migrations`. If you don't happen to see those, you may +also see a ` does not exist` when you load a page, or an error about an `applabel`. Those are generally +indicators that you need to run a database migration. + +You can confirm that you are missing a migration by running: + +``` +./manage.py showmigrations --list +``` + +Migrations that have been applied will have a `[X]` next to them, so make sure they all have one. If any are missing: + +``` +./manage.py migrate +``` From 00b53373dc13ea6b2966a7674aa5788847d5d2c0 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 15 Apr 2020 10:59:30 -0700 Subject: [PATCH 020/424] Added a couple more topics to the troubleshooting guide --- docs/introduction/troubleshooting.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/introduction/troubleshooting.md b/docs/introduction/troubleshooting.md index 17a0a11f5..6c52b9844 100644 --- a/docs/introduction/troubleshooting.md +++ b/docs/introduction/troubleshooting.md @@ -22,3 +22,22 @@ Migrations that have been applied will have a `[X]` next to them, so make sure t ``` ./manage.py migrate ``` + + +## Make sure you're in a virtual environment + +Everyone forgets to activate their virtualenv from time to time. If you get a missing package or some such, ensure that +you've activated your virtualenv: + +``` +source env/bin/activate +``` + +You may need to adapt the above for your particular shell. Also be sure that the virtualenv was created with a compatible +version of Python, and that you installed your dependencies into that virtualenv. + + +## Check your shell + +It's a small development team, and we all use bash. We've seen some issues with people running zsh, fish, and even csh. +You may need to adapt the commands given in the setup guide. \ No newline at end of file From d91c4d6de61c81a6154b9ed0df49ad741c66e404 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 15 Apr 2020 15:36:55 -0700 Subject: [PATCH 021/424] added docs for common exceptions --- docs/api/tom_common/exceptions.rst | 5 +++++ docs/api/tom_common/index.rst | 1 + docs/introduction/index.rst | 2 +- tom_common/exceptions.py | 4 ++++ 4 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 docs/api/tom_common/exceptions.rst diff --git a/docs/api/tom_common/exceptions.rst b/docs/api/tom_common/exceptions.rst new file mode 100644 index 000000000..d679e6dfa --- /dev/null +++ b/docs/api/tom_common/exceptions.rst @@ -0,0 +1,5 @@ +Exceptions +========== + +.. automodule:: tom_common.exceptions + :members: \ No newline at end of file diff --git a/docs/api/tom_common/index.rst b/docs/api/tom_common/index.rst index 154cb04cc..185897299 100644 --- a/docs/api/tom_common/index.rst +++ b/docs/api/tom_common/index.rst @@ -4,5 +4,6 @@ Common .. toctree:: :maxdepth: 2 + exceptions template_tags views \ No newline at end of file diff --git a/docs/introduction/index.rst b/docs/introduction/index.rst index 7abb22e78..f891498f1 100644 --- a/docs/introduction/index.rst +++ b/docs/introduction/index.rst @@ -24,4 +24,4 @@ HTML, CSS, Python, and Django :doc:`Frequently Asked Questions ` - Look here for a potential quick answer to a common question. -:doc:`Troubleshooting ` - Find solutions to common problems or information on how to debug an issue. \ No newline at end of file +:doc:`Troubleshooting ` - Find solutions to common problems or information on how to debug an issue. \ No newline at end of file diff --git a/tom_common/exceptions.py b/tom_common/exceptions.py index f96371ca8..d37e6d49f 100644 --- a/tom_common/exceptions.py +++ b/tom_common/exceptions.py @@ -1,2 +1,6 @@ class ImproperCredentialsException(Exception): + """ + The ImproperCredentialsException should be used when authentication fails with an external service. This exception + is specifically caught by a TOM Toolkit middleware in order to render an appropriate error message. + """ pass From deabb8af0cd2ba49bbab25081443581d71e22231 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 15 Apr 2020 15:44:05 -0700 Subject: [PATCH 022/424] Added docs for exceptions --- docs/advanced/exceptions.md | 9 +++++++++ docs/advanced/index.rst | 6 +++++- docs/index.rst | 3 +++ 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 docs/advanced/exceptions.md diff --git a/docs/advanced/exceptions.md b/docs/advanced/exceptions.md new file mode 100644 index 000000000..f90f0d1f8 --- /dev/null +++ b/docs/advanced/exceptions.md @@ -0,0 +1,9 @@ +# Authentication Exceptions + +The TOM Toolkit offers a few custom exceptions that are documented in the API documentation, but one in particular +should be noted. + +For any modules exposing external services, such as brokers, harvesters, or facilities, a failed authentication should +raise an `ImproperCredentialsException`. Exceptions of this type are caught by the TOM Toolkit's built-in +`ExternalServiceMiddleware`. This middleware will display an error at the top of the page and redirect the user to the +home page. \ No newline at end of file diff --git a/docs/advanced/index.rst b/docs/advanced/index.rst index 3b487b8e1..a20b5d3b3 100644 --- a/docs/advanced/index.rst +++ b/docs/advanced/index.rst @@ -13,6 +13,7 @@ Advanced Topics strategies latex_generation querying + exceptions :doc:`Background Tasks ` - Learn how to set up an asynchronous task library to handle long @@ -32,4 +33,7 @@ custom cadence strategy to automate a series of observations :doc:`LaTeX table generation ` - Learn how to generate LaTeX for certain models and add LaTeX generators for other models -:doc: `Advanced Querying ` - Get a couple of tips on programmatic querying with Django's QuerySet API \ No newline at end of file +:doc: `Advanced Querying ` - Get a couple of tips on programmatic querying with Django's QuerySet API + +:doc: `Authentication exceptions for external services ` - Ensure that your custom external services have + appropriate and visible errors. diff --git a/docs/index.rst b/docs/index.rst index c389a0414..6a5a9631c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -99,6 +99,9 @@ generators for other models. :doc: `Advanced Querying ` - Get a couple of tips on programmatic querying with Django's QuerySet API +:doc: `Authentication exceptions for external services ` - Ensure that your custom external services have + appropriate and visible errors. + Deployment ---------- From 87d32c4165558def88a2f994499ed09211585c18 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 15 Apr 2020 15:46:57 -0700 Subject: [PATCH 023/424] fixed style error --- tom_common/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_common/exceptions.py b/tom_common/exceptions.py index d37e6d49f..c418ec8a2 100644 --- a/tom_common/exceptions.py +++ b/tom_common/exceptions.py @@ -1,6 +1,6 @@ class ImproperCredentialsException(Exception): """ - The ImproperCredentialsException should be used when authentication fails with an external service. This exception + The ImproperCredentialsException should be used when authentication fails with an external service. This exception is specifically caught by a TOM Toolkit middleware in order to render an appropriate error message. """ pass From b181c5991bfe034ff1355c00c533dc575207cf53 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 15 Apr 2020 15:56:17 -0700 Subject: [PATCH 024/424] Updated middleware error message --- tom_common/middleware.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tom_common/middleware.py b/tom_common/middleware.py index bb385ff70..c92a28ce6 100644 --- a/tom_common/middleware.py +++ b/tom_common/middleware.py @@ -19,8 +19,8 @@ def process_exception(self, request, exception): if isinstance(exception, ImproperCredentialsException): msg = ( 'There was a problem authenticating with {}. Please check you have the correct ' - 'credentials entered into your FACILITIES setting. ' - 'https://tomtoolkit.github.io/docs/customsettings#facilities ' + 'credentials the corresponding settings variable. ' + 'https://tomtoolkit.github.io/docs/customsettings ' ).format( str(exception) ) From 2a768fe01db990d99dd8c753d236a8182f52e59c Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 15 Apr 2020 16:03:10 -0700 Subject: [PATCH 025/424] fixing error in syntax of link --- docs/advanced/index.rst | 4 ++-- docs/index.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/advanced/index.rst b/docs/advanced/index.rst index a20b5d3b3..38ddb1389 100644 --- a/docs/advanced/index.rst +++ b/docs/advanced/index.rst @@ -33,7 +33,7 @@ custom cadence strategy to automate a series of observations :doc:`LaTeX table generation ` - Learn how to generate LaTeX for certain models and add LaTeX generators for other models -:doc: `Advanced Querying ` - Get a couple of tips on programmatic querying with Django's QuerySet API +:doc:`Advanced Querying ` - Get a couple of tips on programmatic querying with Django's QuerySet API -:doc: `Authentication exceptions for external services ` - Ensure that your custom external services have +:doc:`Authentication exceptions for external services ` - Ensure that your custom external services have appropriate and visible errors. diff --git a/docs/index.rst b/docs/index.rst index 6a5a9631c..c3c6ebb65 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -97,9 +97,9 @@ custom cadence strategy to automate a series of observations. :doc:`LaTeX table generation ` - Learn how to generate LaTeX for certain models and add LaTeX generators for other models. -:doc: `Advanced Querying ` - Get a couple of tips on programmatic querying with Django's QuerySet API +:doc:`Advanced Querying ` - Get a couple of tips on programmatic querying with Django's QuerySet API -:doc: `Authentication exceptions for external services ` - Ensure that your custom external services have +:doc:`Authentication exceptions for external services ` - Ensure that your custom external services have appropriate and visible errors. Deployment From 8b186b3c5e2fb6ca8634cb3e1dfd9da6d1e3ea29 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 15 Apr 2020 16:09:31 -0700 Subject: [PATCH 026/424] Updated docs to have correct custom settings values --- docs/customization/customsettings.md | 63 ++++++++++++++++++++- tom_catalogs/harvesters/tns.py | 2 +- tom_setup/templates/tom_setup/settings.tmpl | 7 ++- 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/docs/customization/customsettings.md b/docs/customization/customsettings.md index 9646fe6bc..2f06882f6 100644 --- a/docs/customization/customsettings.md +++ b/docs/customization/customsettings.md @@ -5,6 +5,21 @@ The following is a list of TOM Specific settings to be added/edited in your project's `settings.py`. For explanations of Django specific settings, see the [official documentation](https://docs.djangoproject.com/en/2.1/ref/settings/). + +### [ALERT_CREDENTIALS](#alert_credentials) + +Default: + + { + 'TNS': { + 'api_key': '' + } + } + +Credentials for any brokers that require them. At the moment, the only built-in TOM Toolkit broker module that +requires credentials is the TNS. + + ### [AUTH_STRATEGY](#auth_strategy) Default: 'READ_ONLY' @@ -15,13 +30,27 @@ anything. A value of **LOCKED** requires all users to login before viewing any page. Use the [**OPEN_URLS**](#open_urls) setting for adding exemptions. +### [DATA_PROCESSORS](#data_processors) + +Default: + + { + 'photometry': 'tom_dataproducts.processors.photometry_processor.PhotometryProcessor', + 'spectroscopy': 'tom_dataproducts.processors.spectroscopy_processor.SpectroscopyProcessor', + } + +The `DATA_PROCESSORS` dict specifies the subclasses of `DataProcessor` that should be used for processing the +corresponding `data_type`s. + ### [DATA_PRODUCT_TYPES](#data_types) Default: { 'spectroscopy': ('spectroscopy', 'Spectroscopy'), - 'photometry': ('photometry', 'Photometry') + 'photometry': ('photometry', 'Photometry'), + 'spectroscopy': ('spectroscopy', 'Spectroscopy'), + 'image_file': ('image_file', 'Image File') } A list of machine readable, human readable tuples which determine the choices @@ -92,11 +121,21 @@ With an [**AUTH_STRATEGY**](#auth_strategy) value of **LOCKED**, urls in this li visible to unauthenticated users. You might add the homepage ('/'), for example. +### [TARGET_PERMISSIONS_ONLY](#target_permissions_only) + +Default: True + +This settings determines the permissions strategy of the TOM. When set to True, authorization permissions will be set +on Targets and cascade from there--that is, a group that can see a Target can see all ObservationRecords and Data +associated with the Target. When set to False, permissions can be set for a group at the Target level, the +ObservationRecord level, or the DataProduct level. + + ### [TARGET_TYPE](#target_type) Default: No default -Can be either **SIDEREAL** or **NON_SIDEREAL**. This settings determines the +Can be either **SIDEREAL** or **NON_SIDEREAL**. This setting determines the default target type for your TOM. TOMs can still create and work with targets of both types even after this option is set, but setting it to one of the values will optimize the workflow for that target type. @@ -109,7 +148,10 @@ Default: [ 'tom_alerts.brokers.mars.MARSBroker', 'tom_alerts.brokers.lasair.LasairBroker', - 'tom_alerts.brokers.scout.ScoutBroker' + 'tom_alerts.brokers.scout.ScoutBroker', + 'tom_alerts.brokers.tns.TNSBroker', + 'tom_alerts.brokers.antares.ANTARESBroker', + 'tom_alerts.brokers.gaia.GaiaBroker' ] A list of tom alert classes to make available to your TOM. If you have written or @@ -125,6 +167,8 @@ Default: [ 'tom_observations.facilities.lco.LCOFacility', 'tom_observations.facilities.gemini.GEMFacility', + 'tom_observations.facilities.soar.SOARFacility', + 'tom_observations.facilities.lt.LTFacility' ] A list of observation facility classes to make available to your TOM. If you have @@ -147,3 +191,16 @@ Default: A list of TOM harverster classes to make available to your TOM. If you have written or downloaded additional harvester classes you would make them available here. + + +### [TOM_LATEX_PROCESSORS](#tom_latex_processors) + +Default: + + { + 'ObservationGroup': 'tom_publications.processors.latex_processor.ObservationGroupLatexProcessor', + 'TargetList': 'tom_publications.processors.target_list_latex_processor.TargetListLatexProcessor' + } + +A dictionary with the keys being TOM models classes and the values being the modules that should be used to generate +latex tables for those models. diff --git a/tom_catalogs/harvesters/tns.py b/tom_catalogs/harvesters/tns.py index b0c2ef577..4e9d3ffa6 100644 --- a/tom_catalogs/harvesters/tns.py +++ b/tom_catalogs/harvesters/tns.py @@ -12,7 +12,7 @@ TNS_URL = 'https://wis-tns.weizmann.ac.il' try: - TNS_CREDENTIALS = settings.BROKER_CREDENTIALS['TNS'] + TNS_CREDENTIALS = settings.ALERT_CREDENTIALS['TNS'] except (AttributeError, KeyError): TNS_CREDENTIALS = { 'api_key': '' diff --git a/tom_setup/templates/tom_setup/settings.tmpl b/tom_setup/templates/tom_setup/settings.tmpl index d973226ad..ea03fa6b8 100644 --- a/tom_setup/templates/tom_setup/settings.tmpl +++ b/tom_setup/templates/tom_setup/settings.tmpl @@ -246,9 +246,14 @@ TOM_ALERT_CLASSES = [ 'tom_alerts.brokers.lasair.LasairBroker', 'tom_alerts.brokers.scout.ScoutBroker', 'tom_alerts.brokers.tns.TNSBroker', + 'tom_alerts.brokers.antares.ANTARESBroker', + 'tom_alerts.brokers.gaia.GaiaBroker' ] -BROKER_CREDENTIALS = { +ALERT_CREDENTIALS = { + 'TNS': { + 'api_key': '' + } } # Define extra target fields here. Types can be any of "number", "string", "boolean" or "datetime" From 91760259511d20aa513a89e89478f82b0291e9ed Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 15 Apr 2020 16:26:38 -0700 Subject: [PATCH 027/424] Fixed the link, hopefully for real this time --- tom_common/middleware.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tom_common/middleware.py b/tom_common/middleware.py index c92a28ce6..e8ff2662d 100644 --- a/tom_common/middleware.py +++ b/tom_common/middleware.py @@ -18,9 +18,9 @@ def __call__(self, request): def process_exception(self, request, exception): if isinstance(exception, ImproperCredentialsException): msg = ( - 'There was a problem authenticating with {}. Please check you have the correct ' - 'credentials the corresponding settings variable. ' - 'https://tomtoolkit.github.io/docs/customsettings ' + 'There was a problem authenticating with {}. Please check that you have the correct ' + 'credentials in the corresponding settings variable. ' + 'https://tom-toolkit.readthedocs.io/en/stable/customization/customsettings.html ' ).format( str(exception) ) From d04b40e0ae5c35bc7487db9a736b10f82f387dfa Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Thu, 16 Apr 2020 00:09:09 +0000 Subject: [PATCH 028/424] wip: define GenericManualFacility abc --- tom_base/settings.py | 3 +- tom_observations/facilities/manual.py | 197 ++++++++++++++++++++++++++ 2 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 tom_observations/facilities/manual.py diff --git a/tom_base/settings.py b/tom_base/settings.py index d6b06aed2..e685768c1 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -210,7 +210,8 @@ TOM_FACILITY_CLASSES = [ 'tom_observations.facilities.lco.LCOFacility', - 'tom_observations.facilities.gemini.GEMFacility' + 'tom_observations.facilities.gemini.GEMFacility', + 'tom_observations.facilities.manual.GenericManualFacility' ] TOM_CADENCE_STRATEGIES = [ diff --git a/tom_observations/facilities/manual.py b/tom_observations/facilities/manual.py new file mode 100644 index 000000000..9c43d08fa --- /dev/null +++ b/tom_observations/facilities/manual.py @@ -0,0 +1,197 @@ +from abc import abstractmethod +import logging +import requests + +from django.core.files.base import ContentFile + +from tom_observations.facility import GenericObservationFacility, AUTO_THUMBNAILS + +logger = logging.getLogger(__name__) + + +class GenericManualFacility(GenericObservationFacility): + """ + The facility class contains all the logic specific to the facility it is + written for. Some methods are used only internally (starting with an + underscore) but some need to be implemented by all facility classes. + All facilities should inherit from this class which + provides some base functionality. + In order to make use of a facility class, add the path to + ``TOM_FACILITY_CLASSES`` in your ``settings.py``. + + For an implementation example, please see + https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/lco.py + """ + + name = 'MAN' + observation_types = [('IMAGING', 'Imaging'), ('SPECTRA', 'Spectroscopy')] + SITES = {} + + def update_observation_status(self, observation_id): + from tom_observations.models import ObservationRecord + try: + record = ObservationRecord.objects.get(observation_id=observation_id) + status = self.get_observation_status(observation_id) + record.status = status['state'] + record.scheduled_start = status['scheduled_start'] + record.scheduled_end = status['scheduled_end'] + record.save() + except ObservationRecord.DoesNotExist: + raise Exception('No record exists for that observation id') + + def update_all_observation_statuses(self, target=None): + from tom_observations.models import ObservationRecord + failed_records = [] + records = ObservationRecord.objects.filter(facility=self.name) + if target: + records = records.filter(target=target) + records = records.exclude(status__in=self.get_terminal_observing_states()) + for record in records: + try: + self.update_observation_status(record.observation_id) + except Exception as e: + failed_records.append((record.observation_id, str(e))) + return failed_records + + def all_data_products(self, observation_record): + from tom_dataproducts.models import DataProduct + products = {'saved': [], 'unsaved': []} + for product in self.data_products(observation_record.observation_id): + try: + dp = DataProduct.objects.get(product_id=product['id']) + products['saved'].append(dp) + except DataProduct.DoesNotExist: + products['unsaved'].append(product) + # Obtain products uploaded manually by users + user_products = DataProduct.objects.filter( + observation_record_id=observation_record.id, product_id=None + ) + for product in user_products: + products['saved'].append(product) + + # Add any JPEG images created from DataProducts + image_products = DataProduct.objects.filter( + observation_record_id=observation_record.id, data_product_type='image_file' + ) + for product in image_products: + products['saved'].append(product) + return products + + def save_data_products(self, observation_record, product_id=None): + from tom_dataproducts.models import DataProduct + from tom_dataproducts.utils import create_image_dataproduct + final_products = [] + products = self.data_products(observation_record.observation_id, product_id) + + for product in products: + dp, created = DataProduct.objects.get_or_create( + product_id=product['id'], + target=observation_record.target, + observation_record=observation_record, + ) + if created: + product_data = requests.get(product['url']).content + dfile = ContentFile(product_data) + dp.data.save(product['filename'], dfile) + dp.save() + logger.info('Saved new dataproduct: {}'.format(dp.data)) + if AUTO_THUMBNAILS: + create_image_dataproduct(dp) + dp.get_preview() + final_products.append(dp) + return final_products + + @abstractmethod + def get_form(self, observation_type): + """ + This method takes in an observation type and returns the form type that matches it. + """ + pass + + @abstractmethod + def submit_observation(self, observation_payload): + """ + This method takes in the serialized data from the form and actually + submits the observation to the remote api + """ + pass + + @abstractmethod + def validate_observation(self, observation_payload): + """ + Same thing as submit_observation, but a dry run. You can + skip this in different modules by just using "pass" + """ + pass + + @abstractmethod + def get_observation_url(self, observation_id): + """ + Takes an observation id and return the url for which a user + can view the observation at an external location. In this case, + we return a URL to the LCO observation portal's observation + record page. + """ + pass + + def get_flux_constant(self): + """ + Returns the astropy quantity that a facility uses for its spectral flux conversion. + """ + pass + + def get_wavelength_units(self): + """ + Returns the astropy units that a facility uses for its spectral wavelengths + """ + pass + + def is_fits_facility(self, header): + """ + Returns True if the FITS header is from this facility based on valid keywords and associated + values, False otherwise. + """ + return False + + def get_start_end_keywords(self): + """ + Returns the keywords representing the start and end of an observation window for a facility. Defaults to + ``start`` and ``end``. + """ + return ('start', 'end') + + @abstractmethod + def get_terminal_observing_states(self): + """ + Returns the states for which an observation is not expected + to change. + """ + pass + + @abstractmethod + def get_observing_sites(self): + """ + Return a list of dictionaries that contain the information + necessary to be used in the planning (visibility) tool. The + list should contain dictionaries each that contain sitecode, + latitude, longitude and elevation. + """ + pass + + @abstractmethod + def get_observation_status(self, observation_id): + """ + Return the status for a single observation. observation_id should + be able to be used to retrieve the status from the external service. + """ + pass + + @abstractmethod + def data_products(self, observation_id, product_id=None): + """ + Using an observation_id, retrieve a list of the data + products that belong to this observation. In this case, + the LCO module retrieves a list of frames from the LCO + data archive. + """ + pass From ecc65ecddf0684bf7d369125b740de5bffc3b042 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 16 Apr 2020 15:16:14 -0700 Subject: [PATCH 029/424] Added docs for management commands and added FAQ entry for AnonymousUser --- docs/api/management_commands.rst | 25 +++++++++++++++++++ docs/api/modules.rst | 1 + docs/introduction/faqs.md | 8 ++++++ .../management/commands/runbrokerquery.py | 2 +- .../management/commands/updatereduceddata.py | 3 +-- .../commands/runcadencestrategies.py | 2 +- .../management/commands/updatestatus.py | 7 +++++- 7 files changed, 43 insertions(+), 5 deletions(-) create mode 100644 docs/api/management_commands.rst diff --git a/docs/api/management_commands.rst b/docs/api/management_commands.rst new file mode 100644 index 000000000..db32283a3 --- /dev/null +++ b/docs/api/management_commands.rst @@ -0,0 +1,25 @@ +Commands +======== + +********** +tom_alerts +********** + +runbrokerquery.py - Runs saved alert queries and saves the results as Targets. + +**************** +tom_dataproducts +**************** + +downloaddata.py - Downloads available data for all completed observations. + +updatereduceddata - Gets and updates time-series data for alert-generated targets from the original alert source. Can optionally specify a target id. + + +**************** +tom_dataproducts +**************** + +runcadencestrategy.py - Entry point for running cadence strategies. + +updatestatus.py - Updates the status of each observation request in the TOM. Target id can be specified to update the status for all observations for a single target. diff --git a/docs/api/modules.rst b/docs/api/modules.rst index 9d780d96f..4e64e14e1 100644 --- a/docs/api/modules.rst +++ b/docs/api/modules.rst @@ -16,3 +16,4 @@ API Documentation tom_dataproducts/index tom_observations/index tom_targets/index + management_commands \ No newline at end of file diff --git a/docs/introduction/faqs.md b/docs/introduction/faqs.md index e0dc4e22d..4d863ae80 100644 --- a/docs/introduction/faqs.md +++ b/docs/introduction/faqs.md @@ -80,3 +80,11 @@ project. This will make the contents of `newpage.html` available under the path [/newpage/](http://127.0.0.1/newpage/). + + +### Who is AnonymousUser? + +AnonymousUser is a special profile that django-guardian, our permissions library, creates automatically. AnonymousUser +represents an unauthenticated user. The user has no first name, last name, or password, and allows unauthenticated +users to view unprotected pages within your TOM. You can choose to delete the user if you don't want any pages to be +visible without logging in. diff --git a/tom_alerts/management/commands/runbrokerquery.py b/tom_alerts/management/commands/runbrokerquery.py index 9d81f2cf4..dda07ac1c 100644 --- a/tom_alerts/management/commands/runbrokerquery.py +++ b/tom_alerts/management/commands/runbrokerquery.py @@ -6,7 +6,7 @@ class Command(BaseCommand): - help = 'Run saved alert queries and save the results as Targets' + help = 'Runs saved alert queries and saves the results as Targets' def add_arguments(self, parser): parser.add_argument( diff --git a/tom_dataproducts/management/commands/updatereduceddata.py b/tom_dataproducts/management/commands/updatereduceddata.py index 1887eb06e..815c88dac 100644 --- a/tom_dataproducts/management/commands/updatereduceddata.py +++ b/tom_dataproducts/management/commands/updatereduceddata.py @@ -8,12 +8,11 @@ class Command(BaseCommand): - help = 'Gets and updates time-series data for targets from the original source' + help = 'Gets and updates time-series data for alert-generated targets from the original alert source.' def add_arguments(self, parser): parser.add_argument( '--target_id', - help='Gets and updates time-series data for targets from the original source' ) def handle(self, *args, **options): diff --git a/tom_observations/management/commands/runcadencestrategies.py b/tom_observations/management/commands/runcadencestrategies.py index d6815b634..7eb8ce492 100644 --- a/tom_observations/management/commands/runcadencestrategies.py +++ b/tom_observations/management/commands/runcadencestrategies.py @@ -7,7 +7,7 @@ class Command(BaseCommand): - help = 'Entry point for running cadence strategies' + help = 'Entry point for running cadence strategies.' def handle(self, *args, **kwargs): cadenced_groups = ObservationGroup.objects.exclude(cadence_strategy='') diff --git a/tom_observations/management/commands/updatestatus.py b/tom_observations/management/commands/updatestatus.py index 4d911a94b..e80cec991 100644 --- a/tom_observations/management/commands/updatestatus.py +++ b/tom_observations/management/commands/updatestatus.py @@ -6,7 +6,12 @@ class Command(BaseCommand): - help = 'Updates the status of each observation requests in the TOM' + """ + Updates the status of each observation request in the TOM. Target id can be specified to update the status for all + observations for a single target. + """ + + help = 'Updates the status of each observation request in the TOM' def add_arguments(self, parser): parser.add_argument( From b189c482ed7471b6c0277fdc9fe99399db727705 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 17 Apr 2020 09:51:45 -0700 Subject: [PATCH 030/424] Style fix --- tom_observations/management/commands/updatestatus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_observations/management/commands/updatestatus.py b/tom_observations/management/commands/updatestatus.py index e80cec991..6ef98e8e6 100644 --- a/tom_observations/management/commands/updatestatus.py +++ b/tom_observations/management/commands/updatestatus.py @@ -7,7 +7,7 @@ class Command(BaseCommand): """ - Updates the status of each observation request in the TOM. Target id can be specified to update the status for all + Updates the status of each observation request in the TOM. Target id can be specified to update the status for all observations for a single target. """ From 6b9502ea45e36a3607919c2a50c63ddb0a23779a Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Fri, 17 Apr 2020 18:41:50 +0000 Subject: [PATCH 031/424] pep8 compliance (no functional change) --- tom_observations/facilities/manual.py | 2 +- tom_observations/facility.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tom_observations/facilities/manual.py b/tom_observations/facilities/manual.py index 9c43d08fa..ba22c7213 100644 --- a/tom_observations/facilities/manual.py +++ b/tom_observations/facilities/manual.py @@ -158,7 +158,7 @@ def get_start_end_keywords(self): Returns the keywords representing the start and end of an observation window for a facility. Defaults to ``start`` and ``end``. """ - return ('start', 'end') + return 'start', 'end' @abstractmethod def get_terminal_observing_states(self): diff --git a/tom_observations/facility.py b/tom_observations/facility.py index 4267d4766..76e6918c7 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -200,7 +200,7 @@ def get_start_end_keywords(self): Returns the keywords representing the start and end of an observation window for a facility. Defaults to ``start`` and ``end``. """ - return ('start', 'end') + return 'start', 'end' @abstractmethod def get_terminal_observing_states(self): From e0caef1201796835532b87cbc48d2f60a2639371 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Fri, 17 Apr 2020 18:44:25 +0000 Subject: [PATCH 032/424] since self.name is refered to in methods, should be defined --- tom_observations/facility.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tom_observations/facility.py b/tom_observations/facility.py index 76e6918c7..80cd11aab 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -68,6 +68,7 @@ class GenericObservationFacility(ABC): For an implementation example, please see https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/lco.py """ + name = "Generic" # rename in concrete subclasses def update_observation_status(self, observation_id): from tom_observations.models import ObservationRecord From 10f309ac67dbbc95c25a37c21adaa5afcae77d31 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Fri, 17 Apr 2020 19:12:45 +0000 Subject: [PATCH 033/424] make manual.GenericManualFacility a concrete class for the moment raise NotImplemetedError for (formerly) @abstractmethods --- tom_observations/facilities/manual.py | 32 +++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tom_observations/facilities/manual.py b/tom_observations/facilities/manual.py index ba22c7213..327e2a659 100644 --- a/tom_observations/facilities/manual.py +++ b/tom_observations/facilities/manual.py @@ -1,4 +1,3 @@ -from abc import abstractmethod import logging import requests @@ -101,30 +100,29 @@ def save_data_products(self, observation_record, product_id=None): final_products.append(dp) return final_products - @abstractmethod def get_form(self, observation_type): """ This method takes in an observation type and returns the form type that matches it. """ - pass + # TODO: implement me + raise NotImplementedError - @abstractmethod def submit_observation(self, observation_payload): """ This method takes in the serialized data from the form and actually submits the observation to the remote api """ - pass + # TODO: implement me + raise NotImplementedError - @abstractmethod def validate_observation(self, observation_payload): """ Same thing as submit_observation, but a dry run. You can skip this in different modules by just using "pass" """ - pass + # TODO: implement me + raise NotImplementedError - @abstractmethod def get_observation_url(self, observation_id): """ Takes an observation id and return the url for which a user @@ -132,6 +130,8 @@ def get_observation_url(self, observation_id): we return a URL to the LCO observation portal's observation record page. """ + # TODO: implement me + raise NotImplementedError pass def get_flux_constant(self): @@ -160,15 +160,14 @@ def get_start_end_keywords(self): """ return 'start', 'end' - @abstractmethod def get_terminal_observing_states(self): """ Returns the states for which an observation is not expected to change. """ - pass + # TODO: implement me + raise NotImplementedError - @abstractmethod def get_observing_sites(self): """ Return a list of dictionaries that contain the information @@ -176,17 +175,17 @@ def get_observing_sites(self): list should contain dictionaries each that contain sitecode, latitude, longitude and elevation. """ - pass + # TODO: implement me + raise NotImplementedError - @abstractmethod def get_observation_status(self, observation_id): """ Return the status for a single observation. observation_id should be able to be used to retrieve the status from the external service. """ - pass + # TODO: implement me + raise NotImplementedError - @abstractmethod def data_products(self, observation_id, product_id=None): """ Using an observation_id, retrieve a list of the data @@ -194,4 +193,5 @@ def data_products(self, observation_id, product_id=None): the LCO module retrieves a list of frames from the LCO data archive. """ - pass + # TODO: implement me + raise NotImplementedError From c2b158247b64ee15c4e59ac62ae3fe8ea83ad0e1 Mon Sep 17 00:00:00 2001 From: fraserw Date: Fri, 17 Apr 2020 16:31:37 -0700 Subject: [PATCH 034/424] --changes to reflect correct EPHEMERIS obs prep process for all sites --- tom_observations/facilities/lco.py | 283 +++++++++++++++++---- tom_observations/utils.py | 69 ++++- tom_targets/models.py | 2 +- tom_targets/templatetags/targets_extras.py | 39 +-- tom_targets/utils.py | 48 ++-- 5 files changed, 354 insertions(+), 87 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 130b4d569..5e79174f8 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -5,6 +5,7 @@ from crispy_forms.layout import Layout, Div from django.core.cache import cache from astropy import units as u +from astropy.time import Time from tom_observations.facility import GenericObservationForm from tom_common.exceptions import ImproperCredentialsException @@ -13,6 +14,17 @@ Target, REQUIRED_NON_SIDEREAL_FIELDS, REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME ) +from tom_observations.utils import get_radec_ephemeris + +import json +import numpy as np + +import logging +logger = logging.getLogger('submit_testing.log') + +def take_second_element(elem): + return elem[1] + # Determine settings for this module. try: @@ -46,7 +58,6 @@ def make_request(*args, **kwargs): response.raise_for_status() return response - class LCOBaseObservationForm(GenericObservationForm): name = forms.CharField() ipp_value = forms.FloatField() @@ -59,17 +70,43 @@ class LCOBaseObservationForm(GenericObservationForm): choices=(('NORMAL', 'Normal'), ('TARGET_OF_OPPORTUNITY', 'Rapid Response')) ) + site = forms.ChoiceField( + choices = (('all', 'All Sites'), + ('coj','Siding Spring'), + ('cpt','Sutherland'), + ('tfn', 'Teide'), + ('tlv', 'Wise'), + ('lsc','Cerro Tololo'), + ('elp', 'McDonald'), + ('ogg', 'Haleakala')) + #widget=forms.CheckboxSelectMultiple() + ) + + imaging_interval = forms.FloatField(label='Interval (hrs). Will schedule exposure count per interval.') + + + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.fields['proposal'] = forms.ChoiceField(choices=self.proposal_choices()) self.fields['filter'] = forms.ChoiceField(choices=self.filter_choices()) self.fields['instrument_type'] = forms.ChoiceField(choices=self.instrument_choices()) + + self.eph_target = False + target = Target.objects.get(pk=kwargs['initial']['target_id']) + if target.type == Target.NON_SIDEREAL: + if target.scheme == 'EPHEMERIS': + self.eph_target = True + self.helper.layout = Layout( self.common_layout, self.layout(), self.extra_layout() ) + + + def layout(self): return Div( Div( @@ -84,8 +121,10 @@ def layout(self): ) def extra_layout(self): + if self.eph_target: + return Div('site','imaging_interval') # If you just want to add some fields to the end of the form, add them here. - return Div() + #return Div() def _get_instruments(self): cached_instruments = cache.get('lco_instruments') @@ -98,17 +137,17 @@ def _get_instruments(self): ) cached_instruments = {k: v for k, v in response.json().items() if 'SOAR' not in k} cache.set('lco_instruments', cached_instruments) - return cached_instruments def instrument_choices(self): return [(k, v['name']) for k, v in self._get_instruments().items()] def filter_choices(self): - return set([ + return sorted(set([ (f['code'], f['name']) for ins in self._get_instruments().values() for f in ins['optical_elements'].get('filters', []) + ins['optical_elements'].get('slits', []) - ]) + ]),key=take_second_element) + def proposal_choices(self): response = make_request( @@ -181,29 +220,84 @@ def _build_target_fields(self): target_fields['proper_motion_ra'] = target.pm_ra target_fields['proper_motion_dec'] = target.pm_dec target_fields['epoch'] = target.epoch + elif target.type == Target.NON_SIDEREAL: - target_fields['type'] = 'ORBITAL_ELEMENTS' - # Mapping from TOM field names to LCO API field names, for fields - # where there are differences - field_mapping = { - 'inclination': 'orbinc', - 'lng_asc_node': 'longascnode', - 'arg_of_perihelion': 'argofperih', - 'semimajor_axis': 'meandist', - 'mean_anomaly': 'meananom', - 'mean_daily_motion': 'dailymot', - 'epoch_of_elements': 'epochofel', - 'epoch_of_perihelion': 'epochofperih', - } - # The fields to include in the payload depend on the scheme. Add - # only those that are required - fields = (REQUIRED_NON_SIDEREAL_FIELDS - + REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME[target.scheme]) - for field in fields: - lco_field = field_mapping.get(field, field) - target_fields[lco_field] = getattr(target, field) + if self.eph_target: + ephemeris_targets = {} + ephemeris_windows = {} + + eph_json = json.loads(target.eph_json) + + site_selection = self.cleaned_data['site'] + if site_selection !='all': + site_selection = [site_selection] + else: + site_selection = ['coj','cpt','tfn','tlv','lsc','elp','ogg'] + + for site in site_selection: + if site in eph_json.keys(): + ephemeris_targets[site] = [] + ephemeris_windows[site] = [] + (mjd_vals,ra_vals,dec_vals,air_vals,sun_alt_vals) = get_radec_ephemeris(eph_json[site], + self.cleaned_data['start'], + self.cleaned_data['end'], + self.cleaned_data['imaging_interval'], + 'LCO', + site) + if mjd_vals is not None: + for i in range(len(ra_vals)-1): + if (air_vals[i]1.0 and + air_vals[i+1]>1.0 and + sun_alt_vals[i]<-30.0 and + sun_alt_vals[i+1]<-30.0): + + new_target_fields = {} + new_target_fields['type'] = 'ICRS' + new_target_fields['ra'] = (ra_vals[i]+ra_vals[i+1])/2.0 + new_target_fields['dec'] = (dec_vals[i]+dec_vals[i+1])/2.0 + new_target_fields['proper_motion_ra'] = 0.0#target.pm_ra + new_target_fields['proper_motion_dec'] = 0.0#target.pm_dec + new_target_fields['epoch'] = 2000#'2000' + new_target_fields['parallax'] = 0#'2000' + + start = Time(mjd_vals[i], format='mjd') + end = Time(mjd_vals[i+1], format='mjd') + + #print(site,mjd_vals[i],ra_vals[i],dec_vals[i],air_vals[i],start.isot,end.isot,sun_alt_vals[i]) + + #store start and end times in the target for a matter of convenience in passing this information forward to the request builder + new_target_fields['name'] = '{}_{}_{}'.format(target.name,site,i) + ephemeris_targets[site].append(new_target_fields) + ephemeris_windows[site].append([start.isot, end.isot]) + elif mjd_vals is None and sun_alt_vals==-2: + self.add_error(None,'Date range outside range available in the provided ephemeris.') + return (ephemeris_targets,ephemeris_windows) + + else: + target_fields['type'] = 'ORBITAL_ELEMENTS' + # Mapping from TOM field names to LCO API field names, for fields + # where there are differences + field_mapping = { + 'inclination': 'orbinc', + 'lng_asc_node': 'longascnode', + 'arg_of_perihelion': 'argofperih', + 'semimajor_axis': 'meandist', + 'mean_anomaly': 'meananom', + 'mean_daily_motion': 'dailymot', + 'epoch_of_elements': 'epochofel', + 'epoch_of_perihelion': 'epochofperih', + } + # The fields to include in the payload depend on the scheme. Add + # only those that are required + fields = (REQUIRED_NON_SIDEREAL_FIELDS + + REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME[target.scheme]) + for field in fields: + lco_field = field_mapping.get(field, field) + target_fields[lco_field] = getattr(target, field) - return target_fields + return target_fields def _build_instrument_config(self): instrument_config = { @@ -233,28 +327,119 @@ def _build_configuration(self): } } + """ + def _valid_site_instrument(self,site,ins): + if ins == '1M0-SCICAM-SINISTRO': + if site in ['coj','cpt','lsc','elp']: + return True + else: + return False + + return True + """ + def _build_ephemeris_request_parts(self): + (new_targets, new_windows) = self._build_target_fields() + sites = new_targets.keys() + configurations = [] + windows = [] + locations = [] + for site in sites: + #if self._valid_site_instrument(site,self.cleaned_data['instrument_type']): + #print(self._valid_site_instrument(site,self.cleaned_data['instrument_type']),'weeeee') + for i in range(len(new_targets[site])): + single_obs_config = { + 'type': self.instrument_to_type(self.cleaned_data['instrument_type']), + 'instrument_type': self.cleaned_data['instrument_type'], + 'target': new_targets[site][i], + 'instrument_configs': [self._build_instrument_config()], + 'acquisition_config': { + + }, + 'guiding_config': { + + }, + 'constraints': { + 'max_airmass': self.cleaned_data['max_airmass'] + } + } + single_location = {'site': site, + 'telescope_class': self._get_instruments()[self.cleaned_data['instrument_type']]['class']} + single_windows = [{'start': new_windows[site][i][0], + 'end': new_windows[site][i][1]}] + + + configurations.append(single_obs_config) + windows.append(single_windows) + locations.append(single_location) + return (configurations,windows,locations) + + def _build_ephemeris_requests(self): + (configurations,windows,locations) = self._build_ephemeris_request_parts() + requests = [] + for i in range(len(configurations)): + req = {'configurations': [configurations[i]], + 'location': locations[i], + 'windows': windows[i]} + requests.append(req) + return requests + def observation_payload(self): - return { - "name": self.cleaned_data['name'], - "proposal": self.cleaned_data['proposal'], - "ipp_value": self.cleaned_data['ipp_value'], - "operator": "SINGLE", - "observation_type": self.cleaned_data['observation_mode'], - "requests": [ - { - "configurations": [self._build_configuration()], - "windows": [ - { - "start": self.cleaned_data['start'], - "end": self.cleaned_data['end'] + if not self.eph_target: + return { + "name": self.cleaned_data['name'], + "proposal": self.cleaned_data['proposal'], + "ipp_value": self.cleaned_data['ipp_value'], + "operator": "SINGLE", + "observation_type": self.cleaned_data['observation_mode'], + "requests": [ + { + "configurations": [self._build_configuration()], + "windows": [ + { + "start": self.cleaned_data['start'], + "end": self.cleaned_data['end'] + } + ], + "location": { + "telescope_class": self._get_instruments()[self.cleaned_data['instrument_type']]['class'] } - ], - "location": { - "telescope_class": self._get_instruments()[self.cleaned_data['instrument_type']]['class'] } - } - ] - } + ] + } + else: #ephemeris scheme payload creation + #this is inefficient as the request validation is done to check for site+scope + #configuration errors, and then is done again later to check for other errors. + # + #This could be used to estiamte airmass windows instead of using astropy as + #is done in tom_base/utils.py. + obs_module = get_service_class(self.cleaned_data['facility']) + requests = self._build_ephemeris_requests() + errors = obs_module().validate_observation({ + "name": self.cleaned_data['name'], + "proposal": self.cleaned_data['proposal'], + "ipp_value": self.cleaned_data['ipp_value'], + "operator": "MANY", + "observation_type": self.cleaned_data['observation_mode'], + "requests": requests + + }) + valid_requests = [] + for i,e in enumerate(errors['requests']): + if e!={}: + if 'non_field_errors' not in e: + valid_requests.append(requests[i]) + else: + valid_requests.append(requests[i]) + + return { + "name": self.cleaned_data['name'], + "proposal": self.cleaned_data['proposal'], + "ipp_value": self.cleaned_data['ipp_value'], + "operator": "MANY", + "observation_type": self.cleaned_data['observation_mode'], + "requests": valid_requests + + } class LCOImagingObservationForm(LCOBaseObservationForm): @@ -262,10 +447,10 @@ def instrument_choices(self): return [(k, v['name']) for k, v in self._get_instruments().items() if 'IMAGE' in v['type']] def filter_choices(self): - return set([ + return sorted(set([ (f['code'], f['name']) for ins in self._get_instruments().values() for f in ins['optical_elements'].get('filters', []) - ]) + ]),key=take_second_element) class LCOSpectroscopyObservationForm(LCOBaseObservationForm): @@ -289,10 +474,10 @@ def instrument_choices(self): # NRES does not take a slit, and therefore needs an option of None def filter_choices(self): - return set([ + return sorted(set([ (f['code'], f['name']) for ins in self._get_instruments().values() for f in ins['optical_elements'].get('slits', []) - ] + [('None', 'None')]) + ] + [('None', 'None')]),key=take_second_element) def _build_instrument_config(self): instrument_config = super()._build_instrument_config() diff --git a/tom_observations/utils.py b/tom_observations/utils.py index 5131e6e38..a0211036f 100644 --- a/tom_observations/utils.py +++ b/tom_observations/utils.py @@ -1,14 +1,79 @@ -from astropy.coordinates import get_sun, SkyCoord -from astropy import units +from astropy.coordinates import get_sun, SkyCoord, Angle, AltAz +from astropy import units, coordinates from astropy.time import Time from astroplan import Observer, FixedTarget, time_grid_from_range import numpy as np +from scipy import interpolate as interp import logging from tom_observations import facility +#Work-around necessary until the NOAO main service upgrades are complete. +#Would be sufficient to update astropy to v4 which handles this internally. But Wes doesn't want to do this to his work environment yet. +#In future remove the two following lines and require an up to date astropy +from astropy.utils import iers +iers.Conf.iers_auto_url.set('ftp://cddis.gsfc.nasa.gov/pub/products/iers/finals2000A.all') + logger = logging.getLogger(__name__) +d2r = np.pi/180.0 + +def get_radec_ephemeris(eph_json_single, start_time, end_time, interval, observing_facility, observing_site): + observing_facility_class = facility.get_service_class(observing_facility) + sites = observing_facility_class().get_observing_sites() + observer = None + for site_name in sites: + obs_site = sites[site_name] + if obs_site['sitecode'] == observing_site: + + observer = coordinates.EarthLocation(lat=obs_site.get('latitude')*units.deg, + lon=obs_site.get('longitude')*units.deg, + height=obs_site.get('elevation')*units.m) + if observer is None: + #this condition occurs if the facility being requested isn't in the site list provided. + return (None,None,None,None,-1) + ra = [] + dec = [] + mjd = [] + for i in range(len(eph_json_single)): + ra.append(float(eph_json_single[i]['R'])) + dec.append(float(eph_json_single[i]['D'])) + mjd.append(float(eph_json_single[i]['t'])) + ra = np.array(ra) + dec = np.array(dec) + mjd = np.array(mjd) + + fra = interp.interp1d(mjd,ra) + fdec = interp.interp1d(mjd,dec) + + start = Time(start_time) + end = Time(end_time) + + + time_range = time_grid_from_range(time_range=[start, end], time_resolution=interval*units.hour) + tr_mjd = time_range.mjd + #tr_mjd += interval/(24.0*2.0) + + + airmasses = [] + sun_alts = [] + for i in range(len(tr_mjd)): + c = SkyCoord(fra(time_range[i].mjd), fdec(time_range[i].mjd), frame="icrs", unit="deg") + t = Time(tr_mjd[i],format='mjd')#-8*units.h + sun = coordinates.get_sun(t) + altaz = c.transform_to(AltAz(obstime=t,location=observer)) + sun_altaz = sun.transform_to(AltAz(obstime=t,location=observer)) + airmass = altaz.secz + airmasses.append(airmass) + sun_alts.append(sun_altaz.alt.value) + airmasses = np.array(airmasses) + sun_alts = np.array(sun_alts) + + if np.min(tr_mjd)>=np.min(mjd) and np.max(tr_mjd)<=np.max(mjd): + return (tr_mjd,fra(tr_mjd),fdec(tr_mjd),airmasses,sun_alts) + else: + return (None,None,None,None,-2) + def get_sidereal_visibility(target, start_time, end_time, interval, airmass_limit): """ diff --git a/tom_targets/models.py b/tom_targets/models.py index 3839929f3..bd3b3f592 100644 --- a/tom_targets/models.py +++ b/tom_targets/models.py @@ -235,7 +235,7 @@ class Target(models.Model): max_length=50, null=True, blank=True, verbose_name='Centre-Site Name', help_text='Observatory Site Code' ) eph_json = models.TextField( - null=True, blank=True, verbose_name='Ephemeris JSON', help_text='MJD in days, RA and Dec in degress' + null=True, blank=True, verbose_name='Ephemeris JSON', help_text="Don't fill this in by hand unless you know what you are doing." ) class Meta: diff --git a/tom_targets/templatetags/targets_extras.py b/tom_targets/templatetags/targets_extras.py index af53c11b4..02d63bfbc 100644 --- a/tom_targets/templatetags/targets_extras.py +++ b/tom_targets/templatetags/targets_extras.py @@ -184,30 +184,39 @@ def eph_json_to_value_ra(value): """ Returns the middle RA and Dec of the json_ephemeris """ - eph_json = json.loads(value) - keys = list(eph_json.keys()) - k = keys[0] - l = len(eph_json[k][0]) - return( deg_to_sexigesimal(float(eph_json[k][int(l/2)]['R']),'hms')) + if value != 'None': + eph_json = json.loads(value) + keys = list(eph_json.keys()) + k = keys[0] + l = len(eph_json[k][0]) + return( deg_to_sexigesimal(float(eph_json[k][int(l/2)]['R']),'hms')) + else: + return -32768.0 @register.filter def eph_json_to_value_dec(value): """ Returns the middle RA and Dec of the json_ephemeris """ - eph_json = json.loads(value) - keys = list(eph_json.keys()) - k = keys[0] - l = len(eph_json[k][0]) - return( deg_to_sexigesimal(float(eph_json[k][int(l/2)]['D']),'dms')) + if value != 'None': + eph_json = json.loads(value) + keys = list(eph_json.keys()) + k = keys[0] + l = len(eph_json[k][0]) + return( deg_to_sexigesimal(float(eph_json[k][int(l/2)]['D']),'dms')) + else: + return -32768.0 @register.filter def eph_json_to_value_mjd(value): """ Returns the middle RA and Dec of the json_ephemeris """ - eph_json = json.loads(value) - keys = list(eph_json.keys()) - k = keys[0] - l = len(eph_json[k][0]) - return( float(eph_json[k][int(l/2)]['t'])) + if value != 'None': + eph_json = json.loads(value) + keys = list(eph_json.keys()) + k = keys[0] + l = len(eph_json[k][0]) + return( float(eph_json[k][int(l/2)]['t'])) + else: + return -32768.0 diff --git a/tom_targets/utils.py b/tom_targets/utils.py index 905f93318..a8904abf3 100644 --- a/tom_targets/utils.py +++ b/tom_targets/utils.py @@ -5,6 +5,21 @@ from io import StringIO import json +#this dictionary should contain as key entires text sufficient to uniquely identify +#the observatory name from the common English names used by JPL for that site. +#For example, Sunderland is probably unique enough to identify SAAO +#there may be a better way to handle this. +site_names = {'Mauna Kea': '568', + 'Haleakala':'ogg', + 'McDonald':'elp', + 'Tololo': 'lsc', + 'Teide': 'tfn', + 'Sutherland': 'cpt', + 'Wise': 'tlv', + 'Siding Spring': 'coj', + } + + # NOTE: This saves locally. To avoid this, create file buffer. # referenced https://www.codingforentrepreneurs.com/blog/django-queryset-to-csv-files-datasets/ def export_targets(qs): @@ -127,7 +142,8 @@ def import_ephemeris_target(stream): end_ind = 0 for ns in range(num_sites): - centre_site_name = '' + centre_site_name = None + site_name_found = False name = 'custom' jd_inds = None ra_inds = None @@ -135,23 +151,12 @@ def import_ephemeris_target(stream): for i in range(end_ind,len(eph)): if 'Center-site name' in eph[i]: s = eph[i].split(': ')[-1] - if 'Mauna Kea' in s: - centre_site_name = '568' - elif 'Haleakala' in s: - centre_site_name = 'T04' - elif 'McDonald' in s: - centre_site_name = '711' - elif 'Tololo' in s: - centre_site_name = 'W85' - elif 'Teide' in s: - centre_site_name = '954' - elif 'Sunderland' in s: - centre_site_name = 'K91' - elif 'Wise' in s: - centre_site_name = '097' - elif 'Siding Spring' in s: - centre_site_name = 'Q63' - else: + for j in site_names.keys(): + if j in s: + centre_site_name = site_names[j] + site_name_found = True + break + if not site_name_found: centre_site_name = s if 'Target body name' in eph[i]: @@ -160,7 +165,6 @@ def import_ephemeris_target(stream): if jpl_ra_key in eph[i] and jpl_jd_key in eph[i]: ra_inds = [eph[i].index(jpl_ra_key),eph[i].index(jpl_ra_key)+len(jpl_ra_key)] jd_inds = [eph[i].index(jpl_jd_key),eph[i].index(jpl_jd_key)+len(jpl_jd_key)] - if '$$SOE' in eph[i]: if ra_inds is not None and loop_inds[0]==-1: loop_inds[0] = i+1 @@ -171,6 +175,10 @@ def import_ephemeris_target(stream): end_ind = loop_inds[1]+1 + #throw an HTML warning if I cannot understand the centre site name + if not site_name_found: + errors.append(Exception(f'Site name {centre_site_name} not understood.')) + #throw HTML screen of warning if I cannot find the coordinates or ephemerides #here we will put a better error check and correctly thrown warning #for now being lazy @@ -186,7 +194,7 @@ def import_ephemeris_target(stream): for i in range(loop_inds[0],loop_inds[1]): mjds.append(str(float(eph[i][jd_inds[0]:jd_inds[1]])-2400000.5)) s = eph[i][ra_inds[0]:ra_inds[1]].split() - r = 15.0*float(s[0])+float(s[1])/60.0+float(s[2])/3600.0 + r = 15.0*(float(s[0])+float(s[1])/60.0+float(s[2])/3600.0) ras.append("{:11.7f}".format(r)) d = abs(float(s[3]))+float(s[4])/60.0+float(s[5])/3600.0 if '-' in s[3]: From 4bd276ca671d0d8476f408453ef01d5b8ef63d21 Mon Sep 17 00:00:00 2001 From: fraserw Date: Fri, 17 Apr 2020 16:32:24 -0700 Subject: [PATCH 035/424] --adding the ephemeris upload html --- .../tom_targets/target_ephemeris_import.html | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tom_targets/templates/tom_targets/target_ephemeris_import.html diff --git a/tom_targets/templates/tom_targets/target_ephemeris_import.html b/tom_targets/templates/tom_targets/target_ephemeris_import.html new file mode 100644 index 000000000..53714102c --- /dev/null +++ b/tom_targets/templates/tom_targets/target_ephemeris_import.html @@ -0,0 +1,17 @@ +{% extends 'tom_common/base.html' %} +{% load bootstrap4 static %} +{% block title %}Import Targets{% endblock %} +{% block content %} +

Import Targets

+

+ Upload a JPL formatted ephemeris file. + View the example .eph file. +

+
+ {% csrf_token %} + + {% buttons %} + + {% endbuttons %} +
+{% endblock %} From 53b092598d4c6d0d81e403fa8d873ff9f6c0c760 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Mon, 20 Apr 2020 15:53:52 +0000 Subject: [PATCH 036/424] install stub for ManualObservationForm --- tom_observations/facilities/manual.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/tom_observations/facilities/manual.py b/tom_observations/facilities/manual.py index 327e2a659..a8faefcdd 100644 --- a/tom_observations/facilities/manual.py +++ b/tom_observations/facilities/manual.py @@ -1,13 +1,30 @@ import logging import requests +from crispy_forms.layout import Layout, HTML from django.core.files.base import ContentFile -from tom_observations.facility import GenericObservationFacility, AUTO_THUMBNAILS +from tom_observations.facility import GenericObservationFacility, GenericObservationForm, AUTO_THUMBNAILS logger = logging.getLogger(__name__) +class ManualObservationForm(GenericObservationForm): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + target_id = self.initial.get('target_id') + self.helper.inputs.pop() + self.helper.layout = Layout( + HTML(''' +
+

A Manual Observation Form must be defined. +

+
+ '''), + HTML(f'''Back''') + ) + + class GenericManualFacility(GenericObservationFacility): """ The facility class contains all the logic specific to the facility it is @@ -104,8 +121,7 @@ def get_form(self, observation_type): """ This method takes in an observation type and returns the form type that matches it. """ - # TODO: implement me - raise NotImplementedError + return ManualObservationForm def submit_observation(self, observation_payload): """ @@ -175,8 +191,7 @@ def get_observing_sites(self): list should contain dictionaries each that contain sitecode, latitude, longitude and elevation. """ - # TODO: implement me - raise NotImplementedError + return {} def get_observation_status(self, observation_id): """ From ecd924ef8e231080ff8b472a7e7e0921e735df1f Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 20 Apr 2020 10:37:06 -0700 Subject: [PATCH 037/424] Added button for adding existing observations from a facility --- .../partials/existing_observation_button.html | 1 + .../tom_observations/partials/observation_list.html | 6 ++++++ tom_observations/templatetags/observation_extras.py | 5 +++++ tom_targets/templates/tom_targets/target_detail.html | 1 + 4 files changed, 13 insertions(+) create mode 100644 tom_observations/templates/tom_observations/partials/existing_observation_button.html diff --git a/tom_observations/templates/tom_observations/partials/existing_observation_button.html b/tom_observations/templates/tom_observations/partials/existing_observation_button.html new file mode 100644 index 000000000..90c4ea468 --- /dev/null +++ b/tom_observations/templates/tom_observations/partials/existing_observation_button.html @@ -0,0 +1 @@ +Add Observation From Facility \ No newline at end of file diff --git a/tom_observations/templates/tom_observations/partials/observation_list.html b/tom_observations/templates/tom_observations/partials/observation_list.html index 5448c0672..8013053d8 100644 --- a/tom_observations/templates/tom_observations/partials/observation_list.html +++ b/tom_observations/templates/tom_observations/partials/observation_list.html @@ -10,5 +10,11 @@ 		{{ observation.dataproduct_set.count }} View
+ No observations yet for this target. +
diff --git a/tom_observations/templatetags/observation_extras.py b/tom_observations/templatetags/observation_extras.py index c1f5327f0..17eed74d8 100644 --- a/tom_observations/templatetags/observation_extras.py +++ b/tom_observations/templatetags/observation_extras.py @@ -27,6 +27,11 @@ def observing_buttons(target): return {'target': target, 'facilities': facilities} +@register.inclusion_tag('tom_observations/partials/existing_observation_button.html') +def existing_observation_button(target): + return {'target_id': target.id} + + @register.inclusion_tag('tom_observations/partials/observation_type_tabs.html', takes_context=True) def observation_type_tabs(context): """ diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index 885627a3b..3f2d1894d 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -61,6 +61,7 @@

Plan

Observations

Update Observations Status + {% existing_observation_button object %} {% observation_list object %}
From d87e9f1d01ff077e66679214dfed13942374ac37 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Mon, 20 Apr 2020 20:30:19 +0000 Subject: [PATCH 038/424] define BaseManual{Facility, ObservationForm} --- tom_observations/facility.py | 119 ++++++++++++++++++++++++++++++++++- 1 file changed, 118 insertions(+), 1 deletion(-) diff --git a/tom_observations/facility.py b/tom_observations/facility.py index 80cd11aab..4ad4bd47e 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -6,7 +6,7 @@ import requests from crispy_forms.helper import FormHelper -from crispy_forms.layout import Layout, Submit +from crispy_forms.layout import Layout, Submit, Div, HTML from django import forms from django.conf import settings from django.contrib.auth.models import Group @@ -285,3 +285,120 @@ def observation_payload(self): 'target_id': target.id, 'params': self.serialize_parameters() } + + +# TODO: refactor GenericObservationFacility to GenericRoboticFacility (or BaseRoboticFacility) +# TODO: create new BaseObservationFacility from common parts of Base{Manual, Robotic}Facility classes +# TODO: refactor BaseManualFacility to be subclass of BaseObservationFacility + +# +# Manual Observing Base Classes +# +class BaseManualObservationForm(GenericObservationForm): + name = forms.CharField() + observation_id = forms.CharField(required=False) + observation_params = forms.CharField( + widget=forms.TextInput(attrs={'type': 'json'})) + start = forms.CharField(widget=forms.TextInput(attrs={'type': 'date'})) + end = forms.CharField(widget=forms.TextInput(attrs={'type': 'date'})) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.target_id = self.initial.get('target_id') + self.helper.inputs.pop() + self.helper.layout = Layout( + self.common_layout, + self.layout() + ) + + def layout(self): + return Div( + Div( + Div('name', 'observation_id', 'observation_params', 'start', 'end', + css_class='col'), + css_class='form-row'), + HTML(f'''Back''') + ) + + +class BaseManualFacility(ABC): + """ + """ + name = "BaseManual" # rename in concrete subclasses + + @abstractmethod + def get_form(self, observation_type): + """ + This method takes in an observation type and returns the form type that matches it. + """ + pass + + @abstractmethod + def submit_observation(self, observation_payload): + """ + This method takes in the serialized data from the form and actually + submits the observation to manual facility (perhaps my email?) + """ + pass + + @abstractmethod + def validate_observation(self, observation_payload): + """ + Same thing as submit_observation, but a dry run. You can + skip this in different modules by just using "pass" + """ + pass + + def get_flux_constant(self): + """ + Returns the astropy quantity that a facility uses for its spectral flux conversion. + """ + pass + + def get_wavelength_units(self): + """ + Returns the astropy units that a facility uses for its spectral wavelengths + """ + pass + + def is_fits_facility(self, header): + """ + Returns True if the FITS header is from this facility based on valid keywords and associated + values, False otherwise. + """ + return False + + def get_start_end_keywords(self): + """ + Returns the keywords representing the start and end of an observation window for a facility. Defaults to + ``start`` and ``end``. + """ + return 'start', 'end' + + @abstractmethod + def get_terminal_observing_states(self): + """ + Returns the states for which an observation is not expected + to change. + """ + pass + + @abstractmethod + def get_observing_sites(self): + """ + Return a list of dictionaries that contain the information + necessary to be used in the planning (visibility) tool. The + list should contain dictionaries each that contain sitecode, + latitude, longitude and elevation. + """ + pass + + @abstractmethod + def data_products(self, observation_id, product_id=None): + """ + Using an observation_id, retrieve a list of the data + products that belong to this observation. In this case, + the LCO module retrieves a list of frames from the LCO + data archive. + """ + pass From 87d17e1501096effe5bc8fcaaf0b95f90f14b823 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Mon, 20 Apr 2020 20:31:15 +0000 Subject: [PATCH 039/424] wip: implement concrete base class of BaseManualFacility --- tom_observations/facilities/manual.py | 155 ++++---------------------- 1 file changed, 22 insertions(+), 133 deletions(-) diff --git a/tom_observations/facilities/manual.py b/tom_observations/facilities/manual.py index a8faefcdd..74ec21b32 100644 --- a/tom_observations/facilities/manual.py +++ b/tom_observations/facilities/manual.py @@ -1,132 +1,44 @@ import logging -import requests -from crispy_forms.layout import Layout, HTML -from django.core.files.base import ContentFile +from django.conf import settings -from tom_observations.facility import GenericObservationFacility, GenericObservationForm, AUTO_THUMBNAILS +from tom_observations.facility import BaseManualFacility, BaseManualObservationForm logger = logging.getLogger(__name__) +try: + ZZZ_SETTINGS = settings.FACILITIES['ZZZ'] +except KeyError: + ZZZ_SETTINGS = { + } -class ManualObservationForm(GenericObservationForm): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - target_id = self.initial.get('target_id') - self.helper.inputs.pop() - self.helper.layout = Layout( - HTML(''' -
-

A Manual Observation Form must be defined. -

-
- '''), - HTML(f'''Back''') - ) +SITES = { + 'Zero-zero Island': { + 'sitecode': 'zzz', # top-secret observing site on Zero-zero Island + 'latitude': 0.0, + 'longitude': 0.0, + 'elevation': 0.0 + }, +} -class GenericManualFacility(GenericObservationFacility): +class GenericManualFacility(BaseManualFacility): """ - The facility class contains all the logic specific to the facility it is - written for. Some methods are used only internally (starting with an - underscore) but some need to be implemented by all facility classes. - All facilities should inherit from this class which - provides some base functionality. - In order to make use of a facility class, add the path to - ``TOM_FACILITY_CLASSES`` in your ``settings.py``. - - For an implementation example, please see - https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/lco.py """ - name = 'MAN' - observation_types = [('IMAGING', 'Imaging'), ('SPECTRA', 'Spectroscopy')] - SITES = {} - - def update_observation_status(self, observation_id): - from tom_observations.models import ObservationRecord - try: - record = ObservationRecord.objects.get(observation_id=observation_id) - status = self.get_observation_status(observation_id) - record.status = status['state'] - record.scheduled_start = status['scheduled_start'] - record.scheduled_end = status['scheduled_end'] - record.save() - except ObservationRecord.DoesNotExist: - raise Exception('No record exists for that observation id') - - def update_all_observation_statuses(self, target=None): - from tom_observations.models import ObservationRecord - failed_records = [] - records = ObservationRecord.objects.filter(facility=self.name) - if target: - records = records.filter(target=target) - records = records.exclude(status__in=self.get_terminal_observing_states()) - for record in records: - try: - self.update_observation_status(record.observation_id) - except Exception as e: - failed_records.append((record.observation_id, str(e))) - return failed_records - - def all_data_products(self, observation_record): - from tom_dataproducts.models import DataProduct - products = {'saved': [], 'unsaved': []} - for product in self.data_products(observation_record.observation_id): - try: - dp = DataProduct.objects.get(product_id=product['id']) - products['saved'].append(dp) - except DataProduct.DoesNotExist: - products['unsaved'].append(product) - # Obtain products uploaded manually by users - user_products = DataProduct.objects.filter( - observation_record_id=observation_record.id, product_id=None - ) - for product in user_products: - products['saved'].append(product) - - # Add any JPEG images created from DataProducts - image_products = DataProduct.objects.filter( - observation_record_id=observation_record.id, data_product_type='image_file' - ) - for product in image_products: - products['saved'].append(product) - return products - - def save_data_products(self, observation_record, product_id=None): - from tom_dataproducts.models import DataProduct - from tom_dataproducts.utils import create_image_dataproduct - final_products = [] - products = self.data_products(observation_record.observation_id, product_id) - - for product in products: - dp, created = DataProduct.objects.get_or_create( - product_id=product['id'], - target=observation_record.target, - observation_record=observation_record, - ) - if created: - product_data = requests.get(product['url']).content - dfile = ContentFile(product_data) - dp.data.save(product['filename'], dfile) - dp.save() - logger.info('Saved new dataproduct: {}'.format(dp.data)) - if AUTO_THUMBNAILS: - create_image_dataproduct(dp) - dp.get_preview() - final_products.append(dp) - return final_products + name = 'ZZZ' + observation_types = [('IMAGING', 'Imaging')] def get_form(self, observation_type): """ This method takes in an observation type and returns the form type that matches it. """ - return ManualObservationForm + return BaseManualObservationForm def submit_observation(self, observation_payload): """ - This method takes in the serialized data from the form and actually - submits the observation to the remote api + This method takes in the serialized data from the form. + """ # TODO: implement me raise NotImplementedError @@ -139,29 +51,6 @@ def validate_observation(self, observation_payload): # TODO: implement me raise NotImplementedError - def get_observation_url(self, observation_id): - """ - Takes an observation id and return the url for which a user - can view the observation at an external location. In this case, - we return a URL to the LCO observation portal's observation - record page. - """ - # TODO: implement me - raise NotImplementedError - pass - - def get_flux_constant(self): - """ - Returns the astropy quantity that a facility uses for its spectral flux conversion. - """ - pass - - def get_wavelength_units(self): - """ - Returns the astropy units that a facility uses for its spectral wavelengths - """ - pass - def is_fits_facility(self, header): """ Returns True if the FITS header is from this facility based on valid keywords and associated @@ -191,7 +80,7 @@ def get_observing_sites(self): list should contain dictionaries each that contain sitecode, latitude, longitude and elevation. """ - return {} + return SITES def get_observation_status(self, observation_id): """ From 4acb80d805662aa0e7320a90932795849101406d Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Mon, 20 Apr 2020 23:16:25 +0000 Subject: [PATCH 040/424] don't pop the submit button off the ManualForm --- tom_observations/facilities/manual.py | 4 ++-- tom_observations/facility.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tom_observations/facilities/manual.py b/tom_observations/facilities/manual.py index 74ec21b32..7eaef6c58 100644 --- a/tom_observations/facilities/manual.py +++ b/tom_observations/facilities/manual.py @@ -12,7 +12,7 @@ ZZZ_SETTINGS = { } -SITES = { +ZZZ_SITES = { 'Zero-zero Island': { 'sitecode': 'zzz', # top-secret observing site on Zero-zero Island 'latitude': 0.0, @@ -80,7 +80,7 @@ def get_observing_sites(self): list should contain dictionaries each that contain sitecode, latitude, longitude and elevation. """ - return SITES + return ZZZ_SITES def get_observation_status(self, observation_id): """ diff --git a/tom_observations/facility.py b/tom_observations/facility.py index 4ad4bd47e..0eeaec4e3 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -305,7 +305,7 @@ class BaseManualObservationForm(GenericObservationForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.target_id = self.initial.get('target_id') - self.helper.inputs.pop() + self.helper.layout = Layout( self.common_layout, self.layout() From db0e67ff2fef07d913ed900595d2f9e6356ecae4 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 20 Apr 2020 17:18:54 -0700 Subject: [PATCH 041/424] Added button for existing observations --- tom_observations/forms.py | 27 +++++++++++++++++++ .../partials/existing_observation_button.html | 4 ++- .../templatetags/observation_extras.py | 3 ++- .../templates/tom_targets/target_detail.html | 2 +- 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/tom_observations/forms.py b/tom_observations/forms.py index 8fe40ea0b..8728d838e 100644 --- a/tom_observations/forms.py +++ b/tom_observations/forms.py @@ -1,4 +1,7 @@ from django import forms +from django.urls import reverse +from crispy_forms.helper import FormHelper +from crispy_forms.layout import ButtonHolder, Column, Layout, Row, Submit from tom_observations.facility import get_service_classes @@ -11,3 +14,27 @@ class ManualObservationForm(forms.Form): target_id = forms.IntegerField(required=True, widget=forms.HiddenInput()) facility = forms.ChoiceField(choices=facility_choices) observation_id = forms.CharField() + + +class AddExistingObservationForm(forms.Form): + target_id = forms.IntegerField(required=True, widget=forms.HiddenInput()) + facility = forms.ChoiceField(required=True, choices=facility_choices, label=False) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.helper = FormHelper() + self.helper.form_method = 'GET' + self.helper.form_action = reverse('tom_observations:manual') + self.helper.layout = Layout( + 'target_id', + Row( + Column( + 'facility' + ), + Column( + ButtonHolder( + Submit('submit', 'Add Existing Observation') + ) + ) + ) + ) diff --git a/tom_observations/templates/tom_observations/partials/existing_observation_button.html b/tom_observations/templates/tom_observations/partials/existing_observation_button.html index 90c4ea468..300affec7 100644 --- a/tom_observations/templates/tom_observations/partials/existing_observation_button.html +++ b/tom_observations/templates/tom_observations/partials/existing_observation_button.html @@ -1 +1,3 @@ -Add Observation From Facility \ No newline at end of file +{% load crispy_forms_tags %} +

Add an Existing Observation

+{% crispy form %} \ No newline at end of file diff --git a/tom_observations/templatetags/observation_extras.py b/tom_observations/templatetags/observation_extras.py index 17eed74d8..f9477bc8c 100644 --- a/tom_observations/templatetags/observation_extras.py +++ b/tom_observations/templatetags/observation_extras.py @@ -8,6 +8,7 @@ from plotly import offline import plotly.graph_objs as go +from tom_observations.forms import AddExistingObservationForm from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class, get_service_classes from tom_observations.observing_strategy import RunStrategyForm @@ -29,7 +30,7 @@ def observing_buttons(target): @register.inclusion_tag('tom_observations/partials/existing_observation_button.html') def existing_observation_button(target): - return {'target_id': target.id} + return {'form': AddExistingObservationForm(initial={'target_id': target.id})} @register.inclusion_tag('tom_observations/partials/observation_type_tabs.html', takes_context=True) diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index 3f2d1894d..2bb4e4030 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -59,9 +59,9 @@

Plan

{% endif %}
+ {% existing_observation_button object %}

Observations

Update Observations Status - {% existing_observation_button object %} {% observation_list object %}
From e99101957c1498ba9d8163a694e1f9168cf6d94c Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 20 Apr 2020 17:39:01 -0700 Subject: [PATCH 042/424] Small refactor to allow observation association within target detail page --- tom_observations/forms.py | 6 +++++- tom_observations/views.py | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tom_observations/forms.py b/tom_observations/forms.py index 8728d838e..a6856429c 100644 --- a/tom_observations/forms.py +++ b/tom_observations/forms.py @@ -19,11 +19,12 @@ class ManualObservationForm(forms.Form): class AddExistingObservationForm(forms.Form): target_id = forms.IntegerField(required=True, widget=forms.HiddenInput()) facility = forms.ChoiceField(required=True, choices=facility_choices, label=False) + observation_id = forms.CharField(required=True, label=False, + widget=forms.TextInput(attrs={'placeholder': 'Observation ID'})) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.helper = FormHelper() - self.helper.form_method = 'GET' self.helper.form_action = reverse('tom_observations:manual') self.helper.layout = Layout( 'target_id', @@ -31,6 +32,9 @@ def __init__(self, *args, **kwargs): Column( 'facility' ), + Column( + 'observation_id' + ), Column( ButtonHolder( Submit('submit', 'Add Existing Observation') diff --git a/tom_observations/views.py b/tom_observations/views.py index 3f534d8b0..38ac82c00 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -298,10 +298,11 @@ def get(self, request, *args, **kwargs): class ManualObservationCreateView(LoginRequiredMixin, FormView): + # TODO: Add confirmation page for existing conflicting observation """ View for associating a pre-existing observation with a target. Requires authentication. - This view is not currently exposed in the out-of-the-box TOM Toolkit. + The GET view returns a confirmation """ template_name = 'tom_observations/observation_form_manual.html' form_class = ManualObservationForm From e344815df606e08cc7447f226bce83b8620408f9 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 21 Apr 2020 10:28:24 -0700 Subject: [PATCH 043/424] Added confirmation page for manual observation create view --- tom_observations/forms.py | 32 ++++++++- .../existing_observation_confirm.html | 7 ++ .../observation_form_manual.html | 13 ---- tom_observations/views.py | 72 +++++++++---------- 4 files changed, 72 insertions(+), 52 deletions(-) create mode 100644 tom_observations/templates/tom_observations/existing_observation_confirm.html delete mode 100644 tom_observations/templates/tom_observations/observation_form_manual.html diff --git a/tom_observations/forms.py b/tom_observations/forms.py index a6856429c..2a110a6e5 100644 --- a/tom_observations/forms.py +++ b/tom_observations/forms.py @@ -1,7 +1,8 @@ from django import forms -from django.urls import reverse +from django.urls import reverse, reverse_lazy +from crispy_forms.bootstrap import FormActions from crispy_forms.helper import FormHelper -from crispy_forms.layout import ButtonHolder, Column, Layout, Row, Submit +from crispy_forms.layout import Button, ButtonHolder, Column, HTML, Layout, Row, Submit from tom_observations.facility import get_service_classes @@ -21,6 +22,7 @@ class AddExistingObservationForm(forms.Form): facility = forms.ChoiceField(required=True, choices=facility_choices, label=False) observation_id = forms.CharField(required=True, label=False, widget=forms.TextInput(attrs={'placeholder': 'Observation ID'})) + confirm = forms.BooleanField(widget=forms.HiddenInput()) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -28,6 +30,7 @@ def __init__(self, *args, **kwargs): self.helper.form_action = reverse('tom_observations:manual') self.helper.layout = Layout( 'target_id', + 'confirm', Row( Column( 'facility' @@ -42,3 +45,28 @@ def __init__(self, *args, **kwargs): ) ) ) + + +class ConfirmExistingObservationForm(AddExistingObservationForm): + # TODO: Should this inherit from AddExistingObservationForm or be its own thing? + def __init__(self, *args, **kwargs): + target_id = kwargs['data']['target_id'] + super().__init__(*args, **kwargs) + self.fields['facility'].widget = forms.HiddenInput() + self.fields['observation_id'].widget = forms.HiddenInput() + cancel_url = reverse('home') + print(target_id) + if target_id: + cancel_url = reverse('tom_targets:detail', kwargs={'pk': target_id}) + self.helper.layout = Layout( + HTML('''

An observation record already exists in your TOM for this combination of observation ID, + facility, and target. Are you sure you want to create this record?

'''), + 'target_id', + 'facility', + 'observation_id', + 'confirm', + FormActions( + Submit('confirm', 'Confirm'), + HTML(f'Cancel') + ) + ) diff --git a/tom_observations/templates/tom_observations/existing_observation_confirm.html b/tom_observations/templates/tom_observations/existing_observation_confirm.html new file mode 100644 index 000000000..f83e1b5ff --- /dev/null +++ b/tom_observations/templates/tom_observations/existing_observation_confirm.html @@ -0,0 +1,7 @@ +{% extends 'tom_common/base.html' %} +{% load crispy_forms_tags %} + +{% block content %} +

Confirm Observation Record Creation

+{% crispy form %} +{% endblock %} diff --git a/tom_observations/templates/tom_observations/observation_form_manual.html b/tom_observations/templates/tom_observations/observation_form_manual.html deleted file mode 100644 index d209691be..000000000 --- a/tom_observations/templates/tom_observations/observation_form_manual.html +++ /dev/null @@ -1,13 +0,0 @@ -{% extends 'tom_common/base.html' %} -{% load bootstrap4 %} -{% block title %}Manual Observation{% endblock %} -{% block content %} -

Associate an observation id

-
- {% csrf_token %} - {% bootstrap_form form %} - {% buttons %} - - {% endbuttons %} -
-{% endblock %} diff --git a/tom_observations/views.py b/tom_observations/views.py index 38ac82c00..01365668f 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -2,6 +2,9 @@ from urllib.parse import urlparse import json +from crispy_forms.bootstrap import FormActions +from crispy_forms.layout import Button, HTML, Layout, Submit +from django import forms from django.conf import settings from django.contrib import messages from django.contrib.auth.mixins import LoginRequiredMixin @@ -14,7 +17,7 @@ from django.utils.safestring import mark_safe from django.views.generic import View from django.views.generic.detail import DetailView -from django.views.generic.edit import FormView, DeleteView +from django.views.generic.edit import DeleteView, FormView, UpdateView from django.views.generic.list import ListView from guardian.shortcuts import get_objects_for_user, assign_perm from guardian.mixins import PermissionListMixin @@ -23,7 +26,7 @@ from tom_common.mixins import Raise403PermissionRequiredMixin from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm from tom_observations.facility import get_service_class, get_service_classes -from tom_observations.forms import ManualObservationForm +from tom_observations.forms import ConfirmExistingObservationForm from tom_observations.models import ObservationRecord, ObservationGroup, ObservingStrategy from tom_targets.models import Target @@ -274,6 +277,14 @@ def form_valid(self, form): ) +class ObservationUpdateView(LoginRequiredMixin, UpdateView): + """ + This view will eventually allow updating solely the observation id and status, and possibly the parameters + """ + model = ObservationRecord + fields = ['observation_id', 'status', 'scheduled_start', 'scheduled_end'] + + class ObservationGroupCancelView(LoginRequiredMixin, View): def get_context_data(self, *args, **kwargs): @@ -304,20 +315,8 @@ class ManualObservationCreateView(LoginRequiredMixin, FormView): The GET view returns a confirmation """ - template_name = 'tom_observations/observation_form_manual.html' - form_class = ManualObservationForm - - def get_target_id(self): - """ - Gets the id of the target of the observation from the query parameters. - - :returns: target id - :rtype: int - """ - if self.request.method == 'GET': - return self.request.GET.get('target_id') - elif self.request.method == 'POST': - return self.request.POST.get('target_id') + template_name = 'tom_observations/existing_observation_confirm.html' + form_class = ConfirmExistingObservationForm def get_initial(self): """ @@ -326,34 +325,33 @@ def get_initial(self): :returns: initial form data :rtype: dict """ - initial = super().get_initial() - if not self.get_target_id(): - raise Exception('Must provide target_id') - initial['target_id'] = self.get_target_id() - return initial - - def get_target(self): - """ - Gets the ``Target`` associated with the specified target_id from the database. - - :returns: target instance to be associated with an observation - :rtype: Target - """ - return Target.objects.get(pk=self.get_target_id()) + if self.request.method == 'GET': + params = self.request.GET.dict() + params['confirm'] = True + return params def form_valid(self, form): """ Runs after form validation. Creates a new ``ObservationRecord`` associated with the specified target and facility. """ - ObservationRecord.objects.create( - target=self.get_target(), - facility=form.cleaned_data['facility'], - parameters={}, - observation_id=form.cleaned_data['observation_id'] - ) + records = ObservationRecord.objects.filter(target_id=form.cleaned_data['target_id'], + facility=form.cleaned_data['facility'], + observation_id=form.cleaned_data['observation_id']) + + if records and not form.cleaned_data.get('confirm'): + return redirect(reverse('tom_observations:manual') + '?' + self.request.POST.urlencode()) + else: + ObservationRecord.objects.create( + target_id=form.cleaned_data['target_id'], + facility=form.cleaned_data['facility'], + parameters={}, + observation_id=form.cleaned_data['observation_id'] + ) + observation_id = form.cleaned_data['observation_id'] + messages.success(self.request, f'Successfully associated observation record {observation_id}') return redirect(reverse( - 'tom_targets:detail', kwargs={'pk': self.get_target().id}) + 'tom_targets:detail', kwargs={'pk': form.cleaned_data['target_id']}) ) From 539a24901933551c409dab0035c21ffebf334bda Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 21 Apr 2020 10:34:51 -0700 Subject: [PATCH 044/424] A little more documentation --- tom_observations/forms.py | 9 +-------- tom_observations/views.py | 8 +++++++- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tom_observations/forms.py b/tom_observations/forms.py index 2a110a6e5..f410136e7 100644 --- a/tom_observations/forms.py +++ b/tom_observations/forms.py @@ -11,12 +11,6 @@ def facility_choices(): return [(k, k) for k in get_service_classes().keys()] -class ManualObservationForm(forms.Form): - target_id = forms.IntegerField(required=True, widget=forms.HiddenInput()) - facility = forms.ChoiceField(choices=facility_choices) - observation_id = forms.CharField() - - class AddExistingObservationForm(forms.Form): target_id = forms.IntegerField(required=True, widget=forms.HiddenInput()) facility = forms.ChoiceField(required=True, choices=facility_choices, label=False) @@ -50,12 +44,11 @@ def __init__(self, *args, **kwargs): class ConfirmExistingObservationForm(AddExistingObservationForm): # TODO: Should this inherit from AddExistingObservationForm or be its own thing? def __init__(self, *args, **kwargs): - target_id = kwargs['data']['target_id'] super().__init__(*args, **kwargs) self.fields['facility'].widget = forms.HiddenInput() self.fields['observation_id'].widget = forms.HiddenInput() + target_id = kwargs['data']['target_id'] cancel_url = reverse('home') - print(target_id) if target_id: cancel_url = reverse('tom_targets:detail', kwargs={'pk': target_id}) self.helper.layout = Layout( diff --git a/tom_observations/views.py b/tom_observations/views.py index 01365668f..0044f8704 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -313,7 +313,13 @@ class ManualObservationCreateView(LoginRequiredMixin, FormView): """ View for associating a pre-existing observation with a target. Requires authentication. - The GET view returns a confirmation + The GET view returns a confirmation page for adding duplicate ObservationRecords. Two duplicates are any two + ObservationRecords with the same target_id, facility, and observation_id. + + The POST view validates the form and redirects to the confirmation page if the confirm flag isn't set. + + This view is intended to be navigated to via the existing_observation_button templatetag, as the + ConfirmExistingObservationForm has a hidden confirmation checkbox selected by default. """ template_name = 'tom_observations/existing_observation_confirm.html' form_class = ConfirmExistingObservationForm From a443a99c186be9007770d530d36e2f5d1ca5b3a1 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 21 Apr 2020 16:53:30 -0700 Subject: [PATCH 045/424] Fixed bug in confirmation page, WIP for update observationrecord --- tom_observations/facilities/lco.py | 7 +++++++ tom_observations/forms.py | 2 +- .../templates/tom_observations/observationupdate_form.html | 7 +++++++ tom_observations/urls.py | 3 ++- tom_observations/views.py | 6 +++++- 5 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 tom_observations/templates/tom_observations/observationupdate_form.html diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 3aed482da..0aec85bab 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -507,6 +507,13 @@ def get_form(self, observation_type): def get_strategy_form(self, observation_type): return LCOObservingStrategyForm + def get_update_form(self, observation_type): + form = self.get_form(observation_type)() + form.fields['observation_type'].widget = forms.HiddenInput() + form.fields['period'].widget = forms.HiddenInput() + form.fields['jitter'].widget = forms.HiddenInput() + return form + def submit_observation(self, observation_payload): response = make_request( 'POST', diff --git a/tom_observations/forms.py b/tom_observations/forms.py index f410136e7..996a1bf15 100644 --- a/tom_observations/forms.py +++ b/tom_observations/forms.py @@ -16,7 +16,7 @@ class AddExistingObservationForm(forms.Form): facility = forms.ChoiceField(required=True, choices=facility_choices, label=False) observation_id = forms.CharField(required=True, label=False, widget=forms.TextInput(attrs={'placeholder': 'Observation ID'})) - confirm = forms.BooleanField(widget=forms.HiddenInput()) + confirm = forms.BooleanField(widget=forms.HiddenInput(), required=False) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/tom_observations/templates/tom_observations/observationupdate_form.html b/tom_observations/templates/tom_observations/observationupdate_form.html new file mode 100644 index 000000000..abeaf0fbb --- /dev/null +++ b/tom_observations/templates/tom_observations/observationupdate_form.html @@ -0,0 +1,7 @@ +{% extends 'tom_common/base.html' %} +{% load crispy_forms_tags %} +{% block content %} +
+ {% crispy form %} +
+{% endblock %} \ No newline at end of file diff --git a/tom_observations/urls.py b/tom_observations/urls.py index 90793ac6d..5cd4613d9 100644 --- a/tom_observations/urls.py +++ b/tom_observations/urls.py @@ -1,6 +1,6 @@ from django.urls import path -from tom_observations.views import (ManualObservationCreateView, ObservationCreateView, +from tom_observations.views import (ManualObservationCreateView, ObservationCreateView, ObservationUpdateView, ObservationGroupDeleteView, ObservationGroupListView, ObservationListView, ObservationRecordDetailView, ObservingStrategyCreateView, ObservingStrategyDeleteView, ObservingStrategyListView, @@ -17,6 +17,7 @@ path('strategy//delete/', ObservingStrategyDeleteView.as_view(), name='strategy-delete'), path('strategy//', ObservingStrategyUpdateView.as_view(), name='strategy-detail'), path('/create/', ObservationCreateView.as_view(), name='create'), + path('/update/', ObservationUpdateView.as_view(), name='update'), path('/', ObservationRecordDetailView.as_view(), name='detail'), path('groups/list/', ObservationGroupListView.as_view(), name='group-list'), path('groups//delete/', ObservationGroupDeleteView.as_view(), name='group-delete'), diff --git a/tom_observations/views.py b/tom_observations/views.py index 0044f8704..56e0e32bd 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -283,6 +283,11 @@ class ObservationUpdateView(LoginRequiredMixin, UpdateView): """ model = ObservationRecord fields = ['observation_id', 'status', 'scheduled_start', 'scheduled_end'] + template_name = 'tom_observations/observationupdate_form.html' + + def get_form(self): + facility_class = get_service_class(self.object.facility)() + return facility_class.get_update_form(None) class ObservationGroupCancelView(LoginRequiredMixin, View): @@ -309,7 +314,6 @@ def get(self, request, *args, **kwargs): class ManualObservationCreateView(LoginRequiredMixin, FormView): - # TODO: Add confirmation page for existing conflicting observation """ View for associating a pre-existing observation with a target. Requires authentication. From 2f278701e50d2bf65d4a56c7c3f08a538f202563 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Wed, 22 Apr 2020 16:41:11 +0000 Subject: [PATCH 046/424] define observing states for generic manual facility --- tom_observations/facilities/manual.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tom_observations/facilities/manual.py b/tom_observations/facilities/manual.py index 7eaef6c58..fbe05f2b4 100644 --- a/tom_observations/facilities/manual.py +++ b/tom_observations/facilities/manual.py @@ -6,6 +6,13 @@ logger = logging.getLogger(__name__) + +# +# facility properties needed by both the Facility and Form classes +# are candidates for module-level definitions. If the property is just +# for the Facility, put it in the class definition +# + try: ZZZ_SETTINGS = settings.FACILITIES['ZZZ'] except KeyError: @@ -20,6 +27,7 @@ 'elevation': 0.0 }, } +ZZZ_TERMINAL_OBSERVING_STATES = ['Completed'] class GenericManualFacility(BaseManualFacility): @@ -70,8 +78,10 @@ def get_terminal_observing_states(self): Returns the states for which an observation is not expected to change. """ - # TODO: implement me - raise NotImplementedError + return ZZZ_TERMINAL_OBSERVING_STATES + + def get_observation_url(self, observation_id): + return def get_observing_sites(self): """ @@ -97,5 +107,4 @@ def data_products(self, observation_id, product_id=None): the LCO module retrieves a list of frames from the LCO data archive. """ - # TODO: implement me - raise NotImplementedError + return [] From fef552a1ed79bc2b5e8c16a0dfac62b9ef91a257 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Wed, 22 Apr 2020 16:42:07 +0000 Subject: [PATCH 047/424] complete impl of methods of BaseManualFacility see TODOs in this commit for refactoring instructions --- tom_observations/facility.py | 93 ++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/tom_observations/facility.py b/tom_observations/facility.py index 0eeaec4e3..66ae6725b 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -70,6 +70,15 @@ class GenericObservationFacility(ABC): """ name = "Generic" # rename in concrete subclasses +# TODO: 4 methods: update_observation_status, update_all_observation_statuses, +# TODO: all_data_products, and save_data_products are duplicated (via C&P) in +# TODO: BaseManualFacility(ABC) and GenericObservationFacility(ABC) +# TODO: +# TODO: Refactor these methods common into a BaseObservationFacility class +# TODO: and have BaseManualFacility and BaseRoboticFacility inherit from that. +# TODO: (along the way, GenericObservationFacility should be renamed to +# TODO: BaseRoboticFacility) + def update_observation_status(self, observation_id): from tom_observations.models import ObservationRecord try: @@ -326,6 +335,90 @@ class BaseManualFacility(ABC): """ name = "BaseManual" # rename in concrete subclasses +# TODO: 4 methods: update_observation_status, update_all_observation_statuses, +# TODO: all_data_products, and save_data_products are duplicated (via C&P) in +# TODO: BaseManualFacility(ABC) and GenericObservationFacility(ABC) +# TODO: +# TODO: Refactor these methods common into a BaseObservationFacility class +# TODO: and have BaseManualFacility and BaseRoboticFacility inherit from that. +# TODO: (along the way, GenericObservationFacility should be renamed to +# TODO: BaseRoboticFacility) + + def update_observation_status(self, observation_id): + from tom_observations.models import ObservationRecord + try: + record = ObservationRecord.objects.get(observation_id=observation_id) + status = self.get_observation_status(observation_id) + record.status = status['state'] + record.scheduled_start = status['scheduled_start'] + record.scheduled_end = status['scheduled_end'] + record.save() + except ObservationRecord.DoesNotExist: + raise Exception('No record exists for that observation id') + + def update_all_observation_statuses(self, target=None): + from tom_observations.models import ObservationRecord + failed_records = [] + records = ObservationRecord.objects.filter(facility=self.name) + if target: + records = records.filter(target=target) + records = records.exclude(status__in=self.get_terminal_observing_states()) + for record in records: + try: + self.update_observation_status(record.observation_id) + except Exception as e: + failed_records.append((record.observation_id, str(e))) + return failed_records + + def all_data_products(self, observation_record): + from tom_dataproducts.models import DataProduct + products = {'saved': [], 'unsaved': []} + for product in self.data_products(observation_record.observation_id): + try: + dp = DataProduct.objects.get(product_id=product['id']) + products['saved'].append(dp) + except DataProduct.DoesNotExist: + products['unsaved'].append(product) + # Obtain products uploaded manually by users + user_products = DataProduct.objects.filter( + observation_record_id=observation_record.id, product_id=None + ) + for product in user_products: + products['saved'].append(product) + + # Add any JPEG images created from DataProducts + image_products = DataProduct.objects.filter( + observation_record_id=observation_record.id, data_product_type='image_file' + ) + for product in image_products: + products['saved'].append(product) + return products + + def save_data_products(self, observation_record, product_id=None): + from tom_dataproducts.models import DataProduct + from tom_dataproducts.utils import create_image_dataproduct + final_products = [] + products = self.data_products(observation_record.observation_id, product_id) + + for product in products: + dp, created = DataProduct.objects.get_or_create( + product_id=product['id'], + target=observation_record.target, + observation_record=observation_record, + ) + if created: + product_data = requests.get(product['url']).content + dfile = ContentFile(product_data) + dp.data.save(product['filename'], dfile) + dp.save() + logger.info('Saved new dataproduct: {}'.format(dp.data)) + if AUTO_THUMBNAILS: + create_image_dataproduct(dp) + dp.get_preview() + final_products.append(dp) + return final_products + + @abstractmethod def get_form(self, observation_type): """ From dc1ee5b09fa8a9b1effb38d85b87f2eb6e22522d Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Thu, 23 Apr 2020 18:18:27 +0000 Subject: [PATCH 048/424] Add more core fields after feedback from product owner --- tom_observations/facility.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tom_observations/facility.py b/tom_observations/facility.py index 66ae6725b..d35cf5b8f 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -305,16 +305,23 @@ def observation_payload(self): # class BaseManualObservationForm(GenericObservationForm): name = forms.CharField() - observation_id = forms.CharField(required=False) - observation_params = forms.CharField( - widget=forms.TextInput(attrs={'type': 'json'})) start = forms.CharField(widget=forms.TextInput(attrs={'type': 'date'})) - end = forms.CharField(widget=forms.TextInput(attrs={'type': 'date'})) + end = forms.CharField(required=False, widget=forms.TextInput(attrs={'type': 'date'})) + observation_id = forms.CharField(required=False) + filter = forms.CharField(required=False) + grating = forms.CharField(required=False) + instrument = forms.CharField(required=False) + annotation = forms.CharField(required=False, widget=forms.Textarea()) + observation_params = forms.CharField(required=False, + widget=forms.Textarea(attrs={'type': 'json'})) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.target_id = self.initial.get('target_id') + # TODO: get Back button horizontally adjacent to Submit + # self.helper.add_input(('back', 'Back')) # FIXME: this is wrong + self.helper.layout = Layout( self.common_layout, self.layout() @@ -323,7 +330,8 @@ def __init__(self, *args, **kwargs): def layout(self): return Div( Div( - Div('name', 'observation_id', 'observation_params', 'start', 'end', + Div('name', 'observation_id', 'start', 'end', 'filter', 'grating', + 'instrument', 'annotation', 'observation_params', css_class='col'), css_class='form-row'), HTML(f'''Back''') From e77b448ef56f0afd44d443256315e5ef6e2911ce Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Thu, 23 Apr 2020 21:57:15 +0000 Subject: [PATCH 049/424] wip: observation_payload method required in ObservationForm class --- tom_base/settings.py | 4 +++- tom_observations/facilities/manual.py | 9 +++++++-- tom_observations/facility.py | 3 +++ tom_observations/views.py | 3 +++ 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index e685768c1..d763d3d3a 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -211,7 +211,9 @@ TOM_FACILITY_CLASSES = [ 'tom_observations.facilities.lco.LCOFacility', 'tom_observations.facilities.gemini.GEMFacility', - 'tom_observations.facilities.manual.GenericManualFacility' + 'tom_observations.facilities.manual.GenericManualFacility', + # 'tom_observations.facility.BaseManualFacility' + # FIXME: sort out Base/ManualFacility hierarchy ] TOM_CADENCE_STRATEGIES = [ diff --git a/tom_observations/facilities/manual.py b/tom_observations/facilities/manual.py index fbe05f2b4..d3f916465 100644 --- a/tom_observations/facilities/manual.py +++ b/tom_observations/facilities/manual.py @@ -48,8 +48,13 @@ def submit_observation(self, observation_payload): This method takes in the serialized data from the form. """ - # TODO: implement me - raise NotImplementedError + # TODO: finish implementation + print(f'observation_payload: {observation_payload}') + obs_ids = [] + for payload in observation_payload: + obs_ids.append(f'{payload}') + + return obs_ids def validate_observation(self, observation_payload): """ diff --git a/tom_observations/facility.py b/tom_observations/facility.py index d35cf5b8f..0a086a321 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -337,6 +337,9 @@ def layout(self): HTML(f'''Back''') ) + def observation_payload(self): + pass + class BaseManualFacility(ABC): """ diff --git a/tom_observations/views.py b/tom_observations/views.py index 56e0e32bd..6c969fd7f 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -254,6 +254,9 @@ def form_valid(self, form): ) records.append(record) + print(f'records: {records}') + print(f'observation_ids: {observation_ids}') + if len(records) > 1 or form.cleaned_data.get('cadence_strategy'): group_name = form.cleaned_data['name'] observation_group = ObservationGroup.objects.create( From 34716eab7b8bba412751cc18a6bc38b7ccd05261 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 23 Apr 2020 15:44:00 -0700 Subject: [PATCH 050/424] cleaned up inheritance --- tom_observations/facilities/gemini.py | 7 +- tom_observations/facilities/lco.py | 6 +- tom_observations/facilities/manual.py | 4 +- tom_observations/facility.py | 381 ++++++++------------------ 4 files changed, 128 insertions(+), 270 deletions(-) diff --git a/tom_observations/facilities/gemini.py b/tom_observations/facilities/gemini.py index 61e2fe09b..27f592cbd 100644 --- a/tom_observations/facilities/gemini.py +++ b/tom_observations/facilities/gemini.py @@ -5,9 +5,8 @@ from crispy_forms.layout import Layout, Div, HTML from astropy import units as u -from tom_observations.facility import GenericObservationForm +from tom_observations.facility import BaseRoboticObservationFacility, BaseRoboticObservationForm from tom_common.exceptions import ImproperCredentialsException -from tom_observations.facility import GenericObservationFacility from tom_targets.models import Target try: @@ -117,7 +116,7 @@ def get_site(progid, location=False): return site -class GEMObservationForm(GenericObservationForm): +class GEMObservationForm(BaseRoboticObservationForm): """ The GEMObservationForm defines and collects the parameters for the Gemini Target of Opportunity (ToO) observation request API. The Gemini ToO process is described at @@ -412,7 +411,7 @@ def isodatetime(value): return payloads -class GEMFacility(GenericObservationFacility): +class GEMFacility(BaseRoboticObservationFacility): """ The ``GEMFacility`` is the interface to the Gemini Telescope. For information regarding Gemini observing and the available parameters, please see https://www.gemini.edu/sciops/observing-gemini. diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 0aec85bab..c7c1bdb72 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -9,7 +9,7 @@ from tom_common.exceptions import ImproperCredentialsException from tom_observations.cadence import CadenceForm -from tom_observations.facility import GenericObservationFacility, GenericObservationForm, get_service_class +from tom_observations.facility import BaseRoboticObservationFacility, BaseRoboticObservationForm, get_service_class from tom_observations.observing_strategy import GenericStrategyForm from tom_targets.models import Target, REQUIRED_NON_SIDEREAL_FIELDS, REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME @@ -136,7 +136,7 @@ def proposal_choices(self): return choices -class LCOBaseObservationForm(GenericObservationForm, LCOBaseForm, CadenceForm): +class LCOBaseObservationForm(BaseRoboticObservationForm, LCOBaseForm, CadenceForm): name = forms.CharField() ipp_value = forms.FloatField(label='Intra Proposal Priority (IPP factor)', min_value=0.5, @@ -445,7 +445,7 @@ def __init__(self, *args, **kwargs): ) -class LCOFacility(GenericObservationFacility): +class LCOFacility(BaseRoboticObservationFacility): """ The ``LCOFacility`` is the interface to the Las Cumbres Observatory Observation Portal. For information regarding LCO observing and the available parameters, please see https://observe.lco.global/help/. diff --git a/tom_observations/facilities/manual.py b/tom_observations/facilities/manual.py index d3f916465..8a670b592 100644 --- a/tom_observations/facilities/manual.py +++ b/tom_observations/facilities/manual.py @@ -2,7 +2,7 @@ from django.conf import settings -from tom_observations.facility import BaseManualFacility, BaseManualObservationForm +from tom_observations.facility import BaseManualObservationFacility, BaseManualObservationForm logger = logging.getLogger(__name__) @@ -30,7 +30,7 @@ ZZZ_TERMINAL_OBSERVING_STATES = ['Completed'] -class GenericManualFacility(BaseManualFacility): +class TestManualFacility(BaseManualObservationFacility): """ """ diff --git a/tom_observations/facility.py b/tom_observations/facility.py index 0a086a321..cd2543b1f 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -6,7 +6,7 @@ import requests from crispy_forms.helper import FormHelper -from crispy_forms.layout import Layout, Submit, Div, HTML +from crispy_forms.layout import ButtonHolder, Layout, Submit, Div, HTML from django import forms from django.conf import settings from django.contrib.auth.models import Group @@ -55,201 +55,7 @@ def get_service_class(name): raise ImportError('Could not a find a facility with that name. Did you add it to TOM_FACILITY_CLASSES?') -class GenericObservationFacility(ABC): - """ - The facility class contains all the logic specific to the facility it is - written for. Some methods are used only internally (starting with an - underscore) but some need to be implemented by all facility classes. - All facilities should inherit from this class which - provides some base functionality. - In order to make use of a facility class, add the path to - ``TOM_FACILITY_CLASSES`` in your ``settings.py``. - - For an implementation example, please see - https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/lco.py - """ - name = "Generic" # rename in concrete subclasses - -# TODO: 4 methods: update_observation_status, update_all_observation_statuses, -# TODO: all_data_products, and save_data_products are duplicated (via C&P) in -# TODO: BaseManualFacility(ABC) and GenericObservationFacility(ABC) -# TODO: -# TODO: Refactor these methods common into a BaseObservationFacility class -# TODO: and have BaseManualFacility and BaseRoboticFacility inherit from that. -# TODO: (along the way, GenericObservationFacility should be renamed to -# TODO: BaseRoboticFacility) - - def update_observation_status(self, observation_id): - from tom_observations.models import ObservationRecord - try: - record = ObservationRecord.objects.get(observation_id=observation_id) - status = self.get_observation_status(observation_id) - record.status = status['state'] - record.scheduled_start = status['scheduled_start'] - record.scheduled_end = status['scheduled_end'] - record.save() - except ObservationRecord.DoesNotExist: - raise Exception('No record exists for that observation id') - - def update_all_observation_statuses(self, target=None): - from tom_observations.models import ObservationRecord - failed_records = [] - records = ObservationRecord.objects.filter(facility=self.name) - if target: - records = records.filter(target=target) - records = records.exclude(status__in=self.get_terminal_observing_states()) - for record in records: - try: - self.update_observation_status(record.observation_id) - except Exception as e: - failed_records.append((record.observation_id, str(e))) - return failed_records - - def all_data_products(self, observation_record): - from tom_dataproducts.models import DataProduct - products = {'saved': [], 'unsaved': []} - for product in self.data_products(observation_record.observation_id): - try: - dp = DataProduct.objects.get(product_id=product['id']) - products['saved'].append(dp) - except DataProduct.DoesNotExist: - products['unsaved'].append(product) - # Obtain products uploaded manually by users - user_products = DataProduct.objects.filter( - observation_record_id=observation_record.id, product_id=None - ) - for product in user_products: - products['saved'].append(product) - - # Add any JPEG images created from DataProducts - image_products = DataProduct.objects.filter( - observation_record_id=observation_record.id, data_product_type='image_file' - ) - for product in image_products: - products['saved'].append(product) - return products - - def save_data_products(self, observation_record, product_id=None): - from tom_dataproducts.models import DataProduct - from tom_dataproducts.utils import create_image_dataproduct - final_products = [] - products = self.data_products(observation_record.observation_id, product_id) - - for product in products: - dp, created = DataProduct.objects.get_or_create( - product_id=product['id'], - target=observation_record.target, - observation_record=observation_record, - ) - if created: - product_data = requests.get(product['url']).content - dfile = ContentFile(product_data) - dp.data.save(product['filename'], dfile) - dp.save() - logger.info('Saved new dataproduct: {}'.format(dp.data)) - if AUTO_THUMBNAILS: - create_image_dataproduct(dp) - dp.get_preview() - final_products.append(dp) - return final_products - - @abstractmethod - def get_form(self, observation_type): - """ - This method takes in an observation type and returns the form type that matches it. - """ - pass - - @abstractmethod - def submit_observation(self, observation_payload): - """ - This method takes in the serialized data from the form and actually - submits the observation to the remote api - """ - pass - - @abstractmethod - def validate_observation(self, observation_payload): - """ - Same thing as submit_observation, but a dry run. You can - skip this in different modules by just using "pass" - """ - pass - - @abstractmethod - def get_observation_url(self, observation_id): - """ - Takes an observation id and return the url for which a user - can view the observation at an external location. In this case, - we return a URL to the LCO observation portal's observation - record page. - """ - pass - - def get_flux_constant(self): - """ - Returns the astropy quantity that a facility uses for its spectral flux conversion. - """ - pass - - def get_wavelength_units(self): - """ - Returns the astropy units that a facility uses for its spectral wavelengths - """ - pass - - def is_fits_facility(self, header): - """ - Returns True if the FITS header is from this facility based on valid keywords and associated - values, False otherwise. - """ - return False - - def get_start_end_keywords(self): - """ - Returns the keywords representing the start and end of an observation window for a facility. Defaults to - ``start`` and ``end``. - """ - return 'start', 'end' - - @abstractmethod - def get_terminal_observing_states(self): - """ - Returns the states for which an observation is not expected - to change. - """ - pass - - @abstractmethod - def get_observing_sites(self): - """ - Return a list of dictionaries that contain the information - necessary to be used in the planning (visibility) tool. The - list should contain dictionaries each that contain sitecode, - latitude, longitude and elevation. - """ - pass - - @abstractmethod - def get_observation_status(self, observation_id): - """ - Return the status for a single observation. observation_id should - be able to be used to retrieve the status from the external service. - """ - pass - - @abstractmethod - def data_products(self, observation_id, product_id=None): - """ - Using an observation_id, retrieve a list of the data - products that belong to this observation. In this case, - the LCO module retrieves a list of frames from the LCO - data archive. - """ - pass - - -class GenericObservationForm(forms.Form): +class BaseObservationForm(forms.Form): """ This is the class that is responsible for displaying the observation request form. Facility classes that provide a form should subclass this form. It provides @@ -296,6 +102,10 @@ def observation_payload(self): } +class BaseRoboticObservationForm(BaseObservationForm): + pass + + # TODO: refactor GenericObservationFacility to GenericRoboticFacility (or BaseRoboticFacility) # TODO: create new BaseObservationFacility from common parts of Base{Manual, Robotic}Facility classes # TODO: refactor BaseManualFacility to be subclass of BaseObservationFacility @@ -303,15 +113,11 @@ def observation_payload(self): # # Manual Observing Base Classes # -class BaseManualObservationForm(GenericObservationForm): +class BaseManualObservationForm(BaseObservationForm): name = forms.CharField() start = forms.CharField(widget=forms.TextInput(attrs={'type': 'date'})) end = forms.CharField(required=False, widget=forms.TextInput(attrs={'type': 'date'})) observation_id = forms.CharField(required=False) - filter = forms.CharField(required=False) - grating = forms.CharField(required=False) - instrument = forms.CharField(required=False) - annotation = forms.CharField(required=False, widget=forms.Textarea()) observation_params = forms.CharField(required=False, widget=forms.Textarea(attrs={'type': 'json'})) @@ -328,23 +134,110 @@ def __init__(self, *args, **kwargs): ) def layout(self): + self.helper.inputs = [] + return Div( Div( Div('name', 'observation_id', 'start', 'end', 'filter', 'grating', 'instrument', 'annotation', 'observation_params', css_class='col'), css_class='form-row'), - HTML(f'''Back''') + ButtonHolder( + Submit('submit', 'Submit'), + HTML(f'''Back''') + ) ) def observation_payload(self): + print(self.cleaned_data) + return self.serialize_parameters() + + +class BaseFacility(ABC): + name = 'Generic' + + @abstractmethod + def get_form(self, observation_type): + """ + This method takes in an observation type and returns the form type that matches it. + """ + pass + + @abstractmethod + def submit_observation(self, observation_payload): + """ + This method takes in the serialized data from the form and actually + submits the observation to the remote api + """ + pass + + @abstractmethod + def validate_observation(self, observation_payload): + """ + Same thing as submit_observation, but a dry run. You can + skip this in different modules by just using "pass" + """ + pass + + def get_flux_constant(self): + """ + Returns the astropy quantity that a facility uses for its spectral flux conversion. + """ + pass + + def get_wavelength_units(self): + """ + Returns the astropy units that a facility uses for its spectral wavelengths + """ + pass + + def is_fits_facility(self, header): + """ + Returns True if the FITS header is from this facility based on valid keywords and associated + values, False otherwise. + """ + return False + + def get_start_end_keywords(self): + """ + Returns the keywords representing the start and end of an observation window for a facility. Defaults to + ``start`` and ``end``. + """ + return 'start', 'end' + + @abstractmethod + def get_terminal_observing_states(self): + """ + Returns the states for which an observation is not expected + to change. + """ + pass + + @abstractmethod + def get_observing_sites(self): + """ + Return a list of dictionaries that contain the information + necessary to be used in the planning (visibility) tool. The + list should contain dictionaries each that contain sitecode, + latitude, longitude and elevation. + """ pass -class BaseManualFacility(ABC): +class BaseRoboticObservationFacility(BaseFacility): """ + The facility class contains all the logic specific to the facility it is + written for. Some methods are used only internally (starting with an + underscore) but some need to be implemented by all facility classes. + All facilities should inherit from this class which + provides some base functionality. + In order to make use of a facility class, add the path to + ``TOM_FACILITY_CLASSES`` in your ``settings.py``. + + For an implementation example, please see + https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/lco.py """ - name = "BaseManual" # rename in concrete subclasses + name = "Generic" # rename in concrete subclasses # TODO: 4 methods: update_observation_status, update_all_observation_statuses, # TODO: all_data_products, and save_data_products are duplicated (via C&P) in @@ -429,71 +322,22 @@ def save_data_products(self, observation_record, product_id=None): final_products.append(dp) return final_products - - @abstractmethod - def get_form(self, observation_type): - """ - This method takes in an observation type and returns the form type that matches it. - """ - pass - - @abstractmethod - def submit_observation(self, observation_payload): - """ - This method takes in the serialized data from the form and actually - submits the observation to manual facility (perhaps my email?) - """ - pass - @abstractmethod - def validate_observation(self, observation_payload): - """ - Same thing as submit_observation, but a dry run. You can - skip this in different modules by just using "pass" - """ - pass - - def get_flux_constant(self): - """ - Returns the astropy quantity that a facility uses for its spectral flux conversion. - """ - pass - - def get_wavelength_units(self): + def get_observation_url(self, observation_id): """ - Returns the astropy units that a facility uses for its spectral wavelengths + Takes an observation id and return the url for which a user + can view the observation at an external location. In this case, + we return a URL to the LCO observation portal's observation + record page. """ pass - def is_fits_facility(self, header): - """ - Returns True if the FITS header is from this facility based on valid keywords and associated - values, False otherwise. - """ - return False - - def get_start_end_keywords(self): - """ - Returns the keywords representing the start and end of an observation window for a facility. Defaults to - ``start`` and ``end``. - """ - return 'start', 'end' - - @abstractmethod - def get_terminal_observing_states(self): - """ - Returns the states for which an observation is not expected - to change. - """ - pass @abstractmethod - def get_observing_sites(self): + def get_observation_status(self, observation_id): """ - Return a list of dictionaries that contain the information - necessary to be used in the planning (visibility) tool. The - list should contain dictionaries each that contain sitecode, - latitude, longitude and elevation. + Return the status for a single observation. observation_id should + be able to be used to retrieve the status from the external service. """ pass @@ -506,3 +350,18 @@ def data_products(self, observation_id, product_id=None): data archive. """ pass + + +class BaseManualObservationFacility(BaseFacility): + """ + """ + name = 'GenericManual' # rename in concrete subclasses + +# TODO: 4 methods: update_observation_status, update_all_observation_statuses, +# TODO: all_data_products, and save_data_products are duplicated (via C&P) in +# TODO: BaseManualFacility(ABC) and GenericObservationFacility(ABC) +# TODO: +# TODO: Refactor these methods common into a BaseObservationFacility class +# TODO: and have BaseManualFacility and BaseRoboticFacility inherit from that. +# TODO: (along the way, GenericObservationFacility should be renamed to +# TODO: BaseRoboticFacility) From 1e510c18313c1f032d94e517e1964797133ed5e8 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 23 Apr 2020 16:17:38 -0700 Subject: [PATCH 051/424] Fixed buttons to be in common layout --- tom_observations/facilities/gemini.py | 13 ++++--- tom_observations/facilities/lco.py | 9 ++--- tom_observations/facility.py | 52 ++++++++++++++++----------- 3 files changed, 42 insertions(+), 32 deletions(-) diff --git a/tom_observations/facilities/gemini.py b/tom_observations/facilities/gemini.py index 27f592cbd..506552e92 100644 --- a/tom_observations/facilities/gemini.py +++ b/tom_observations/facilities/gemini.py @@ -253,10 +253,15 @@ class GEMObservationForm(BaseRoboticObservationForm): label='UT Timing Window Start [Date Time]') window_duration = forms.IntegerField(required=False, min_value=1, label='Timing Window Duration [hr]') - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.helper.layout = Layout( - self.common_layout, + # def __init__(self, *args, **kwargs): + # super().__init__(*args, **kwargs) + # self.helper.layout = Layout( + # self.common_layout,, + # self.button_layout() + # ) + + def layout(self): + return Div( HTML('Observation Parameters'), Div( Div( diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index c7c1bdb72..a2df2ce06 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -162,12 +162,11 @@ def __init__(self, *args, **kwargs): self.helper.layout = Layout( self.common_layout, self.layout(), - self.extra_layout(), - self.cadence_layout + self.cadence_layout, + self.button_layout() ) def layout(self): - return Div( Div( Div( @@ -196,10 +195,6 @@ def layout(self): ), ) - def extra_layout(self): - # If you just want to add some fields to the end of the form, add them here. - return Div() - def clean_start(self): start = self.cleaned_data['start'] return parse(start).isoformat() diff --git a/tom_observations/facility.py b/tom_observations/facility.py index cd2543b1f..cd8ec1672 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -75,7 +75,6 @@ class BaseObservationForm(forms.Form): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.helper = FormHelper() - self.helper.add_input(Submit('submit', 'Submit')) if settings.TARGET_PERMISSIONS_ONLY: self.common_layout = Layout('facility', 'target_id', 'observation_type') else: @@ -83,6 +82,21 @@ def __init__(self, *args, **kwargs): required=False, widget=forms.CheckboxSelectMultiple) self.common_layout = Layout('facility', 'target_id', 'observation_type', 'groups') + self.helper.layout = Layout( + self.common_layout, + self.layout(), + self.button_layout() + ) + + def layout(self): + return + + def button_layout(self): + target_id = self.initial.get('target_id') + return ButtonHolder( + Submit('submit', 'Submit'), + HTML(f'''Back''') + ) def serialize_parameters(self): parameters = copy.deepcopy(self.cleaned_data) @@ -121,31 +135,28 @@ class BaseManualObservationForm(BaseObservationForm): observation_params = forms.CharField(required=False, widget=forms.Textarea(attrs={'type': 'json'})) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.target_id = self.initial.get('target_id') + # def __init__(self, *args, **kwargs): + # super().__init__(*args, **kwargs) + # self.target_id = self.initial.get('target_id') - # TODO: get Back button horizontally adjacent to Submit - # self.helper.add_input(('back', 'Back')) # FIXME: this is wrong + # # TODO: Make the layout common to the parent class - self.helper.layout = Layout( - self.common_layout, - self.layout() - ) + # self.helper.layout = Layout( + # self.common_layout, + # self.layout() + # ) def layout(self): - self.helper.inputs = [] + # self.helper.inputs = [] return Div( - Div( - Div('name', 'observation_id', 'start', 'end', 'filter', 'grating', - 'instrument', 'annotation', 'observation_params', - css_class='col'), - css_class='form-row'), - ButtonHolder( - Submit('submit', 'Submit'), - HTML(f'''Back''') - ) + Div('name', 'observation_id', 'start', 'end', 'observation_params', + css_class='col'), + css_class='form-row', + # ButtonHolder( + # Submit('submit', 'Submit'), + # HTML(f'''Back''') + # ) ) def observation_payload(self): @@ -332,7 +343,6 @@ def get_observation_url(self, observation_id): """ pass - @abstractmethod def get_observation_status(self, observation_id): """ From 075f61ec016bafc84a3bf8f4831c5d78e2cedc34 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 23 Apr 2020 16:19:11 -0700 Subject: [PATCH 052/424] Updated facility name in settings.py --- tom_base/settings.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index d763d3d3a..afbe1a9f3 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -211,9 +211,7 @@ TOM_FACILITY_CLASSES = [ 'tom_observations.facilities.lco.LCOFacility', 'tom_observations.facilities.gemini.GEMFacility', - 'tom_observations.facilities.manual.GenericManualFacility', - # 'tom_observations.facility.BaseManualFacility' - # FIXME: sort out Base/ManualFacility hierarchy + 'tom_observations.facilities.manual.TestManualFacility', ] TOM_CADENCE_STRATEGIES = [ From 2e6327fbc3513ad661dc767f4c60961251eebf59 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 23 Apr 2020 16:21:58 -0700 Subject: [PATCH 053/424] Hiding reactive cadence text --- tom_observations/cadence.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tom_observations/cadence.py b/tom_observations/cadence.py index b7dd3a425..68878d52a 100644 --- a/tom_observations/cadence.py +++ b/tom_observations/cadence.py @@ -199,21 +199,21 @@ def __init__(self, *args, **kwargs): # If cadence strategy or cadence frequency aren't set, this is a normal observation and the widgets shouldn't # be rendered if not (self.initial.get('cadence_strategy') or self.initial.get('cadence_frequency')): - self.fields['cadence_strategy'].widget = forms.HiddenInput() - self.fields['cadence_frequency'].widget = forms.HiddenInput() - self.cadence_layout = Layout( - Div( - HTML('

Reactive cadencing parameters. Leave blank if no reactive cadencing is desired.

'), - ), - Div( + self.cadence_layout = Layout() + else: + self.cadence_layout = Layout( Div( - 'cadence_strategy', - css_class='col' + HTML('

Reactive cadencing parameters. Leave blank if no reactive cadencing is desired.

'), ), Div( - 'cadence_frequency', - css_class='col' - ), - css_class='form-row' + Div( + 'cadence_strategy', + css_class='col' + ), + Div( + 'cadence_frequency', + css_class='col' + ), + css_class='form-row' + ) ) - ) From 2bd96b67ff10f0ed601713acbaa91e2403bb33e9 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 23 Apr 2020 16:25:19 -0700 Subject: [PATCH 054/424] Cleaned up manual facility form --- tom_observations/facility.py | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/tom_observations/facility.py b/tom_observations/facility.py index cd8ec1672..0a8fa4a67 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -132,31 +132,17 @@ class BaseManualObservationForm(BaseObservationForm): start = forms.CharField(widget=forms.TextInput(attrs={'type': 'date'})) end = forms.CharField(required=False, widget=forms.TextInput(attrs={'type': 'date'})) observation_id = forms.CharField(required=False) - observation_params = forms.CharField(required=False, - widget=forms.Textarea(attrs={'type': 'json'})) - - # def __init__(self, *args, **kwargs): - # super().__init__(*args, **kwargs) - # self.target_id = self.initial.get('target_id') - - # # TODO: Make the layout common to the parent class - - # self.helper.layout = Layout( - # self.common_layout, - # self.layout() - # ) + observation_params = forms.CharField(required=False, widget=forms.Textarea(attrs={'type': 'json'})) def layout(self): - # self.helper.inputs = [] - return Div( - Div('name', 'observation_id', 'start', 'end', 'observation_params', - css_class='col'), - css_class='form-row', - # ButtonHolder( - # Submit('submit', 'Submit'), - # HTML(f'''Back''') - # ) + Div('name', 'observation_id'), + Div( + Div('start', css_class='col'), + Div('end', css_class='col'), + css_class='form-row' + ), + Div('observation_params') ) def observation_payload(self): From 1e14a077cc4a2fc827309c8ff35cf7f800c09d03 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 23 Apr 2020 16:48:34 -0700 Subject: [PATCH 055/424] Aliased old classes and added release notes doc --- docs/releasenotes.md | 8 ++++++++ tom_observations/facility.py | 32 ++++++++------------------------ 2 files changed, 16 insertions(+), 24 deletions(-) create mode 100644 docs/releasenotes.md diff --git a/docs/releasenotes.md b/docs/releasenotes.md new file mode 100644 index 000000000..eaeb58459 --- /dev/null +++ b/docs/releasenotes.md @@ -0,0 +1,8 @@ +### 1.5.0 + +- Introduced a manual facility interface for classical observing. + + +#### What to watch out for + +- For facility implementers: in order to support a Manual Facility Interface, the team created a `BaseObservationFacility` and two abstract implementations of it, `BaseRoboticObservationFacility` and `BaseManualObservationFacility`. `BaseRoboticObservationFacility` was aliased as `GenericObservationFacility` to support backwards compatibility, but will be removed in 2.0. \ No newline at end of file diff --git a/tom_observations/facility.py b/tom_observations/facility.py index 0a8fa4a67..6d01c143c 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -120,8 +120,10 @@ class BaseRoboticObservationForm(BaseObservationForm): pass -# TODO: refactor GenericObservationFacility to GenericRoboticFacility (or BaseRoboticFacility) -# TODO: create new BaseObservationFacility from common parts of Base{Manual, Robotic}Facility classes +# This aliasing exists to support backwards compatibility +GenericObservationForm = BaseRoboticObservationForm + + # TODO: refactor BaseManualFacility to be subclass of BaseObservationFacility # @@ -145,10 +147,6 @@ def layout(self): Div('observation_params') ) - def observation_payload(self): - print(self.cleaned_data) - return self.serialize_parameters() - class BaseFacility(ABC): name = 'Generic' @@ -236,15 +234,6 @@ class BaseRoboticObservationFacility(BaseFacility): """ name = "Generic" # rename in concrete subclasses -# TODO: 4 methods: update_observation_status, update_all_observation_statuses, -# TODO: all_data_products, and save_data_products are duplicated (via C&P) in -# TODO: BaseManualFacility(ABC) and GenericObservationFacility(ABC) -# TODO: -# TODO: Refactor these methods common into a BaseObservationFacility class -# TODO: and have BaseManualFacility and BaseRoboticFacility inherit from that. -# TODO: (along the way, GenericObservationFacility should be renamed to -# TODO: BaseRoboticFacility) - def update_observation_status(self, observation_id): from tom_observations.models import ObservationRecord try: @@ -348,16 +337,11 @@ def data_products(self, observation_id, product_id=None): pass +# This aliasing exists to support backwards compatibility +GenericObservationFacility = BaseRoboticObservationFacility + + class BaseManualObservationFacility(BaseFacility): """ """ name = 'GenericManual' # rename in concrete subclasses - -# TODO: 4 methods: update_observation_status, update_all_observation_statuses, -# TODO: all_data_products, and save_data_products are duplicated (via C&P) in -# TODO: BaseManualFacility(ABC) and GenericObservationFacility(ABC) -# TODO: -# TODO: Refactor these methods common into a BaseObservationFacility class -# TODO: and have BaseManualFacility and BaseRoboticFacility inherit from that. -# TODO: (along the way, GenericObservationFacility should be renamed to -# TODO: BaseRoboticFacility) From 2d02058c49868d181a8e7296f5bea49f520b9015 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 23 Apr 2020 17:10:01 -0700 Subject: [PATCH 056/424] Updated some TODOs --- tom_observations/facilities/lco.py | 1 - tom_observations/facilities/manual.py | 4 ++-- tom_observations/facility.py | 18 ++++++++++-------- tom_observations/forms.py | 2 +- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index a2df2ce06..4d6750315 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -205,7 +205,6 @@ def clean_end(self): def is_valid(self): super().is_valid() - # TODO this is a bit leaky and should be done without the need of get_service_class obs_module = get_service_class(self.cleaned_data['facility']) errors = obs_module().validate_observation(self.observation_payload()) if errors: diff --git a/tom_observations/facilities/manual.py b/tom_observations/facilities/manual.py index 8a670b592..8b94dba55 100644 --- a/tom_observations/facilities/manual.py +++ b/tom_observations/facilities/manual.py @@ -48,7 +48,8 @@ def submit_observation(self, observation_payload): This method takes in the serialized data from the form. """ - # TODO: finish implementation + # TODO: update to generate ID from payload, potentially move to super class + # TODO: explore adding logic to send email to tom-demo print(f'observation_payload: {observation_payload}') obs_ids = [] for payload in observation_payload: @@ -61,7 +62,6 @@ def validate_observation(self, observation_payload): Same thing as submit_observation, but a dry run. You can skip this in different modules by just using "pass" """ - # TODO: implement me raise NotImplementedError def is_fits_facility(self, header): diff --git a/tom_observations/facility.py b/tom_observations/facility.py index 6d01c143c..11f646032 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -55,6 +55,9 @@ def get_service_class(name): raise ImportError('Could not a find a facility with that name. Did you add it to TOM_FACILITY_CLASSES?') +# TODO: Ensure docstrings are up to date + + class BaseObservationForm(forms.Form): """ This is the class that is responsible for displaying the observation request form. @@ -98,6 +101,10 @@ def button_layout(self): HTML(f'''Back''') ) + def is_valid(self): + # TODO: Make this call the validate_observation method in facility + super().is_valid() + def serialize_parameters(self): parameters = copy.deepcopy(self.cleaned_data) parameters.pop('groups', None) @@ -124,11 +131,6 @@ class BaseRoboticObservationForm(BaseObservationForm): GenericObservationForm = BaseRoboticObservationForm -# TODO: refactor BaseManualFacility to be subclass of BaseObservationFacility - -# -# Manual Observing Base Classes -# class BaseManualObservationForm(BaseObservationForm): name = forms.CharField() start = forms.CharField(widget=forms.TextInput(attrs={'type': 'date'})) @@ -148,7 +150,7 @@ def layout(self): ) -class BaseFacility(ABC): +class BaseObservationFacility(ABC): name = 'Generic' @abstractmethod @@ -219,7 +221,7 @@ def get_observing_sites(self): pass -class BaseRoboticObservationFacility(BaseFacility): +class BaseRoboticObservationFacility(BaseObservationFacility): """ The facility class contains all the logic specific to the facility it is written for. Some methods are used only internally (starting with an @@ -341,7 +343,7 @@ def data_products(self, observation_id, product_id=None): GenericObservationFacility = BaseRoboticObservationFacility -class BaseManualObservationFacility(BaseFacility): +class BaseManualObservationFacility(BaseObservationFacility): """ """ name = 'GenericManual' # rename in concrete subclasses diff --git a/tom_observations/forms.py b/tom_observations/forms.py index 996a1bf15..aab78a7a6 100644 --- a/tom_observations/forms.py +++ b/tom_observations/forms.py @@ -42,7 +42,7 @@ def __init__(self, *args, **kwargs): class ConfirmExistingObservationForm(AddExistingObservationForm): - # TODO: Should this inherit from AddExistingObservationForm or be its own thing? + # TODO: Attempt to put this logic in ManualObservationCreateView.get_form def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.fields['facility'].widget = forms.HiddenInput() From a78d048fb6489f481e6b39366260b4f3d938a89c Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 23 Apr 2020 17:21:36 -0700 Subject: [PATCH 057/424] Fixed validation issue, added working submit_observation --- tom_observations/facilities/manual.py | 7 +++++-- tom_observations/facility.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tom_observations/facilities/manual.py b/tom_observations/facilities/manual.py index 8b94dba55..523d82b35 100644 --- a/tom_observations/facilities/manual.py +++ b/tom_observations/facilities/manual.py @@ -1,4 +1,5 @@ import logging +from datetime import datetime from django.conf import settings @@ -52,8 +53,10 @@ def submit_observation(self, observation_payload): # TODO: explore adding logic to send email to tom-demo print(f'observation_payload: {observation_payload}') obs_ids = [] - for payload in observation_payload: - obs_ids.append(f'{payload}') + # for payload in observation_payload: + # obs_ids.append(f'{payload}') + + obs_ids.append(datetime.now()) return obs_ids diff --git a/tom_observations/facility.py b/tom_observations/facility.py index 11f646032..eec60f4bb 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -103,7 +103,7 @@ def button_layout(self): def is_valid(self): # TODO: Make this call the validate_observation method in facility - super().is_valid() + return super().is_valid() def serialize_parameters(self): parameters = copy.deepcopy(self.cleaned_data) From a339f5ab69f0fb7141dc6981900923b19bfdfaf7 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 23 Apr 2020 17:28:46 -0700 Subject: [PATCH 058/424] Moved all_data_products to BaseObservationFacility, added some TODOs --- tom_observations/facilities/manual.py | 11 ------ tom_observations/facility.py | 48 +++++++++++++-------------- tom_observations/views.py | 1 + 3 files changed, 25 insertions(+), 35 deletions(-) diff --git a/tom_observations/facilities/manual.py b/tom_observations/facilities/manual.py index 523d82b35..039c44b10 100644 --- a/tom_observations/facilities/manual.py +++ b/tom_observations/facilities/manual.py @@ -88,9 +88,6 @@ def get_terminal_observing_states(self): """ return ZZZ_TERMINAL_OBSERVING_STATES - def get_observation_url(self, observation_id): - return - def get_observing_sites(self): """ Return a list of dictionaries that contain the information @@ -100,14 +97,6 @@ def get_observing_sites(self): """ return ZZZ_SITES - def get_observation_status(self, observation_id): - """ - Return the status for a single observation. observation_id should - be able to be used to retrieve the status from the external service. - """ - # TODO: implement me - raise NotImplementedError - def data_products(self, observation_id, product_id=None): """ Using an observation_id, retrieve a list of the data diff --git a/tom_observations/facility.py b/tom_observations/facility.py index eec60f4bb..1e6dcceab 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -153,6 +153,30 @@ def layout(self): class BaseObservationFacility(ABC): name = 'Generic' + def all_data_products(self, observation_record): + from tom_dataproducts.models import DataProduct + products = {'saved': [], 'unsaved': []} + for product in self.data_products(observation_record.observation_id): + try: + dp = DataProduct.objects.get(product_id=product['id']) + products['saved'].append(dp) + except DataProduct.DoesNotExist: + products['unsaved'].append(product) + # Obtain products uploaded manually by users + user_products = DataProduct.objects.filter( + observation_record_id=observation_record.id, product_id=None + ) + for product in user_products: + products['saved'].append(product) + + # Add any JPEG images created from DataProducts + image_products = DataProduct.objects.filter( + observation_record_id=observation_record.id, data_product_type='image_file' + ) + for product in image_products: + products['saved'].append(product) + return products + @abstractmethod def get_form(self, observation_type): """ @@ -262,30 +286,6 @@ def update_all_observation_statuses(self, target=None): failed_records.append((record.observation_id, str(e))) return failed_records - def all_data_products(self, observation_record): - from tom_dataproducts.models import DataProduct - products = {'saved': [], 'unsaved': []} - for product in self.data_products(observation_record.observation_id): - try: - dp = DataProduct.objects.get(product_id=product['id']) - products['saved'].append(dp) - except DataProduct.DoesNotExist: - products['unsaved'].append(product) - # Obtain products uploaded manually by users - user_products = DataProduct.objects.filter( - observation_record_id=observation_record.id, product_id=None - ) - for product in user_products: - products['saved'].append(product) - - # Add any JPEG images created from DataProducts - image_products = DataProduct.objects.filter( - observation_record_id=observation_record.id, data_product_type='image_file' - ) - for product in image_products: - products['saved'].append(product) - return products - def save_data_products(self, observation_record, product_id=None): from tom_dataproducts.models import DataProduct from tom_dataproducts.utils import create_image_dataproduct diff --git a/tom_observations/views.py b/tom_observations/views.py index 6c969fd7f..8b19a0928 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -256,6 +256,7 @@ def form_valid(self, form): print(f'records: {records}') print(f'observation_ids: {observation_ids}') + # TODO: redirect to observation list for multiple observations, observation detail otherwise if len(records) > 1 or form.cleaned_data.get('cadence_strategy'): group_name = form.cleaned_data['name'] From 0326b145640130189477dd7fe314ea92ef89b27d Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 24 Apr 2020 11:02:26 -0700 Subject: [PATCH 059/424] Consolidated forms --- tom_observations/forms.py | 29 ++--------------------------- tom_observations/views.py | 29 +++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/tom_observations/forms.py b/tom_observations/forms.py index aab78a7a6..0c1665151 100644 --- a/tom_observations/forms.py +++ b/tom_observations/forms.py @@ -1,8 +1,7 @@ from django import forms -from django.urls import reverse, reverse_lazy -from crispy_forms.bootstrap import FormActions +from django.urls import reverse from crispy_forms.helper import FormHelper -from crispy_forms.layout import Button, ButtonHolder, Column, HTML, Layout, Row, Submit +from crispy_forms.layout import ButtonHolder, Column, Layout, Row, Submit from tom_observations.facility import get_service_classes @@ -39,27 +38,3 @@ def __init__(self, *args, **kwargs): ) ) ) - - -class ConfirmExistingObservationForm(AddExistingObservationForm): - # TODO: Attempt to put this logic in ManualObservationCreateView.get_form - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.fields['facility'].widget = forms.HiddenInput() - self.fields['observation_id'].widget = forms.HiddenInput() - target_id = kwargs['data']['target_id'] - cancel_url = reverse('home') - if target_id: - cancel_url = reverse('tom_targets:detail', kwargs={'pk': target_id}) - self.helper.layout = Layout( - HTML('''

An observation record already exists in your TOM for this combination of observation ID, - facility, and target. Are you sure you want to create this record?

'''), - 'target_id', - 'facility', - 'observation_id', - 'confirm', - FormActions( - Submit('confirm', 'Confirm'), - HTML(f'Cancel') - ) - ) diff --git a/tom_observations/views.py b/tom_observations/views.py index 8b19a0928..8a108955f 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -26,7 +26,7 @@ from tom_common.mixins import Raise403PermissionRequiredMixin from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm from tom_observations.facility import get_service_class, get_service_classes -from tom_observations.forms import ConfirmExistingObservationForm +from tom_observations.forms import AddExistingObservationForm from tom_observations.models import ObservationRecord, ObservationGroup, ObservingStrategy from tom_targets.models import Target @@ -330,7 +330,32 @@ class ManualObservationCreateView(LoginRequiredMixin, FormView): ConfirmExistingObservationForm has a hidden confirmation checkbox selected by default. """ template_name = 'tom_observations/existing_observation_confirm.html' - form_class = ConfirmExistingObservationForm + form_class = AddExistingObservationForm + + def get_form(self): + form = super().get_form() + form.fields['facility'].widget = forms.HiddenInput() + form.fields['observation_id'].widget = forms.HiddenInput() + if self.request.method == 'GET': + target_id = self.request.GET.get('target_id') + elif self.request.method == 'POST': + target_id = self.request.POST.get('target_id') + cancel_url = reverse('home') + if target_id: + cancel_url = reverse('tom_targets:detail', kwargs={'pk': target_id}) + form.helper.layout = Layout( + HTML('''

An observation record already exists in your TOM for this combination of observation ID, + facility, and target. Are you sure you want to create this record?

'''), + 'target_id', + 'facility', + 'observation_id', + 'confirm', + FormActions( + Submit('confirm', 'Confirm'), + HTML(f'Cancel') + ) + ) + return form def get_initial(self): """ From d4b7bf02758e23e69fd69215ba0bae84fe991ba1 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Fri, 24 Apr 2020 18:05:45 +0000 Subject: [PATCH 060/424] refactor: ZZZ, Test renamed to Example --- tom_base/settings.py | 2 +- tom_observations/facilities/manual.py | 22 +++++++++++----------- tom_observations/facility.py | 6 +++--- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index afbe1a9f3..4e5174c54 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -211,7 +211,7 @@ TOM_FACILITY_CLASSES = [ 'tom_observations.facilities.lco.LCOFacility', 'tom_observations.facilities.gemini.GEMFacility', - 'tom_observations.facilities.manual.TestManualFacility', + 'tom_observations.facilities.manual.ExampleManualFacility', ] TOM_CADENCE_STRATEGIES = [ diff --git a/tom_observations/facilities/manual.py b/tom_observations/facilities/manual.py index 039c44b10..ec2676e4d 100644 --- a/tom_observations/facilities/manual.py +++ b/tom_observations/facilities/manual.py @@ -15,28 +15,28 @@ # try: - ZZZ_SETTINGS = settings.FACILITIES['ZZZ'] + EXAMPLE_MANUAL_SETTINGS = settings.FACILITIES['EXAMPLE_MANUAL'] except KeyError: - ZZZ_SETTINGS = { + EXAMPLE_MANUAL_SETTINGS = { } -ZZZ_SITES = { - 'Zero-zero Island': { - 'sitecode': 'zzz', # top-secret observing site on Zero-zero Island +EXAMPLE_SITES = { + 'Example Manual Facility': { + 'sitecode': 'Example', 'latitude': 0.0, 'longitude': 0.0, 'elevation': 0.0 }, } -ZZZ_TERMINAL_OBSERVING_STATES = ['Completed'] +EXAMPLE_TERMINAL_OBSERVING_STATES = ['Completed'] -class TestManualFacility(BaseManualObservationFacility): +class ExampleManualFacility(BaseManualObservationFacility): """ """ - name = 'ZZZ' - observation_types = [('IMAGING', 'Imaging')] + name = 'Example' + observation_types = [('OBSERVATION', 'Manual Observation')] def get_form(self, observation_type): """ @@ -86,7 +86,7 @@ def get_terminal_observing_states(self): Returns the states for which an observation is not expected to change. """ - return ZZZ_TERMINAL_OBSERVING_STATES + return EXAMPLE_TERMINAL_OBSERVING_STATES def get_observing_sites(self): """ @@ -95,7 +95,7 @@ def get_observing_sites(self): list should contain dictionaries each that contain sitecode, latitude, longitude and elevation. """ - return ZZZ_SITES + return EXAMPLE_SITES def data_products(self, observation_id, product_id=None): """ diff --git a/tom_observations/facility.py b/tom_observations/facility.py index 1e6dcceab..b33cd595f 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -151,7 +151,7 @@ def layout(self): class BaseObservationFacility(ABC): - name = 'Generic' + name = 'BaseObservation' def all_data_products(self, observation_record): from tom_dataproducts.models import DataProduct @@ -258,7 +258,7 @@ class BaseRoboticObservationFacility(BaseObservationFacility): For an implementation example, please see https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/lco.py """ - name = "Generic" # rename in concrete subclasses + name = "BaseRobotic" # rename in concrete subclasses def update_observation_status(self, observation_id): from tom_observations.models import ObservationRecord @@ -346,4 +346,4 @@ def data_products(self, observation_id, product_id=None): class BaseManualObservationFacility(BaseObservationFacility): """ """ - name = 'GenericManual' # rename in concrete subclasses + name = 'BaseManual' # rename in concrete subclasses From 3e0df8de4001e0df2dbc44c0b372b32a0d33af0d Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Fri, 24 Apr 2020 18:06:34 +0000 Subject: [PATCH 061/424] implement get_observation_url for ManualFacility This is a temporary fix b/x Manual Facilities shouldn't need to implement get_observation_utl but at the moment this is required by ObervationRecord model. --- tom_observations/facilities/manual.py | 3 +++ tom_observations/facility.py | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/tom_observations/facilities/manual.py b/tom_observations/facilities/manual.py index ec2676e4d..f0c6d72c2 100644 --- a/tom_observations/facilities/manual.py +++ b/tom_observations/facilities/manual.py @@ -105,3 +105,6 @@ def data_products(self, observation_id, product_id=None): data archive. """ return [] + + def get_observation_url(self, observation_id): + return \ No newline at end of file diff --git a/tom_observations/facility.py b/tom_observations/facility.py index b33cd595f..0ad6eb85a 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -244,6 +244,16 @@ def get_observing_sites(self): """ pass + @abstractmethod + def get_observation_url(self, observation_id): + """ + Takes an observation id and return the url for which a user + can view the observation at an external location. In this case, + we return a URL to the LCO observation portal's observation + record page. + """ + pass + class BaseRoboticObservationFacility(BaseObservationFacility): """ @@ -310,16 +320,6 @@ def save_data_products(self, observation_record, product_id=None): final_products.append(dp) return final_products - @abstractmethod - def get_observation_url(self, observation_id): - """ - Takes an observation id and return the url for which a user - can view the observation at an external location. In this case, - we return a URL to the LCO observation portal's observation - record page. - """ - pass - @abstractmethod def get_observation_status(self, observation_id): """ From 59ca762ad7435528ffad878b0dc45a018914bfec Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 24 Apr 2020 11:19:43 -0700 Subject: [PATCH 062/424] Fixed code style issues --- tom_observations/facilities/manual.py | 2 +- tom_observations/facility.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tom_observations/facilities/manual.py b/tom_observations/facilities/manual.py index f0c6d72c2..9a3a40f04 100644 --- a/tom_observations/facilities/manual.py +++ b/tom_observations/facilities/manual.py @@ -107,4 +107,4 @@ def data_products(self, observation_id, product_id=None): return [] def get_observation_url(self, observation_id): - return \ No newline at end of file + return diff --git a/tom_observations/facility.py b/tom_observations/facility.py index 0ad6eb85a..e690ec6ac 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -98,7 +98,8 @@ def button_layout(self): target_id = self.initial.get('target_id') return ButtonHolder( Submit('submit', 'Submit'), - HTML(f'''Back''') + HTML(f''' + Back''') ) def is_valid(self): From 23810237519dac8ad946df6b5e9e5baa93bd370c Mon Sep 17 00:00:00 2001 From: fraserw Date: Fri, 24 Apr 2020 11:56:39 -0700 Subject: [PATCH 063/424] --added a missed file, fixed bug in requests for single facilities --- tom_observations/facilities/lco.py | 31 ++++++++++++++----- .../migrations/0018_auto_20200423_2018.py | 28 +++++++++++++++++ 2 files changed, 51 insertions(+), 8 deletions(-) create mode 100644 tom_targets/migrations/0018_auto_20200423_2018.py diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 6f4856129..5d48f785a 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -505,28 +505,43 @@ def observation_payload(self): #is done in tom_base/utils.py. obs_module = get_service_class(self.cleaned_data['facility']) requests = self._build_ephemeris_requests() + locations = [] + for j in range(len(requests)): + if requests[j]['location'] not in locations: + locations.append(requests[j]['location']) + if len(locations)>1: + operator = "MANY" + else: + operator = "SINGLE" + errors = obs_module().validate_observation({ "name": self.cleaned_data['name'], "proposal": self.cleaned_data['proposal'], "ipp_value": self.cleaned_data['ipp_value'], - "operator": "MANY", + "operator": operator, "observation_type": self.cleaned_data['observation_mode'], "requests": requests }) - valid_requests = [] - for i,e in enumerate(errors['requests']): - if e!={}: - if 'non_field_errors' not in e: + print(requests,len(requests)) + print(errors) + print() + if len(errors)>0: + valid_requests = [] + for i,e in enumerate(errors['requests']): + if e!={}: + if 'non_field_errors' not in e: + valid_requests.append(requests[i]) + else: valid_requests.append(requests[i]) - else: - valid_requests.append(requests[i]) + else: + valid_requests = requests return { "name": self.cleaned_data['name'], "proposal": self.cleaned_data['proposal'], "ipp_value": self.cleaned_data['ipp_value'], - "operator": "MANY", + "operator": operator, "observation_type": self.cleaned_data['observation_mode'], "requests": valid_requests diff --git a/tom_targets/migrations/0018_auto_20200423_2018.py b/tom_targets/migrations/0018_auto_20200423_2018.py new file mode 100644 index 000000000..dba8caeb9 --- /dev/null +++ b/tom_targets/migrations/0018_auto_20200423_2018.py @@ -0,0 +1,28 @@ +# Generated by Django 2.2.5 on 2020-04-23 20:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tom_targets', '0017_auto_20200130_2350'), + ] + + operations = [ + migrations.AddField( + model_name='target', + name='centsite', + field=models.CharField(blank=True, help_text='Observatory Site Code', max_length=50, null=True, verbose_name='Centre-Site Name'), + ), + migrations.AddField( + model_name='target', + name='eph_json', + field=models.TextField(blank=True, help_text="Don't fill this in by hand unless you know what you are doing.", null=True, verbose_name='Ephemeris JSON'), + ), + migrations.AlterField( + model_name='target', + name='scheme', + field=models.CharField(blank=True, choices=[('MPC_MINOR_PLANET', 'MPC Minor Planet'), ('MPC_COMET', 'MPC Comet'), ('JPL_MAJOR_PLANET', 'JPL Major Planet'), ('EPHEMERIS', 'Custom Ephemeris')], default='', max_length=50, verbose_name='Orbital Element Scheme'), + ), + ] From bf80f005fe3e6714fd468efc8f7469392c11dc62 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Sat, 25 Apr 2020 00:32:37 +0000 Subject: [PATCH 064/424] don't display View at observatory button without a url --- tom_observations/facilities/manual.py | 2 +- .../tom_observations/observationrecord_detail.html | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tom_observations/facilities/manual.py b/tom_observations/facilities/manual.py index f0c6d72c2..9b0cf83ae 100644 --- a/tom_observations/facilities/manual.py +++ b/tom_observations/facilities/manual.py @@ -107,4 +107,4 @@ def data_products(self, observation_id, product_id=None): return [] def get_observation_url(self, observation_id): - return \ No newline at end of file + return '' diff --git a/tom_observations/templates/tom_observations/observationrecord_detail.html b/tom_observations/templates/tom_observations/observationrecord_detail.html index 360ed5d62..6ab45a136 100644 --- a/tom_observations/templates/tom_observations/observationrecord_detail.html +++ b/tom_observations/templates/tom_observations/observationrecord_detail.html @@ -7,7 +7,12 @@ {% block content %}
-

{{ object }} View at observatory »

+ +

{{ object }} + {% if object.url %} + View at observatory » + {% endif %} +

Created: {{ object.created }} Modified: {{ object.modified }}

Status: {{ object.status }}

From a0a5a40c44ecc1ce1a53c019cbc4573798a5b3f2 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Mon, 27 Apr 2020 17:04:45 +0000 Subject: [PATCH 065/424] add Observation ID to obs detail, group elements with my-auto class my-auto meaning margin(m);top and bottom(y)-size(auto) for ref. see Bootstrap Spacing docs. --- .../tom_observations/observationrecord_detail.html | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tom_observations/templates/tom_observations/observationrecord_detail.html b/tom_observations/templates/tom_observations/observationrecord_detail.html index 6ab45a136..b8d001b15 100644 --- a/tom_observations/templates/tom_observations/observationrecord_detail.html +++ b/tom_observations/templates/tom_observations/observationrecord_detail.html @@ -13,8 +13,11 @@

{{ object }} View at observatory » {% endif %}

-

Created: {{ object.created }} Modified: {{ object.modified }}

-

Status: {{ object.status }}

+
+

Observation ID: {{ object.observation_id }}

+

Created: {{ object.created }} Modified: {{ object.modified }}

+

Status: {{ object.status }}

+

{% upload_dataproduct object %}
From 7e6ac0d726899f33bae3b4e313a8eadbc9d8189d Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Mon, 27 Apr 2020 17:04:45 +0000 Subject: [PATCH 066/424] add Observation ID to obs detail, group elements with my-auto class my-auto meaning margin(m);top and bottom(y)-size(auto) for ref. see Bootstrap Spacing docs. --- .../tom_observations/observationrecord_detail.html | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tom_observations/templates/tom_observations/observationrecord_detail.html b/tom_observations/templates/tom_observations/observationrecord_detail.html index 6ab45a136..aa9b02804 100644 --- a/tom_observations/templates/tom_observations/observationrecord_detail.html +++ b/tom_observations/templates/tom_observations/observationrecord_detail.html @@ -6,15 +6,17 @@ {% endblock %} {% block content %}
-
- +

{{ object }} {% if object.url %} View at observatory » {% endif %}

-

Created: {{ object.created }} Modified: {{ object.modified }}

-

Status: {{ object.status }}

+
+

Observation ID: {{ object.observation_id }}

+

Created: {{ object.created }} Modified: {{ object.modified }}

+

Status: {{ object.status }}

+

{% upload_dataproduct object %}
From 18e5ea4b1907ff875fd098e86fd9f639fab227e6 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Mon, 27 Apr 2020 19:08:50 +0000 Subject: [PATCH 067/424] supply a decent default observation_id in ExampleManualFacility --- tom_observations/facilities/manual.py | 31 +++++++++++++++++++++------ 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/tom_observations/facilities/manual.py b/tom_observations/facilities/manual.py index 9b0cf83ae..e4319901b 100644 --- a/tom_observations/facilities/manual.py +++ b/tom_observations/facilities/manual.py @@ -1,9 +1,11 @@ +import json import logging -from datetime import datetime + from django.conf import settings from tom_observations.facility import BaseManualObservationFacility, BaseManualObservationForm +from tom_targets.models import Target logger = logging.getLogger(__name__) @@ -48,15 +50,30 @@ def submit_observation(self, observation_payload): """ This method takes in the serialized data from the form. + The BaseManualObservationForm(BaseObservationForm) does not require an observation_id. + In this example, if no observation_id is given, we construct one to return from the + other required form fields. + """ - # TODO: update to generate ID from payload, potentially move to super class # TODO: explore adding logic to send email to tom-demo - print(f'observation_payload: {observation_payload}') - obs_ids = [] - # for payload in observation_payload: - # obs_ids.append(f'{payload}') - obs_ids.append(datetime.now()) + obs_ids = [] + # params comes as JSON string, to turn it back into a dictionary + obs_params = json.loads(observation_payload['params']) + + # if the Observation id was supplied then use it + if obs_params['observation_id']: + obs_ids.append(obs_params['observation_id']) + else: + # observation_id was empty string, so construct reasonable default + # such as name:target-facility-start + target = Target.objects.get(pk=observation_payload['target_id']).name + obs_name = obs_params['name'] + facility = obs_params['facility'] + start = obs_params[self.get_start_end_keywords()[0]] + + obs_id = f'{obs_name}:{target}-{facility}-{start}' + obs_ids.append(obs_id) return obs_ids From bae9a0a194f60290a3805d6a3d23922abe7ff54d Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 27 Apr 2020 15:21:40 -0700 Subject: [PATCH 068/424] Renamed existing observation form --- ...observation_button.html => existing_observation_form.html} | 0 tom_observations/templatetags/observation_extras.py | 4 ++-- tom_targets/templates/tom_targets/target_detail.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename tom_observations/templates/tom_observations/partials/{existing_observation_button.html => existing_observation_form.html} (100%) diff --git a/tom_observations/templates/tom_observations/partials/existing_observation_button.html b/tom_observations/templates/tom_observations/partials/existing_observation_form.html similarity index 100% rename from tom_observations/templates/tom_observations/partials/existing_observation_button.html rename to tom_observations/templates/tom_observations/partials/existing_observation_form.html diff --git a/tom_observations/templatetags/observation_extras.py b/tom_observations/templatetags/observation_extras.py index f9477bc8c..2ecb8f00b 100644 --- a/tom_observations/templatetags/observation_extras.py +++ b/tom_observations/templatetags/observation_extras.py @@ -28,8 +28,8 @@ def observing_buttons(target): return {'target': target, 'facilities': facilities} -@register.inclusion_tag('tom_observations/partials/existing_observation_button.html') -def existing_observation_button(target): +@register.inclusion_tag('tom_observations/partials/existing_observation_form.html') +def existing_observation_form(target): return {'form': AddExistingObservationForm(initial={'target_id': target.id})} diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index 2bb4e4030..b8670e10f 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -59,7 +59,7 @@

Plan

{% endif %}
- {% existing_observation_button object %} + {% existing_observation_form object %}

Observations

Update Observations Status {% observation_list object %} From 8440547114668787a76c637b1f845d41711888d3 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 27 Apr 2020 17:02:17 -0700 Subject: [PATCH 069/424] Added new update observation id form to observation detail page --- tom_observations/facilities/lco.py | 7 ------ tom_observations/facility.py | 2 +- tom_observations/forms.py | 25 +++++++++++++++++++ .../observationrecord_detail.html | 3 +++ .../partials/update_observation_id_form.html | 2 ++ .../templatetags/observation_extras.py | 7 +++++- tom_observations/urls.py | 4 +-- tom_observations/views.py | 16 ++++++------ 8 files changed, 46 insertions(+), 20 deletions(-) create mode 100644 tom_observations/templates/tom_observations/partials/update_observation_id_form.html diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 4d6750315..9870734f9 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -501,13 +501,6 @@ def get_form(self, observation_type): def get_strategy_form(self, observation_type): return LCOObservingStrategyForm - def get_update_form(self, observation_type): - form = self.get_form(observation_type)() - form.fields['observation_type'].widget = forms.HiddenInput() - form.fields['period'].widget = forms.HiddenInput() - form.fields['jitter'].widget = forms.HiddenInput() - return form - def submit_observation(self, observation_payload): response = make_request( 'POST', diff --git a/tom_observations/facility.py b/tom_observations/facility.py index e690ec6ac..5a5570da8 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -269,7 +269,7 @@ class BaseRoboticObservationFacility(BaseObservationFacility): For an implementation example, please see https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/lco.py """ - name = "BaseRobotic" # rename in concrete subclasses + name = 'BaseRobotic' # rename in concrete subclasses def update_observation_status(self, observation_id): from tom_observations.models import ObservationRecord diff --git a/tom_observations/forms.py b/tom_observations/forms.py index 0c1665151..fc24825c7 100644 --- a/tom_observations/forms.py +++ b/tom_observations/forms.py @@ -11,6 +11,7 @@ def facility_choices(): class AddExistingObservationForm(forms.Form): + "" target_id = forms.IntegerField(required=True, widget=forms.HiddenInput()) facility = forms.ChoiceField(required=True, choices=facility_choices, label=False) observation_id = forms.CharField(required=True, label=False, @@ -38,3 +39,27 @@ def __init__(self, *args, **kwargs): ) ) ) + + +class UpdateObservationId(forms.Form): + obsr_id = forms.IntegerField(required=True, widget=forms.HiddenInput()) + observation_id = forms.CharField(required=True, label=False, + widget=forms.TextInput(attrs={'placeholder': 'Observation ID'})) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.helper = FormHelper() + self.helper.form_action = reverse('tom_observations:update', kwargs={'pk': self.initial.get('obsr_id')}) + self.helper.layout = Layout( + 'obsr_id', + Row( + Column( + 'observation_id' + ), + Column( + ButtonHolder( + Submit('submit', 'Update Observation Id') + ), + ) + ) + ) diff --git a/tom_observations/templates/tom_observations/observationrecord_detail.html b/tom_observations/templates/tom_observations/observationrecord_detail.html index 360ed5d62..df84712f0 100644 --- a/tom_observations/templates/tom_observations/observationrecord_detail.html +++ b/tom_observations/templates/tom_observations/observationrecord_detail.html @@ -8,6 +8,9 @@

{{ object }} View at observatory »

+ {% if editable %} +

{% update_observation_id_form object %}

+ {% endif %}

Created: {{ object.created }} Modified: {{ object.modified }}

Status: {{ object.status }}

diff --git a/tom_observations/templates/tom_observations/partials/update_observation_id_form.html b/tom_observations/templates/tom_observations/partials/update_observation_id_form.html new file mode 100644 index 000000000..96b515abc --- /dev/null +++ b/tom_observations/templates/tom_observations/partials/update_observation_id_form.html @@ -0,0 +1,2 @@ +{% load crispy_forms_tags %} +{% crispy form %} \ No newline at end of file diff --git a/tom_observations/templatetags/observation_extras.py b/tom_observations/templatetags/observation_extras.py index 2ecb8f00b..75dad8903 100644 --- a/tom_observations/templatetags/observation_extras.py +++ b/tom_observations/templatetags/observation_extras.py @@ -8,7 +8,7 @@ from plotly import offline import plotly.graph_objs as go -from tom_observations.forms import AddExistingObservationForm +from tom_observations.forms import AddExistingObservationForm, UpdateObservationId from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class, get_service_classes from tom_observations.observing_strategy import RunStrategyForm @@ -33,6 +33,11 @@ def existing_observation_form(target): return {'form': AddExistingObservationForm(initial={'target_id': target.id})} +@register.inclusion_tag('tom_observations/partials/update_observation_id_form.html') +def update_observation_id_form(obsr): + return {'form': UpdateObservationId(initial={'obsr_id': obsr.id, 'observation_id': obsr.observation_id})} + + @register.inclusion_tag('tom_observations/partials/observation_type_tabs.html', takes_context=True) def observation_type_tabs(context): """ diff --git a/tom_observations/urls.py b/tom_observations/urls.py index 5cd4613d9..d36b61eba 100644 --- a/tom_observations/urls.py +++ b/tom_observations/urls.py @@ -1,6 +1,6 @@ from django.urls import path -from tom_observations.views import (ManualObservationCreateView, ObservationCreateView, ObservationUpdateView, +from tom_observations.views import (ManualObservationCreateView, ObservationCreateView, ObservationRecordUpdateView, ObservationGroupDeleteView, ObservationGroupListView, ObservationListView, ObservationRecordDetailView, ObservingStrategyCreateView, ObservingStrategyDeleteView, ObservingStrategyListView, @@ -17,7 +17,7 @@ path('strategy//delete/', ObservingStrategyDeleteView.as_view(), name='strategy-delete'), path('strategy//', ObservingStrategyUpdateView.as_view(), name='strategy-detail'), path('/create/', ObservationCreateView.as_view(), name='create'), - path('/update/', ObservationUpdateView.as_view(), name='update'), + path('/update/', ObservationRecordUpdateView.as_view(), name='update'), path('/', ObservationRecordDetailView.as_view(), name='detail'), path('groups/list/', ObservationGroupListView.as_view(), name='group-list'), path('groups//delete/', ObservationGroupDeleteView.as_view(), name='group-delete'), diff --git a/tom_observations/views.py b/tom_observations/views.py index 8a108955f..ebb4e91e1 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -3,7 +3,7 @@ import json from crispy_forms.bootstrap import FormActions -from crispy_forms.layout import Button, HTML, Layout, Submit +from crispy_forms.layout import HTML, Layout, Submit from django import forms from django.conf import settings from django.contrib import messages @@ -25,7 +25,7 @@ from tom_common.hints import add_hint from tom_common.mixins import Raise403PermissionRequiredMixin from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm -from tom_observations.facility import get_service_class, get_service_classes +from tom_observations.facility import get_service_class, get_service_classes, BaseManualObservationFacility from tom_observations.forms import AddExistingObservationForm from tom_observations.models import ObservationRecord, ObservationGroup, ObservingStrategy from tom_targets.models import Target @@ -254,8 +254,6 @@ def form_valid(self, form): ) records.append(record) - print(f'records: {records}') - print(f'observation_ids: {observation_ids}') # TODO: redirect to observation list for multiple observations, observation detail otherwise if len(records) > 1 or form.cleaned_data.get('cadence_strategy'): @@ -281,17 +279,16 @@ def form_valid(self, form): ) -class ObservationUpdateView(LoginRequiredMixin, UpdateView): +class ObservationRecordUpdateView(LoginRequiredMixin, UpdateView): """ This view will eventually allow updating solely the observation id and status, and possibly the parameters """ model = ObservationRecord - fields = ['observation_id', 'status', 'scheduled_start', 'scheduled_end'] + fields = ['observation_id'] template_name = 'tom_observations/observationupdate_form.html' - def get_form(self): - facility_class = get_service_class(self.object.facility)() - return facility_class.get_update_form(None) + def get_success_url(self): + return reverse('tom_observations:detail', kwargs={'pk': self.get_object().id}) class ObservationGroupCancelView(LoginRequiredMixin, View): @@ -428,6 +425,7 @@ def get_context_data(self, *args, **kwargs): context = super().get_context_data(*args, **kwargs) context['form'] = AddProductToGroupForm() service_class = get_service_class(self.object.facility) + context['editable'] = isinstance(service_class(), BaseManualObservationFacility) context['data_products'] = service_class().all_data_products(self.object) context['can_be_cancelled'] = self.object.status not in service_class().get_terminal_observing_states() newest_image = None From 413c8463b496252a6b463424f84db5c1026e0b7f Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 27 Apr 2020 17:46:47 -0700 Subject: [PATCH 070/424] Updated docstrings for base classes --- tom_observations/facility.py | 56 ++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/tom_observations/facility.py b/tom_observations/facility.py index 5a5570da8..ea8a58989 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -61,12 +61,9 @@ def get_service_class(name): class BaseObservationForm(forms.Form): """ This is the class that is responsible for displaying the observation request form. - Facility classes that provide a form should subclass this form. It provides - some base shared functionality. Extra fields are provided below. - The layout is handled by Django crispy forms which allows customizability of the - form layout without needing to write html templates: - https://django-crispy-forms.readthedocs.io/en/d-0/layouts.html - See the documentation on Django forms for more information. + This form is meant to be subclassed by more specific BaseForm classes that represent a + form for a particular type of facility. For implementing your own form, please look to + the other BaseObservationForms. For an implementation example please see https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/lco.py#L132 @@ -125,6 +122,20 @@ def observation_payload(self): class BaseRoboticObservationForm(BaseObservationForm): + """ + This is the class that is responsible for displaying the observation request form. + Facility classes that provide a form should subclass this form. It provides + some base shared functionality. Extra fields are provided below. + The layout is handled by Django crispy forms which allows customizability of the + form layout without needing to write html templates: + https://django-crispy-forms.readthedocs.io/en/d-0/layouts.html + See the documentation on Django forms for more information. + + This specific class is intended for use with robotic facilities, such as LCO, Gemini, and SOAR. + + For an implementation example please see + https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/lco.py#L132 + """ pass @@ -133,6 +144,20 @@ class BaseRoboticObservationForm(BaseObservationForm): class BaseManualObservationForm(BaseObservationForm): + """ + This is the class that is responsible for displaying the observation request form. + Facility classes that provide a form should subclass this form. It provides + some base shared functionality. Extra fields are provided below. + The layout is handled by Django crispy forms which allows customizability of the + form layout without needing to write html templates: + https://django-crispy-forms.readthedocs.io/en/d-0/layouts.html + See the documentation on Django forms for more information. + + This specific class is intended for use with classical-style manual facilities. + + For an implementation example please see + https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/lco.py#L132 + """ name = forms.CharField() start = forms.CharField(widget=forms.TextInput(attrs={'type': 'date'})) end = forms.CharField(required=False, widget=forms.TextInput(attrs={'type': 'date'})) @@ -152,6 +177,12 @@ def layout(self): class BaseObservationFacility(ABC): + """ + This is the class that is responsible for defining the base facility class. + This form is meant to be subclassed by more specific BaseFacility classes that represent a + form for a particular type of facility. For implementing your own form, please look to + the other BaseObservationFacilities. + """ name = 'BaseObservation' def all_data_products(self, observation_record): @@ -266,6 +297,8 @@ class BaseRoboticObservationFacility(BaseObservationFacility): In order to make use of a facility class, add the path to ``TOM_FACILITY_CLASSES`` in your ``settings.py``. + This specific class is intended for use with robotic facilities, such as LCO, Gemini, and SOAR. + For an implementation example, please see https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/lco.py """ @@ -346,5 +379,16 @@ def data_products(self, observation_id, product_id=None): class BaseManualObservationFacility(BaseObservationFacility): """ + The facility class contains all the logic specific to the facility it is + written for. Some methods are used only internally (starting with an + underscore) but some need to be implemented by all facility classes. + All facilities should inherit from this class which + provides some base functionality. + In order to make use of a facility class, add the path to + ``TOM_FACILITY_CLASSES`` in your ``settings.py``. + + This specific class is intended for use with classical-style manual facilities. + + TODO: Add an implementation example. """ name = 'BaseManual' # rename in concrete subclasses From 3787950c1ef590e625c497f33432d2631308a995 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 28 Apr 2020 02:29:24 +0000 Subject: [PATCH 071/424] rewrite test_get_observation_form to pass 'fake form input' is the help_text of the single field of the FakeFacilityForm but does not appear in the response.content and so assertContains fails. 'FakeFacility' serves a similar purpose as 'fake form input'. --- tom_observations/tests/tests.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tom_observations/tests/tests.py b/tom_observations/tests/tests.py index 7bbd0b826..efd1f06a5 100644 --- a/tom_observations/tests/tests.py +++ b/tom_observations/tests/tests.py @@ -69,13 +69,10 @@ def test_update_observations(self): self.assertContains(response, 'COMPLETED') def test_get_observation_form(self): - response = self.client.get( - '{}?target_id={}'.format( - reverse('tom_observations:create', kwargs={'facility': 'FakeFacility'}), - self.target.id - ) - ) - self.assertContains(response, 'fake form input') + url = f"{reverse('tom_observations:create', kwargs={'facility': 'FakeFacility'})}?target_id={self.target.id}" + response = self.client.get(url) + # self.assertContains(response, 'fake form input') + self.assertContains(response, 'FakeFacility') def test_add_observations_to_group(self): obs_group = ObservationGroup.objects.create(name='testgroup') From f2a2215a515eebf037d38d0d0208c447711c50ef Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 28 Apr 2020 03:14:59 +0000 Subject: [PATCH 072/424] rename FakeFacility to FakeRoboticFacility (nfc) --- tom_dataproducts/tests/tests.py | 26 +++++++++---------- tom_observations/tests/tests.py | 44 ++++++++++++++++----------------- tom_observations/tests/utils.py | 8 +++--- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index 0248d45af..f5ae0bdd5 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -15,7 +15,7 @@ from astropy.table import Table import numpy as np -from tom_observations.tests.utils import FakeFacility +from tom_observations.tests.utils import FakeRoboticFacility from tom_observations.tests.factories import TargetFactory, ObservingRecordFactory from tom_dataproducts.models import DataProduct, is_fits_image_file from tom_dataproducts.forms import DataProductUploadForm @@ -39,14 +39,14 @@ def mock_is_fits_image_file(filename): return True -@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeFacility'], TARGET_PERMISSIONS_ONLY=True) +@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=True) @patch('tom_dataproducts.models.DataProduct.get_preview', return_value='/no-image.jpg') class Views(TestCase): def setUp(self): self.target = TargetFactory.create() self.observation_record = ObservingRecordFactory.create( target_id=self.target.id, - facility=FakeFacility.name, + facility=FakeRoboticFacility.name, parameters='{}' ) self.data_product = DataProduct.objects.create( @@ -73,10 +73,10 @@ def test_get_dataproducts(self, dp_mock): def test_save_dataproduct(self, dp_mock): mock_return = [DataProduct(product_id='testdpid', data=SimpleUploadedFile('afile.fits', b'afile'))] - with patch.object(FakeFacility, 'save_data_products', return_value=mock_return) as mock: + with patch.object(FakeRoboticFacility, 'save_data_products', return_value=mock_return) as mock: response = self.client.post( reverse('dataproducts:save', kwargs={'pk': self.observation_record.id}), - data={'facility': 'FakeFacility', 'products': ['testdpid']}, + data={'facility': 'FakeRoboticFacility', 'products': ['testdpid']}, follow=True ) self.assertTrue(mock.called) @@ -146,14 +146,14 @@ def test_create_jpeg(self, dp_mock): self.assertEqual(products.count(), 1) -@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeFacility'], TARGET_PERMISSIONS_ONLY=False) +@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=False) @patch('tom_dataproducts.models.DataProduct.get_preview', return_value='/no-image.jpg') class TestViewsWithPermissions(TestCase): def setUp(self): self.target = TargetFactory.create() self.observation_record = ObservingRecordFactory.create( target_id=self.target.id, - facility=FakeFacility.name, + facility=FakeRoboticFacility.name, parameters='{}' ) self.data_product = DataProduct.objects.create( @@ -188,7 +188,7 @@ def test_dataproduct_list_unauthorized(self, dp_mock): response = self.client.get(reverse('tom_dataproducts:list')) self.assertNotContains(response, 'afile.fits') - @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeFacility'], + @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=False) def test_upload_data_extended_permissions(self, dp_mock): group = Group.objects.create(name='permitted') @@ -213,14 +213,14 @@ def test_upload_data_extended_permissions(self, dp_mock): self.assertNotContains(response, 'afile.fits') -@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeFacility'], TARGET_PERMISSIONS_ONLY=True) +@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=True) @patch('tom_dataproducts.views.run_data_processor') class TestUploadDataProducts(TestCase): def setUp(self): self.target = TargetFactory.create() self.observation_record = ObservingRecordFactory.create( target_id=self.target.id, - facility=FakeFacility.name, + facility=FakeRoboticFacility.name, parameters='{}' ) self.data_product = DataProduct.objects.create( @@ -264,7 +264,7 @@ def test_upload_data_for_observation(self, run_data_processor_mock): follow=True ) self.assertContains(response, 'Successfully uploaded: {0}/{1}/bfile.fits'.format( - self.target.name, FakeFacility.name) + self.target.name, FakeRoboticFacility.name) ) @@ -309,7 +309,7 @@ def setUp(self): self.target = TargetFactory.create() self.observation_record = ObservingRecordFactory.create( target_id=self.target.id, - facility=FakeFacility.name, + facility=FakeRoboticFacility.name, parameters='{}' ) self.spectroscopy_form_data = { @@ -377,7 +377,7 @@ def test_deserialize_spectrum_invalid(self): self.serializer.deserialize(json.dumps({'invalid_key': 'value'})) -@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeFacility']) +@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility']) class TestDataProcessor(TestCase): def setUp(self): self.target = TargetFactory.create() diff --git a/tom_observations/tests/tests.py b/tom_observations/tests/tests.py index efd1f06a5..fcc3c48ec 100644 --- a/tom_observations/tests/tests.py +++ b/tom_observations/tests/tests.py @@ -11,20 +11,20 @@ from .factories import ObservingRecordFactory, ObservingStrategyFactory, TargetFactory, TargetNameFactory from tom_observations.utils import get_astroplan_sun_and_time, get_sidereal_visibility -from tom_observations.tests.utils import FakeFacility +from tom_observations.tests.utils import FakeRoboticFacility from tom_observations.models import ObservationRecord, ObservationGroup, ObservingStrategy from tom_targets.models import Target from guardian.shortcuts import assign_perm -@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeFacility'], TARGET_PERMISSIONS_ONLY=True) +@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=True) class TestObservationViews(TestCase): def setUp(self): self.target = TargetFactory.create() self.target_name = TargetNameFactory.create(target=self.target) self.observation_record = ObservingRecordFactory.create( target_id=self.target.id, - facility=FakeFacility.name, + facility=FakeRoboticFacility.name, parameters='{}' ) user = User.objects.create_user(username='vincent_adultman', password='important') @@ -53,7 +53,7 @@ def test_observation_detail(self): ) self.assertEqual(response.status_code, 200) self.assertContains( - response, FakeFacility().get_observation_url(self.observation_record.observation_id) + response, FakeRoboticFacility().get_observation_url(self.observation_record.observation_id) ) def test_observation_detail_unauthorized(self): @@ -69,10 +69,10 @@ def test_update_observations(self): self.assertContains(response, 'COMPLETED') def test_get_observation_form(self): - url = f"{reverse('tom_observations:create', kwargs={'facility': 'FakeFacility'})}?target_id={self.target.id}" + url = f"{reverse('tom_observations:create', kwargs={'facility': 'FakeRoboticFacility'})}?target_id={self.target.id}" response = self.client.get(url) # self.assertContains(response, 'fake form input') - self.assertContains(response, 'FakeFacility') + self.assertContains(response, 'FakeRoboticFacility') def test_add_observations_to_group(self): obs_group = ObservationGroup.objects.create(name='testgroup') @@ -103,11 +103,11 @@ def test_submit_observation(self): form_data = { 'target_id': self.target.id, 'test_input': 'gnomes', - 'facility': 'FakeFacility', + 'facility': 'FakeRoboticFacility', } self.client.post( '{}?target_id={}'.format( - reverse('tom_observations:create', kwargs={'facility': 'FakeFacility'}), + reverse('tom_observations:create', kwargs={'facility': 'FakeRoboticFacility'}), self.target.id ), data=form_data, @@ -116,14 +116,14 @@ def test_submit_observation(self): self.assertTrue(ObservationRecord.objects.filter(observation_id='fakeid').exists()) -@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeFacility'], TARGET_PERMISSIONS_ONLY=False) +@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=False) class TestObservationViewsRowLevelPermissions(TestCase): def setUp(self): self.target = TargetFactory.create() self.target_name = TargetNameFactory.create(target=self.target) self.observation_record = ObservingRecordFactory.create( target_id=self.target.id, - facility=FakeFacility.name, + facility=FakeRoboticFacility.name, parameters='{}' ) user = User.objects.create_user(username='vincent_adultman', password='important') @@ -154,7 +154,7 @@ def test_observation_detail(self): ) self.assertEqual(response.status_code, 200) self.assertContains( - response, FakeFacility().get_observation_url(self.observation_record.observation_id) + response, FakeRoboticFacility().get_observation_url(self.observation_record.observation_id) ) def test_observation_detail_unauthorized(self): @@ -165,12 +165,12 @@ def test_observation_detail_unauthorized(self): self.assertEqual(response.status_code, 404) -@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeFacility']) +@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility']) class TestObservationGroupViews(TestCase): pass -@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeFacility']) +@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility']) class TestObservingStrategyViews(TestCase): def setUp(self): self.observing_strategy = ObservingStrategyFactory.create(name='Test Strategy') @@ -185,7 +185,7 @@ def test_observing_strategy_list(self): ) def test_observing_strategy_create(self): - response = self.client.get(reverse('tom_observations:strategy-create', kwargs={'facility': 'FakeFacility'})) + response = self.client.get(reverse('tom_observations:strategy-create', kwargs={'facility': 'FakeRoboticFacility'})) self.assertContains(response, 'Strategy name') def test_observing_strategy_delete(self): @@ -199,23 +199,23 @@ def test_observing_strategy_delete(self): class TestUpdatingObservations(TestCase): def setUp(self): self.t1 = TargetFactory.create() - self.or1 = ObservingRecordFactory.create(target_id=self.t1.id, facility='FakeFacility', status='PENDING') + self.or1 = ObservingRecordFactory.create(target_id=self.t1.id, facility='FakeRoboticFacility', status='PENDING') self.or2 = ObservingRecordFactory.create(target_id=self.t1.id, status='COMPLETED') - self.or3 = ObservingRecordFactory.create(target_id=self.t1.id, facility='FakeFacility', status='PENDING') + self.or3 = ObservingRecordFactory.create(target_id=self.t1.id, facility='FakeRoboticFacility', status='PENDING') self.t2 = TargetFactory.create() self.or4 = ObservingRecordFactory.create(target_id=self.t2.id, status='PENDING') # Tests that only 2 of the three created observing records are updated, as # the third is in a completed state def test_update_all_observations_for_facility(self): - with mock.patch.object(FakeFacility, 'update_observation_status') as uos_mock: - FakeFacility().update_all_observation_statuses() + with mock.patch.object(FakeRoboticFacility, 'update_observation_status') as uos_mock: + FakeRoboticFacility().update_all_observation_statuses() self.assertEquals(uos_mock.call_count, 2) # Tests that only the observing records associated with the given target are updated def test_update_individual_target_observations_for_facility(self): - with mock.patch.object(FakeFacility, 'update_observation_status', return_value='COMPLETED') as uos_mock: - FakeFacility().update_all_observation_statuses(target=self.t1) + with mock.patch.object(FakeRoboticFacility, 'update_observation_status', return_value='COMPLETED') as uos_mock: + FakeRoboticFacility().update_all_observation_statuses(target=self.t1) self.assertEquals(uos_mock.call_count, 2) @@ -270,10 +270,10 @@ def test_get_visibility_invalid_params(self): @mock.patch('tom_observations.utils.facility.get_service_classes') def test_get_visibility_sidereal(self, mock_facility): - mock_facility.return_value = {'Fake Facility': FakeFacility} + mock_facility.return_value = {'Fake Robotic Facility': FakeRoboticFacility} end = self.start + timedelta(minutes=60) airmass = get_sidereal_visibility(self.target, self.start, end, self.interval, self.airmass_limit) - airmass_data = airmass['(Fake Facility) Siding Spring'][1] + airmass_data = airmass['(Fake Robotic Facility) Siding Spring'][1] expected_airmass = [ 1.2619096566629477, 1.2648181328558852, 1.2703522349950636, 1.2785703053923894, 1.2895601364316183, 1.3034413026227516, 1.3203684217446099 diff --git a/tom_observations/tests/utils.py b/tom_observations/tests/utils.py index 5109915d2..63c66adaa 100644 --- a/tom_observations/tests/utils.py +++ b/tom_observations/tests/utils.py @@ -3,7 +3,7 @@ from django.utils import timezone from astropy import units -from tom_observations.facility import GenericObservationFacility, GenericObservationForm +from tom_observations.facility import BaseRoboticObservationFacility, GenericObservationForm from tom_observations.observing_strategy import GenericStrategyForm # Site data matches built-in pyephem observer data for Los Angeles @@ -30,9 +30,9 @@ class FakeFacilityStrategyForm(GenericStrategyForm): pass -class FakeFacility(GenericObservationFacility): - name = 'FakeFacility' - observation_types = [('FakeFacility Observation', 'OBSERVATION')] +class FakeRoboticFacility(BaseRoboticObservationFacility): + name = 'FakeRoboticFacility' + observation_types = [('FakeRoboticFacility Observation', 'OBSERVATION')] def get_form(self, observation_type): return FakeFacilityForm From f08761f1ae489da124969354d6df518a5263ecfb Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 28 Apr 2020 03:34:12 +0000 Subject: [PATCH 073/424] name method signiture match parent class method --- tom_observations/tests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_observations/tests/utils.py b/tom_observations/tests/utils.py index 63c66adaa..362dba9b2 100644 --- a/tom_observations/tests/utils.py +++ b/tom_observations/tests/utils.py @@ -46,7 +46,7 @@ def get_observing_sites(self): def get_observation_url(self, observation_id): return '' - def data_products(self, observation_record): + def data_products(self, observation_id, product_id=None): return [{'id': 'testdpid'}] def get_observation_status(self, observation_id): From aa216a1a80e27dbc9117d9233f97aa721c533b3f Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 28 Apr 2020 04:13:33 +0000 Subject: [PATCH 074/424] add FakeManualFacility and test --- tom_observations/tests/tests.py | 15 ++++++++-- tom_observations/tests/utils.py | 52 +++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/tom_observations/tests/tests.py b/tom_observations/tests/tests.py index fcc3c48ec..82fd180b4 100644 --- a/tom_observations/tests/tests.py +++ b/tom_observations/tests/tests.py @@ -17,7 +17,8 @@ from guardian.shortcuts import assign_perm -@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=True) +@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility', + 'tom_observations.tests.utils.FakeManualFacility'], TARGET_PERMISSIONS_ONLY=True) class TestObservationViews(TestCase): def setUp(self): self.target = TargetFactory.create() @@ -99,7 +100,7 @@ def test_remove_observations_from_group(self): obs_group.refresh_from_db() self.assertNotIn(self.observation_record, obs_group.observation_records.all()) - def test_submit_observation(self): + def test_submit_observation_robotic(self): form_data = { 'target_id': self.target.id, 'test_input': 'gnomes', @@ -115,6 +116,16 @@ def test_submit_observation(self): ) self.assertTrue(ObservationRecord.objects.filter(observation_id='fakeid').exists()) + def test_submit_observation_manual(self): + form_data = { + 'target_id': self.target.id, + 'test_input': 'elves', + 'facility': 'FakeManualFacility', + } + url = f"{reverse('tom_observations:create', kwargs={'facility': 'FakeManualFacility'})}?target_id={self.target.id}" + self.client.post(url, data=form_data, follow=True) + self.assertTrue(ObservationRecord.objects.filter(observation_id='fakeid').exists()) + @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=False) class TestObservationViewsRowLevelPermissions(TestCase): diff --git a/tom_observations/tests/utils.py b/tom_observations/tests/utils.py index 362dba9b2..5c6ab48a0 100644 --- a/tom_observations/tests/utils.py +++ b/tom_observations/tests/utils.py @@ -4,6 +4,7 @@ from astropy import units from tom_observations.facility import BaseRoboticObservationFacility, GenericObservationForm +from tom_observations.facility import BaseManualObservationFacility from tom_observations.observing_strategy import GenericStrategyForm # Site data matches built-in pyephem observer data for Los Angeles @@ -70,3 +71,54 @@ def get_wavelength_units(self): def validate_observation(self, observation_payload): return True + + +class FakeManualFacility(BaseManualObservationFacility): + name = 'FakeManualFacility' + observation_types = [('FakeManualFacility Observation', 'OBSERVATION')] + + def get_form(self, observation_type): + return FakeFacilityForm + + def get_strategy_form(self, observation_type): + return FakeFacilityStrategyForm + + def get_observing_sites(self): + return SITES + + def get_observation_url(self, observation_id): + return '' + + def data_products(self, observation_id, product_id=None): + return [{'id': 'testdpid'}] + + def get_observation_status(self, observation_id): + return { + 'state': 'COMPLETED', + 'scheduled_start': timezone.now() + timedelta(hours=1), + 'scheduled_end': timezone.now() + timedelta(hours=2) + } + + def get_terminal_observing_states(self): + return ['COMPLETED', 'FAILED', 'CANCELED', 'WINDOW_EXPIRED'] + + def submit_observation(self, payload): + return ['fakeid'] + + def get_flux_constant(self): + return units.erg / units.angstrom + + def get_wavelength_units(self): + return units.angstrom + + def validate_observation(self, observation_payload): + return True + + # TOOD: this method does not belong to this Subclass of BaseObservationFacility + # it's only here to satisfy tests.test_update_observations() which makes a (now) + # invalid assumption that all facilities are robotic and have this method + # The underlying problem is that when an ObservationRecord gets it's facility + # class, it assumes that it's a BaseRoboticFacility subclass. + def update_all_observation_statuses(self, target): + return [] + From 90699707860337b89cea53d98b48a912869b69f9 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 28 Apr 2020 09:18:37 -0700 Subject: [PATCH 075/424] Updated documentation and renamed a view --- tom_observations/facilities/gemini.py | 7 ------- tom_observations/forms.py | 9 +++++++-- tom_observations/templatetags/observation_extras.py | 9 +++++++++ tom_observations/urls.py | 4 ++-- tom_observations/views.py | 8 ++++---- 5 files changed, 22 insertions(+), 15 deletions(-) diff --git a/tom_observations/facilities/gemini.py b/tom_observations/facilities/gemini.py index 506552e92..fe776841d 100644 --- a/tom_observations/facilities/gemini.py +++ b/tom_observations/facilities/gemini.py @@ -253,13 +253,6 @@ class GEMObservationForm(BaseRoboticObservationForm): label='UT Timing Window Start [Date Time]') window_duration = forms.IntegerField(required=False, min_value=1, label='Timing Window Duration [hr]') - # def __init__(self, *args, **kwargs): - # super().__init__(*args, **kwargs) - # self.helper.layout = Layout( - # self.common_layout,, - # self.button_layout() - # ) - def layout(self): return Div( HTML('Observation Parameters'), diff --git a/tom_observations/forms.py b/tom_observations/forms.py index fc24825c7..db1e434cc 100644 --- a/tom_observations/forms.py +++ b/tom_observations/forms.py @@ -11,7 +11,9 @@ def facility_choices(): class AddExistingObservationForm(forms.Form): - "" + """ + This form is used for adding existing API-based observations to a Target object. + """ target_id = forms.IntegerField(required=True, widget=forms.HiddenInput()) facility = forms.ChoiceField(required=True, choices=facility_choices, label=False) observation_id = forms.CharField(required=True, label=False, @@ -21,7 +23,7 @@ class AddExistingObservationForm(forms.Form): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.helper = FormHelper() - self.helper.form_action = reverse('tom_observations:manual') + self.helper.form_action = reverse('tom_observations:add-existing') self.helper.layout = Layout( 'target_id', 'confirm', @@ -42,6 +44,9 @@ def __init__(self, *args, **kwargs): class UpdateObservationId(forms.Form): + """ + This form is used for updating the observation ID on an ObservationRecord object. + """ obsr_id = forms.IntegerField(required=True, widget=forms.HiddenInput()) observation_id = forms.CharField(required=True, label=False, widget=forms.TextInput(attrs={'placeholder': 'Observation ID'})) diff --git a/tom_observations/templatetags/observation_extras.py b/tom_observations/templatetags/observation_extras.py index 75dad8903..f3bbe5013 100644 --- a/tom_observations/templatetags/observation_extras.py +++ b/tom_observations/templatetags/observation_extras.py @@ -30,11 +30,17 @@ def observing_buttons(target): @register.inclusion_tag('tom_observations/partials/existing_observation_form.html') def existing_observation_form(target): + """ + Renders a form for adding an existing API-based observation to a Target. + """ return {'form': AddExistingObservationForm(initial={'target_id': target.id})} @register.inclusion_tag('tom_observations/partials/update_observation_id_form.html') def update_observation_id_form(obsr): + """ + Renders a form for updating the observation ID for an ObservationRecord. + """ return {'form': UpdateObservationId(initial={'obsr_id': obsr.id, 'observation_id': obsr.observation_id})} @@ -57,6 +63,9 @@ def observation_type_tabs(context): @register.inclusion_tag('tom_observations/partials/facility_observation_form.html') def facility_observation_form(target, facility, observation_type): + """ + Displays a form for submitting an observation for a specific facility and observation type, e.g., imaging. + """ facility_class = get_service_class(facility)() initial_fields = { 'target_id': target.id, diff --git a/tom_observations/urls.py b/tom_observations/urls.py index d36b61eba..21d43afb1 100644 --- a/tom_observations/urls.py +++ b/tom_observations/urls.py @@ -1,6 +1,6 @@ from django.urls import path -from tom_observations.views import (ManualObservationCreateView, ObservationCreateView, ObservationRecordUpdateView, +from tom_observations.views import (AddExistingObservationView, ObservationCreateView, ObservationRecordUpdateView, ObservationGroupDeleteView, ObservationGroupListView, ObservationListView, ObservationRecordDetailView, ObservingStrategyCreateView, ObservingStrategyDeleteView, ObservingStrategyListView, @@ -9,7 +9,7 @@ app_name = 'tom_observations' urlpatterns = [ - path('manual/', ManualObservationCreateView.as_view(), name='manual'), + path('add/', AddExistingObservationView.as_view(), name='add-existing'), path('list/', ObservationListView.as_view(), name='list'), path('strategy/list/', ObservingStrategyListView.as_view(), name='strategy-list'), path('strategy//create/', ObservingStrategyCreateView.as_view(), name='strategy-create'), diff --git a/tom_observations/views.py b/tom_observations/views.py index ebb4e91e1..2e60f1d93 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -281,7 +281,7 @@ def form_valid(self, form): class ObservationRecordUpdateView(LoginRequiredMixin, UpdateView): """ - This view will eventually allow updating solely the observation id and status, and possibly the parameters + This view allows for the updating of the observation id, which will eventually be expanded to more fields. """ model = ObservationRecord fields = ['observation_id'] @@ -314,7 +314,7 @@ def get(self, request, *args, **kwargs): return redirect(referer) -class ManualObservationCreateView(LoginRequiredMixin, FormView): +class AddExistingObservationView(LoginRequiredMixin, FormView): """ View for associating a pre-existing observation with a target. Requires authentication. @@ -324,7 +324,7 @@ class ManualObservationCreateView(LoginRequiredMixin, FormView): The POST view validates the form and redirects to the confirmation page if the confirm flag isn't set. This view is intended to be navigated to via the existing_observation_button templatetag, as the - ConfirmExistingObservationForm has a hidden confirmation checkbox selected by default. + AddExistingObservationForm has a hidden confirmation checkbox selected by default. """ template_name = 'tom_observations/existing_observation_confirm.html' form_class = AddExistingObservationForm @@ -376,7 +376,7 @@ def form_valid(self, form): observation_id=form.cleaned_data['observation_id']) if records and not form.cleaned_data.get('confirm'): - return redirect(reverse('tom_observations:manual') + '?' + self.request.POST.urlencode()) + return redirect(reverse('tom_observations:add-existing') + '?' + self.request.POST.urlencode()) else: ObservationRecord.objects.create( target_id=form.cleaned_data['target_id'], From 5e6cdb1d8501588de408bff76dd46c9d05b409c1 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 28 Apr 2020 16:48:16 +0000 Subject: [PATCH 076/424] refactor: pycodestyle corrections (nfc) --- tom_dataproducts/tests/tests.py | 9 ++++++--- tom_observations/facility.py | 4 ++-- tom_observations/tests/tests.py | 15 ++++++++++----- tom_observations/tests/utils.py | 1 - 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index f5ae0bdd5..1df903366 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -39,7 +39,8 @@ def mock_is_fits_image_file(filename): return True -@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=True) +@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], + TARGET_PERMISSIONS_ONLY=True) @patch('tom_dataproducts.models.DataProduct.get_preview', return_value='/no-image.jpg') class Views(TestCase): def setUp(self): @@ -146,7 +147,8 @@ def test_create_jpeg(self, dp_mock): self.assertEqual(products.count(), 1) -@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=False) +@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], + TARGET_PERMISSIONS_ONLY=False) @patch('tom_dataproducts.models.DataProduct.get_preview', return_value='/no-image.jpg') class TestViewsWithPermissions(TestCase): def setUp(self): @@ -213,7 +215,8 @@ def test_upload_data_extended_permissions(self, dp_mock): self.assertNotContains(response, 'afile.fits') -@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=True) +@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], + TARGET_PERMISSIONS_ONLY=True) @patch('tom_dataproducts.views.run_data_processor') class TestUploadDataProducts(TestCase): def setUp(self): diff --git a/tom_observations/facility.py b/tom_observations/facility.py index ea8a58989..8a9ee4884 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -61,8 +61,8 @@ def get_service_class(name): class BaseObservationForm(forms.Form): """ This is the class that is responsible for displaying the observation request form. - This form is meant to be subclassed by more specific BaseForm classes that represent a - form for a particular type of facility. For implementing your own form, please look to + This form is meant to be subclassed by more specific BaseForm classes that represent a + form for a particular type of facility. For implementing your own form, please look to the other BaseObservationForms. For an implementation example please see diff --git a/tom_observations/tests/tests.py b/tom_observations/tests/tests.py index 82fd180b4..183c420f4 100644 --- a/tom_observations/tests/tests.py +++ b/tom_observations/tests/tests.py @@ -18,7 +18,8 @@ @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility', - 'tom_observations.tests.utils.FakeManualFacility'], TARGET_PERMISSIONS_ONLY=True) + 'tom_observations.tests.utils.FakeManualFacility'], + TARGET_PERMISSIONS_ONLY=True) class TestObservationViews(TestCase): def setUp(self): self.target = TargetFactory.create() @@ -70,7 +71,8 @@ def test_update_observations(self): self.assertContains(response, 'COMPLETED') def test_get_observation_form(self): - url = f"{reverse('tom_observations:create', kwargs={'facility': 'FakeRoboticFacility'})}?target_id={self.target.id}" + url = f"{reverse('tom_observations:create', kwargs={'facility': 'FakeRoboticFacility'})}" \ + f"?target_id={self.target.id}" response = self.client.get(url) # self.assertContains(response, 'fake form input') self.assertContains(response, 'FakeRoboticFacility') @@ -122,12 +124,14 @@ def test_submit_observation_manual(self): 'test_input': 'elves', 'facility': 'FakeManualFacility', } - url = f"{reverse('tom_observations:create', kwargs={'facility': 'FakeManualFacility'})}?target_id={self.target.id}" + url = f"{reverse('tom_observations:create', kwargs={'facility': 'FakeManualFacility'})}" \ + f"?target_id={self.target.id}" self.client.post(url, data=form_data, follow=True) self.assertTrue(ObservationRecord.objects.filter(observation_id='fakeid').exists()) -@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=False) +@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], + TARGET_PERMISSIONS_ONLY=False) class TestObservationViewsRowLevelPermissions(TestCase): def setUp(self): self.target = TargetFactory.create() @@ -196,7 +200,8 @@ def test_observing_strategy_list(self): ) def test_observing_strategy_create(self): - response = self.client.get(reverse('tom_observations:strategy-create', kwargs={'facility': 'FakeRoboticFacility'})) + response = self.client.get(reverse('tom_observations:strategy-create', + kwargs={'facility': 'FakeRoboticFacility'})) self.assertContains(response, 'Strategy name') def test_observing_strategy_delete(self): diff --git a/tom_observations/tests/utils.py b/tom_observations/tests/utils.py index 5c6ab48a0..93c1d7cb5 100644 --- a/tom_observations/tests/utils.py +++ b/tom_observations/tests/utils.py @@ -121,4 +121,3 @@ def validate_observation(self, observation_payload): # class, it assumes that it's a BaseRoboticFacility subclass. def update_all_observation_statuses(self, target): return [] - From 0b71180a09ba7099bbcbaf43fd24698f893349f6 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 28 Apr 2020 19:47:43 +0000 Subject: [PATCH 077/424] remove Example manual facility; add SOAR --- tom_base/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index 4e5174c54..6acb0b174 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -211,7 +211,7 @@ TOM_FACILITY_CLASSES = [ 'tom_observations.facilities.lco.LCOFacility', 'tom_observations.facilities.gemini.GEMFacility', - 'tom_observations.facilities.manual.ExampleManualFacility', + 'tom_observations.facilities.soar.SOARFacility', ] TOM_CADENCE_STRATEGIES = [ From be725ca791bd952e8cb94316ecad1a06a5654c26 Mon Sep 17 00:00:00 2001 From: fraserw Date: Tue, 28 Apr 2020 14:13:40 -0700 Subject: [PATCH 078/424] --code style cleanup --- tom_observations/facilities/lco.py | 185 +++++++++++++++-------------- tom_targets/models.py | 2 +- tom_targets/utils.py | 56 +++++---- 3 files changed, 123 insertions(+), 120 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 5d48f785a..5a46d18e5 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -11,9 +11,14 @@ from tom_common.exceptions import ImproperCredentialsException from tom_observations.cadence import CadenceForm -from tom_observations.facility import GenericObservationFacility, GenericObservationForm, get_service_class +from tom_observations.facility import ( + GenericObservationFacility, GenericObservationForm, get_service_class + ) from tom_observations.observing_strategy import GenericStrategyForm -from tom_targets.models import Target, REQUIRED_NON_SIDEREAL_FIELDS, REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME +from tom_targets.models import ( + Target, REQUIRED_NON_SIDEREAL_FIELDS, + REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME + ) from tom_targets.models import ( Target, REQUIRED_NON_SIDEREAL_FIELDS, REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME @@ -22,9 +27,6 @@ import json import numpy as np -def take_second_element(elem): - return elem[1] - # Determine settings for this module. try: @@ -93,6 +95,10 @@ def take_second_element(elem): """ +def take_second_element(elem): + return elem[1] + + def make_request(*args, **kwargs): response = requests.request(*args, **kwargs) if 400 <= response.status_code < 500: @@ -108,26 +114,31 @@ class LCOBaseForm(forms.Form): max_airmass = forms.FloatField() site = forms.ChoiceField( - choices = (('all', 'All Sites'), - ('coj','Siding Spring'), - ('cpt','Sutherland'), - ('tfn', 'Teide'), - ('tlv', 'Wise'), - ('lsc','Cerro Tololo'), - ('elp', 'McDonald'), - ('ogg', 'Haleakala')) - #widget=forms.CheckboxSelectMultiple() + choices=(('all', 'All Sites'), + ('coj', 'Siding Spring'), + ('cpt', 'Sutherland'), + ('tfn', 'Teide'), + ('tlv', 'Wise'), + ('lsc', 'Cerro Tololo'), + ('elp', 'McDonald'), + ('ogg', 'Haleakala')) ) - imaging_interval = forms.FloatField(label='Interval (hrs). Will schedule exposure count per interval.') - - + imaging_interval = forms.FloatField( + label='Interval (hrs). Will schedule exposure count per interval.' + ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields['proposal'] = forms.ChoiceField(choices=self.proposal_choices()) - self.fields['filter'] = forms.ChoiceField(choices=self.filter_choices()) - self.fields['instrument_type'] = forms.ChoiceField(choices=self.instrument_choices()) + self.fields['proposal'] = forms.ChoiceField( + choices=self.proposal_choices() + ) + self.fields['filter'] = forms.ChoiceField( + choices=self.filter_choices() + ) + self.fields['instrument_type'] = forms.ChoiceField( + choices=self.instrument_choices() + ) self.eph_target = False target = Target.objects.get(pk=kwargs['initial']['target_id']) @@ -155,8 +166,7 @@ def filter_choices(self): return sorted(set([ (f['code'], f['name']) for ins in self._get_instruments().values() for f in ins['optical_elements'].get('filters', []) + ins['optical_elements'].get('slits', []) - ]),key=take_second_element) - + ]), key=take_second_element) def proposal_choices(self): response = make_request( @@ -182,13 +192,17 @@ class LCOBaseObservationForm(GenericObservationForm, LCOBaseForm, CadenceForm): help_text=end_help) exposure_count = forms.IntegerField(min_value=1) exposure_time = forms.FloatField(min_value=0.1, - widget=forms.TextInput(attrs={'placeholder': 'Seconds'}), + widget=forms.TextInput( + attrs={'placeholder': 'Seconds'} + ), help_text=exposure_time_help) max_airmass = forms.FloatField(help_text=max_airmass_help) period = forms.FloatField(required=False) jitter = forms.FloatField(required=False) observation_mode = forms.ChoiceField( - choices=(('NORMAL', 'Normal'), ('TARGET_OF_OPPORTUNITY', 'Rapid Response')), + choices=(('NORMAL', 'Normal'), + ('TARGET_OF_OPPORTUNITY', 'Rapid Response') + ), help_text=observation_mode_help ) @@ -234,7 +248,7 @@ def layout(self): def extra_layout(self): # If you just want to add some fields to the end of the form, add them here. if self.eph_target: - return Div('site','imaging_interval') + return Div('site', 'imaging_interval') return Div() def clean_start(self): @@ -305,56 +319,61 @@ def _build_target_fields(self): eph_json = json.loads(target.eph_json) site_selection = self.cleaned_data['site'] - if site_selection !='all': + if site_selection != 'all': site_selection = [site_selection] else: - site_selection = ['coj','cpt','tfn','tlv','lsc','elp','ogg'] + site_selection = ['coj', + 'cpt', + 'tfn', + 'tlv', + 'lsc', + 'elp', + 'ogg'] for site in site_selection: if site in eph_json.keys(): ephemeris_targets[site] = [] ephemeris_windows[site] = [] - (mjd_vals,ra_vals,dec_vals,air_vals,sun_alt_vals) = get_radec_ephemeris(eph_json[site], - self.cleaned_data['start'], - self.cleaned_data['end'], - self.cleaned_data['imaging_interval'], - 'LCO', - site) + (mjd_vals, ra_vals, dec_vals, air_vals, sun_alt_vals) = get_radec_ephemeris(eph_json[site], + self.cleaned_data['start'], + self.cleaned_data['end'], + self.cleaned_data['imaging_interval'], + 'LCO', + site) if mjd_vals is not None: for i in range(len(ra_vals)-1): - if (air_vals[i]1.0 and - air_vals[i+1]>1.0 and - sun_alt_vals[i]<-30.0 and - sun_alt_vals[i+1]<-30.0): + if (air_vals[i] < float(self.cleaned_data['max_airmass']) and + air_vals[i+1] < float(self.cleaned_data['max_airmass']) and + air_vals[i] > 1.0 and + air_vals[i+1] > 1.0 and + sun_alt_vals[i] < -30.0 and + sun_alt_vals[i+1] < -30.0): new_target_fields = {} new_target_fields['type'] = 'ICRS' new_target_fields['ra'] = (ra_vals[i]+ra_vals[i+1])/2.0 new_target_fields['dec'] = (dec_vals[i]+dec_vals[i+1])/2.0 - new_target_fields['proper_motion_ra'] = 0.0#target.pm_ra - new_target_fields['proper_motion_dec'] = 0.0#target.pm_dec - new_target_fields['epoch'] = 2000#'2000' - new_target_fields['parallax'] = 0#'2000' + new_target_fields['proper_motion_ra'] = 0.0 + new_target_fields['proper_motion_dec'] = 0.0 + new_target_fields['epoch'] = 2000 + new_target_fields['parallax'] = 0 start = Time(mjd_vals[i], format='mjd') end = Time(mjd_vals[i+1], format='mjd') - #print(site,mjd_vals[i],ra_vals[i],dec_vals[i],air_vals[i],start.isot,end.isot,sun_alt_vals[i]) - - #store start and end times in the target for a matter of convenience in passing this information forward to the request builder - new_target_fields['name'] = '{}_{}_{}'.format(target.name,site,i) + # store start and end times in the target for a matter of convenience in passing this information forward to the request builder + new_target_fields['name'] = '{}_{}_{}'.format(target.name, site, i) ephemeris_targets[site].append(new_target_fields) ephemeris_windows[site].append([start.isot, end.isot]) elif mjd_vals is None and sun_alt_vals==-2: - self.add_error(None,'Date range outside range available in the provided ephemeris.') - return (ephemeris_targets,ephemeris_windows) + self.add_error(None, 'Date range outside range available in the provided ephemeris.') + + return (ephemeris_targets, ephemeris_windows) else: target_fields['type'] = 'ORBITAL_ELEMENTS' - # Mapping from TOM field names to LCO API field names, for fields - # where there are differences + # Mapping from TOM field names to LCO API field names, + # for fields where there are differences field_mapping = { 'inclination': 'orbinc', 'lng_asc_node': 'longascnode', @@ -365,8 +384,8 @@ def _build_target_fields(self): 'epoch_of_elements': 'epochofel', 'epoch_of_perihelion': 'epochofperih', } - # The fields to include in the payload depend on the scheme. Add - # only those that are required + # The fields to include in the payload depend on the scheme. + # Add only those that are required fields = (REQUIRED_NON_SIDEREAL_FIELDS + REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME[target.scheme]) for field in fields: @@ -403,7 +422,6 @@ def _build_configuration(self): } } - def _build_ephemeris_request_parts(self): (new_targets, new_windows) = self._build_target_fields() sites = new_targets.keys() @@ -411,8 +429,6 @@ def _build_ephemeris_request_parts(self): windows = [] locations = [] for site in sites: - #if self._valid_site_instrument(site,self.cleaned_data['instrument_type']): - #print(self._valid_site_instrument(site,self.cleaned_data['instrument_type']),'weeeee') for i in range(len(new_targets[site])): single_obs_config = { 'type': self.instrument_to_type(self.cleaned_data['instrument_type']), @@ -429,25 +445,25 @@ def _build_ephemeris_request_parts(self): 'max_airmass': self.cleaned_data['max_airmass'] } } - single_location = {'site': site, - 'telescope_class': self._get_instruments()[self.cleaned_data['instrument_type']]['class']} + single_location = {'site': site, + 'telescope_class': self._get_instruments()[self.cleaned_data['instrument_type']]['class']} single_windows = [{'start': new_windows[site][i][0], - 'end': new_windows[site][i][1]}] - + 'end': new_windows[site][i][1]} + ] configurations.append(single_obs_config) windows.append(single_windows) locations.append(single_location) - return (configurations,windows,locations) + return (configurations, windows, locations) def _build_ephemeris_requests(self): - (configurations,windows,locations) = self._build_ephemeris_request_parts() + (configurations, windows, locations) = self._build_ephemeris_request_parts() requests = [] for i in range(len(configurations)): - req = {'configurations': [configurations[i]], - 'location': locations[i], - 'windows': windows[i]} - requests.append(req) + req = {'configurations': [configurations[i]], + 'location': locations[i], + 'windows': windows[i]} + requests.append(req) return requests def _expand_cadence_request(self, payload): @@ -481,35 +497,27 @@ def observation_payload(self): { "start": self.cleaned_data['start'], "end": self.cleaned_data['end'] - } - ] - } - ] + }]}] } if self.cleaned_data.get('period') and self.cleaned_data.get('jitter'): payload = self._expand_cadence_request(payload) return payload - - - - - - - else: #ephemeris scheme payload creation - #this is inefficient as the request validation is done to check for site+scope - #configuration errors, and then is done again later to check for other errors. + else: + # ephemeris scheme payload creation + # this is inefficient as the request validation is done to check for site+scope + # configuration errors, and then is done again later to check for other errors. # - #This could be used to estiamte airmass windows instead of using astropy as - #is done in tom_base/utils.py. + # This could be used to estiamte airmass windows instead of using astropy as + # is done in tom_base/utils.py. obs_module = get_service_class(self.cleaned_data['facility']) requests = self._build_ephemeris_requests() locations = [] for j in range(len(requests)): if requests[j]['location'] not in locations: locations.append(requests[j]['location']) - if len(locations)>1: + if len(locations) > 1: operator = "MANY" else: operator = "SINGLE" @@ -523,13 +531,10 @@ def observation_payload(self): "requests": requests }) - print(requests,len(requests)) - print(errors) - print() - if len(errors)>0: + if len(errors) > 0: valid_requests = [] - for i,e in enumerate(errors['requests']): - if e!={}: + for i, e in enumerate(errors['requests']): + if e != {}: if 'non_field_errors' not in e: valid_requests.append(requests[i]) else: @@ -556,7 +561,7 @@ def filter_choices(self): return sorted(set([ (f['code'], f['name']) for ins in self._get_instruments().values() for f in ins['optical_elements'].get('filters', []) - ]),key=take_second_element) + ]), key=take_second_element) class LCOSpectroscopyObservationForm(LCOBaseObservationForm): @@ -598,7 +603,7 @@ def filter_choices(self): return sorted(set([ (f['code'], f['name']) for ins in self._get_instruments().values() for f in ins['optical_elements'].get('slits', []) - ] + [('None', 'None')]),key=take_second_element) + ] + [('None', 'None')]), key=take_second_element) def _build_instrument_config(self): instrument_config = super()._build_instrument_config() diff --git a/tom_targets/models.py b/tom_targets/models.py index 45b482312..d6203564f 100644 --- a/tom_targets/models.py +++ b/tom_targets/models.py @@ -30,7 +30,7 @@ 'MPC_COMET': ['perihdist', 'epoch_of_perihelion', 'inclination', 'lng_asc_node', 'arg_of_perihelion', 'eccentricity'], 'MPC_MINOR_PLANET': ['mean_anomaly', 'semimajor_axis', 'inclination', 'lng_asc_node', 'arg_of_perihelion', 'eccentricity'], 'JPL_MAJOR_PLANET': ['mean_daily_motion', 'mean_anomaly', 'semimajor_axis', 'inclination', 'lng_asc_node', 'arg_of_perihelion', 'eccentricity'], - 'EPHEMERIS':['eph_json'] + 'EPHEMERIS': ['eph_json'] } diff --git a/tom_targets/utils.py b/tom_targets/utils.py index a8509b616..1c8f4d3b6 100644 --- a/tom_targets/utils.py +++ b/tom_targets/utils.py @@ -5,13 +5,13 @@ from io import StringIO import json -#this dictionary should contain as key entires text sufficient to uniquely identify -#the observatory name from the common English names used by JPL for that site. -#For example, Sunderland is probably unique enough to identify SAAO -#there may be a better way to handle this. +# this dictionary should contain as key entires text sufficient to uniquely +# identify the observatory name from the common English names used by JPL for +# that site. For example, Sunderland is probably unique enough to identify SAAO +# there may be a better way to handle this. site_names = {'Mauna Kea': '568', - 'Haleakala':'ogg', - 'McDonald':'elp', + 'Haleakala': 'ogg', + 'McDonald': 'elp', 'Tololo': 'lsc', 'Teide': 'tfn', 'Sutherland': 'cpt', @@ -111,6 +111,7 @@ def import_targets(targets): return {'targets': targets, 'errors': errors} + def import_ephemeris_target(stream): """ Reads in a custom ephemeris from provided file stream. @@ -118,14 +119,12 @@ def import_ephemeris_target(stream): Currently only reads in the first site-code ephemeris. """ - - #need to make robust to input date type - #need to make robuest to input coordinate type + # TO-DO: need to make robust to input date type + # TO-DO: need to make robust to input coordinate type errors = [] targets = [] - jpl_ra_key = 'R.A._____(ICRF)_____DEC' jpl_jd_key = 'Date_________JDUT' @@ -134,12 +133,11 @@ def import_ephemeris_target(stream): num_sites = 0 for i in range(len(eph)): if 'Center-site name' in eph[i]: - num_sites+=1 + num_sites += 1 - if num_sites!=8: + if num_sites != 8: errors.append(Warning('WARNING: Provided file does not have ephemerides for all 7 LCO sites.')) - eph_json = {} end_ind = 0 for ns in range(num_sites): @@ -149,8 +147,8 @@ def import_ephemeris_target(stream): name = 'custom' jd_inds = None ra_inds = None - loop_inds = [-1,-1] - for i in range(end_ind,len(eph)): + loop_inds = [-1, -1] + for i in range(end_ind, len(eph)): if 'Center-site name' in eph[i]: s = eph[i].split(': ')[-1] for j in site_names.keys(): @@ -165,26 +163,26 @@ def import_ephemeris_target(stream): name = "-".join(eph[i].split(': ')[1].split('{source')[0].split()) if jpl_ra_key in eph[i] and jpl_jd_key in eph[i]: - ra_inds = [eph[i].index(jpl_ra_key),eph[i].index(jpl_ra_key)+len(jpl_ra_key)] - jd_inds = [eph[i].index(jpl_jd_key),eph[i].index(jpl_jd_key)+len(jpl_jd_key)] + ra_inds = [eph[i].index(jpl_ra_key), eph[i].index(jpl_ra_key)+len(jpl_ra_key)] + jd_inds = [eph[i].index(jpl_jd_key), eph[i].index(jpl_jd_key)+len(jpl_jd_key)] if '$$SOE' in eph[i]: - if ra_inds is not None and loop_inds[0]==-1: + if ra_inds is not None and loop_inds[0] == -1: loop_inds[0] = i+1 if '$$EOE' in eph[i]: - if ra_inds is not None and loop_inds[0]!=-1: + if ra_inds is not None and loop_inds[0] != -1: loop_inds[1] = i break end_ind = loop_inds[1]+1 - #throw an HTML warning if I cannot understand the centre site name + # throw an HTML warning if I cannot understand the centre site name if not site_name_found: errors.append(Exception(f'Site name {centre_site_name} not understood.')) - #throw HTML screen of warning if I cannot find the coordinates or ephemerides - #here we will put a better error check and correctly thrown warning - #for now being lazy - if loop_inds == [-1,-1] or ra_inds is None or jd_inds is None: + # throw HTML screen of warning if I cannot find the coordinates or + # ephemerides. TO-DO: put a better error check and correctly thrown + # warning for now being lazy + if loop_inds == [-1, -1] or ra_inds is None or jd_inds is None: errors.append(Exception('We were not able to understand that ephemeris file.')) mjds = [] @@ -193,19 +191,19 @@ def import_ephemeris_target(stream): R = 0.0 D = 0.0 n = 0.0 - for i in range(loop_inds[0],loop_inds[1]): + for i in range(loop_inds[0], loop_inds[1]): mjds.append(str(float(eph[i][jd_inds[0]:jd_inds[1]])-2400000.5)) s = eph[i][ra_inds[0]:ra_inds[1]].split() r = 15.0*(float(s[0])+float(s[1])/60.0+float(s[2])/3600.0) ras.append("{:11.7f}".format(r)) d = abs(float(s[3]))+float(s[4])/60.0+float(s[5])/3600.0 if '-' in s[3]: - d*=-1.0 + d *= -1.0 decs.append("{:10.6f}".format(d)) - R+=r - D+=d - n+=1.0 + R += r + D += d + n += 1.0 eph_json[centre_site_name] = [] for i in range(len(ras)): From 26276d9efe42759bcea13d3b185730ad67637855 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 29 Apr 2020 15:18:33 -0700 Subject: [PATCH 079/424] Updating SOAR documentation for AEON-specific information --- docs/advanced/observation_module.md | 36 ++++++++++++++++++----------- docs/releasenotes.md | 2 ++ 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/docs/advanced/observation_module.md b/docs/advanced/observation_module.md index 9713a7bd6..bce4bcb64 100644 --- a/docs/advanced/observation_module.md +++ b/docs/advanced/observation_module.md @@ -1,10 +1,12 @@ # Writing an observation module to interface with observatories This guide will walk you through how to create a custom observation facility -module using some mocked up endpoints to simulate a real observatory interface. +module using some mocked up endpoints to simulate a real observatory interface. It will also +provide information on creating a custom manual observation facility for tracking observations that +were not created through an API. You can use this example as the foundation to build an observing facility module -to connect to a real observatory. +to connect to a real observatory or track observations on non-API supported facilities. Be sure you've followed the [Getting Started](/introduction/getting_started) guide before continuing onto this tutorial. @@ -25,6 +27,9 @@ You should have a working TOM already. You can start where the [Getting Started](/introduction/getting_started) guide leaves off. You should also be familiar with the observing facility's API that you would like to work with. + +## Creating a custom robotic facility + ### Defining the minimal implementation Within any existing module in your TOM you should create a new python module @@ -47,14 +52,14 @@ We'll place our `myfacility.py` file inside the `mytom` directory, next to `settings.py`. For now, copy the following lines into `myfacility.py`: ```python -from tom_observations.facility import GenericObservationFacility, GenericObservationForm +from tom_observations.facility import BaseRoboticObservationFacility, BaseRoboticObservationForm -class MyObservationFacilityForm(GenericObservationForm): +class MyObservationFacilityForm(BaseRoboticObservationForm): pass -class MyObservationFacility(GenericObservationFacility): +class MyObservationFacility(BaseRoboticObservationFacility): name = 'MyFacility' observation_types = [('OBSERVATION', 'Custom Observation')] ``` @@ -77,7 +82,7 @@ Now go ahead and view a target in your TOM, you should see something like this: This means our new observation facility module has been successfully loaded. -### GenericObservationFacility and GenericObservationForm +### BaseRoboticObservationFacility and BaseRoboticObservationForm You will have noticed our module consists of two classes that inherit from two other classes. @@ -85,12 +90,12 @@ other classes. `MyObservationFacility` is the class that will contain the "business logic" for interacting with the remote observatory. This includes methods to submit observations, check observation status, etc. It inherits from -`GenericObservationFacility`, which contains some functionality that all +`BaseRoboticObservationFacility`, which contains some functionality that all observation facility classes will want. `MyObservationFacilityForm` is the class that will display a GUI form for our users to create an observation. We can submit observations programmatically, but it -is also nice to have a GUI for our users to use. The `GenericObservationForm` +is also nice to have a GUI for our users to use. The `BaseRoboticObservationForm` class, just like the previous super class, contains logic and layout that all observation facility form classes should contain. @@ -109,7 +114,7 @@ To start, let's define new functions in `MyObservationFacility` for each missing function like so: ```python -class MyObservationFacility(GenericObservationFacility): +class MyObservationFacility(BaseRoboticObservationFacility): name = 'MyFacility' observation_types = [('OBSERVATION', 'Custom Observation')] @@ -185,13 +190,13 @@ submit the observation request: ```python from django import forms -from tom_observations.facility import GenericObservationFacility, GenericObservationForm +from tom_observations.facility import BaseRoboticObservationFacility, BaseRoboticObservationForm -class MyObservationFacilityForm(GenericObservationForm): +class MyObservationFacilityForm(BaseRoboticObservationForm): exposure_time = forms.IntegerField() exposure_count = forms.IntegerField() -class MyObservationFacility(GenericObservationFacility): +class MyObservationFacility(BaseRoboticObservationFacility): name = 'MyFacility' observation_types = [('OBSERVATION', 'Custom Observation')] @@ -259,7 +264,7 @@ Modeling our `SITES` on the one defined for we can easily put new sites into the airmass plots: ```python -class MyObservationFacility(GenericObservationFacility): +class MyObservationFacility(BaseRoboticObservationFacility): name = 'MyFacility' observation_types = [('OBSERVATION', 'Custom Observation')] @@ -287,3 +292,8 @@ API-accessible, you can still add them to your TOM's airmass plots to judge what targets to observe when. Happy developing! + + +## Creating a custom manual facility + + diff --git a/docs/releasenotes.md b/docs/releasenotes.md index eaeb58459..a32d21687 100644 --- a/docs/releasenotes.md +++ b/docs/releasenotes.md @@ -1,6 +1,8 @@ ### 1.5.0 - Introduced a manual facility interface for classical observing. +- Introduced a view and corresponding form to add existing API-based observations to a Target. +- Introduced a view and corresponding form to update an existing manual observation with an API-based observation ID. #### What to watch out for From 487b27ce4649e0acf7c9fa31f76c688141371372 Mon Sep 17 00:00:00 2001 From: fraserw Date: Tue, 5 May 2020 14:56:03 -0700 Subject: [PATCH 080/424] --fixed code review issues. --- tom_observations/facilities/lco.py | 7 +---- tom_observations/utils.py | 30 ++++++++-------------- tom_targets/templatetags/targets_extras.py | 9 ++++--- 3 files changed, 16 insertions(+), 30 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 5a46d18e5..27160ac6b 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -19,13 +19,8 @@ Target, REQUIRED_NON_SIDEREAL_FIELDS, REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME ) -from tom_targets.models import ( - Target, REQUIRED_NON_SIDEREAL_FIELDS, - REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME -) from tom_observations.utils import get_radec_ephemeris import json -import numpy as np # Determine settings for this module. @@ -365,7 +360,7 @@ def _build_target_fields(self): new_target_fields['name'] = '{}_{}_{}'.format(target.name, site, i) ephemeris_targets[site].append(new_target_fields) ephemeris_windows[site].append([start.isot, end.isot]) - elif mjd_vals is None and sun_alt_vals==-2: + elif mjd_vals is None and sun_alt_vals == -2: self.add_error(None, 'Date range outside range available in the provided ephemeris.') return (ephemeris_targets, ephemeris_windows) diff --git a/tom_observations/utils.py b/tom_observations/utils.py index a0211036f..219356b7f 100644 --- a/tom_observations/utils.py +++ b/tom_observations/utils.py @@ -8,16 +8,9 @@ from tom_observations import facility -#Work-around necessary until the NOAO main service upgrades are complete. -#Would be sufficient to update astropy to v4 which handles this internally. But Wes doesn't want to do this to his work environment yet. -#In future remove the two following lines and require an up to date astropy -from astropy.utils import iers -iers.Conf.iers_auto_url.set('ftp://cddis.gsfc.nasa.gov/pub/products/iers/finals2000A.all') logger = logging.getLogger(__name__) -d2r = np.pi/180.0 - def get_radec_ephemeris(eph_json_single, start_time, end_time, interval, observing_facility, observing_site): observing_facility_class = facility.get_service_class(observing_facility) sites = observing_facility_class().get_observing_sites() @@ -30,8 +23,8 @@ def get_radec_ephemeris(eph_json_single, start_time, end_time, interval, observi lon=obs_site.get('longitude')*units.deg, height=obs_site.get('elevation')*units.m) if observer is None: - #this condition occurs if the facility being requested isn't in the site list provided. - return (None,None,None,None,-1) + # this condition occurs if the facility being requested isn't in the site list provided. + return (None, None, None, None, -1) ra = [] dec = [] mjd = [] @@ -43,36 +36,33 @@ def get_radec_ephemeris(eph_json_single, start_time, end_time, interval, observi dec = np.array(dec) mjd = np.array(mjd) - fra = interp.interp1d(mjd,ra) - fdec = interp.interp1d(mjd,dec) + fra = interp.interp1d(mjd, ra) + fdec = interp.interp1d(mjd, dec) start = Time(start_time) end = Time(end_time) - time_range = time_grid_from_range(time_range=[start, end], time_resolution=interval*units.hour) tr_mjd = time_range.mjd - #tr_mjd += interval/(24.0*2.0) - airmasses = [] sun_alts = [] for i in range(len(tr_mjd)): c = SkyCoord(fra(time_range[i].mjd), fdec(time_range[i].mjd), frame="icrs", unit="deg") - t = Time(tr_mjd[i],format='mjd')#-8*units.h + t = Time(tr_mjd[i], format='mjd') sun = coordinates.get_sun(t) - altaz = c.transform_to(AltAz(obstime=t,location=observer)) - sun_altaz = sun.transform_to(AltAz(obstime=t,location=observer)) + altaz = c.transform_to(AltAz(obstime=t, location=observer)) + sun_altaz = sun.transform_to(AltAz(obstime=t, location=observer)) airmass = altaz.secz airmasses.append(airmass) sun_alts.append(sun_altaz.alt.value) airmasses = np.array(airmasses) sun_alts = np.array(sun_alts) - if np.min(tr_mjd)>=np.min(mjd) and np.max(tr_mjd)<=np.max(mjd): - return (tr_mjd,fra(tr_mjd),fdec(tr_mjd),airmasses,sun_alts) + if np.min(tr_mjd) >= np.min(mjd) and np.max(tr_mjd) <= np.max(mjd): + return (tr_mjd, fra(tr_mjd), fdec(tr_mjd), airmasses, sun_alts) else: - return (None,None,None,None,-2) + return (None, None, None, None, -2) def get_sidereal_visibility(target, start_time, end_time, interval, airmass_limit): diff --git a/tom_targets/templatetags/targets_extras.py b/tom_targets/templatetags/targets_extras.py index 80e25193d..6f632cfd8 100644 --- a/tom_targets/templatetags/targets_extras.py +++ b/tom_targets/templatetags/targets_extras.py @@ -248,6 +248,7 @@ def aladin(target): """ return {'target': target} + @register.filter def eph_json_to_value_ra(value): """ @@ -257,8 +258,8 @@ def eph_json_to_value_ra(value): eph_json = json.loads(value) keys = list(eph_json.keys()) k = keys[0] - l = len(eph_json[k][0]) - return( deg_to_sexigesimal(float(eph_json[k][int(l/2)]['R']),'hms')) + eph_len = len(eph_json[k][0]) + return deg_to_sexigesimal(float(eph_json[k][int(eph_len/2)]['R']),'hms') else: return -32768.0 @@ -272,7 +273,7 @@ def eph_json_to_value_dec(value): keys = list(eph_json.keys()) k = keys[0] l = len(eph_json[k][0]) - return( deg_to_sexigesimal(float(eph_json[k][int(l/2)]['D']),'dms')) + return deg_to_sexigesimal(float(eph_json[k][int(l/2)]['D']),'dms') else: return -32768.0 @@ -286,6 +287,6 @@ def eph_json_to_value_mjd(value): keys = list(eph_json.keys()) k = keys[0] l = len(eph_json[k][0]) - return( float(eph_json[k][int(l/2)]['t'])) + return round(float(eph_json[k][int(l/2)]['t']),5) else: return -32768.0 From 946d8678ec188ac21700bdbcf049ba3836c4d134 Mon Sep 17 00:00:00 2001 From: fraserw Date: Tue, 5 May 2020 14:59:15 -0700 Subject: [PATCH 081/424] --oops. Missed a couple style fixes --- tom_targets/templatetags/targets_extras.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tom_targets/templatetags/targets_extras.py b/tom_targets/templatetags/targets_extras.py index 6f632cfd8..888008f82 100644 --- a/tom_targets/templatetags/targets_extras.py +++ b/tom_targets/templatetags/targets_extras.py @@ -259,10 +259,11 @@ def eph_json_to_value_ra(value): keys = list(eph_json.keys()) k = keys[0] eph_len = len(eph_json[k][0]) - return deg_to_sexigesimal(float(eph_json[k][int(eph_len/2)]['R']),'hms') + return deg_to_sexigesimal(float(eph_json[k][int(eph_len/2)]['R']), 'hms') else: return -32768.0 + @register.filter def eph_json_to_value_dec(value): """ @@ -272,11 +273,12 @@ def eph_json_to_value_dec(value): eph_json = json.loads(value) keys = list(eph_json.keys()) k = keys[0] - l = len(eph_json[k][0]) - return deg_to_sexigesimal(float(eph_json[k][int(l/2)]['D']),'dms') + eph_len = len(eph_json[k][0]) + return deg_to_sexigesimal(float(eph_json[k][int(eph_len/2)]['D']), 'dms') else: return -32768.0 + @register.filter def eph_json_to_value_mjd(value): """ @@ -286,7 +288,7 @@ def eph_json_to_value_mjd(value): eph_json = json.loads(value) keys = list(eph_json.keys()) k = keys[0] - l = len(eph_json[k][0]) - return round(float(eph_json[k][int(l/2)]['t']),5) + eph_len = len(eph_json[k][0]) + return round(float(eph_json[k][int(eph_len/2)]['t']), 5) else: return -32768.0 From e119fdd1941f3db24e61463b44da969cc4827cbf Mon Sep 17 00:00:00 2001 From: fraserw Date: Wed, 6 May 2020 14:07:11 -0700 Subject: [PATCH 082/424] --further style fixes --- tom_observations/facilities/lco.py | 30 +++++++++++++++++------------- tom_observations/utils.py | 1 + tom_targets/models.py | 13 +++++++++---- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 27160ac6b..827998228 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -329,20 +329,21 @@ def _build_target_fields(self): if site in eph_json.keys(): ephemeris_targets[site] = [] ephemeris_windows[site] = [] - (mjd_vals, ra_vals, dec_vals, air_vals, sun_alt_vals) = get_radec_ephemeris(eph_json[site], - self.cleaned_data['start'], - self.cleaned_data['end'], - self.cleaned_data['imaging_interval'], - 'LCO', - site) + (mjd_vals, ra_vals, dec_vals, air_vals, sun_alt_vals) = \ + get_radec_ephemeris(eph_json[site], + self.cleaned_data['start'], + self.cleaned_data['end'], + self.cleaned_data['imaging_interval'], + 'LCO', + site) if mjd_vals is not None: for i in range(len(ra_vals)-1): if (air_vals[i] < float(self.cleaned_data['max_airmass']) and - air_vals[i+1] < float(self.cleaned_data['max_airmass']) and - air_vals[i] > 1.0 and - air_vals[i+1] > 1.0 and - sun_alt_vals[i] < -30.0 and - sun_alt_vals[i+1] < -30.0): + air_vals[i+1] < float(self.cleaned_data['max_airmass']) and + air_vals[i] > 1.0 and + air_vals[i+1] > 1.0 and + sun_alt_vals[i] < -30.0 and + sun_alt_vals[i+1] < -30.0): new_target_fields = {} new_target_fields['type'] = 'ICRS' @@ -356,7 +357,9 @@ def _build_target_fields(self): start = Time(mjd_vals[i], format='mjd') end = Time(mjd_vals[i+1], format='mjd') - # store start and end times in the target for a matter of convenience in passing this information forward to the request builder + # store start and end times in the target for + # a matter of convenience in passing this + # information forward to the request builder new_target_fields['name'] = '{}_{}_{}'.format(target.name, site, i) ephemeris_targets[site].append(new_target_fields) ephemeris_windows[site].append([start.isot, end.isot]) @@ -440,8 +443,9 @@ def _build_ephemeris_request_parts(self): 'max_airmass': self.cleaned_data['max_airmass'] } } + i_type = self.cleaned_data['instrument_type'] single_location = {'site': site, - 'telescope_class': self._get_instruments()[self.cleaned_data['instrument_type']]['class']} + 'telescope_class': self._get_instruments()[i_type]['class']} single_windows = [{'start': new_windows[site][i][0], 'end': new_windows[site][i][1]} ] diff --git a/tom_observations/utils.py b/tom_observations/utils.py index 219356b7f..2111ea990 100644 --- a/tom_observations/utils.py +++ b/tom_observations/utils.py @@ -11,6 +11,7 @@ logger = logging.getLogger(__name__) + def get_radec_ephemeris(eph_json_single, start_time, end_time, interval, observing_facility, observing_site): observing_facility_class = facility.get_service_class(observing_facility) sites = observing_facility_class().get_observing_sites() diff --git a/tom_targets/models.py b/tom_targets/models.py index d6203564f..2c940a3ba 100644 --- a/tom_targets/models.py +++ b/tom_targets/models.py @@ -27,9 +27,13 @@ # Additional non-sidereal fields that are required for specific orbital element # schemes REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME = { - 'MPC_COMET': ['perihdist', 'epoch_of_perihelion', 'inclination', 'lng_asc_node', 'arg_of_perihelion', 'eccentricity'], - 'MPC_MINOR_PLANET': ['mean_anomaly', 'semimajor_axis', 'inclination', 'lng_asc_node', 'arg_of_perihelion', 'eccentricity'], - 'JPL_MAJOR_PLANET': ['mean_daily_motion', 'mean_anomaly', 'semimajor_axis', 'inclination', 'lng_asc_node', 'arg_of_perihelion', 'eccentricity'], + 'MPC_COMET': ['perihdist', 'epoch_of_perihelion', 'inclination', + 'lng_asc_node', 'arg_of_perihelion', 'eccentricity'], + 'MPC_MINOR_PLANET': ['mean_anomaly', 'semimajor_axis', 'inclination', + 'lng_asc_node', 'arg_of_perihelion', 'eccentricity'], + 'JPL_MAJOR_PLANET': ['mean_daily_motion', 'mean_anomaly', 'semimajor_axis', + 'inclination', 'lng_asc_node', 'arg_of_perihelion', + 'eccentricity'], 'EPHEMERIS': ['eph_json'] } @@ -235,7 +239,8 @@ class Target(models.Model): max_length=50, null=True, blank=True, verbose_name='Centre-Site Name', help_text='Observatory Site Code' ) eph_json = models.TextField( - null=True, blank=True, verbose_name='Ephemeris JSON', help_text="Don't fill this in by hand unless you know what you are doing." + null=True, blank=True, verbose_name='Ephemeris JSON', + help_text="Don't fill this in by hand unless you know what you are doing." ) class Meta: From 28f2062299ea7b9bb0895eb94e104d06968bdf17 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 7 May 2020 13:11:28 -0700 Subject: [PATCH 083/424] added note to SOAR module --- tom_observations/facilities/soar.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tom_observations/facilities/soar.py b/tom_observations/facilities/soar.py index da24471b4..947c840c4 100644 --- a/tom_observations/facilities/soar.py +++ b/tom_observations/facilities/soar.py @@ -89,6 +89,9 @@ class SOARFacility(LCOFacility): """ The ``SOARFacility`` is the interface to the SOAR Telescope. For information regarding SOAR observing and the available parameters, please see http://www.ctio.noao.edu/soar/content/observing-soar. + + Please note that SOAR is only available in AEON-mode. It also uses the LCO API key, so to use this module, the + LCO dictionary in FACILITIES in `settings.py` will need to be completed. """ name = 'SOAR' From 9fb16fbc26e23446f913a86db0e7104dc2cca3ed Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 15 May 2020 15:40:32 -0700 Subject: [PATCH 084/424] Attempting alpha release --- .travis.yml | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2f9a53151..c9b21b61e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,64 @@ before_install: - pip uninstall -y numpy install: - pip install -I -r requirements.txt pycodestyle -script: - - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - - python3 manage.py test +# script: +# - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 +# - python3 manage.py test + +# before_deploy: +# if ! [[ $TRAVIS_BRANCH ]] + +stages: + - style_check + - test + - name: deploy_alpha + if: branch = development + +jobs: + - stage: style_check + script: pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 + - stage: test + script: python3 manage.py test + - stage: deploy_alpha + deploy: + provider: pypi + branch: development + on: + tags: false + user: "__token__" + password: + secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" + +# # Deploy alpha releases to PyPi on pushes/merges to dev branch +# deploy: +# provider: pypi +# branch: development +# before_deploy: +# - ./before_deploy_development.sh +# on: +# tags: false +# user: "__token__" +# password: +# secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" + +# # Deploy release candidates to PyPi on pushes/merges to master branch +# deploy: +# provider: pypi +# branch: master +# skip_existing: true +# on: +# tags: false +# user: "__token__" +# password: +# secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" + +# # Deploy to PyPi on tags of master branch +# deploy: +# provider: pypi +# branch: master +# skip_existing: true +# on: +# tags: true +# user: "__token__" +# password: +# secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" From ddb3dfd36b29e89a09476d4a9678f1ad26c70f88 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 15 May 2020 15:48:21 -0700 Subject: [PATCH 085/424] another try --- .travis.yml | 66 ++++++++++++++++++++++++++--------------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/.travis.yml b/.travis.yml index c9b21b61e..4e14d27e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,45 +9,45 @@ before_install: - pip uninstall -y numpy install: - pip install -I -r requirements.txt pycodestyle -# script: -# - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 -# - python3 manage.py test +script: + - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 + - python3 manage.py test # before_deploy: # if ! [[ $TRAVIS_BRANCH ]] -stages: - - style_check - - test - - name: deploy_alpha - if: branch = development +# stages: +# - style_check +# - test +# - name: deploy_alpha +# if: branch = development -jobs: - - stage: style_check - script: pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - - stage: test - script: python3 manage.py test - - stage: deploy_alpha - deploy: - provider: pypi - branch: development - on: - tags: false - user: "__token__" - password: - secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" +# jobs: +# - stage: style_check +# script: pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 +# - stage: test +# script: python3 manage.py test +# - stage: deploy_alpha +# deploy: +# provider: pypi +# branch: development +# on: +# tags: false +# user: "__token__" +# password: +# secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" -# # Deploy alpha releases to PyPi on pushes/merges to dev branch -# deploy: -# provider: pypi -# branch: development -# before_deploy: -# - ./before_deploy_development.sh -# on: -# tags: false -# user: "__token__" -# password: -# secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" +# Deploy alpha releases to PyPi on pushes/merges to dev branch +deploy: + provider: pypi + branch: development + before_deploy: + - ./before_deploy_development.sh + on: + tags: false + user: "__token__" + password: + secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" # # Deploy release candidates to PyPi on pushes/merges to master branch # deploy: From cae5c3b6573d50a70f9558b75d0e21ff4459e9dc Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 15 May 2020 15:51:06 -0700 Subject: [PATCH 086/424] ignoring style checks for the moments --- .travis.yml | 66 ++++++++++++++++++++++++++--------------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4e14d27e0..155c7b309 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,45 +9,45 @@ before_install: - pip uninstall -y numpy install: - pip install -I -r requirements.txt pycodestyle -script: - - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - - python3 manage.py test +# script: +# - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 +# - python3 manage.py test # before_deploy: # if ! [[ $TRAVIS_BRANCH ]] -# stages: -# - style_check -# - test -# - name: deploy_alpha -# if: branch = development +stages: + # - style_check + - test + - name: deploy_alpha + if: branch = development -# jobs: -# - stage: style_check -# script: pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 -# - stage: test -# script: python3 manage.py test -# - stage: deploy_alpha -# deploy: -# provider: pypi -# branch: development -# on: -# tags: false -# user: "__token__" -# password: -# secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" +jobs: + - stage: style_check + script: pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 + - stage: test + script: python3 manage.py test + - stage: deploy_alpha + deploy: + provider: pypi + branch: development + on: + tags: false + user: "__token__" + password: + secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" -# Deploy alpha releases to PyPi on pushes/merges to dev branch -deploy: - provider: pypi - branch: development - before_deploy: - - ./before_deploy_development.sh - on: - tags: false - user: "__token__" - password: - secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" +# # Deploy alpha releases to PyPi on pushes/merges to dev branch +# deploy: +# provider: pypi +# branch: development +# before_deploy: +# - ./before_deploy_development.sh +# on: +# tags: false +# user: "__token__" +# password: +# secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" # # Deploy release candidates to PyPi on pushes/merges to master branch # deploy: From 0d8b8454eea1e92fccae54aa3deb5cb6064ed397 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 15 May 2020 15:56:37 -0700 Subject: [PATCH 087/424] Moving away from jobs --- .travis.yml | 68 +++++++++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/.travis.yml b/.travis.yml index 155c7b309..4fc9bf393 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,45 +9,47 @@ before_install: - pip uninstall -y numpy install: - pip install -I -r requirements.txt pycodestyle -# script: -# - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 -# - python3 manage.py test +script: + - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 + - python3 manage.py test # before_deploy: # if ! [[ $TRAVIS_BRANCH ]] -stages: - # - style_check - - test - - name: deploy_alpha - if: branch = development +# stages: +# - style_check +# - test +# - name: deploy_alpha +# if: branch = development -jobs: - - stage: style_check - script: pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - - stage: test - script: python3 manage.py test - - stage: deploy_alpha - deploy: - provider: pypi - branch: development - on: - tags: false - user: "__token__" - password: - secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" +# jobs: +# - stage: style_check +# script: pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 +# - stage: test +# script: python3 manage.py test +# - stage: deploy_alpha +# deploy: +# provider: pypi +# branch: development +# on: +# tags: false +# user: "__token__" +# password: +# secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" -# # Deploy alpha releases to PyPi on pushes/merges to dev branch -# deploy: -# provider: pypi -# branch: development -# before_deploy: -# - ./before_deploy_development.sh -# on: -# tags: false -# user: "__token__" -# password: -# secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" +# Deploy alpha releases to PyPi on pushes/merges to dev branch +deploy: + provider: pypi + branch: development + before_deploy: + - ./before_deploy_development.sh + on: + tags: false + skip_existing: + true: + user: "__token__" + password: + secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" # # Deploy release candidates to PyPi on pushes/merges to master branch # deploy: From c52f1fcbb7d64dd89db9cff2ff40d8116a063de6 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 18 May 2020 08:26:57 -0700 Subject: [PATCH 088/424] temporarily removing style checks --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4fc9bf393..bbeb45f2d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ before_install: install: - pip install -I -r requirements.txt pycodestyle script: - - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 + # - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - python3 manage.py test # before_deploy: From c8593c42e4f50a10962104c39c203caecc55e6ff Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 18 May 2020 09:02:35 -0700 Subject: [PATCH 089/424] attempted refactor --- .travis.yml | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index bbeb45f2d..6e05b049e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,10 @@ language: python python: - '3.6' - '3.7' + - '3.8' +os: + - linux + - osx dist: xenial sudo: true before_install: @@ -9,19 +13,28 @@ before_install: - pip uninstall -y numpy install: - pip install -I -r requirements.txt pycodestyle -script: - # - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - - python3 manage.py test + +stages: + - style_check + - test + # - name: deploy_alpha + # if: branch = development + +jobs: + include: + - stage: test + - python: 3.8 + script: + - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 + - stage: test + script: + - python3 manage.py test + # - stage: deploy + # before_deploy: # if ! [[ $TRAVIS_BRANCH ]] -# stages: -# - style_check -# - test -# - name: deploy_alpha -# if: branch = development - # jobs: # - stage: style_check # script: pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 From 4592ca441435a3ccd37e4cfdb5de294bc39627c5 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 18 May 2020 09:30:23 -0700 Subject: [PATCH 090/424] removing osx --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6e05b049e..57cf6329e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,12 +5,13 @@ python: - '3.8' os: - linux - - osx dist: xenial sudo: true + before_install: - sudo apt-get install -y gfortran - pip uninstall -y numpy + install: - pip install -I -r requirements.txt pycodestyle @@ -22,7 +23,7 @@ stages: jobs: include: - - stage: test + - stage: style_check - python: 3.8 script: - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 From 5745dd84bc3d5af882eb5a71129c0e1523a77c93 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 18 May 2020 09:36:27 -0700 Subject: [PATCH 091/424] More travis refactoring and fixed style error --- .travis.yml | 11 ++++++----- .../templatetags/observation_extras.py | 18 +++++++++--------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 57cf6329e..e6207aafd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,10 +10,7 @@ sudo: true before_install: - sudo apt-get install -y gfortran - - pip uninstall -y numpy - -install: - - pip install -I -r requirements.txt pycodestyle + - pip uninstall -y numpy stages: - style_check @@ -24,10 +21,14 @@ stages: jobs: include: - stage: style_check - - python: 3.8 + name: "Code style check" + python: 3.8 + install: + - pip install -I -r requirements.txt pycodestyle script: - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - stage: test + name: "Unit tests" script: - python3 manage.py test # - stage: deploy diff --git a/tom_observations/templatetags/observation_extras.py b/tom_observations/templatetags/observation_extras.py index f3bbe5013..1986a1557 100644 --- a/tom_observations/templatetags/observation_extras.py +++ b/tom_observations/templatetags/observation_extras.py @@ -175,27 +175,27 @@ def observation_distribution(observations): data = [ dict( - lon=[l[0] for l in locations_no_status], - lat=[l[1] for l in locations_no_status], - text=[l[2] for l in locations_no_status], + lon=[location[0] for location in locations_no_status], + lat=[location[1] for location in locations_no_status], + text=[location[2] for location in locations_no_status], hoverinfo='lon+lat+text', mode='markers', marker=dict(color='rgba(90, 90, 90, .8)'), type='scattergeo' ), dict( - lon=[l[0] for l in locations_non_terminal], - lat=[l[1] for l in locations_non_terminal], - text=[l[2] for l in locations_non_terminal], + lon=[location[0] for location in locations_non_terminal], + lat=[location[1] for location in locations_non_terminal], + text=[location[2] for location in locations_non_terminal], hoverinfo='lon+lat+text', mode='markers', marker=dict(color='rgba(152, 0, 0, .8)'), type='scattergeo' ), dict( - lon=[l[0] for l in locations_terminal], - lat=[l[1] for l in locations_terminal], - text=[l[2] for l in locations_terminal], + lon=[location[0] for location in locations_terminal], + lat=[location[1] for location in locations_terminal], + text=[location[2] for location in locations_terminal], hoverinfo='lon+lat+text', mode='markers', marker=dict(color='rgba(0, 152, 0, .8)'), From 1fccdc179c313c0c33074b5c469141c7dc87db33 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 18 May 2020 09:40:03 -0700 Subject: [PATCH 092/424] more style fixes --- tom_targets/templatetags/targets_extras.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tom_targets/templatetags/targets_extras.py b/tom_targets/templatetags/targets_extras.py index f6ec0e90f..95fcee842 100644 --- a/tom_targets/templatetags/targets_extras.py +++ b/tom_targets/templatetags/targets_extras.py @@ -173,9 +173,9 @@ def target_distribution(targets): locations = targets.filter(type=Target.SIDEREAL).values_list('ra', 'dec', 'name') data = [ dict( - lon=[l[0] for l in locations], - lat=[l[1] for l in locations], - text=[l[2] for l in locations], + lon=[location[0] for location in locations], + lat=[location[1] for location in locations], + text=[location[2] for location in locations], hoverinfo='lon+lat+text', mode='markers', type='scattergeo' From ba3b82c94761c5db57150641b75ff24db9dfe832 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 18 May 2020 09:54:27 -0700 Subject: [PATCH 093/424] attempting a tag-based dev deployment --- .travis.yml | 47 ++++++++++++++++------------------------------- setup.py | 5 ++++- 2 files changed, 20 insertions(+), 32 deletions(-) diff --git a/.travis.yml b/.travis.yml index e6207aafd..f9f2ff2a1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,11 +24,13 @@ jobs: name: "Code style check" python: 3.8 install: - - pip install -I -r requirements.txt pycodestyle + - pip install -I pycodestyle script: - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - stage: test name: "Unit tests" + install: + - pip install -I -r requirements.txt pycodestyle script: - python3 manage.py test # - stage: deploy @@ -37,29 +39,23 @@ jobs: # before_deploy: # if ! [[ $TRAVIS_BRANCH ]] -# jobs: -# - stage: style_check -# script: pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 -# - stage: test -# script: python3 manage.py test -# - stage: deploy_alpha -# deploy: -# provider: pypi -# branch: development -# on: -# tags: false -# user: "__token__" -# password: -# secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" - -# Deploy alpha releases to PyPi on pushes/merges to dev branch +# Deploy alpha releases to PyPi on tags of dev branch deploy: provider: pypi branch: development - before_deploy: - - ./before_deploy_development.sh + skip_existing: true + on: + tags: true + user: "__token__" + password: + secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" + +# Deploy to PyPi on tags of master branch +deploy: + provider: pypi + branch: master on: - tags: false + tags: true skip_existing: true: user: "__token__" @@ -76,14 +72,3 @@ deploy: # user: "__token__" # password: # secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" - -# # Deploy to PyPi on tags of master branch -# deploy: -# provider: pypi -# branch: master -# skip_existing: true -# on: -# tags: true -# user: "__token__" -# password: -# secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" diff --git a/setup.py b/setup.py index 4eb32d0d8..68461f972 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='tomtoolkit', - version='1.4.0', + # version='1.4.0', description='The TOM Toolkit and base modules', long_description=long_description, long_description_content_type='text/markdown', @@ -26,6 +26,8 @@ ], keywords=['tomtoolkit', 'astronomy', 'astrophysics', 'cosmology', 'science', 'fits', 'observatory'], packages=find_packages(), + use_scm_version=True, + setup_requires=['setuptools_scm'], install_requires=[ 'django>=2.2', # TOM Toolkit requires db math functions 'django-bootstrap4==1.1.1', @@ -45,6 +47,7 @@ # 'matplotlib', 'pillow==7.1.0', 'fits2image==0.4.3', + 'setuptools-scm', 'specutils==1.0', 'dataclasses; python_version < "3.7"', ], From 08158d8733f88b08898e847fe88b5340cfe2cf51 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 18 May 2020 13:36:22 -0700 Subject: [PATCH 094/424] attempting a different api token --- .travis.yml | 6 +++--- setup.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f9f2ff2a1..6c213b062 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ jobs: - stage: test name: "Unit tests" install: - - pip install -I -r requirements.txt pycodestyle + - pip install -I -r requirements.txt script: - python3 manage.py test # - stage: deploy @@ -48,7 +48,7 @@ deploy: tags: true user: "__token__" password: - secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" + secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" # Deploy to PyPi on tags of master branch deploy: @@ -60,7 +60,7 @@ deploy: true: user: "__token__" password: - secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" + secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" # # Deploy release candidates to PyPi on pushes/merges to master branch # deploy: diff --git a/setup.py b/setup.py index 68461f972..a3e29eaad 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,6 @@ # 'matplotlib', 'pillow==7.1.0', 'fits2image==0.4.3', - 'setuptools-scm', 'specutils==1.0', 'dataclasses; python_version < "3.7"', ], From f327c067ca9156ea3f961aa79e66a8d127763ba5 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 18 May 2020 13:50:46 -0700 Subject: [PATCH 095/424] Attempting to build wheel as well --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 6c213b062..1bbea76cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,6 +49,7 @@ deploy: user: "__token__" password: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" + distributions: "sdist bdist_wheel" # Deploy to PyPi on tags of master branch deploy: From 760dc07eca3785f4e8d7df969aa223f77c0ba8e7 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 18 May 2020 14:05:15 -0700 Subject: [PATCH 096/424] trying something new --- .travis.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1bbea76cc..b0db37823 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,9 +12,9 @@ before_install: - sudo apt-get install -y gfortran - pip uninstall -y numpy -stages: - - style_check - - test +# stages: +# - style_check +# - test # - name: deploy_alpha # if: branch = development @@ -33,7 +33,6 @@ jobs: - pip install -I -r requirements.txt script: - python3 manage.py test - # - stage: deploy # before_deploy: @@ -55,10 +54,9 @@ deploy: deploy: provider: pypi branch: master + skip_existing: true on: tags: true - skip_existing: - true: user: "__token__" password: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" From 1af571c3358adae701ee68a29869e5a9268da388 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 18 May 2020 16:10:25 -0700 Subject: [PATCH 097/424] adding wheel dependency --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a3e29eaad..69336ca00 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ long_description_content_type='text/markdown', url='https://tomtoolkit.github.io', author='TOM Toolkit Project', - author_email='ariba@lco.global', + author_email='dcollom@lco.global', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Science/Research', @@ -27,7 +27,7 @@ keywords=['tomtoolkit', 'astronomy', 'astrophysics', 'cosmology', 'science', 'fits', 'observatory'], packages=find_packages(), use_scm_version=True, - setup_requires=['setuptools_scm'], + setup_requires=['setuptools_scm', 'wheel'], install_requires=[ 'django>=2.2', # TOM Toolkit requires db math functions 'django-bootstrap4==1.1.1', From cb4050d7ca0d5f736abc5e9aea3bbd8a3fa3fe9f Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 18 May 2020 16:17:15 -0700 Subject: [PATCH 098/424] Adding skip_cleanup --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index b0db37823..023a7fede 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,6 +43,7 @@ deploy: provider: pypi branch: development skip_existing: true + skip_cleanup: true on: tags: true user: "__token__" @@ -60,6 +61,7 @@ deploy: user: "__token__" password: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" + distributions: "sdist bdist_wheel" # # Deploy release candidates to PyPi on pushes/merges to master branch # deploy: From f489ccb1b6818a22e382036391eddcb1f6b9844f Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 18 May 2020 17:29:21 -0700 Subject: [PATCH 099/424] Added development README --- README-dev.md | 28 ++++++++++++++++++++++++++++ README.md | 3 +++ setup.py | 2 -- 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 README-dev.md diff --git a/README-dev.md b/README-dev.md new file mode 100644 index 000000000..205c3e583 --- /dev/null +++ b/README-dev.md @@ -0,0 +1,28 @@ +# TOM Toolkit +[![Build Status](https://travis-ci.org/TOMToolkit/tom_base.svg?branch=master)](https://travis-ci.org/TOMToolkit/tom_base) +[Documentation](https://tom-toolkit.readthedocs.io/en/latest/) + +![logo](tom_common/static/tom_common/img/logo-color.png) + +This README-dev is intended for maintainers of the repository for information on releases, standards, and anything that +isn't pertinent to the wider community. + +## Deployment +The [PyPi](https://pypi.org/project/tomtoolkit/) package is kept under the Las Cumbres Observatory PyPi account. The +development and master branches are deployed automatically by TravisCI upon tagging either branch. + +In order to trigger a PyPi deployment of either development or master, the branch must be given an annotated tag that +matches the correct version format. The version formats are as follows: + +| | Development | Master | All other branches | +|-------------|--------------|--------------|--------------------| +| Tagged | Push to PyPi | Push to PyPi | No effect | +| Not tagged | No effect | No effect | No effect | + +Tagged branches must follow the following [semantic versioning syntax](https://semver.org/): + +| | Development | Master | +|---|---------------|--------| +| | x.y.z-alpha.w | x.y.z | + +Following deployment of a release, a Github Release is created, and this should be filled in with the relevant release notes. diff --git a/README.md b/README.md index 6629968c3..7f8a1cf72 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,9 @@ have a [contribution guide](https://tom-toolkit.readthedocs.io/en/latest/contrib you might find helpful. We are particularly interested in the contribution of observation and alert modules. +## Developer information +For development information targeted at the maintainers of the project, please see [README-dev.md](README-dev.md). + ## Plugins diff --git a/setup.py b/setup.py index 69336ca00..39bdb2981 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,6 @@ setup( name='tomtoolkit', - # version='1.4.0', description='The TOM Toolkit and base modules', long_description=long_description, long_description_content_type='text/markdown', @@ -44,7 +43,6 @@ 'astropy==4.0', 'astroplan==0.6', 'plotly==4.6.0', - # 'matplotlib', 'pillow==7.1.0', 'fits2image==0.4.3', 'specutils==1.0', From 1243b1604b15aa1c7a968dc305279ef2f7e3f8e3 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 08:48:26 -0700 Subject: [PATCH 100/424] Added tag validation and attempting to clean up build --- .travis.yml | 37 ++++++++----------------------------- 1 file changed, 8 insertions(+), 29 deletions(-) diff --git a/.travis.yml b/.travis.yml index 023a7fede..419075cb7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,34 +9,22 @@ dist: xenial sudo: true before_install: - - sudo apt-get install -y gfortran - - pip uninstall -y numpy - -# stages: -# - style_check -# - test - # - name: deploy_alpha - # if: branch = development + - sudo apt-get install -y gfortran + - pip uninstall -y numpy jobs: include: - - stage: style_check - name: "Code style check" + - name: "Code style check" python: 3.8 install: - pip install -I pycodestyle script: - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - - stage: test - name: "Unit tests" + - name: "Unit tests" install: - pip install -I -r requirements.txt script: - python3 manage.py test - - -# before_deploy: -# if ! [[ $TRAVIS_BRANCH ]] # Deploy alpha releases to PyPi on tags of dev branch deploy: @@ -45,10 +33,11 @@ deploy: skip_existing: true skip_cleanup: true on: - tags: true + tags: true + condition: "$TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$" user: "__token__" password: - secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" + secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" distributions: "sdist bdist_wheel" # Deploy to PyPi on tags of master branch @@ -58,18 +47,8 @@ deploy: skip_existing: true on: tags: true + condition: "$TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$" user: "__token__" password: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" distributions: "sdist bdist_wheel" - -# # Deploy release candidates to PyPi on pushes/merges to master branch -# deploy: -# provider: pypi -# branch: master -# skip_existing: true -# on: -# tags: false -# user: "__token__" -# password: -# secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" From 0bb7341836bb9ede526a6df6f8cf4465987207e0 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 08:51:33 -0700 Subject: [PATCH 101/424] Simplifying the script even more --- .travis.yml | 19 ++++++------------- README-dev.md | 3 ++- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 419075cb7..d2f726e68 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,19 +12,12 @@ before_install: - sudo apt-get install -y gfortran - pip uninstall -y numpy -jobs: - include: - - name: "Code style check" - python: 3.8 - install: - - pip install -I pycodestyle - script: - - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - - name: "Unit tests" - install: - - pip install -I -r requirements.txt - script: - - python3 manage.py test +install: + - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 + - pip install -I -r requirements.txt pycodestyle + +script: + - python3 manage.py test # Deploy alpha releases to PyPi on tags of dev branch deploy: diff --git a/README-dev.md b/README-dev.md index 205c3e583..81b6e60df 100644 --- a/README-dev.md +++ b/README-dev.md @@ -19,7 +19,8 @@ matches the correct version format. The version formats are as follows: | Tagged | Push to PyPi | Push to PyPi | No effect | | Not tagged | No effect | No effect | No effect | -Tagged branches must follow the following [semantic versioning syntax](https://semver.org/): +Tagged branches must follow the [semantic versioning syntax](https://semver.org/). Tagged versions will not be +deployed unless they match the validation regex. The version format is as follows: | | Development | Master | |---|---------------|--------| From 716b88cb51783cf643349df5addaa74bf2a3e5d2 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 08:54:28 -0700 Subject: [PATCH 102/424] Commenting out regex validations --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d2f726e68..80273036f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,7 @@ deploy: skip_cleanup: true on: tags: true - condition: "$TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$" + # condition: "$TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$" user: "__token__" password: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" @@ -40,7 +40,7 @@ deploy: skip_existing: true on: tags: true - condition: "$TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$" + # condition: "$TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$" user: "__token__" password: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" From beeb477c5fa705ee8036e6b9178205be5875d865 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 08:59:41 -0700 Subject: [PATCH 103/424] Moved pycodestyle to the correct spot --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 80273036f..ac99b87f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,10 +13,10 @@ before_install: - pip uninstall -y numpy install: - - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - pip install -I -r requirements.txt pycodestyle script: + - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - python3 manage.py test # Deploy alpha releases to PyPi on tags of dev branch From b9469be92dcb889e51f49f5c50edb4592bf873d4 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 09:22:04 -0700 Subject: [PATCH 104/424] Attempting to use stages again --- .travis.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ac99b87f0..9f3aec813 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,9 +15,14 @@ before_install: install: - pip install -I -r requirements.txt pycodestyle -script: - - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - - python3 manage.py test +jobs: + include: + - stage: "Style checks" + python: 3.8 + name: "Style checks" + script: pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 + - stage: "Unit tests" + script: python3 manage.py test # Deploy alpha releases to PyPi on tags of dev branch deploy: From 3ae0c72ad92ea26ae04a0756d573aaf299593fda Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 09:26:43 -0700 Subject: [PATCH 105/424] Moving install to stages --- .travis.yml | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9f3aec813..48647956a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,23 +6,35 @@ python: os: - linux dist: xenial -sudo: true - -before_install: - - sudo apt-get install -y gfortran - - pip uninstall -y numpy - -install: - - pip install -I -r requirements.txt pycodestyle +sudo: true jobs: include: - stage: "Style checks" python: 3.8 name: "Style checks" + install: pip install -I pycodestyle script: pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - stage: "Unit tests" + before_install: + - sudo apt-get install -y gfortran + - pip uninstall -y numpy + install: pip install -I -r requirements.txt script: python3 manage.py test + - stage: "Deploy development" + deploy: + provider: pypi + branch: development + skip_existing: true + skip_cleanup: true + on: + tags: true + # condition: "$TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$" + user: "__token__" + password: + secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" + distributions: "sdist bdist_wheel" + # Deploy alpha releases to PyPi on tags of dev branch deploy: From 66d4d2eda51d9e814e18e02c9a7b2d177eb4b144 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 09:45:58 -0700 Subject: [PATCH 106/424] Updating tag regex validation --- .travis.yml | 40 ++++++++++++---------------------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/.travis.yml b/.travis.yml index 48647956a..4ab08c6e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,35 +6,18 @@ python: os: - linux dist: xenial -sudo: true +sudo: true -jobs: - include: - - stage: "Style checks" - python: 3.8 - name: "Style checks" - install: pip install -I pycodestyle - script: pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - - stage: "Unit tests" - before_install: - - sudo apt-get install -y gfortran - - pip uninstall -y numpy - install: pip install -I -r requirements.txt - script: python3 manage.py test - - stage: "Deploy development" - deploy: - provider: pypi - branch: development - skip_existing: true - skip_cleanup: true - on: - tags: true - # condition: "$TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$" - user: "__token__" - password: - secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" - distributions: "sdist bdist_wheel" +before_install: + - sudo apt-get install -y gfortran + - pip uninstall -y numpy +install: + - pip install -I -r requirements.txt pycodestyle + +script: + - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 + - python3 manage.py test # Deploy alpha releases to PyPi on tags of dev branch deploy: @@ -44,7 +27,8 @@ deploy: skip_cleanup: true on: tags: true - # condition: "$TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$" + condition: + - tag =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$ user: "__token__" password: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" From f823dc7278ae672a37c274ef57adc132097014a8 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 09:54:36 -0700 Subject: [PATCH 107/424] testing out new deploy block --- .travis.yml | 65 +++++++++++++++++------------------------------------ 1 file changed, 21 insertions(+), 44 deletions(-) diff --git a/.travis.yml b/.travis.yml index 023a7fede..e72ed561b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,67 +9,44 @@ dist: xenial sudo: true before_install: - - sudo apt-get install -y gfortran - - pip uninstall -y numpy + - sudo apt-get install -y gfortran + - pip uninstall -y numpy -# stages: -# - style_check -# - test - # - name: deploy_alpha - # if: branch = development +install: + - pip install -I -r requirements.txt pycodestyle -jobs: - include: - - stage: style_check - name: "Code style check" - python: 3.8 - install: - - pip install -I pycodestyle - script: - - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - - stage: test - name: "Unit tests" - install: - - pip install -I -r requirements.txt - script: - - python3 manage.py test - +script: + - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 + - python3 manage.py test -# before_deploy: -# if ! [[ $TRAVIS_BRANCH ]] - -# Deploy alpha releases to PyPi on tags of dev branch +# Deploy alpha releases to PyPi on tags of dev branch and full releases on tags of master branch deploy: provider: pypi - branch: development + branches: + only: + - development + - master skip_existing: true skip_cleanup: true - on: - tags: true - user: "__token__" - password: - secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" - distributions: "sdist bdist_wheel" - -# Deploy to PyPi on tags of master branch -deploy: - provider: pypi - branch: master - skip_existing: true on: tags: true + condition: + - branch = development AND tag =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$ + - branch = master AND tag =~ ^[0-9]+\.[0-9]+\.[0-9]+$ user: "__token__" password: - secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" + secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" distributions: "sdist bdist_wheel" -# # Deploy release candidates to PyPi on pushes/merges to master branch +# # Deploy to PyPi on tags of master branch # deploy: # provider: pypi # branch: master # skip_existing: true # on: -# tags: false +# tags: true +# # condition: "$TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$" # user: "__token__" # password: -# secure: "QK0IxvFGrZsA4mJK/MWQ34kXmpDEpH1ckuq1HPP/Kr3p/11VgEsNWwkkXY8kFiOS35S073XYJo4VrUUxZ9xeRNIvG1JgySfqiO+GGUsp3Lx4AP7pOcsb9NfgSVt8ekq/HS1/gzJaFuXvx7CZk3aR/RLKWoHZ1PHzxq2010QGtIRcMMKXOtk6pIE8rveoTe9hHG+cwdEDvuI7q485y9Wop1nN58QcFxIvf8DBhdXDXov50iWleLQzPXIDShuYJGQMfK68JUqGXZrXdEA6oyGU+VGTGgyXPPRvRQLZqZtXsI3Ex8fwyaFN7z1BtiJXgnp84viA03lAlLofTSAwFkS/1krAvz9LKTzv/HwynkcMZtNAFWBytaugFpkLbSA3S+jDnVUgMP6RE4dQSo39l3+LDGyl04u2rgy8GcWJJdlup0Z4NFhKKIm6eBaZw6Yzedk+VVahglEHFlOnWZejvWoz1m57Sj54eh2TKeLXT7wVnVeI4vEl3WdBO2/MZagfWfTjRai3N/TPoq7kNlkgUPKM960VTbG0jVupm+IVYmLCKTtkcpWQbX+Rk8kZv/ujQYg/cqTvJr5rjL7OvxEurzXMSDDi0BmCEFyq5WbYWRi93hgsNSph9HrDZl5VAIKEnEgbL7FvAslX+b1OwPMJh8DukSWVZs2a+UYxOEyd1BCpxZo=" +# secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" +# distributions: "sdist bdist_wheel" From c75ab5266b7180b4864a29f812a101ff75bcaf2f Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 10:01:49 -0700 Subject: [PATCH 108/424] Attempting to resolve syntax issues --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e72ed561b..25f071141 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,8 +31,7 @@ deploy: on: tags: true condition: - - branch = development AND tag =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$ - - branch = master AND tag =~ ^[0-9]+\.[0-9]+\.[0-9]+$ + - (branch = development AND tag =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$) OR (branch = master AND tag =~ ^[0-9]+\.[0-9]+\.[0-9]+$) user: "__token__" password: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" From e4dae420c94c33571f348190e6720a27289aeb40 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 13:35:21 -0700 Subject: [PATCH 109/424] maybe a better deploy block --- .travis.yml | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/.travis.yml b/.travis.yml index 25f071141..52e5a6894 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,31 +21,25 @@ script: # Deploy alpha releases to PyPi on tags of dev branch and full releases on tags of master branch deploy: - provider: pypi - branches: - only: - - development - - master + - provider: pypi + branch: development skip_existing: true - skip_cleanup: true + cleanup: false on: tags: true - condition: - - (branch = development AND tag =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$) OR (branch = master AND tag =~ ^[0-9]+\.[0-9]+\.[0-9]+$) + condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$) OR (branch = master AND ) user: "__token__" password: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" distributions: "sdist bdist_wheel" - -# # Deploy to PyPi on tags of master branch -# deploy: -# provider: pypi -# branch: master -# skip_existing: true -# on: -# tags: true -# # condition: "$TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$" -# user: "__token__" -# password: -# secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" -# distributions: "sdist bdist_wheel" + - provider: pypi + branch: master + skip_existing: true + cleanup: false + on: + tags: true + condition: "$TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$" + user: "__token__" + password: + secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" + distributions: "sdist bdist_wheel" From be5743ba878fba82ab6000a075f02adb05294426 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 13:53:21 -0700 Subject: [PATCH 110/424] Adding codacy badge and first run at coveralls --- .coveralls.yml | 1 + README.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 .coveralls.yml diff --git a/.coveralls.yml b/.coveralls.yml new file mode 100644 index 000000000..9000930ee --- /dev/null +++ b/.coveralls.yml @@ -0,0 +1 @@ +repo_token: hBI4O62Oa5RfM8pZrImGRdil2JWsJDcy4 \ No newline at end of file diff --git a/README.md b/README.md index 7f8a1cf72..5d9ad1cca 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # TOM Toolkit [![Build Status](https://travis-ci.org/TOMToolkit/tom_base.svg?branch=master)](https://travis-ci.org/TOMToolkit/tom_base) +[![Codacy Badge](https://api.codacy.com/project/badge/Grade/9846cee7c4904cae8864525101030169)](https://www.codacy.com/gh/observatorycontrolsystem/observation-portal?utm_source=github.com&utm_medium=referral&utm_content=observatorycontrolsystem/observation-portal&utm_campaign=Badge_Grade) [Documentation](https://tom-toolkit.readthedocs.io/en/latest/) ![logo](tom_common/static/tom_common/img/logo-color.png) From ffabf975145ba717f491529421b1980981979a1f Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 14:01:22 -0700 Subject: [PATCH 111/424] Removing bugging condition --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 52e5a6894..31e6f1462 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,7 @@ deploy: cleanup: false on: tags: true - condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$) OR (branch = master AND ) + condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$ user: "__token__" password: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" From c31b58f4f09aa83083313ed1c82a9e3f13347f5c Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 14:02:24 -0700 Subject: [PATCH 112/424] Removing conditions entirely --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 31e6f1462..b1e018ebb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,7 @@ deploy: cleanup: false on: tags: true - condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$ + # condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$ user: "__token__" password: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" @@ -38,7 +38,7 @@ deploy: cleanup: false on: tags: true - condition: "$TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$" + # condition: "$TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$" user: "__token__" password: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" From a20245292b4ae45d694610a2ecb27aabb4ad6981 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 14:08:56 -0700 Subject: [PATCH 113/424] Attempting to fix conditions --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b1e018ebb..825f6175d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,7 @@ deploy: cleanup: false on: tags: true - # condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$ + condition: "$TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$" user: "__token__" password: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" @@ -38,7 +38,7 @@ deploy: cleanup: false on: tags: true - # condition: "$TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$" + condition: "$TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$" user: "__token__" password: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" From e3a1753eda95fe21dd793b2ce010e5cca15b3e38 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 14:12:22 -0700 Subject: [PATCH 114/424] removing quotes --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 825f6175d..ad1ef0f05 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,8 @@ deploy: cleanup: false on: tags: true - condition: "$TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$" + python: 3.8 + condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$ user: "__token__" password: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" @@ -38,7 +39,8 @@ deploy: cleanup: false on: tags: true - condition: "$TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$" + python: 3.8 + condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$ user: "__token__" password: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" From 0fd2cb6d9ac53f092d9c9395b4acd174b841a39d Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 15:17:13 -0700 Subject: [PATCH 115/424] Trying to move deployments to stages --- .travis.yml | 78 +++++++++++++++++++++++++++++++++++------------------ README.md | 2 ++ 2 files changed, 54 insertions(+), 26 deletions(-) diff --git a/.travis.yml b/.travis.yml index ad1ef0f05..d84fd7594 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,32 +16,58 @@ install: - pip install -I -r requirements.txt pycodestyle script: - - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 + # - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - python3 manage.py test # Deploy alpha releases to PyPi on tags of dev branch and full releases on tags of master branch -deploy: - - provider: pypi - branch: development - skip_existing: true - cleanup: false - on: - tags: true - python: 3.8 - condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$ - user: "__token__" - password: - secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" - distributions: "sdist bdist_wheel" - - provider: pypi - branch: master - skip_existing: true - cleanup: false - on: - tags: true - python: 3.8 - condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$ - user: "__token__" - password: - secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" - distributions: "sdist bdist_wheel" +jobs: + include: + - stage: "Deploy Development" + deploy: + provider: pypi + branch: development + skip_existing: true + cleanup: false + on: + tags: true + python: 3.8 + condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$ + user: "__token__" + password: + secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" + distributions: "sdist bdist_wheel" + # - provider: releases + # api_key: "" + # branch: + # - development + # on: + # tags: true + # file_glob: true + # file: dist/* + # cleanup: false + # draft: true + # prerelease: true + - stage: "Deploy Master" + deploy: + provider: pypi + branch: master + skip_existing: true + cleanup: false + on: + tags: true + python: 3.8 + condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$ + user: "__token__" + password: + secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" + distributions: "sdist bdist_wheel" + # - provider: releases + # api_key: "" + # branch: + # - master + # on: + # tags: true + # file_glob: true + # file: dist/* + # cleanup: false + # draft: true diff --git a/README.md b/README.md index 5d9ad1cca..1e5f581f5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # TOM Toolkit [![Build Status](https://travis-ci.org/TOMToolkit/tom_base.svg?branch=master)](https://travis-ci.org/TOMToolkit/tom_base) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/9846cee7c4904cae8864525101030169)](https://www.codacy.com/gh/observatorycontrolsystem/observation-portal?utm_source=github.com&utm_medium=referral&utm_content=observatorycontrolsystem/observation-portal&utm_campaign=Badge_Grade) +[![Coverage Status](https://coveralls.io/repos/github/TOMToolkit/tom_base/badge.svg?branch=master)](https://coveralls.io/github/TOMToolkit/tom_base?branch=master) +[![Documentation Status](https://readthedocs.org/projects/tom-toolkit/badge/?version=stable)](https://tom-toolkit.readthedocs.io/en/stable/?badge=stable) [Documentation](https://tom-toolkit.readthedocs.io/en/latest/) ![logo](tom_common/static/tom_common/img/logo-color.png) From 08267bf8c4ff4b42690eb16221c8bec43dc508f3 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 15:20:10 -0700 Subject: [PATCH 116/424] Moving tests to stage --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index d84fd7594..7ec33c6dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ python: - '3.8' os: - linux -dist: xenial +dist: bionic sudo: true before_install: @@ -15,13 +15,13 @@ before_install: install: - pip install -I -r requirements.txt pycodestyle -script: - # - pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - - python3 manage.py test - # Deploy alpha releases to PyPi on tags of dev branch and full releases on tags of master branch jobs: include: + - stage: "Style Checks" + script: pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 + - stage: "Test" + script: python3 manage.py test - stage: "Deploy Development" deploy: provider: pypi From e39582c43592bc84e679325419321d7aacf1ce91 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 15:25:27 -0700 Subject: [PATCH 117/424] Adding script skip --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 7ec33c6dd..3717c2260 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,14 +15,18 @@ before_install: install: - pip install -I -r requirements.txt pycodestyle +script: skip + # Deploy alpha releases to PyPi on tags of dev branch and full releases on tags of master branch jobs: include: - stage: "Style Checks" + python: 3.8 script: pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - stage: "Test" script: python3 manage.py test - stage: "Deploy Development" + script: skip deploy: provider: pypi branch: development @@ -48,6 +52,7 @@ jobs: # draft: true # prerelease: true - stage: "Deploy Master" + script: skip deploy: provider: pypi branch: master From e496fe2d76588ac47cecf8612e5f6527e75d6a3d Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 15:32:12 -0700 Subject: [PATCH 118/424] Fixing up python versions --- .travis.yml | 57 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3717c2260..1d1fcd2ab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,9 +23,12 @@ jobs: - stage: "Style Checks" python: 3.8 script: pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 + - stage: "Test" script: python3 manage.py test + - stage: "Deploy Development" + python: 3.8 script: skip deploy: provider: pypi @@ -34,24 +37,29 @@ jobs: cleanup: false on: tags: true - python: 3.8 condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$ user: "__token__" password: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" distributions: "sdist bdist_wheel" - # - provider: releases - # api_key: "" - # branch: - # - development - # on: - # tags: true - # file_glob: true - # file: dist/* - # cleanup: false - # draft: true - # prerelease: true + + # - stage: "Deploy Development" + # python: 3.8 + # deploy: + # provider: releases + # api_key: "" + # branch: + # - development + # on: + # tags: true + # file_glob: true + # file: dist/* + # cleanup: false + # draft: true + # prerelease: true + - stage: "Deploy Master" + python: 3.8 script: skip deploy: provider: pypi @@ -60,19 +68,22 @@ jobs: cleanup: false on: tags: true - python: 3.8 condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$ user: "__token__" password: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" distributions: "sdist bdist_wheel" - # - provider: releases - # api_key: "" - # branch: - # - master - # on: - # tags: true - # file_glob: true - # file: dist/* - # cleanup: false - # draft: true + + # - stage: "Deploy Master" + # python: 3.8 + # deploy: + # provider: releases + # api_key: "" + # branch: + # - master + # on: + # tags: true + # file_glob: true + # file: dist/* + # cleanup: false + # draft: true From 14fae0f85c7c991493d5b92badeac0cef2ad43f6 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 15:42:59 -0700 Subject: [PATCH 119/424] More cleanup --- .travis.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1d1fcd2ab..e87435781 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,21 +12,17 @@ before_install: - sudo apt-get install -y gfortran - pip uninstall -y numpy -install: - - pip install -I -r requirements.txt pycodestyle - -script: skip +script: + - python3 manage.py test # Deploy alpha releases to PyPi on tags of dev branch and full releases on tags of master branch jobs: include: - stage: "Style Checks" python: 3.8 + install: pip install -I pycodestyle script: pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - - stage: "Test" - script: python3 manage.py test - - stage: "Deploy Development" python: 3.8 script: skip From d50f513f63c8998f79cb315f6cb9925deaa83ec2 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 16:25:12 -0700 Subject: [PATCH 120/424] Attempting to move matrix into jobs --- .travis.yml | 44 +++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index e87435781..3721e8098 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,30 +1,43 @@ language: python -python: - - '3.6' - - '3.7' - - '3.8' -os: - - linux -dist: bionic -sudo: true +# python: +# - '3.8' +# - '3.7' +# - '3.6' +# os: +# - linux +# dist: bionic -before_install: - - sudo apt-get install -y gfortran - - pip uninstall -y numpy +# before_install: +# - sudo apt-get install -y gfortran +# - pip uninstall -y numpy -script: - - python3 manage.py test +# script: +# - python3 manage.py test # Deploy alpha releases to PyPi on tags of dev branch and full releases on tags of master branch jobs: + python: + - 3.8 + - 3.7 + - 3.6 + os: + - linux + dist: bionic + + before_install: + - sudo apt-get install -y gfortran + - pip uninstall -y numpy + include: - stage: "Style Checks" - python: 3.8 install: pip install -I pycodestyle script: pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 + - stage: "Unit Tests" + script: python3 manage.py test + + # Deploy alpha releases to PyPi and generate draft release on Github Releases - stage: "Deploy Development" - python: 3.8 script: skip deploy: provider: pypi @@ -54,6 +67,7 @@ jobs: # draft: true # prerelease: true + # Deploy full releases to PyPi and generate draft release on Github Releases - stage: "Deploy Master" python: 3.8 script: skip From d64937680a59b22ab0dbbac9238d61031a85789d Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 16:30:29 -0700 Subject: [PATCH 121/424] Moving build matrix back out, adding macos tests --- .travis.yml | 46 +++++++++++++++++----------------------------- 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3721e8098..ea8f1e9c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,42 +1,30 @@ language: python -# python: -# - '3.8' -# - '3.7' -# - '3.6' -# os: -# - linux -# dist: bionic +python: + - '3.8' + - '3.7' + - '3.6' +os: + - linux +dist: bionic -# before_install: -# - sudo apt-get install -y gfortran -# - pip uninstall -y numpy +before_install: + - sudo apt-get install -y gfortran + - pip uninstall -y numpy -# script: -# - python3 manage.py test +script: + - python3 manage.py test -# Deploy alpha releases to PyPi on tags of dev branch and full releases on tags of master branch jobs: - python: - - 3.8 - - 3.7 - - 3.6 - os: - - linux - dist: bionic - - before_install: - - sudo apt-get install -y gfortran - - pip uninstall -y numpy - include: - stage: "Style Checks" install: pip install -I pycodestyle script: pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - - stage: "Unit Tests" - script: python3 manage.py test + - stage: "Tests on MacOS" + os: osx + language: shell - # Deploy alpha releases to PyPi and generate draft release on Github Releases + # Deploy alpha releases to PyPi and generate draft release on Github Releases (incomplete) - stage: "Deploy Development" script: skip deploy: @@ -67,7 +55,7 @@ jobs: # draft: true # prerelease: true - # Deploy full releases to PyPi and generate draft release on Github Releases + # Deploy full releases to PyPi and generate draft release on Github Releases (incomplete) - stage: "Deploy Master" python: 3.8 script: skip From 20b70e505ae72ff0d876ed113bac459db84a69f1 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 16:36:21 -0700 Subject: [PATCH 122/424] Fixing validation warnings and attempting to order stages --- .travis.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index ea8f1e9c1..eb048ddbc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,13 +14,19 @@ before_install: script: - python3 manage.py test +stages: + - "Style Checks" + - "test" + - "Deploy Development" + - "Deploy Master" + jobs: include: - stage: "Style Checks" install: pip install -I pycodestyle script: pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 - - stage: "Tests on MacOS" + - stage: "test" os: osx language: shell @@ -29,13 +35,13 @@ jobs: script: skip deploy: provider: pypi - branch: development skip_existing: true cleanup: false on: + branch: development tags: true condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$ - user: "__token__" + username: "__token__" password: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" distributions: "sdist bdist_wheel" @@ -61,13 +67,13 @@ jobs: script: skip deploy: provider: pypi - branch: master skip_existing: true cleanup: false on: + branch: master tags: true condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$ - user: "__token__" + username: "__token__" password: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" distributions: "sdist bdist_wheel" From 08b98789a082b25d2933f33c7d71cf887d9eaf65 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 16:40:18 -0700 Subject: [PATCH 123/424] Attempting to get macos working --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index eb048ddbc..58ee4210a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,8 @@ os: dist: bionic before_install: - - sudo apt-get install -y gfortran - - pip uninstall -y numpy + - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get install -y gfortran + - pip uninstall -y numpy script: - python3 manage.py test From 2a3cf591e25dbd40674ad0aef3bd43b05b95661c Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 16:43:34 -0700 Subject: [PATCH 124/424] fixing bash script syntax and skipping deploy stages for non-matching branches --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 58ee4210a..dc8a5734b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ os: dist: bionic before_install: - - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get install -y gfortran + - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get install -y gfortran; fi - pip uninstall -y numpy script: @@ -32,6 +32,7 @@ jobs: # Deploy alpha releases to PyPi and generate draft release on Github Releases (incomplete) - stage: "Deploy Development" + if: branch = development script: skip deploy: provider: pypi @@ -63,6 +64,7 @@ jobs: # Deploy full releases to PyPi and generate draft release on Github Releases (incomplete) - stage: "Deploy Master" + if: branch = master python: 3.8 script: skip deploy: From 540faf200eaa779fedd75f65be8c7c4db06136ed Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 16:50:08 -0700 Subject: [PATCH 125/424] Adding explicit install to try to fix macos tests --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index dc8a5734b..405f93b41 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,8 +11,9 @@ before_install: - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get install -y gfortran; fi - pip uninstall -y numpy -script: - - python3 manage.py test +install: pip install -r requirements.txt + +script: python3 manage.py test stages: - "Style Checks" From 4807d31436c1b0c0b6c37dffbf0484cd5beffc92 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 17:01:30 -0700 Subject: [PATCH 126/424] More macos debugging --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 405f93b41..5675282b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,8 @@ os: dist: bionic before_install: - - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get install -y gfortran; fi + # - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get install -y gfortran; fi + - sudo apt-get install -y gfortran - pip uninstall -y numpy install: pip install -r requirements.txt @@ -30,6 +31,7 @@ jobs: - stage: "test" os: osx language: shell + before_install: pip3 install -U pip # Deploy alpha releases to PyPi and generate draft release on Github Releases (incomplete) - stage: "Deploy Development" From 0a6237d6655a5461a2e40d32a049456ac117ee1b Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 17:08:35 -0700 Subject: [PATCH 127/424] removing conditional linux install --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5675282b9..b4d3bd9e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,6 @@ os: dist: bionic before_install: - # - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get install -y gfortran; fi - sudo apt-get install -y gfortran - pip uninstall -y numpy From 00e90b2fdf9bdd5250cf18546a0d64babb0a7e04 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 17:12:35 -0700 Subject: [PATCH 128/424] Attempting to get coveralls to pick up the branch --- .coveralls.yml | 1 - .travis.yml | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 .coveralls.yml diff --git a/.coveralls.yml b/.coveralls.yml deleted file mode 100644 index 9000930ee..000000000 --- a/.coveralls.yml +++ /dev/null @@ -1 +0,0 @@ -repo_token: hBI4O62Oa5RfM8pZrImGRdil2JWsJDcy4 \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index b4d3bd9e4..cc58d4054 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,8 @@ install: pip install -r requirements.txt script: python3 manage.py test +after_success: coveralls + stages: - "Style Checks" - "test" From d3c8594d056980fc2760e4689082f6296ae32f87 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 17:29:48 -0700 Subject: [PATCH 129/424] Hopefully implemented automated release to Github Releases --- .travis.yml | 56 +++++++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index cc58d4054..867636fbf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,20 +51,21 @@ jobs: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" distributions: "sdist bdist_wheel" - # - stage: "Deploy Development" - # python: 3.8 - # deploy: - # provider: releases - # api_key: "" - # branch: - # - development - # on: - # tags: true - # file_glob: true - # file: dist/* - # cleanup: false - # draft: true - # prerelease: true + - stage: "Deploy Development" + python: 3.8 + deploy: + provider: releases + api_key: + secure: "V5t/Rk0jK6ggR6Oc2sk6Kr8+mC4vcKD4Vh1/CVhFoB+/QJTJR4wR1ZzdmxSTHVqyYRWMhtbQg4GEenVu3CCE6Aqcpb8FuXagBGDjrydYMvBz0QV0tGpc0QfAGFgihoMxRXYYy8x0qJoImpIgweIcBP6qCIbWlHNn7uSbJgQAcpHI1KP5//SJbiZ5+h1tYa3AImfhuRIjtw5X5XCkbNe04DpJZZrR7et4BnZwjBKmkPBHAoDhfuGzA5PF1S6jHCpWIddhRQ60v6T7yp8jcBEn5yvQSmusfUju61GI0+irFV8LksaWMaLFkxdR2ne8DtHGuXoRs8G/hOLlYZGBMM/JkV2VcuX9F61vspQW2bDrNgAzMEHknYKeTamvt5HV+zYly7X/OnBMAdbDl3ZOjP5h7zYjdeDza+t6dG1+iJKfCjz5dekT20NDVlwJvIMGhqxNOxDRfnlHopoTT/16/Jd7SPDOY2gpDoZxA8uPGZJR52y6T8tZqrR/tamoXRZk5WR4zeIW2rFK85hZwmz59dWQywlhzcIxMHhBcH6r6O3p1NwFA1LPIzgEQHKig/5DxFk52reU70fA9L2nZ1AHSES2RtG9kDQZLx7gbnOaOGx7Jbd6mECFMzliEBKiSnYxN404bNVAoTBEwcuxTho9zJRccQVeIdTD617K4sAqPWWhFwk=" + branch: + - development + on: + tags: true + file_glob: true + file: dist/* + cleanup: false + draft: true + prerelease: true # Deploy full releases to PyPi and generate draft release on Github Releases (incomplete) - stage: "Deploy Master" @@ -84,16 +85,17 @@ jobs: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" distributions: "sdist bdist_wheel" - # - stage: "Deploy Master" - # python: 3.8 - # deploy: - # provider: releases - # api_key: "" - # branch: - # - master - # on: - # tags: true - # file_glob: true - # file: dist/* - # cleanup: false - # draft: true + - stage: "Deploy Master" + python: 3.8 + deploy: + provider: releases + api_key: + secure: "V5t/Rk0jK6ggR6Oc2sk6Kr8+mC4vcKD4Vh1/CVhFoB+/QJTJR4wR1ZzdmxSTHVqyYRWMhtbQg4GEenVu3CCE6Aqcpb8FuXagBGDjrydYMvBz0QV0tGpc0QfAGFgihoMxRXYYy8x0qJoImpIgweIcBP6qCIbWlHNn7uSbJgQAcpHI1KP5//SJbiZ5+h1tYa3AImfhuRIjtw5X5XCkbNe04DpJZZrR7et4BnZwjBKmkPBHAoDhfuGzA5PF1S6jHCpWIddhRQ60v6T7yp8jcBEn5yvQSmusfUju61GI0+irFV8LksaWMaLFkxdR2ne8DtHGuXoRs8G/hOLlYZGBMM/JkV2VcuX9F61vspQW2bDrNgAzMEHknYKeTamvt5HV+zYly7X/OnBMAdbDl3ZOjP5h7zYjdeDza+t6dG1+iJKfCjz5dekT20NDVlwJvIMGhqxNOxDRfnlHopoTT/16/Jd7SPDOY2gpDoZxA8uPGZJR52y6T8tZqrR/tamoXRZk5WR4zeIW2rFK85hZwmz59dWQywlhzcIxMHhBcH6r6O3p1NwFA1LPIzgEQHKig/5DxFk52reU70fA9L2nZ1AHSES2RtG9kDQZLx7gbnOaOGx7Jbd6mECFMzliEBKiSnYxN404bNVAoTBEwcuxTho9zJRccQVeIdTD617K4sAqPWWhFwk=" + branch: + - master + on: + tags: true + file_glob: true + file: dist/* + cleanup: false + draft: true From 41abb3d730320d6e520c2cdbca441eeea87943d2 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 17:42:12 -0700 Subject: [PATCH 130/424] Continuing to try to fix code coverage --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 867636fbf..a6994b2af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,9 +11,9 @@ before_install: - sudo apt-get install -y gfortran - pip uninstall -y numpy -install: pip install -r requirements.txt +install: pip install -r requirements.txt coverage coveralls -script: python3 manage.py test +script: coverage run manage.py test after_success: coveralls From 1ea23d1228606962611dd69458bc0d6ec2b28e44 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 19 May 2020 18:01:51 -0700 Subject: [PATCH 131/424] skipping tests for Github release job --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index a6994b2af..c24c5be9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,6 +53,7 @@ jobs: - stage: "Deploy Development" python: 3.8 + script: skip deploy: provider: releases api_key: @@ -87,6 +88,7 @@ jobs: - stage: "Deploy Master" python: 3.8 + script: skip deploy: provider: releases api_key: From f2d27f7ad4c013b44d1766456893e5cc06cd4521 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 20 May 2020 14:31:20 -0700 Subject: [PATCH 132/424] Fixing a codacy issue in order to test travis --- tom_publications/tests.py | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 tom_publications/tests.py diff --git a/tom_publications/tests.py b/tom_publications/tests.py deleted file mode 100644 index 7ce503c2d..000000000 --- a/tom_publications/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. From 210f247dca5ae1d29eb49e30a83aef8e0183db64 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 20 May 2020 20:40:30 -0700 Subject: [PATCH 133/424] skipping github releases code if not the correct branch --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index c24c5be9b..93d85447b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,6 +52,7 @@ jobs: distributions: "sdist bdist_wheel" - stage: "Deploy Development" + if: branch = development python: 3.8 script: skip deploy: @@ -87,6 +88,7 @@ jobs: distributions: "sdist bdist_wheel" - stage: "Deploy Master" + if: branch = master python: 3.8 script: skip deploy: From db8be64be5af4c29d85712e27a5abc3be4612418 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 20 May 2020 21:07:18 -0700 Subject: [PATCH 134/424] Updating conditional stages --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 93d85447b..0a0aab2f9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ jobs: # Deploy alpha releases to PyPi and generate draft release on Github Releases (incomplete) - stage: "Deploy Development" - if: branch = development + if: (tag IS present) AND (branch = development) script: skip deploy: provider: pypi @@ -52,7 +52,7 @@ jobs: distributions: "sdist bdist_wheel" - stage: "Deploy Development" - if: branch = development + if: (tag IS present) AND (branch = development) python: 3.8 script: skip deploy: @@ -71,7 +71,7 @@ jobs: # Deploy full releases to PyPi and generate draft release on Github Releases (incomplete) - stage: "Deploy Master" - if: branch = master + if: (tag IS present) AND (branch = master) python: 3.8 script: skip deploy: @@ -88,7 +88,7 @@ jobs: distributions: "sdist bdist_wheel" - stage: "Deploy Master" - if: branch = master + if: (tag IS present) AND (branch = master) python: 3.8 script: skip deploy: From 6ff715347728938e92f3526c6f096d74aa4fa046 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 21 May 2020 08:16:17 -0700 Subject: [PATCH 135/424] Fixing branch conditional syntax --- .travis.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0a0aab2f9..0f196dd56 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,11 +57,10 @@ jobs: script: skip deploy: provider: releases - api_key: + token: secure: "V5t/Rk0jK6ggR6Oc2sk6Kr8+mC4vcKD4Vh1/CVhFoB+/QJTJR4wR1ZzdmxSTHVqyYRWMhtbQg4GEenVu3CCE6Aqcpb8FuXagBGDjrydYMvBz0QV0tGpc0QfAGFgihoMxRXYYy8x0qJoImpIgweIcBP6qCIbWlHNn7uSbJgQAcpHI1KP5//SJbiZ5+h1tYa3AImfhuRIjtw5X5XCkbNe04DpJZZrR7et4BnZwjBKmkPBHAoDhfuGzA5PF1S6jHCpWIddhRQ60v6T7yp8jcBEn5yvQSmusfUju61GI0+irFV8LksaWMaLFkxdR2ne8DtHGuXoRs8G/hOLlYZGBMM/JkV2VcuX9F61vspQW2bDrNgAzMEHknYKeTamvt5HV+zYly7X/OnBMAdbDl3ZOjP5h7zYjdeDza+t6dG1+iJKfCjz5dekT20NDVlwJvIMGhqxNOxDRfnlHopoTT/16/Jd7SPDOY2gpDoZxA8uPGZJR52y6T8tZqrR/tamoXRZk5WR4zeIW2rFK85hZwmz59dWQywlhzcIxMHhBcH6r6O3p1NwFA1LPIzgEQHKig/5DxFk52reU70fA9L2nZ1AHSES2RtG9kDQZLx7gbnOaOGx7Jbd6mECFMzliEBKiSnYxN404bNVAoTBEwcuxTho9zJRccQVeIdTD617K4sAqPWWhFwk=" - branch: - - development on: + branch: development tags: true file_glob: true file: dist/* @@ -93,11 +92,10 @@ jobs: script: skip deploy: provider: releases - api_key: + token: secure: "V5t/Rk0jK6ggR6Oc2sk6Kr8+mC4vcKD4Vh1/CVhFoB+/QJTJR4wR1ZzdmxSTHVqyYRWMhtbQg4GEenVu3CCE6Aqcpb8FuXagBGDjrydYMvBz0QV0tGpc0QfAGFgihoMxRXYYy8x0qJoImpIgweIcBP6qCIbWlHNn7uSbJgQAcpHI1KP5//SJbiZ5+h1tYa3AImfhuRIjtw5X5XCkbNe04DpJZZrR7et4BnZwjBKmkPBHAoDhfuGzA5PF1S6jHCpWIddhRQ60v6T7yp8jcBEn5yvQSmusfUju61GI0+irFV8LksaWMaLFkxdR2ne8DtHGuXoRs8G/hOLlYZGBMM/JkV2VcuX9F61vspQW2bDrNgAzMEHknYKeTamvt5HV+zYly7X/OnBMAdbDl3ZOjP5h7zYjdeDza+t6dG1+iJKfCjz5dekT20NDVlwJvIMGhqxNOxDRfnlHopoTT/16/Jd7SPDOY2gpDoZxA8uPGZJR52y6T8tZqrR/tamoXRZk5WR4zeIW2rFK85hZwmz59dWQywlhzcIxMHhBcH6r6O3p1NwFA1LPIzgEQHKig/5DxFk52reU70fA9L2nZ1AHSES2RtG9kDQZLx7gbnOaOGx7Jbd6mECFMzliEBKiSnYxN404bNVAoTBEwcuxTho9zJRccQVeIdTD617K4sAqPWWhFwk=" - branch: - - master on: + branch: development tags: true file_glob: true file: dist/* From 2867b2a34f723cdf25ead639e42b1d0ef3560d2a Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 21 May 2020 08:24:26 -0700 Subject: [PATCH 136/424] removing branch condition on deployment, leaving tag condition --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0f196dd56..1dd125000 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ jobs: # Deploy alpha releases to PyPi and generate draft release on Github Releases (incomplete) - stage: "Deploy Development" - if: (tag IS present) AND (branch = development) + if: tag IS present script: skip deploy: provider: pypi @@ -52,7 +52,7 @@ jobs: distributions: "sdist bdist_wheel" - stage: "Deploy Development" - if: (tag IS present) AND (branch = development) + if: tag IS present python: 3.8 script: skip deploy: @@ -70,7 +70,7 @@ jobs: # Deploy full releases to PyPi and generate draft release on Github Releases (incomplete) - stage: "Deploy Master" - if: (tag IS present) AND (branch = master) + if: tag IS present python: 3.8 script: skip deploy: @@ -87,7 +87,7 @@ jobs: distributions: "sdist bdist_wheel" - stage: "Deploy Master" - if: (tag IS present) AND (branch = master) + if: tag IS present python: 3.8 script: skip deploy: From 9b57d8296f02ef4990e085d42513f1635cebd6b0 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 25 May 2020 15:20:29 -0700 Subject: [PATCH 137/424] Made LCO observing strategy permissions-friendly --- tom_observations/facilities/lco.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 9870734f9..e9167b770 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -420,6 +420,8 @@ def _build_instrument_config(self): class LCOObservingStrategyForm(GenericStrategyForm, LCOBaseForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + if self.fields.get('groups'): + self.fields.pop('groups') for field in self.fields: if field != 'strategy_name': self.fields[field].required = False From 8fc0fc50e546310ba62c618056d815b454a571e3 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 25 May 2020 15:26:24 -0700 Subject: [PATCH 138/424] Updating link to Travis build for .com migration --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1e5f581f5..e7ef6aee6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # TOM Toolkit -[![Build Status](https://travis-ci.org/TOMToolkit/tom_base.svg?branch=master)](https://travis-ci.org/TOMToolkit/tom_base) +[![Build Status](https://travis-ci.com/TOMToolkit/tom_base.svg?branch=master)](https://travis-ci.org/TOMToolkit/tom_base) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/9846cee7c4904cae8864525101030169)](https://www.codacy.com/gh/observatorycontrolsystem/observation-portal?utm_source=github.com&utm_medium=referral&utm_content=observatorycontrolsystem/observation-portal&utm_campaign=Badge_Grade) [![Coverage Status](https://coveralls.io/repos/github/TOMToolkit/tom_base/badge.svg?branch=master)](https://coveralls.io/github/TOMToolkit/tom_base?branch=master) [![Documentation Status](https://readthedocs.org/projects/tom-toolkit/badge/?version=stable)](https://tom-toolkit.readthedocs.io/en/stable/?badge=stable) From f31d1b4b41478d417dbae61400b13bbea79c5956 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 25 May 2020 16:43:31 -0700 Subject: [PATCH 139/424] cleaned up observing strategy code --- tom_observations/facilities/lco.py | 4 ++-- tom_observations/observing_strategy.py | 1 + tom_observations/templatetags/observation_extras.py | 6 ++++-- tom_targets/views.py | 2 -- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index e9167b770..4295e3519 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -420,8 +420,8 @@ def _build_instrument_config(self): class LCOObservingStrategyForm(GenericStrategyForm, LCOBaseForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if self.fields.get('groups'): - self.fields.pop('groups') + for field_name in ['groups', 'target_id']: + self.fields.pop(field_name, None) for field in self.fields: if field != 'strategy_name': self.fields[field].required = False diff --git a/tom_observations/observing_strategy.py b/tom_observations/observing_strategy.py index b8d1344f5..81d4267c4 100644 --- a/tom_observations/observing_strategy.py +++ b/tom_observations/observing_strategy.py @@ -46,6 +46,7 @@ class RunStrategyForm(forms.Form): observing_strategy = forms.ModelChoiceField(queryset=ObservingStrategy.objects.all()) cadence_strategy = forms.ChoiceField( choices=[('', '')] + [(k, k) for k in get_cadence_strategies().keys()], + required=False ) cadence_frequency = forms.IntegerField( required=False, diff --git a/tom_observations/templatetags/observation_extras.py b/tom_observations/templatetags/observation_extras.py index 1986a1557..ae71377dc 100644 --- a/tom_observations/templatetags/observation_extras.py +++ b/tom_observations/templatetags/observation_extras.py @@ -137,10 +137,12 @@ def observingstrategy_from_record(obsr): Renders a button that will pre-populate and observing strategy form with parameters from the specified ``ObservationRecord``. """ - params = urlencode(obsr.parameters_as_dict) + obs_params = obsr.parameters_as_dict + obs_params.pop('target_id') + strategy_params = urlencode(obs_params) return { 'facility': obsr.facility, - 'params': params + 'params': strategy_params } diff --git a/tom_targets/views.py b/tom_targets/views.py index ed53f88b9..5cf5b184c 100644 --- a/tom_targets/views.py +++ b/tom_targets/views.py @@ -358,9 +358,7 @@ def get(self, request, *args, **kwargs): run_strategy_form = RunStrategyForm(request.GET) if run_strategy_form.is_valid(): obs_strat = ObservingStrategy.objects.get(pk=run_strategy_form.cleaned_data['observing_strategy'].id) - target_id = kwargs.get('pk', None) params = urlencode(obs_strat.parameters_as_dict) - params += urlencode(request.GET) return redirect( reverse('tom_observations:create', args=(obs_strat.facility,)) + f'?target_id={self.get_object().id}&' + params) From be5bf6011cdc4ce7ddc1d20d999f63595534f055 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 25 May 2020 17:06:18 -0700 Subject: [PATCH 140/424] Fixed bug in observing strategy button templatetag --- tom_observations/templatetags/observation_extras.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_observations/templatetags/observation_extras.py b/tom_observations/templatetags/observation_extras.py index ae71377dc..e9b7ceaae 100644 --- a/tom_observations/templatetags/observation_extras.py +++ b/tom_observations/templatetags/observation_extras.py @@ -138,7 +138,7 @@ def observingstrategy_from_record(obsr): ``ObservationRecord``. """ obs_params = obsr.parameters_as_dict - obs_params.pop('target_id') + obs_params.pop('target_id', None) strategy_params = urlencode(obs_params) return { 'facility': obsr.facility, From ebdc332470716cda6d1ff50f1931462587be099a Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Mon, 1 Jun 2020 19:28:18 +0000 Subject: [PATCH 141/424] define abstract method for Facility status --- tom_observations/facility.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tom_observations/facility.py b/tom_observations/facility.py index 8a9ee4884..698ad93b2 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -269,10 +269,24 @@ def get_terminal_observing_states(self): @abstractmethod def get_observing_sites(self): """ - Return a list of dictionaries that contain the information + Return an iterable of dictionaries that contain the information necessary to be used in the planning (visibility) tool. The - list should contain dictionaries each that contain sitecode, - latitude, longitude and elevation. + iterable should contain dictionaries each that contain sitecode, + latitude, longitude and elevation. This is the static information + about a site. + """ + pass + + @abstractmethod + def get_facility_status(self): + """ + Return a dictionary hierarchy of the form: + + facility_dict = {'code': 'XYZ', 'sites': [ site_dict, ... ]} + site_dict = {'code': 'XYZ', 'telescopes': [ telescope_dict, ... ]} + telescope_dict = {'code': 'XYZ', 'status': 'AVAILABILITY'} + + See lco.py for a concrete implementation example """ pass From e6ea4980f03c7ae08ff14b1c488d88d231f69aaa Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Mon, 1 Jun 2020 19:30:25 +0000 Subject: [PATCH 142/424] provide stub implementations of get_facility_status --- tom_observations/facilities/gemini.py | 3 +++ tom_observations/facilities/lt.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/tom_observations/facilities/gemini.py b/tom_observations/facilities/gemini.py index fe776841d..b94895fec 100644 --- a/tom_observations/facilities/gemini.py +++ b/tom_observations/facilities/gemini.py @@ -458,6 +458,9 @@ def validate_observation(clz, observation_payload): return errors + def get_facility_status(self): + return {} + @classmethod def get_observation_url(clz, observation_id): # return PORTAL_URL + '/requests/' + observation_id diff --git a/tom_observations/facilities/lt.py b/tom_observations/facilities/lt.py index 328b8cb59..32e4dba68 100644 --- a/tom_observations/facilities/lt.py +++ b/tom_observations/facilities/lt.py @@ -32,6 +32,9 @@ def submit_observation(self, observation_payload): def validate_observation(self, observation_payload): return + def get_facility_status(self): + return {} + def get_observation_url(self, observation_id): return From c22c607f09d189c8e1d85fc417b23f9c289603ac Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Mon, 1 Jun 2020 19:31:04 +0000 Subject: [PATCH 143/424] reference implementation of get_facility_status this implementation is LCO specific in that it uses the observe.lco.global/api/telescope_states/ endpoint for raw status data. Other implementations will be Facility specific in how the status data is obtained, but should return the same data struture as documented in the doc string. --- tom_observations/facilities/lco.py | 87 ++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 9870734f9..3892c92fa 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -564,6 +564,93 @@ def get_failed_observing_states(self): def get_observing_sites(self): return self.SITES + def get_facility_status(self): + """Get the telescope_states from the LCO API endpoint and simply + transform the returned JSON into the following dictionary hierarchy + for use by the facility_status.html template partial. + + facility_dict = {'code': 'LCO', 'sites': [ site_dict, ... ]} + site_dict = {'code': 'XYZ', 'telescopes': [ telescope_dict, ... ]} + telescope_dict = {'code': 'XYZ', 'status': 'AVAILABILITY'} + + Here's an example of the returned dictionary + ``` + literal_facility_status_example = { + 'code': 'LCO', + 'sites': [ + { + 'code': 'BPL', + 'telescopes': [ + { + 'code': 'bpl.doma.1m0a', + 'status': 'AVAILABLE' + }, + ], + }, + { + 'code': 'ELP', + 'telescopes': [ + { + 'code': 'elp.doma.1m0a', + 'status': 'AVAILABLE' + }, + { + 'code': 'elp.domb.1m0a', + 'status': 'AVAILABLE' + }, + ] + } + ] + } + ``` + + :return: facility_dict + """ + + # make the request to the LCO API for the telescope_states + response = make_request( + 'GET', + PORTAL_URL + '/api/telescope_states/', + headers=self._portal_headers() + ) + telescope_states = response.json() + + # Now, transform the telescopes_state dictionary in a dictionary suitable + # for the facility_status.html template partial. + + # set up the return value to be populated by the for loop below + facility_status = { + 'code': 'LCO', + 'sites': [] + } + + for telescope_key, telescope_value in telescope_states.items(): + [site_code, enclosure_code, telescope_code] = telescope_key.split('.') + + # extract this telescope and it's status from the response + telescope = { + 'code': telescope_code, + 'status': telescope_value[0]['event_type'] + } + + # get the site dictionary from the facilities list of sites + # filter by site_code and provide a default (None) for new sites + site = next((site_ix for site_ix in facility_status['sites'] + if site_ix['code'] == site_code), None) + # create the site if it's new and not yet in the facility_status['sites] list + if site is None: + new_site = { + 'code': site_code, + 'telescopes': [] + } + facility_status['sites'].append(new_site) + site = new_site + + # Now, add the telescope to the site's telescopes + site['telescopes'].append(telescope) + + return facility_status + def get_observation_status(self, observation_id): response = make_request( 'GET', From 39dd11ce681a81843187f1a3b399fef6160b6cac Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Mon, 1 Jun 2020 19:34:22 +0000 Subject: [PATCH 144/424] partial template and inclusion_tag for facility_status --- .../partials/facility_status.html | 34 +++++++++++++++++++ .../templatetags/observation_extras.py | 17 ++++++++++ 2 files changed, 51 insertions(+) create mode 100644 tom_observations/templates/tom_observations/partials/facility_status.html diff --git a/tom_observations/templates/tom_observations/partials/facility_status.html b/tom_observations/templates/tom_observations/partials/facility_status.html new file mode 100644 index 000000000..d06f50832 --- /dev/null +++ b/tom_observations/templates/tom_observations/partials/facility_status.html @@ -0,0 +1,34 @@ +{% load tom_common_extras %} +
+
+ Facility Status +
+ + + + + + + + + + + {% for facility in facilities %} + {% for site in facility.sites %} + {% for telescope in site.telescopes %} + + + + + + + {% endfor %} + {% endfor %} + {% empty %} + + + + {% endfor %} + +
FacilitySiteTelescopeStatus
{{ facility.code }}{{ site.code }}{{ telescope.code }}{{ telescope.status }}
Facility status unknown.
+
\ No newline at end of file diff --git a/tom_observations/templatetags/observation_extras.py b/tom_observations/templatetags/observation_extras.py index f3bbe5013..127585da4 100644 --- a/tom_observations/templatetags/observation_extras.py +++ b/tom_observations/templatetags/observation_extras.py @@ -232,3 +232,20 @@ def observation_distribution(observations): } figure = offline.plot(go.Figure(data=data, layout=layout), output_type='div', show_link=False) return {'figure': figure} + + +@register.inclusion_tag('tom_observations/partials/facility_status.html') +def facility_status(): + """ + Collect the facility status from the registered facilities and pass them + to the facility_status.html partial template. + See lco.py Facility implementation for example. + :return: + """ + + facility_statuses = [] + for facility_code, facility_class in get_service_classes().items(): + status = facility_class().get_facility_status() + facility_statuses.append(status) + + return {'facilities': facility_statuses} From 1c8b42db9e84f27266830a47c6e2804b5ba28193 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Mon, 1 Jun 2020 19:35:00 +0000 Subject: [PATCH 145/424] add facility_status as a tab to the target_detail template --- tom_targets/templates/tom_targets/target_detail.html | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index 057754413..569fc95d6 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -44,6 +44,9 @@
  • Spectroscopy
    • +
  • + Facility Status +
    • @@ -81,6 +84,10 @@

    Observations

    {% spectroscopy_for_target target %}
    +
    + {% facility_status %} +
    + {% comments_enabled as comments_are_enabled %}
    Comments
    From 6c7ed46170b30a3b30db396c377725425e99f5d4 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Mon, 1 Jun 2020 22:38:34 +0000 Subject: [PATCH 146/424] improve doc string --- tom_observations/facility.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tom_observations/facility.py b/tom_observations/facility.py index 698ad93b2..6beea8628 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -280,15 +280,23 @@ def get_observing_sites(self): @abstractmethod def get_facility_status(self): """ - Return a dictionary hierarchy of the form: + Returns a dictionary describing the current availability of the Facility + telescopes. This is intended to be useful in observation planning. + The top-level (Facility) dictionary has a list of sites. Each site + is represented by a site dictionary which has a list of telescopes. + Each telescope has an identifier (code) and an status string. - facility_dict = {'code': 'XYZ', 'sites': [ site_dict, ... ]} - site_dict = {'code': 'XYZ', 'telescopes': [ telescope_dict, ... ]} - telescope_dict = {'code': 'XYZ', 'status': 'AVAILABILITY'} + The dictionary hierarchy is of the form: - See lco.py for a concrete implementation example + `facility_dict = {'code': 'XYZ', 'sites': [ site_dict, ... ]}` + where + `site_dict = {'code': 'XYZ', 'telescopes': [ telescope_dict, ... ]}` + where + `telescope_dict = {'code': 'XYZ', 'status': 'AVAILABILITY'}` + + See lco.py for a concrete implementation example. """ - pass + return {} @abstractmethod def get_observation_url(self, observation_id): From aa1fe86f325fa64927db11ea252184407bc3f357 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Mon, 1 Jun 2020 22:39:45 +0000 Subject: [PATCH 147/424] get_facility_status is not a required abstract method. It's optional --- tom_observations/facilities/gemini.py | 3 --- tom_observations/facilities/lt.py | 3 --- tom_observations/facility.py | 1 - 3 files changed, 7 deletions(-) diff --git a/tom_observations/facilities/gemini.py b/tom_observations/facilities/gemini.py index b94895fec..fe776841d 100644 --- a/tom_observations/facilities/gemini.py +++ b/tom_observations/facilities/gemini.py @@ -458,9 +458,6 @@ def validate_observation(clz, observation_payload): return errors - def get_facility_status(self): - return {} - @classmethod def get_observation_url(clz, observation_id): # return PORTAL_URL + '/requests/' + observation_id diff --git a/tom_observations/facilities/lt.py b/tom_observations/facilities/lt.py index 32e4dba68..328b8cb59 100644 --- a/tom_observations/facilities/lt.py +++ b/tom_observations/facilities/lt.py @@ -32,9 +32,6 @@ def submit_observation(self, observation_payload): def validate_observation(self, observation_payload): return - def get_facility_status(self): - return {} - def get_observation_url(self, observation_id): return diff --git a/tom_observations/facility.py b/tom_observations/facility.py index 6beea8628..1c7bd087a 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -277,7 +277,6 @@ def get_observing_sites(self): """ pass - @abstractmethod def get_facility_status(self): """ Returns a dictionary describing the current availability of the Facility From 0ca91105fd10b145c984f407039e8f131cd31e84 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Mon, 1 Jun 2020 22:41:26 +0000 Subject: [PATCH 148/424] the _key (vs. _code) has the useful enclosure info --- tom_observations/facilities/lco.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index b08a4c3b8..7b4859dd7 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -631,7 +631,7 @@ def get_facility_status(self): # extract this telescope and it's status from the response telescope = { - 'code': telescope_code, + 'code': telescope_key, 'status': telescope_value[0]['event_type'] } From 1d652775dcd9896c3c4dca59e03657f2746aae48 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 2 Jun 2020 00:47:59 +0000 Subject: [PATCH 149/424] define and document get_facility_weather_urls --- tom_observations/facility.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tom_observations/facility.py b/tom_observations/facility.py index 1c7bd087a..b0f39eed2 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -277,6 +277,19 @@ def get_observing_sites(self): """ pass + def get_facility_weather_urls(self): + """ + Returns a dictionary containing a URL for weather information + for each site in the Facility SITES. This is intended to be useful + in observation planning. + + `facility_weather = {'code': 'XYZ', 'sites': [ site_dict, ... ]}` + where + `site_dict = {'code': 'XYZ', 'weather_url': 'http://path/to/weather'}` + + """ + return {} + def get_facility_status(self): """ Returns a dictionary describing the current availability of the Facility From dcc2ea525bf62d659d70b545a16f0ce4c1115986 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 2 Jun 2020 00:48:33 +0000 Subject: [PATCH 150/424] LCO implementation of get_weather_facility_urls --- tom_observations/facilities/lco.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 7b4859dd7..61a22f053 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -453,6 +453,7 @@ class LCOFacility(BaseRoboticObservationFacility): # planning tool. All entries should contain latitude, longitude, elevation # and a code. # TODO: Flip sitecode and site name + # TODO: Why is tlv not represented here? SITES = { 'Siding Spring': { 'sitecode': 'coj', @@ -566,6 +567,25 @@ def get_failed_observing_states(self): def get_observing_sites(self): return self.SITES + def get_facility_weather_urls(self): + """ + `facility_weather_urls = {'code': 'XYZ', 'sites': [ site_dict, ... ]}` + where + `site_dict = {'code': 'XYZ', 'weather_url': 'http://path/to/weather'}` + """ + # TODO: manually add a weather url for tlv + facility_weather_urls = { + 'code': 'LCO', + 'sites': [ + { + 'code': site['sitecode'], + 'weather_url': f'https://weather.lco.global/#/{site["sitecode"]}' + } + for site in self.SITES.values()] + } + + return facility_weather_urls + def get_facility_status(self): """Get the telescope_states from the LCO API endpoint and simply transform the returned JSON into the following dictionary hierarchy From 4abb85795d8b5356c868412421b9ef3c35012a77 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 2 Jun 2020 00:49:05 +0000 Subject: [PATCH 151/424] plumb weather_urls through inclusion_tab to partial template --- .../tom_observations/partials/facility_status.html | 2 ++ tom_observations/templatetags/observation_extras.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tom_observations/templates/tom_observations/partials/facility_status.html b/tom_observations/templates/tom_observations/partials/facility_status.html index d06f50832..0bba6052c 100644 --- a/tom_observations/templates/tom_observations/partials/facility_status.html +++ b/tom_observations/templates/tom_observations/partials/facility_status.html @@ -10,6 +10,7 @@ Site Telescope Status + Weather URL @@ -21,6 +22,7 @@ {{ site.code }} {{ telescope.code }} {{ telescope.status }} + link {% endfor %} {% endfor %} diff --git a/tom_observations/templatetags/observation_extras.py b/tom_observations/templatetags/observation_extras.py index 132833b36..348f7ff61 100644 --- a/tom_observations/templatetags/observation_extras.py +++ b/tom_observations/templatetags/observation_extras.py @@ -247,7 +247,17 @@ def facility_status(): facility_statuses = [] for facility_code, facility_class in get_service_classes().items(): - status = facility_class().get_facility_status() + facility = facility_class() + weather_urls = facility.get_facility_weather_urls() + status = facility.get_facility_status() + + # add the weather_url to the site dictionary + for site in status.get('sites', []): + url = next((site_url['weather_url'] for site_url in weather_urls.get('sites', []) + if site_url['code'] == site['code']), None) + if url is not None: + site['weather_url'] = url + facility_statuses.append(status) return {'facilities': facility_statuses} From e45736ced028976b899e13dac4101ddc6a1a8325 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 2 Jun 2020 01:27:21 +0000 Subject: [PATCH 152/424] clean up codacy complaints --- tom_observations/facilities/lco.py | 4 ++-- tom_observations/templatetags/observation_extras.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 61a22f053..c7368792c 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -586,6 +586,7 @@ def get_facility_weather_urls(self): return facility_weather_urls + @staticmethod def get_facility_status(self): """Get the telescope_states from the LCO API endpoint and simply transform the returned JSON into the following dictionary hierarchy @@ -628,7 +629,6 @@ def get_facility_status(self): :return: facility_dict """ - # make the request to the LCO API for the telescope_states response = make_request( 'GET', @@ -647,7 +647,7 @@ def get_facility_status(self): } for telescope_key, telescope_value in telescope_states.items(): - [site_code, enclosure_code, telescope_code] = telescope_key.split('.') + [site_code, _, telescope_code] = telescope_key.split('.') # extract this telescope and it's status from the response telescope = { diff --git a/tom_observations/templatetags/observation_extras.py b/tom_observations/templatetags/observation_extras.py index 348f7ff61..0237d9fa2 100644 --- a/tom_observations/templatetags/observation_extras.py +++ b/tom_observations/templatetags/observation_extras.py @@ -246,7 +246,7 @@ def facility_status(): """ facility_statuses = [] - for facility_code, facility_class in get_service_classes().items(): + for _, facility_class in get_service_classes().items(): facility = facility_class() weather_urls = facility.get_facility_weather_urls() status = facility.get_facility_status() From c671643869be2531538b34131999f5fad44dae9f Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 2 Jun 2020 01:50:49 +0000 Subject: [PATCH 153/424] @staticmethod or self: choose one only --- tom_observations/facilities/lco.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index c7368792c..8e9c1650c 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -586,7 +586,6 @@ def get_facility_weather_urls(self): return facility_weather_urls - @staticmethod def get_facility_status(self): """Get the telescope_states from the LCO API endpoint and simply transform the returned JSON into the following dictionary hierarchy From 143c02af7a6679f80551b0f63c0c558efd411bbd Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 2 Jun 2020 02:05:52 +0000 Subject: [PATCH 154/424] more codacy nonsense --- tom_observations/facilities/lco.py | 10 +++++----- tom_observations/facility.py | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 8e9c1650c..e78cf8719 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -586,7 +586,8 @@ def get_facility_weather_urls(self): return facility_weather_urls - def get_facility_status(self): + @staticmethod + def get_facility_status(): """Get the telescope_states from the LCO API endpoint and simply transform the returned JSON into the following dictionary hierarchy for use by the facility_status.html template partial. @@ -595,8 +596,8 @@ def get_facility_status(self): site_dict = {'code': 'XYZ', 'telescopes': [ telescope_dict, ... ]} telescope_dict = {'code': 'XYZ', 'status': 'AVAILABILITY'} - Here's an example of the returned dictionary - ``` + Here's an example of the returned dictionary: + literal_facility_status_example = { 'code': 'LCO', 'sites': [ @@ -624,7 +625,6 @@ def get_facility_status(self): } ] } - ``` :return: facility_dict """ @@ -646,7 +646,7 @@ def get_facility_status(self): } for telescope_key, telescope_value in telescope_states.items(): - [site_code, _, telescope_code] = telescope_key.split('.') + [site_code, _, _] = telescope_key.split('.') # extract this telescope and it's status from the response telescope = { diff --git a/tom_observations/facility.py b/tom_observations/facility.py index b0f39eed2..1681dbf89 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -290,7 +290,8 @@ def get_facility_weather_urls(self): """ return {} - def get_facility_status(self): + @staticmethod + def get_facility_status(): """ Returns a dictionary describing the current availability of the Facility telescopes. This is intended to be useful in observation planning. From 9d5e5e7b2c8525b9549070c82426fd3efdd67b36 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 2 Jun 2020 02:17:43 +0000 Subject: [PATCH 155/424] revert last changes: codacy nonsense --- tom_observations/facilities/lco.py | 3 +-- tom_observations/facility.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index e78cf8719..b76b08546 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -586,8 +586,7 @@ def get_facility_weather_urls(self): return facility_weather_urls - @staticmethod - def get_facility_status(): + def get_facility_status(self): """Get the telescope_states from the LCO API endpoint and simply transform the returned JSON into the following dictionary hierarchy for use by the facility_status.html template partial. diff --git a/tom_observations/facility.py b/tom_observations/facility.py index 1681dbf89..b0f39eed2 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -290,8 +290,7 @@ def get_facility_weather_urls(self): """ return {} - @staticmethod - def get_facility_status(): + def get_facility_status(self): """ Returns a dictionary describing the current availability of the Facility telescopes. This is intended to be useful in observation planning. From 034fd804fa2ad817ba55823f46b37a883b744504 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Fri, 5 Jun 2020 19:55:31 +0000 Subject: [PATCH 156/424] first draft of some deployment workflow instructions --- README-dev.md | 47 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/README-dev.md b/README-dev.md index 81b6e60df..98ed0726d 100644 --- a/README-dev.md +++ b/README-dev.md @@ -1,9 +1,3 @@ -# TOM Toolkit -[![Build Status](https://travis-ci.org/TOMToolkit/tom_base.svg?branch=master)](https://travis-ci.org/TOMToolkit/tom_base) -[Documentation](https://tom-toolkit.readthedocs.io/en/latest/) - -![logo](tom_common/static/tom_common/img/logo-color.png) - This README-dev is intended for maintainers of the repository for information on releases, standards, and anything that isn't pertinent to the wider community. @@ -27,3 +21,44 @@ deployed unless they match the validation regex. The version format is as follow | | x.y.z-alpha.w | x.y.z | Following deployment of a release, a Github Release is created, and this should be filled in with the relevant release notes. + +## Deployment Workflow + _**This section of this document is a work-in-progress**_ +#### Pre-release deployment +* _meet pre-deployment criteria documented [here]()_. +* merge to `development` +* `git tag -a x.y.z-alpha.w -m "x.y.z-aplha.w"` +* `git push --tags` +* This causes Travis to create a draft release in GitHub and push to PyPI +* Edit the release notes in GitHub; Update, edit; repeat until satisfied. Release notes should contain: + * Links to Read the Docs API (docstring) docs + * Links to Read the Docs higher level docs + * Link to Tom Demo feature demonstration + * what else? + + For example: TODO: _insert example here_ +* When satisfied, `Publish Release` Repo watchers are notified by email. +* deploy `tom-demo-dev` with new features demonstrated, pulling `tom_base-x.y.z-alpha.w` from PyPI + + +#### Public release deployment + +* Create PR: `master <- development` +* Merge PR +* `git tag -a x.y.z -m "Release x.y.z"` +* `git push --tags` Triggers Travis to: + * build, build + * push release to PyPI + * create GitHub draft release +* Update Release Notes in GitHub draft release. (This should be the accumulation of the all + the development-release release notes: For example, release notes for releases x.y.z-alpha.1, + x.y.z-alpha.2, etc. should be combined into release notes for release x.y.z. +* Publish Release +* Post notification to Slack, Tom Toolkit workspace, #general channel. (In the future, we hope to +have automated release notification to a dedicated #releases slack channel). + +### Preview Read the Docs doc strings +* `cd /path/to/tom_base/docs` +* `pip install -r requirements.txt # make sure sphinx is installed to your venv` +* `make html # make clean first, if things are weird` +* point a browser to the html files in `./_build/html/` to proof read before deployment \ No newline at end of file From 91aff66afc3e49533245ae94c19c2c6c160a13b9 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 5 Jun 2020 15:24:30 -0700 Subject: [PATCH 157/424] Updated Django version to address <3.0.7 security vulnerability --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 39bdb2981..e22994cea 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ use_scm_version=True, setup_requires=['setuptools_scm', 'wheel'], install_requires=[ - 'django>=2.2', # TOM Toolkit requires db math functions + 'django>=3.0.7', # TOM Toolkit requires db math functions 'django-bootstrap4==1.1.1', 'django-extensions==2.2.9', 'django-filter==2.2.0', From ddd77abfc7b04451d4ad38497fe700b3c829d514 Mon Sep 17 00:00:00 2001 From: Bryan Miller Date: Fri, 5 Jun 2020 22:51:49 -0400 Subject: [PATCH 158/424] Add support for new noteTitle API field. Form reorganization. --- tom_observations/facilities/gemini.py | 42 +++++++++++++++++---------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/tom_observations/facilities/gemini.py b/tom_observations/facilities/gemini.py index fe776841d..e74ca795d 100644 --- a/tom_observations/facilities/gemini.py +++ b/tom_observations/facilities/gemini.py @@ -140,6 +140,7 @@ class GEMObservationForm(BaseRoboticObservationForm): ra - target RA [J2000], format 'HH:MM:SS.SS' dec - target Dec[J2000], format 'DD:MM:SS.SSS' mags - target magnitude information (optional) + noteTitle - title for the note, "Finding Chart" if not provided (optional) note - text to include in a "Finding Chart" note (optional) posangle - position angle [degrees E of N], defaults to 0 (optional) exptime - exposure time [seconds], if not given then value in template used (optional) @@ -201,12 +202,14 @@ class GEMObservationForm(BaseRoboticObservationForm): # Form fields obsid = forms.MultipleChoiceField(choices=obs_choices()) ready = forms.ChoiceField(initial='true', choices=(('true', 'Yes'), ('false', 'No'))) - brightness = forms.FloatField(required=False, label='Target brightness') + brightness = forms.FloatField(required=False, label='Target Brightness') brightness_system = forms.ChoiceField(required=False, initial='AB', + label='Brightness System', choices=(('Vega', 'Vega'), ('AB', 'AB'), ('Jy', 'Jy'))) brightness_band = forms.ChoiceField(required=False, initial='r', + label='Brightness Band', choices=(('u', 'u'), ('U', 'U'), ('B', 'B'), ('g', 'g'), ('V', 'V'), ('UC', 'UC'), ('r', 'r'), ('R', 'R'), ('i', 'i'), ('I', 'I'), ('z', 'z'), ('Y', 'Y'), ('J', 'J'), ('H', 'H'), ('K', 'K'), @@ -215,12 +218,13 @@ class GEMObservationForm(BaseRoboticObservationForm): max_value=360., required=False, initial=0.0, - label='Position Angle in degrees [0-360]') + label='Position Angle [0-360]') - exptimes = forms.CharField(required=False, label='Exptime [sec]. If multiple, comma separate') + exptimes = forms.CharField(required=False, label='Exptime [s], comma separate') - group = forms.CharField(required=False) - note = forms.CharField(required=False) + group = forms.CharField(required=False, label='Group Name') + notetitle = forms.CharField(required=False, initial='Finding Chart', label='Note Title') + note = forms.CharField(required=False, label='Note Text') eltype = forms.ChoiceField(required=False, label='Airmass/Hour Angle Constraint', choices=(('none', 'None'), ('airmass', 'Airmass'), ('hourAngle', 'Hour Angle'))) @@ -250,12 +254,18 @@ class GEMObservationForm(BaseRoboticObservationForm): ('PWFS2', 'PWFS2'), ('AOWFS', 'AOWFS'))) # GS probe (PWFS1/PWFS2/OIWFS/AOWFS) window_start = forms.CharField(required=False, widget=forms.TextInput(attrs={'type': 'date'}), - label='UT Timing Window Start [Date Time]') + label='Timing Window [Date Time]') window_duration = forms.IntegerField(required=False, min_value=1, label='Timing Window Duration [hr]') def layout(self): return Div( HTML('Observation Parameters'), + HTML('

    Select the Obsids of one or more templates.
    ' + 'Setting Ready=No will keep the new observation(s) On Hold.
    ' + 'If a value is not set, then the template default is used.
    ' + 'If setting Exptime, then provide a list of values if selecting more than one Obsid.

    '), + HTML('

    '), + HTML('

    '), Div( Div( 'obsid', @@ -266,27 +276,28 @@ def layout(self): css_class='col' ), Div( - 'group', + 'notetitle', css_class='col' ), css_class='form-row' ), Div( Div( - 'posangle', 'brightness', 'eltype', 'window_start', + 'posangle', 'brightness', 'eltype', 'group', css_class='col' ), Div( - 'exptimes', 'brightness_band', 'elmin', 'window_duration', + 'exptimes', 'brightness_band', 'elmin', 'window_start', css_class='col' ), Div( - 'note', 'brightness_system', 'elmax', + 'note', 'brightness_system', 'elmax', 'window_duration', css_class='col' ), css_class='form-row' ), - HTML('Optional Guide Star Parameters: If any one of Name/RA/Dec is given, then all must be.'), + HTML('Optional Guide Star Parameters'), + HTML('

    If any one of Name/RA/Dec is given, then all must be.

    '), Div( Div( 'gstarg', 'gsbrightness', 'gsprobe', @@ -348,18 +359,19 @@ def isodatetime(value): obsnum = obs[ii+1:] payload = { "prog": progid, - # "password": self.cleaned_data['userkey'], "password": GEM_SETTINGS['api_key'][get_site(obs)], - # "email": self.cleaned_data['email'], "email": GEM_SETTINGS['user_email'], "obsnum": obsnum, "target": target.name, "ra": target.ra, "dec": target.dec, - "note": self.cleaned_data['note'], "ready": self.cleaned_data['ready'] } + if self.cleaned_data['notetitle'] != 'Finding Chart' or self.cleaned_data['note'] != '': + payload["noteTitle"] = self.cleaned_data['notetitle'] + payload["note"] = self.cleaned_data['note'] + if self.cleaned_data['brightness'] is not None: smags = str(self.cleaned_data['brightness']).strip() + '/' + \ self.cleaned_data['brightness_band'] + '/' + \ @@ -412,7 +424,7 @@ def isodatetime(value): class GEMFacility(BaseRoboticObservationFacility): """ The ``GEMFacility`` is the interface to the Gemini Telescope. For information regarding Gemini observing and the - available parameters, please see https://www.gemini.edu/sciops/observing-gemini. + available parameters, please see https://www.gemini.edu/observing/start-here """ name = 'GEM' From 3a8489cae2d2b43c27c23ccbda91f1f4df6617d5 Mon Sep 17 00:00:00 2001 From: Bryan Miller Date: Fri, 5 Jun 2020 23:07:34 -0400 Subject: [PATCH 159/424] Update HTML formatting --- tom_observations/facilities/gemini.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tom_observations/facilities/gemini.py b/tom_observations/facilities/gemini.py index e74ca795d..d90dd8c4c 100644 --- a/tom_observations/facilities/gemini.py +++ b/tom_observations/facilities/gemini.py @@ -260,12 +260,10 @@ class GEMObservationForm(BaseRoboticObservationForm): def layout(self): return Div( HTML('Observation Parameters'), - HTML('

    Select the Obsids of one or more templates.
    ' - 'Setting Ready=No will keep the new observation(s) On Hold.
    ' - 'If a value is not set, then the template default is used.
    ' - 'If setting Exptime, then provide a list of values if selecting more than one Obsid.

    '), - HTML('

    '), - HTML('

    '), + HTML('

    Select the Obsids of one or more templates.
    '), + HTML('Setting Ready=No will keep the new observation(s) On Hold.
    '), + HTML('If a value is not set, then the template default is used.
    '), + HTML('If setting Exptime, then provide a list of values if selecting more than one Obsid.

    '), Div( Div( 'obsid', From c8f08903b4ff7e4ca7411cd781e963f2c7854651 Mon Sep 17 00:00:00 2001 From: David Collom Date: Sat, 6 Jun 2020 20:56:37 -0700 Subject: [PATCH 160/424] WIP doc refactor --- docs/contributing.md | 4 ++++ docs/index.rst | 39 +++++++++++++++++++++++++-------------- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/docs/contributing.md b/docs/contributing.md index 2e9ce6d69..2e13a4984 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -67,3 +67,7 @@ We recommend that you use a linter, as all pull requests must pass a `pycodestyl ``` pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 ``` + +### Documentation + +We require any new features to diff --git a/docs/index.rst b/docs/index.rst index c3c6ebb65..fa7519be0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,26 +16,40 @@ Introduction The TOM (Target and Observation Manager) Toolkit project was started in early 2018 with the goal of simplifying the development of next generation software for the rapidly evolving field of astronomy. Read more :doc:`about TOMs` and the motivation for them. -Interested in seeing what a TOM can do? Take a look at our `demonstration TOM `_, where we show off the features of the TOM Toolkit. - -Are you looking to run a TOM of your own? This documentation is a good place to get started. The source code for the project is also available on Github. - -Start with the :doc:`introduction` if you are new to using the TOM Toolkit. - -If you'd like to know what we're working on, check out the `TOM Toolkit project board `_. - -:doc:`Architecture ` - This document describes the architecture of the TOM Toolkit at a +:doc:`TOM Toolkit Architecture ` - This document describes the architecture of the TOM Toolkit at a high level. Read this first if you're interested in how the TOM Toolkit works. -:doc:`Getting Started ` - First steps for getting a TOM up and running. +:doc:`Getting Started with the TOM Toolkit` - First steps for getting a TOM up and running. -:doc:`Workflow ` - The general workflow used with TOMs. +:doc:`TOM Workflow ` - The general workflow used with TOMs. :doc:`Programming Resources ` - Resources for learning the core components of the TOM Toolkit: HTML, CSS, Python, and Django :doc:`Frequently Asked Questions ` - Look here for a potential quick answer to a common question. + +Targets +------- + +:doc:`Adding Custom Target Fields ` - Learn how to add custom fields to your TOM Targets if the +defaults do not suffice. + +`Target API ` - Take a look at available properties of Targets and + + + + + + +Interested in seeing what a TOM can do? Take a look at our `demonstration TOM `_, where we show off the features of the TOM Toolkit. + +Are you looking to run a TOM of your own? This documentation is a good place to get started. The source code for the project is also available on Github. + +Start with the :doc:`introduction` if you are new to using the TOM Toolkit. + +If you'd like to know what we're working on, check out the `TOM Toolkit project board `_. + :doc:`Troubleshooting ` - Find solutions to common problems or information on how to debug an issue. Extending and Customizing @@ -55,9 +69,6 @@ the data you need. :doc:`Adding new Pages to your TOM ` - Learn how to add entirely new pages to your TOM, displaying static html pages or dynamic database-driven content. -:doc:`Adding Custom Target Fields ` - Learn how to add custom fields to your TOM Targets if the -defaults do not suffice. - :doc:`Adding Custom Data Processing ` - Learn how you can process data into your TOM from uploaded data products. From 1c9d189f69eaba257bfdad2ba367564e0f455d14 Mon Sep 17 00:00:00 2001 From: David Collom Date: Sat, 6 Jun 2020 20:57:09 -0700 Subject: [PATCH 161/424] Added tests for form and improvements to data cleaning --- setup.py | 1 + tom_alerts/brokers/antares.py | 9 ++++ tom_alerts/brokers/gaia.py | 92 +++++++++++++++++++++++++++------- tom_alerts/tests/tests_gaia.py | 82 ++++++++++++++++++++++++++++++ 4 files changed, 167 insertions(+), 17 deletions(-) create mode 100644 tom_alerts/tests/tests_gaia.py diff --git a/setup.py b/setup.py index 39bdb2981..4166effbf 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ use_scm_version=True, setup_requires=['setuptools_scm', 'wheel'], install_requires=[ + 'beautifulsoup4==4.9.1', 'django>=2.2', # TOM Toolkit requires db math functions 'django-bootstrap4==1.1.1', 'django-extensions==2.2.9', diff --git a/tom_alerts/brokers/antares.py b/tom_alerts/brokers/antares.py index e9eda7cff..c03b7e16b 100644 --- a/tom_alerts/brokers/antares.py +++ b/tom_alerts/brokers/antares.py @@ -21,3 +21,12 @@ def __init__(self, *args, **kwargs): class ANTARESBroker(GenericBroker): name = 'ANTARES' form = ANTARESQueryForm + + def fetch_alerts(self, parameters): + return iter([]) + + def process_reduced_data(self, target, alert=None): + return + + def to_generic_alert(self, alert): + return GenericAlert() diff --git a/tom_alerts/brokers/gaia.py b/tom_alerts/brokers/gaia.py index 5f038bbb7..bc7038f14 100644 --- a/tom_alerts/brokers/gaia.py +++ b/tom_alerts/brokers/gaia.py @@ -1,18 +1,23 @@ -from tom_alerts.alerts import GenericQueryForm, GenericAlert, GenericBroker -from tom_alerts.models import BrokerQuery -from tom_targets.models import Target -from tom_dataproducts.models import ReducedDatum from dateutil.parser import parse -from django import forms +import json +from os import path +import re +import requests +from requests.exceptions import HTTPError + from astropy.coordinates import SkyCoord from astropy.time import Time, TimezoneInfo import astropy.units as u -import requests -from requests.exceptions import HTTPError -import json -from os import path +from bs4 import BeautifulSoup +from django import forms + +from tom_alerts.alerts import GenericQueryForm, GenericAlert, GenericBroker +from tom_alerts.models import BrokerQuery +from tom_dataproducts.models import ReducedDatum +from tom_targets.models import Target BROKER_URL = 'http://gsaweb.ast.cam.ac.uk/alerts/alertsindex' +BASE_BROKER_URL = 'http://gsaweb.ast.cam.ac.uk' class GaiaQueryForm(GenericQueryForm): @@ -23,21 +28,72 @@ class GaiaQueryForm(GenericQueryForm): help_text='RA,Dec,radius in degrees' ) + def clean_cone(self): + cone = self.cleaned_data['cone'] + if cone: + cone_params = cone.split(',') + if len(cone_params) != 3: + raise forms.ValidationError('Cone search parameters must be in the format \'RA,Dec,Radius\'.') + return cone + def clean(self): - if len(self.cleaned_data['target_name']) == 0 and \ - len(self.cleaned_data['cone']) == 0: - raise forms.ValidationError( - "Please enter either a target name or cone search parameters" - ) + super().clean() + if not (self.cleaned_data.get('target_name') or self.cleaned_data.get('cone')): + raise forms.ValidationError('Please enter either a target name or cone search parameters.') + elif self.cleaned_data.get('target_name') and self.cleaned_data.get('cone'): + raise forms.ValidationError('Please only enter one of target name or cone search parameters.') class GaiaBroker(GenericBroker): name = 'Gaia' form = GaiaQueryForm + # def fetch_alerts(self, parameters): + # response = requests.get(f'{BASE_BROKER_URL}/alerts/alertsindex') + # response.raise_for_status() + + # soup = BeautifulSoup(response.content) + # script_tags = soup.find_all('script') + # alerts = [] + + # alerts_pattern = re.compile(r'var alerts = (.*?);') + # for script in script_tags: + # alerts = alerts_pattern.match(str(script.string).strip()) + # if alerts: + # break + + # print(alerts[0]) + # alert_list = json.loads(alerts[0].replace('var_alerts = ', '').replace(';', '')) + + # cone_params = parameters.get('cone').split(',') + # parameters['cone_ra'] = float(cone_params[0]) + # parameters['cone_dec'] = float(cone_params[1]) + # parameters['cone_radius'] = float(cone_params[2])*u.deg + # parameters['cone_centre'] = SkyCoord(float(cone_params[0]), + # float(cone_params[1]), + # frame="icrs", unit="deg") + + # filtered_alerts = [] + # if parameters.get('target_name'): + # for alert in alert_list: + # if parameters['target_name'] in alert['name']: + # filtered_alerts.append(alert) + + # elif 'cone_radius' in parameters.keys(): + # for alert in alert_list: + # c = SkyCoord(float(alert['ra']), float(alert['dec']), + # frame="icrs", unit="deg") + # if parameters['cone_centre'].separation(c) <= parameters['cone_radius']: + # filtered_alerts.append(alert) + + # else: + # filtered_alerts = alert_list + + # return iter(filtered_alerts) + def fetch_alerts(self, parameters): """Must return an iterator""" - response = requests.get(BROKER_URL) + response = requests.get(f'{BASE_BROKER_URL}/alerts/alertsindex') response.raise_for_status() html_data = response.text.split('\n') @@ -58,8 +114,7 @@ def fetch_alerts(self, parameters): frame="icrs", unit="deg") filtered_alerts = [] - if parameters['target_name'] is not None and \ - len(parameters['target_name']) > 0: + if parameters.get('target_name'): for alert in alert_list: if parameters['target_name'] in alert['name']: filtered_alerts.append(alert) @@ -87,6 +142,8 @@ def fetch_alert(self, target_name): def to_generic_alert(self, alert): timestamp = parse(alert['obstime']) + alert_link = alert.get('per_alert', {})['link'] + url = f'{BASE_BROKER_URL}/{alert_link}' url = BROKER_URL.replace('/alerts/alertsindex', alert['per_alert']['link']) return GenericAlert( @@ -103,6 +160,7 @@ def to_generic_alert(self, alert): def process_reduced_data(self, target, alert=None): base_url = BROKER_URL.replace('/alertsindex', '/alert') + query_url = f'{BASE_BROKER_URL}/alert' if not alert: try: diff --git a/tom_alerts/tests/tests_gaia.py b/tom_alerts/tests/tests_gaia.py new file mode 100644 index 000000000..7c1a30f91 --- /dev/null +++ b/tom_alerts/tests/tests_gaia.py @@ -0,0 +1,82 @@ +from requests import Response + +from django.test import TestCase, override_settings +from django.forms import ValidationError +from unittest import mock + +from tom_alerts.brokers.gaia import GaiaQueryForm + +@override_settings(TOM_ALERT_CLASSES=['tom_alerts.brokers.gaia.GaiaBroker']) +class TestGaiaQueryForm(TestCase): + def setUp(self): + self.base_form_params = {'query_name': 'Test Query', 'broker': 'Gaia'} + + def test_with_required_params(self): + self.base_form_params['target_name'] = 'Test Target' + form = GaiaQueryForm(self.base_form_params) + self.assertTrue(form.is_valid()) + + def test_no_query_name(self): + self.base_form_params['target_name'] = 'Test Target' + self.base_form_params.pop('query_name') + form = GaiaQueryForm(self.base_form_params) + self.assertFalse(form.is_valid()) + self.assertIn('This field is required.', form.errors.get('query_name')) + + def test_no_target_name_or_cone(self): + form = GaiaQueryForm(self.base_form_params) + self.assertFalse(form.is_valid()) + with self.assertRaises(ValidationError): + form.clean() + self.assertIn('Please enter either a target name or cone search parameters.', form.errors.get('__all__')) + + def test_both_target_name_and_cone(self): + self.base_form_params['target_name'] = 'Test Target' + self.base_form_params['cone'] = '10,20,3' + form = GaiaQueryForm(self.base_form_params) + self.assertFalse(form.is_valid()) + with self.assertRaises(ValidationError): + form.clean() + self.assertIn('Please only enter one of target name or cone search parameters.', form.errors.get('__all__')) + + def test_cone(self): + self.base_form_params['cone'] = '10,20,3' + form = GaiaQueryForm(self.base_form_params) + self.assertTrue(form.is_valid()) + + def test_cone_invalid_format(self): + self.base_form_params['cone'] = '10' + form = GaiaQueryForm(self.base_form_params) + self.assertFalse(form.is_valid()) + with self.assertRaises(ValidationError): + form.clean() + self.assertIn('Cone search parameters must be in the format \'RA,Dec,Radius\'.', form.errors.get('cone')) + + +@override_settings(TOM_ALERT_CLASSES=['tom_alerts.brokers.gaia.GaiaBroker']) +class TestGaiaBroker(TestCase): + def setUp(self): + self.test_html = """ + + + """ + + @mock.patch('tom_alerts.brokers.mars.requests.get') + def test_fetch_alerts(self, mock_requests_get): + mock_response = Response() + mock_response._content = self.test_html + mock_response.status_code = 200 + mock_requests_get.return_value = mock_response + + # (\[{)(.*?)(}\]) \ No newline at end of file From 95051a7652cdc638d09b028b792aa7b3346b9eed Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 8 Jun 2020 07:22:28 -0700 Subject: [PATCH 162/424] Updated release notes --- docs/releasenotes.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/releasenotes.md b/docs/releasenotes.md index a32d21687..b439e8340 100644 --- a/docs/releasenotes.md +++ b/docs/releasenotes.md @@ -1,3 +1,16 @@ +### 1.6.1 + +- This release pins the Django version in order to address a security vulnerability. + +#### What to watch out for + +- The Django version is now pinned at 3.0.7, where previously it allowed >=2.2. You'll need to ensure that any custom code is compatible with Django >=3.0.7. + +### 1.6.0 + +- New methods expand the Facility API to support reporting Facility status and weather: `get_facility_status()` and `get_facility_weather_url()`. When these methods are implemented by a Facility provider, this information can be made available in your TOM. +- A new template tag, `facility_status()`, is available to present this information. + ### 1.5.0 - Introduced a manual facility interface for classical observing. From 6a7359b715f672a41221bfc0b06bfb8af41ce9f0 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 8 Jun 2020 17:21:08 -0700 Subject: [PATCH 163/424] Completed refactor according to Rachel's suggested format --- docs/advanced/exceptions.md | 9 - docs/advanced/index.rst | 39 --- docs/advanced/scripts.md | 194 --------------- .../create_broker.md | 4 - docs/brokers/index.rst | 23 ++ docs/{customization => code}/automation.md | 0 docs/{advanced => code}/backgroundtasks.md | 0 docs/{advanced => code}/custom_code.md | 0 docs/code/index.rst | 26 ++ docs/{advanced => code}/querying.md | 0 .../customsettings.md | 6 +- docs/{advanced => common}/latex_generation.md | 0 .../permissions.md => common/permissions.rst} | 83 ++++--- docs/{ => common}/releasenotes.md | 2 + docs/common/scripts.rst | 215 ++++++++++++++++ docs/conf.py | 2 + docs/customization/common_customizations.rst | 10 - docs/customization/index.rst | 46 +--- docs/deployment/index.rst | 6 +- docs/index.rst | 124 +++------- docs/{ => introduction}/about.md | 0 docs/{ => introduction}/contributing.md | 0 docs/introduction/faqs.md | 17 +- docs/introduction/support.rst | 29 +++ ...tomarchitecture.md => tomarchitecture.rst} | 229 +++++++++--------- docs/introduction/troubleshooting.md | 2 +- .../customizing_data_processing.md | 0 docs/managing_data/index.rst | 18 ++ .../plotting_data.md | 0 .../customize_observations.md | 0 docs/observing/index.rst | 29 +++ .../observation_module.md | 0 docs/{advanced => observing}/strategies.md | 0 docs/support.md | 21 -- docs/targets/index.rst | 23 ++ .../target_fields.md | 0 36 files changed, 587 insertions(+), 570 deletions(-) delete mode 100644 docs/advanced/exceptions.md delete mode 100644 docs/advanced/index.rst delete mode 100644 docs/advanced/scripts.md rename docs/{customization => brokers}/create_broker.md (97%) create mode 100644 docs/brokers/index.rst rename docs/{customization => code}/automation.md (100%) rename docs/{advanced => code}/backgroundtasks.md (100%) rename docs/{advanced => code}/custom_code.md (100%) create mode 100644 docs/code/index.rst rename docs/{advanced => code}/querying.md (100%) rename docs/{customization => common}/customsettings.md (96%) rename docs/{advanced => common}/latex_generation.md (100%) rename docs/{customization/permissions.md => common/permissions.rst} (52%) rename docs/{ => common}/releasenotes.md (98%) create mode 100644 docs/common/scripts.rst delete mode 100644 docs/customization/common_customizations.rst rename docs/{ => introduction}/about.md (100%) rename docs/{ => introduction}/contributing.md (100%) create mode 100644 docs/introduction/support.rst rename docs/introduction/{tomarchitecture.md => tomarchitecture.rst} (53%) rename docs/{customization => managing_data}/customizing_data_processing.md (100%) create mode 100644 docs/managing_data/index.rst rename docs/{customization => managing_data}/plotting_data.md (100%) rename docs/{customization => observing}/customize_observations.md (100%) create mode 100644 docs/observing/index.rst rename docs/{advanced => observing}/observation_module.md (100%) rename docs/{advanced => observing}/strategies.md (100%) delete mode 100644 docs/support.md create mode 100644 docs/targets/index.rst rename docs/{customization => targets}/target_fields.md (100%) diff --git a/docs/advanced/exceptions.md b/docs/advanced/exceptions.md deleted file mode 100644 index f90f0d1f8..000000000 --- a/docs/advanced/exceptions.md +++ /dev/null @@ -1,9 +0,0 @@ -# Authentication Exceptions - -The TOM Toolkit offers a few custom exceptions that are documented in the API documentation, but one in particular -should be noted. - -For any modules exposing external services, such as brokers, harvesters, or facilities, a failed authentication should -raise an `ImproperCredentialsException`. Exceptions of this type are caught by the TOM Toolkit's built-in -`ExternalServiceMiddleware`. This middleware will display an error at the top of the page and redirect the user to the -home page. \ No newline at end of file diff --git a/docs/advanced/index.rst b/docs/advanced/index.rst deleted file mode 100644 index 38ddb1389..000000000 --- a/docs/advanced/index.rst +++ /dev/null @@ -1,39 +0,0 @@ -*************** -Advanced Topics -*************** - -.. toctree:: - :maxdepth: 1 - :hidden: - - backgroundtasks - observation_module - custom_code - scripts - strategies - latex_generation - querying - exceptions - - -:doc:`Background Tasks ` - Learn how to set up an asynchronous task library to handle long -running and/or concurrent functions. - -:doc:`Building a TOM Observation Facility Module ` - Learn to build a module which will -allow your TOM to submit observation requests to observatories. - -:doc:`Running Custom Code Hooks ` - Learn how to run your own scripts when certain actions happen -within your TOM (for example, an observation completes). - -:doc:`Scripting your TOM with Jupyter Notebooks ` - Use a Jupyter notebook (or just a python -console/scripts) to interact directly with your TOM. - -:doc:`Observing and cadence strategies ` - Learn about observing and cadence strategies and how to write a -custom cadence strategy to automate a series of observations - -:doc:`LaTeX table generation ` - Learn how to generate LaTeX for certain models and add LaTeX generators for other models - -:doc:`Advanced Querying ` - Get a couple of tips on programmatic querying with Django's QuerySet API - -:doc:`Authentication exceptions for external services ` - Ensure that your custom external services have - appropriate and visible errors. diff --git a/docs/advanced/scripts.md b/docs/advanced/scripts.md deleted file mode 100644 index 16227490a..000000000 --- a/docs/advanced/scripts.md +++ /dev/null @@ -1,194 +0,0 @@ -# Scripting your TOM with a Jupyter Notebook - -The TOM provides a graphical interface to perform many tasks, but there are some -tasks where writing code to interact with your TOM's data and functions may be -desirable. In this tutorial we will explore how to interact with a TOM with code, -_programmatically_, using a Jupyter notebook. - - -First install JupyterLab into your TOM's virtualenv: - - pip install jupyterlab - -Then launch the notebook server: - - ./manage.py shell_plus --notebook - - -The notebook interface should open in your browser. Everything is the same as a -standard Jupyter Notebook, with the exception of an additional option under the -"new" menu. When creating a new notebook that interacts with your TOM, you should -use the `Django Shell-Plus` option instead of the regular Python 3 option. This will -open the notebook with the correct Django context loaded: - -![](/_static/jupyterdoc/newnotebook.png) - -Create a new notebook. Now that it's open, we can use it just like any other -Notebook. - -### The API Documentation - -When working with the TOM programmatically, you'll often reference the [API -documentation](/api/modules), which is a reference -to the code of the TOM Toolkit itself. Since you will be using these classes and -functions, it would be a good idea to familiarize yourself with it. - -### Creating Targets - -We create targets by using the `Target` model and the `create` method. Let's -create a target for M51 using the bare necessary information: - -```python -In [1]: from tom_targets.models import Target - ...: t = Target.objects.create(name='m51', type='SIDEREAL', ra=123.3, dec=23.3) - ...: print(t) - ...: -Target post save hook: Messier 51 created: True -Messier 51 -``` - -If we wish to populate any extra fields that we've defined in `settings.EXTRA_FIELDS`, we can do now do that. We can also give our new target additional names, which can be used for searching in the UI: - -```python -In [3]: t.save(extras={'foo': 42, - 'bar': 'baz'}, - names=['Messier 51']) - ...: print(t.extra_fields) -Target post save hook: Messier 51 created: False -Out [3]: {'bar': 'baz', 'foo': 42.0} -``` - -Now we should have a target in our database for M51. We can fetch it now, or -anytime later: - -```python -In [9]: target = Target.objects.get(name='m51') - -In [10]: print(target) -Messier 51 -``` - -We can access attributes of our target: - -```python -In [13]: target.ra -Out[13]: 123.3 - -In [14]: target.future_observations -Out[14]: [] - -In [15]: target.names -Out[15]: ['m51', 'Messier 51'] -``` - -And if we tire of it, we can delete it entirely: - -```python -In [15]: target.delete() -Out[15]: -(1, - {'tom_targets.TargetExtra': 2, - 'tom_targets.TargetList_targets': 0, - 'tom_dataproducts.ReducedDatum': 0, - 'tom_targets.Target': 1}) -``` -See the [django documentation on making -queries](https://docs.djangoproject.com/en/2.2/topics/db/queries/) -for more examples of what can be done with objects in our database. - - -### Submitting observations - -Now that we have a target, we can submit an observation request using our -notebook, too. - -Let's make some imports: - -```python -In [16]: -from tom_targets.models import Target -from tom_observations.facilities.lco import LCOFacility, LCOBaseObservationForm -``` - -And since we are submitting to LCO, we will instantiate an LCO observation form: - -```python -In [17]: -form = LCOBaseObservationForm({ - 'name': 'Programmatic Observation', - 'proposal': 'LCOEngineering', - 'ipp_value': 1.05, - 'start': '2019-08-09T00:00:00', - 'end': '2019-08-10T00:00:00', - 'filter': 'R', - 'instrument_type': '1M0-SCICAM-SINISTRO', - 'exposure_count': 1, - 'exposure_time': 20, - 'max_airmass': 4.0, - 'observation_mode': 'NORMAL', - 'target_id': target.id, - 'facility': 'LCO' -}) -``` - -Is the form valid? - -```python -In [18]: form.is_valid() -Out[18]: true -``` - -Let's submit the request: - -```python -In [19]: observation_ids = LCOFacility().submit_observation(form.observation_payload()) - print(observation_ids) -Out[19]: [123456789] -``` - -And create records for them: - -```python -In [20]: from tom_observations.models import ObservationRecord -In [21]: -for observation_id in observation_ids: - record = ObservationRecord.objects.create( - target=target, - facility='LCO', - parameters=form.serialize_parameters(), - observation_id=observation_id - ) - print(record) -Out[20]: M51 @ LCO -``` - -Now when we check our TOM interface, we should see that our target, M51, has a -pending observation! - -### Saving DataProducts - -It may be that we have some data we want to associate with our target. In that case, we'll need to create a -`DataProduct`. However, one field on the `DataProduct` is the `data` field--the TOM Toolkit expects a -`django.core.files.File` object, so we need to create one first, then create our `DataProduct`. - -```python -In [22]: from tom_dataproducts.models import DataProduct -In [23]: from django.core.files import File -In [24]: f = File(open('path/to/file.png')) -In [25]: -dp = DataProduct.objects.create( - target=target, - data_product_type='image_file', - data=f -) -print(dp.data.name) -Out[25]: 'm51/none/file.png' -``` - -### More possibilities - -These are just a few examples of what's possible using the TOM's programmatic API. -In fact, you have complete control over your data when using this api. The best -way to learn what is possible is by [exploring the API docs](/api/modules) and by -[browsing the source code](https://github.com/tomtoolkit/tom_base) -of the TOM Toolkit project. diff --git a/docs/customization/create_broker.md b/docs/brokers/create_broker.md similarity index 97% rename from docs/customization/create_broker.md rename to docs/brokers/create_broker.md index aa84fd4e0..0307f52ca 100644 --- a/docs/customization/create_broker.md +++ b/docs/brokers/create_broker.md @@ -12,10 +12,6 @@ alert broker. Be sure you've followed the [Getting Started](/introduction/getting_started) guide before continuing onto this tutorial. -### What is an Alert Broker Module? -A TOM Toolkit Alert Broker Module is an object which contains the logic for querying a remote broker -(e.g [MARS](https://mars.lco.global)), and transforming the returned data into TOM Toolkit Targets. - #### TOM Alerts module The TOM Alerts module is a Django app which provides the methods and classes needed to create a custom TOM alert broker module. A module may be created to ingest diff --git a/docs/brokers/index.rst b/docs/brokers/index.rst new file mode 100644 index 000000000..61d4fafae --- /dev/null +++ b/docs/brokers/index.rst @@ -0,0 +1,23 @@ +Brokers +======= + +.. toctree:: + :maxdepth: 2 + :hidden: + + create_broker + ../api/tom_alerts/brokers + ../api/tom_alerts/views + + +What is an Alert Broker Module? +------------------------------- + +A TOM Toolkit Alert Broker Module is an object which contains the logic for querying a remote broker +(e.g `MARS `_), and transforming the returned data into TOM Toolkit Targets. + +:doc:`Creating an Alert Broker ` - Learn how to add a custom broker module to query for targets from your favorite broker. + +:doc:`Broker Modules <../api/tom_alerts/brokers>` - Take a look at the supported brokers. + +:doc:`Broker Views <../api/tom_alerts/views>` - Familiarize yourself with the available Broker Views. diff --git a/docs/customization/automation.md b/docs/code/automation.md similarity index 100% rename from docs/customization/automation.md rename to docs/code/automation.md diff --git a/docs/advanced/backgroundtasks.md b/docs/code/backgroundtasks.md similarity index 100% rename from docs/advanced/backgroundtasks.md rename to docs/code/backgroundtasks.md diff --git a/docs/advanced/custom_code.md b/docs/code/custom_code.md similarity index 100% rename from docs/advanced/custom_code.md rename to docs/code/custom_code.md diff --git a/docs/code/index.rst b/docs/code/index.rst new file mode 100644 index 000000000..4ada5532e --- /dev/null +++ b/docs/code/index.rst @@ -0,0 +1,26 @@ +Interacting with your TOM through code +====================================== + +.. toctree:: + :maxdepth: 2 + :hidden: + + querying + automation + backgroundtasks + custom_code + ../common/scripts + +:doc:`Advanced Querying ` - Get a couple of tips on programmatic querying with Django's QuerySet API + +:doc:`Automating Tasks ` - Run commands automatically to keep your TOM working even when you +aren’t + +:doc:`Background Tasks ` - Learn how to set up an asynchronous task library to handle long +running and/or concurrent functions. + +:doc:`Running Custom Code Hooks ` - Learn how to run your own scripts when certain actions happen +within your TOM (for example, an observation completes). + +:doc:`Scripting your TOM with Jupyter Notebooks <../common/scripts>` - Use a Jupyter notebook (or just a python +console/scripts) to interact directly with your TOM. diff --git a/docs/advanced/querying.md b/docs/code/querying.md similarity index 100% rename from docs/advanced/querying.md rename to docs/code/querying.md diff --git a/docs/customization/customsettings.md b/docs/common/customsettings.md similarity index 96% rename from docs/customization/customsettings.md rename to docs/common/customsettings.md index 2f06882f6..33b8d6f14 100644 --- a/docs/customization/customsettings.md +++ b/docs/common/customsettings.md @@ -63,7 +63,7 @@ Default: [] A list of extra fields to add to your targets. These can be used if the predefined target fields do not match your needs. Please see the documentation on [Adding -Custom Fields to Targets](/customization/target_fields) for an explanation of how to use +Custom Fields to Targets](/targets/target_fields) for an explanation of how to use this feature. @@ -110,7 +110,7 @@ Default: A dictionary of action, method code hooks to run. These hooks allow running arbitrary python code when specific actions happen within a TOM, such as an observation changing state. See the documentation on [Running Custom Code on -Actions in your TOM](/advanced/custom_code) for more details and available hooks. +Actions in your TOM](/code/custom_code) for more details and available hooks. ### [OPEN_URLS](#open_urls) @@ -157,7 +157,7 @@ Default: A list of tom alert classes to make available to your TOM. If you have written or downloaded additional alert classes you would make them available here. If you'd like to write your own alert module please see the documentation on [Creating an -Alert Module for the TOM Toolkit](/customization/create_broker). +Alert Module for the TOM Toolkit](/brokers/create_broker). ### [TOM_FACILITY_CLASSES](#tom_facility_classes) diff --git a/docs/advanced/latex_generation.md b/docs/common/latex_generation.md similarity index 100% rename from docs/advanced/latex_generation.md rename to docs/common/latex_generation.md diff --git a/docs/customization/permissions.md b/docs/common/permissions.rst similarity index 52% rename from docs/customization/permissions.md rename to docs/common/permissions.rst index a66c91ca8..7350afa94 100644 --- a/docs/customization/permissions.md +++ b/docs/common/permissions.rst @@ -1,17 +1,17 @@ The Permissions System ---- +====================== The permissions system is built on top of -[django-guardian](https://django-guardian.readthedocs.io/en/stable/). It has been +`django-guardian `_. It has been kept as simple as possible, but TOM developers may extend the capabilities if needed. The TOM Toolkit provides a permissions system that can be used in two different modes. The mode is controlled by the -`TARGET_PERMISSIONS_ONLY` boolean in `settings.py`. +``TARGET_PERMISSIONS_ONLY`` boolean in ``settings.py``. First Mode -- Permissions on Targets and Observation Records ---- +------------------------------------------------------------ The first mode limits the targets that a user or a group of users can access. This may be helpful if you have many @@ -23,6 +23,8 @@ PI in the TOM, via the users page. To add a group, simply use the "Add Group" button found at the top of the groups table: +.. image:: /_static/permissions_doc/addgroup.png + ![](/_static/permissions_doc/addgroup.png) Modifying a group will allow you to change it's name and add/remove users. @@ -30,6 +32,7 @@ Modifying a group will allow you to change it's name and add/remove users. When a user adds or modifies a target, they are able to choose the groups to assign to the target: +.. image:: /_static/permissions_doc/targetgroups.png ![](/_static/permissions_doc/targetgroups.png) @@ -42,64 +45,66 @@ have the ability to remove users for the Public group, however. Second Mode -- Permissions on most objects ---- +------------------------------------------ The second permissions mode is an expanded version of the first. Observation records and data products can be restricted to certain groups, and children of those objects will have the same restrictions--that is, all data products of an observation record will share its permissions, and all reduced datums of a data product will share its permissions. -A note about toggling `TARGET_PERMISSIONS_ONLY` ---- +A note about toggling ``TARGET_PERMISSIONS_ONLY`` +------------------------------------------------- -It must be noted that while `TARGET_PERMISSIONS_ONLY` is set to `True`, no permissions will be set on any objects other -than targets. This means that if your TOM is used with `TARGET_PERMISSIONS_ONLY`, and `TARGET_PERMISSIONS_ONLY` is +It must be noted that while ``TARGET_PERMISSIONS_ONLY`` is set to ``True``, no permissions will be set on any objects other +than targets. This means that if your TOM is used with ``TARGET_PERMISSIONS_ONLY``, and ``TARGET_PERMISSIONS_ONLY`` is disabled after the fact, all permissions will need to be configured manually. Manual permissions modification ---- +------------------------------- -If you want to disable `TARGET_PERMISSIONS_ONLY` after adding any data, you'll need to do so on your own. We encourage you to read the documention on django-guardian linked above, but here's an example of a bulk permissions assignment for +If you want to disable ``TARGET_PERMISSIONS_ONLY`` after adding any data, you'll need to do so on your own. We encourage you to read the documention on django-guardian linked above, but here's an example of a bulk permissions assignment for a target: -```python ->>> from django.contrib.auth.models import Group, User ->>> from guardian.shortcuts import assign_perm ->>> from tom_targets.models import Target ->>> user = User.objects.filter(username='jaire_alexander').first() ->>> groups = user.groups.all() ->>> targets = Target.objects.all() ->>> for group in groups: -... assign_perm('tom_targets.view_target', group, targets) -... assign_perm('tom_targets.change_target', group, targets) -... assign_perm('tom_targets.delete_target', group, targets) -``` +.. code-block:: python + + >>> from django.contrib.auth.models import Group, User + >>> from guardian.shortcuts import assign_perm + >>> from tom_targets.models import Target + >>> user = User.objects.filter(username='jaire_alexander').first() + >>> groups = user.groups.all() + >>> targets = Target.objects.all() + >>> for group in groups: + ... assign_perm('tom_targets.view_target', group, targets) + ... assign_perm('tom_targets.change_target', group, targets) + ... assign_perm('tom_targets.delete_target', group, targets) The above code will allow all users in the groups that the example user belongs to to view, modify, and delete all targets. This example can be expanded to the other model-related permissions in the TOM. Below is a brief list of the permissions-enabled models with their permission names: -`Targets`: +``Targets``: + +* ``tom_targets.view_target`` +* ``tom_targets.change_target`` +* ``tom_targets.delete_target`` + +``TargetLists``: -* `tom_targets.view_target` -* `tom_targets.change_target` -* `tom_targets.delete_target` +* ``tom_targets.view_targetlist`` +* ``tom_targets.delete_targetlist`` -`TargetLists`: +``ObservationRecords``: -* `tom_targets.view_targetlist` -* `tom_targets.delete_targetlist` +* ``tom_observations.view_observationrecord`` -`ObservationRecords`: +``ObservationGroups``: -* `tom_observations.view_observationrecord` +* ``tom_observations.view_observationgroup`` -`ObservationGroups`: +``DataProducts``: -* `tom_observations.view_observationgroup` +* ``tom_dataproducts.view_dataproduct`` +* ``tom_dataproducts.delete_dataproduct`` -`DataProducts`: -* `tom_dataproducts.view_dataproduct` -* `tom_dataproducts.delete_dataproduct` +``ReducedDatum``: -`ReducedDatum`: -* `tom_dataproducts.view_reduceddatum` \ No newline at end of file +* ``tom_dataproducts.view_reduceddatum`` \ No newline at end of file diff --git a/docs/releasenotes.md b/docs/common/releasenotes.md similarity index 98% rename from docs/releasenotes.md rename to docs/common/releasenotes.md index b439e8340..faa4665ae 100644 --- a/docs/releasenotes.md +++ b/docs/common/releasenotes.md @@ -1,3 +1,5 @@ +# Release Notes + ### 1.6.1 - This release pins the Django version in order to address a security vulnerability. diff --git a/docs/common/scripts.rst b/docs/common/scripts.rst new file mode 100644 index 000000000..a6ffb0d4e --- /dev/null +++ b/docs/common/scripts.rst @@ -0,0 +1,215 @@ +Scripting your TOM with a Jupyter Notebook +------------------------------------------ + +The TOM provides a graphical interface to perform many tasks, but there are some +tasks where writing code to interact with your TOM's data and functions may be +desirable. In this tutorial we will explore how to interact with a TOM with code, +programmatically, using a Jupyter notebook. + + +First install JupyterLab into your TOM's virtualenv: + +.. code-block:: + + pip install jupyterlab + +Then launch the notebook server: + +.. code-block:: + + ./manage.py shell_plus --notebook + + + +The notebook interface should open in your browser. Everything is the same as a +standard Jupyter Notebook, with the exception of an additional option under the +"new" menu. When creating a new notebook that interacts with your TOM, you should +use the `Django Shell-Plus` option instead of the regular Python 3 option. This will +open the notebook with the correct Django context loaded: + +.. image:: /_static/jupyterdoc/newnotebook.png + +Create a new notebook. Now that it's open, we can use it just like any other +Notebook. + +The API Documentation +===================== + +When working with the TOM programmatically, you'll often reference the :doc:`API +documentation `, which is a reference +to the code of the TOM Toolkit itself. Since you will be using these classes and +functions, it would be a good idea to familiarize yourself with it. + +.. _creating-targets-programmatically: + +Creating Targets +================ + +We create targets by using the ``Target`` model and the ``create`` method. Let's +create a target for M51 using the bare necessary information: + +.. code-block:: python + + In [1]: from tom_targets.models import Target + ...: t = Target.objects.create(name='m51', type='SIDEREAL', ra=123.3, dec=23.3) + ...: print(t) + ...: + Target post save hook: Messier 51 created: True + Messier 51 + +If we wish to populate any extra fields that we've defined in ``settings.EXTRA_FIELDS``, we can do now do that. We can also give our new target additional names, which can be used for searching in the UI: + +.. code-block:: python + + In [3]: t.save(extras={'foo': 42, + 'bar': 'baz'}, + names=['Messier 51']) + ...: print(t.extra_fields) + Target post save hook: Messier 51 created: False + Out [3]: {'bar': 'baz', 'foo': 42.0} + +Now we should have a target in our database for M51. We can fetch it now, or +anytime later: + +.. code-block:: python + + In [9]: target = Target.objects.get(name='m51') + + In [10]: print(target) + Messier 51 + +We can access attributes of our target: + +.. code-block:: python + + In [13]: target.ra + Out[13]: 123.3 + + In [14]: target.future_observations + Out[14]: [] + + In [15]: target.names + Out[15]: ['m51', 'Messier 51'] + +And if we tire of it, we can delete it entirely: + +.. code-block:: python + + In [15]: target.delete() + Out[15]: + (1, + {'tom_targets.TargetExtra': 2, + 'tom_targets.TargetList_targets': 0, + 'tom_dataproducts.ReducedDatum': 0, + 'tom_targets.Target': 1}) + +See the `django documentation on making +queries `_ +for more examples of what can be done with objects in our database. + +.. _creating-observations-programmatically: + +Submitting observations +======================= + +Now that we have a target, we can submit an observation request using our +notebook, too. + +Let's make some imports: + +.. code-block:: python + + In [16]: + from tom_targets.models import Target + from tom_observations.facilities.lco import LCOFacility, LCOBaseObservationForm + + +And since we are submitting to LCO, we will instantiate an LCO observation form: + +.. code-block:: python + + In [17]: + form = LCOBaseObservationForm({ + 'name': 'Programmatic Observation', + 'proposal': 'LCOEngineering', + 'ipp_value': 1.05, + 'start': '2019-08-09T00:00:00', + 'end': '2019-08-10T00:00:00', + 'filter': 'R', + 'instrument_type': '1M0-SCICAM-SINISTRO', + 'exposure_count': 1, + 'exposure_time': 20, + 'max_airmass': 4.0, + 'observation_mode': 'NORMAL', + 'target_id': target.id, + 'facility': 'LCO' + }) + +Is the form valid? + +.. code-block:: python + + In [18]: form.is_valid() + Out[18]: true + + +Let's submit the request: + +.. code-block:: python + + In [19]: observation_ids = LCOFacility().submit_observation(form.observation_payload()) + print(observation_ids) + Out[19]: [123456789] + + +And create records for them: + +.. code-block:: python + + In [20]: from tom_observations.models import ObservationRecord + In [21]: + for observation_id in observation_ids: + record = ObservationRecord.objects.create( + target=target, + facility='LCO', + parameters=form.serialize_parameters(), + observation_id=observation_id + ) + print(record) + Out[20]: M51 @ LCO + + +Now when we check our TOM interface, we should see that our target, M51, has a +pending observation! + +Saving DataProducts +=================== + +It may be that we have some data we want to associate with our target. In that case, we'll need to create a +``DataProduct``. However, one field on the ``DataProduct`` is the ``data`` field--the TOM Toolkit expects a +``django.core.files.File`` object, so we need to create one first, then create our ``DataProduct``. + + +.. code-block:: python + + In [22]: from tom_dataproducts.models import DataProduct + In [23]: from django.core.files import File + In [24]: f = File(open('path/to/file.png')) + In [25]: + dp = DataProduct.objects.create( + target=target, + data_product_type='image_file', + data=f + ) + print(dp.data.name) + Out[25]: 'm51/none/file.png' + + +More possibilities +================== + +These are just a few examples of what's possible using the TOM's programmatic API. +In fact, you have complete control over your data when using this api. The best +way to learn what is possible is by :doc:`exploring the API docs ` and by +`browsing the source code `_ +of the TOM Toolkit project. diff --git a/docs/conf.py b/docs/conf.py index b323de72d..9abcbc30b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -101,6 +101,8 @@ 'github_button': 'false', } +pygments_style = 'sphinx' + # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". diff --git a/docs/customization/common_customizations.rst b/docs/customization/common_customizations.rst deleted file mode 100644 index 83cfd3ef3..000000000 --- a/docs/customization/common_customizations.rst +++ /dev/null @@ -1,10 +0,0 @@ -********************* -Common Customizations -********************* - -When starting a new TOM, we're sure there are a few things a user might want to change right away. Fortunately, we've anticipated that! Here are some guides to get you started: - -* Not happy with the appearance? Jump straight in with :doc:`customizing TOM Templates `. You may want to take a look at the available Template Tags in each modules' respective `API Documentation `_ to see what you can do. -* Need another view? Take a peek at :doc:`Adding Pages `. -* Want to automate something? Look at the :doc:`Automation Guide `. Feeling bold? Set up :doc:`background tasks <../advanced/backgroundtasks>`. -* SQLite not meeting your needs? Check out Django's documentation on `databases `_. \ No newline at end of file diff --git a/docs/customization/index.rst b/docs/customization/index.rst index 87eb76722..6244fea60 100644 --- a/docs/customization/index.rst +++ b/docs/customization/index.rst @@ -1,54 +1,22 @@ -*********** -Customizing -*********** +Customization +============= .. toctree:: - :maxdepth: 1 + :maxdepth: 2 :hidden: - customsettings customize_templates - customize_template_tags adding_pages - target_fields - customizing_data_processing - create_broker - customize_observations - plotting_data - permissions - automation + customize_template_tags -Start here to learn how to customize the look and feel of your TOM or add new functionality. -:doc:`Custom Settings ` - Settings available to the TOM Toolkit which you may want to -configure. +Start here to learn how to customize the look and feel of your TOM or add new functionality. :doc:`Customizing TOM Templates ` - Learn how to override built in TOM templates to change the look and feel of your TOM. -:doc:`Customizing Template Tag ` - Learn how to write your own template tags to display -the data you need. - :doc:`Adding new Pages to your TOM ` - Learn how to add entirely new pages to your TOM, displaying static html pages or dynamic database-driven content. -:doc:`Adding Custom Target Fields ` - Learn how to add custom fields to your TOM Targets if the -defaults do not suffice. - -:doc:`Adding Custom Data Processing ` - Learn how you can process data into your -TOM from uploaded data products. - -:doc:`Building a TOM Alert Broker ` - Learn how to build an Alert Broker module to add new -sources of targets to your TOM. - -:doc:`Changing Request Submission Behavior ` - Learn how to customize the LCO -Observation Module in order to add additional parameters to observation requests sent to the LCO Network. - -:doc:`Creating Plots from TOM Data ` - Learn how to create plots using plot.ly and your TOM -data to display anywhere in your TOM. - -:doc:`The Permissions System ` - Use the permissions system to limit access to targets in your -TOM. - -:doc:`Automating Tasks ` - Run commands automatically to keep your TOM working even when you -aren’t \ No newline at end of file +:doc:`Customizing Template Tag ` - Learn how to write your own template tags to display +the data you need. diff --git a/docs/deployment/index.rst b/docs/deployment/index.rst index 3d7ea89af..8b7e6f5fb 100644 --- a/docs/deployment/index.rst +++ b/docs/deployment/index.rst @@ -1,8 +1,8 @@ -Deployment -========== +Deploying your TOM Online +========================= .. toctree:: - :maxdepth: 1 + :maxdepth: 2 :hidden: deployment_tips diff --git a/docs/index.rst b/docs/index.rst index fa7519be0..040c81e27 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,15 +6,15 @@ Welcome to the TOM Toolkit's documentation! :hidden: introduction/index - customization/index - advanced/index - deployment/index - introduction/faqs + introduction/about + introduction/support + introduction/troubleshooting + Introduction ------------ -The TOM (Target and Observation Manager) Toolkit project was started in early 2018 with the goal of simplifying the development of next generation software for the rapidly evolving field of astronomy. Read more :doc:`about TOMs` and the motivation for them. +The TOM (Target and Observation Manager) Toolkit project was started in early 2018 with the goal of simplifying the development of next generation software for the rapidly evolving field of astronomy. Read more :doc:`about TOMs` and the motivation for them. :doc:`TOM Toolkit Architecture ` - This document describes the architecture of the TOM Toolkit at a high level. Read this first if you're interested in how the TOM Toolkit works. @@ -28,114 +28,63 @@ HTML, CSS, Python, and Django :doc:`Frequently Asked Questions ` - Look here for a potential quick answer to a common question. - -Targets -------- - -:doc:`Adding Custom Target Fields ` - Learn how to add custom fields to your TOM Targets if the -defaults do not suffice. - -`Target API ` - Take a look at available properties of Targets and - - - - - +:doc:`Troubleshooting ` - Find solutions to common problems or information on how to debug an issue. Interested in seeing what a TOM can do? Take a look at our `demonstration TOM `_, where we show off the features of the TOM Toolkit. -Are you looking to run a TOM of your own? This documentation is a good place to get started. The source code for the project is also available on Github. - -Start with the :doc:`introduction` if you are new to using the TOM Toolkit. - If you'd like to know what we're working on, check out the `TOM Toolkit project board `_. -:doc:`Troubleshooting ` - Find solutions to common problems or information on how to debug an issue. - -Extending and Customizing -------------------------- -Start here to learn how to customize the look and feel of your TOM or add new functionality. +Topics +------ -:doc:`Custom Settings ` - Settings available to the TOM Toolkit which you may want to -configure. - -:doc:`Customizing TOM Templates ` - Learn how to override built in TOM templates to -change the look and feel of your TOM. - -:doc:`Customizing Template Tag ` - Learn how to write your own template tags to display -the data you need. - -:doc:`Adding new Pages to your TOM ` - Learn how to add entirely new pages to your TOM, -displaying static html pages or dynamic database-driven content. - -:doc:`Adding Custom Data Processing ` - Learn how you can process data into your -TOM from uploaded data products. - -:doc:`Building a TOM Alert Broker ` - Learn how to build an Alert Broker module to add new -sources of targets to your TOM. - -:doc:`Changing Request Submission Behavior ` - Learn how to customize the LCO -Observation Module in order to add additional parameters to observation requests sent to the LCO Network. - -:doc:`Creating Plots from TOM Data ` - Learn how to create plots using plot.ly and your TOM -data to display anywhere in your TOM. - -:doc:`The Permissions System ` - Use the permissions system to limit access to targets in your -TOM. - -:doc:`Automating Tasks ` - Run commands automatically to keep your TOM working even when you -aren’t - -Advanced Topics ---------------- - -:doc:`Background Tasks ` - Learn how to set up an asynchronous task library to handle long -running and/or concurrent functions. +.. toctree:: + :maxdepth: 2 + :hidden: -:doc:`Building a TOM Observation Facility Module ` - Learn to build a module which will -allow your TOM to submit observation requests to observatories. + targets/index + brokers/index + observing/index + managing_data/index + customization/index + common/permissions + common/latex_generation + code/index + deployment/index + common/customsettings -:doc:`Running Custom Code Hooks ` - Learn how to run your own scripts when certain actions happen -within your TOM (for example, an observation completes). -:doc:`Scripting your TOM with Jupyter Notebooks ` - Use a Jupyter notebook (or just a python -console/scripts) to interact directly with your TOM. +:doc:`Targets ` - Learn all about how to manage Targets in a TOM. -:doc:`Observing and cadence strategies ` - Learn about observing and cadence strategies and how to write a -custom cadence strategy to automate a series of observations. +:doc:`Brokers ` - Find out about querying brokers in the TOM, which are available, and writing your own. -:doc:`LaTeX table generation ` - Learn how to generate LaTeX for certain models and add LaTeX -generators for other models. +:doc:`Observing ` - Tutorials on submitting observations, customizing submission, and the available facilities. -:doc:`Advanced Querying ` - Get a couple of tips on programmatic querying with Django's QuerySet API +:doc:`Managing Data ` - Customize plots, upload data, and even integrate a data reduction pipeline. -:doc:`Authentication exceptions for external services ` - Ensure that your custom external services have - appropriate and visible errors. +:doc:`Customization ` - Customize and create new views in your TOM. -Deployment ----------- +:doc:`The Permissions System ` - Use the permissions system to limit access to targets in your TOM. -Once you’ve got a TOM up and running on your machine, you’ll probably want to deploy it somewhere so it is permanently -accessible by you and your colleagues. +:doc:`LaTeX Generation ` -:doc:`General Deployment Tips ` - Read this first before deploying your TOM for others to use. +:doc:`Interacting with your TOM through code ` -:doc:`Deploy to Heroku ` - Heroku is a PaaS that allows you to publicly deploy your web applications without the need for managing the infrastructure yourself. +:doc:`Deploying your TOM Online ` - Resources for deploying your TOM to a cloud provider -:doc:`Using Amazon S3 to Store Data for a TOM ` - Enable storing data on the cloud storage service Amazon S3 instead of your local disk. +:doc:`TOM Settings ` - Reference and description for the available settings values to be added to/edited in your project's ``settings.py``. Contributing ------------ If you find an issue, you need help with your TOM, you have a useful idea, or you wrote a module you'd like to be -included in the TOM Toolkit, start with the :doc:`Contribution Guide `. +included in the TOM Toolkit, start with the :doc:`Contribution Guide `. Support ------- Looking for help? Want to request a feature? Have questions about Github Issues? Take a look at the :doc:`support guide -`. +`. If you just need an idea, checkout out the :doc:`examples` of existing TOMs built with the TOM Toolkit. @@ -143,10 +92,9 @@ If you just need an idea, checkout out the :doc:`examples` of existing :maxdepth: 1 :hidden: - contributing - support examples - about + introduction/contributing + common/releasenotes Github API Documentation @@ -173,4 +121,4 @@ About the TOM Toolkit The TOM Toolkit is managed by Las Cumbres Observatory, with generous financial support from the `Heising-Simons Foundation `_ and the `Zegar Family Foundation `_. -Read about the project and the motivations behind it on the :doc:`About page `. +Read about the project and the motivations behind it on the :doc:`About page `. diff --git a/docs/about.md b/docs/introduction/about.md similarity index 100% rename from docs/about.md rename to docs/introduction/about.md diff --git a/docs/contributing.md b/docs/introduction/contributing.md similarity index 100% rename from docs/contributing.md rename to docs/introduction/contributing.md diff --git a/docs/introduction/faqs.md b/docs/introduction/faqs.md index 4d863ae80..fb1931b1e 100644 --- a/docs/introduction/faqs.md +++ b/docs/introduction/faqs.md @@ -15,7 +15,7 @@ notebook server: Under the new notebook menu, choose "Django Shell-Plus". This will create a new notebook in the correct TOM context. -There is also a [tutorial](/advanced/scripts) on interacting with your TOM using +There is also a [tutorial](../common/scripts) on interacting with your TOM using Jupyter notebooks. ### What are tags on the Target form? @@ -26,12 +26,12 @@ You can then search for targets via tags on the target list page, by entering th target detail pages. If you'd like to have more control over extra target data, see the documentation -on [Adding Custom Target Fields](/customization/target_fields). +on [Adding Custom Target Fields](../targets/target_fields). ### I try to observe a target with LCO but get an error. You might not have added your LCO api key to your settings file under the -`FACILITIES` settings. See [Custom Settings](/customization/customsettings#facilities) for +`FACILITIES` settings. See [Custom Settings](../uncategorized/customsettings#facilities) for more details. ### How do I create a super user (PI)? @@ -48,13 +48,13 @@ in as a superuser by visiting the admin page for users: ### My science requires more parameters than are provided by the TOM Toolkit. It is possible to add additional parameters to your targets within the TOM. See -the documentation on [Adding Custom Target Fields](/customization/target_fields). +the documentation on [Adding Custom Target Fields](../targets/target_fields). ### Yuck! My TOM is ugly. How do I change how it looks? You have a few options. If you'd like to rearrange the layout or information on the page, you can follow the tutorial on -[Customizing your TOM](/customization/customize_templates). If you'd like to modify colors, +[Customizing your TOM](../customization/customize_templates). If you'd like to modify colors, typography, etc you'll want to use CSS. [W3Schools](https://www.w3schools.com/Css/) is a good resource if you are unfamiliar with Cascading Style Sheets. @@ -88,3 +88,10 @@ AnonymousUser is a special profile that django-guardian, our permissions library represents an unauthenticated user. The user has no first name, last name, or password, and allows unauthenticated users to view unprotected pages within your TOM. You can choose to delete the user if you don't want any pages to be visible without logging in. + +### How can I display an error message when authentication to an external facility fails? + +For any modules exposing external services, such as brokers, harvesters, or facilities, a failed authentication should +raise an `ImproperCredentialsException`. Exceptions of this type are caught by the TOM Toolkit's built-in +`ExternalServiceMiddleware`. This middleware will display an error at the top of the page and redirect the user to the +home page. diff --git a/docs/introduction/support.rst b/docs/introduction/support.rst new file mode 100644 index 000000000..5a8b98d23 --- /dev/null +++ b/docs/introduction/support.rst @@ -0,0 +1,29 @@ +Getting Support +=============== + +This page will go over the process for reporting issues, requesting features, and getting +support for the TOM Toolkit. + +Check the FAQ +------------- + +Take a look at our :doc:`Frequently Asked Questions page ` for a potential quick answer to a common query. + +Reporting Issues +---------------- + +Issue reporting can be done via the `Github issues page `_ +of the tom_base project. Reporting an issue requires a Github account, but provides an easy way for +developers to ask follow-up questions about an issue in order to resolve it. + +Please include as much detail as possible, as well as the steps taken that trigger the issue, and be sure to tag it with the "bug" tag! + +Requesting Features +------------------- + +Like issues, feature and enhancement requests should be done via the same `Github issues page `_ of the tom_base project. This is also a great place to see what's being worked on and what's already been requested, which will allow you to voice support for a backlogged feature to be reprioritized. + +Support +------- + +If you're looking for help with some aspect of your TOM, the `Github issues page `_ is once again the place to go. The "question" or "help wanted" tags should be very useful when looking for support, and the TOM Toolkit developers are more than happy to provide the help necessary to get your TOM running. You may also want to peruse the `Closed Issues `_, where someone may have already had (and solved!) your problem. \ No newline at end of file diff --git a/docs/introduction/tomarchitecture.md b/docs/introduction/tomarchitecture.rst similarity index 53% rename from docs/introduction/tomarchitecture.md rename to docs/introduction/tomarchitecture.rst index 883b77076..86920d4b9 100644 --- a/docs/introduction/tomarchitecture.md +++ b/docs/introduction/tomarchitecture.rst @@ -1,9 +1,9 @@ TOM Software Architecture -------------------------- +************************* The goal of the TOM Toolkit is to make developing TOMs as easy as possible while providing the flexibility needed to tailor each TOM to its specific science -case. The motivation for the TOM Toolkit is discussed on the [about](https://tomtoolkit.github.io/about) +case. The motivation for the TOM Toolkit is discussed on the :doc:`about ` page. The TOM Toolkit (referred to as "the toolkit") provides a framework for @@ -14,15 +14,15 @@ Web-based technologies allow developers to create rich user interfaces, simplify distribution and choose from a huge variety of programming languages and frameworks. -[Python](https://python.org) has become the go-to language for many in +`Python `_ has become the go-to language for many in science. Fortunately, Python also enjoys widespread popularity in web development communities. This provides a unique opportunity for "Pythonistas" to develop scientific codebases which integrate seamlessly with web-based technologies. One need look no further than the success of the -[Jupyter](https://jupyter.org) project to see evidence of this. +`Jupyter `_ project to see evidence of this. There has been a lot of development surrounding Python and the web in the last -two decades. One framework in particular, [Django](https://djangoproject.com) +two decades. One framework in particular, `Django `_ has emerged as one of the more popular choices for web development. Django is well known for its maturity, ease of use and modularity. @@ -30,51 +30,38 @@ Instead of reinventing the wheel, it often makes sense to build on the proven work of others. Thus, it was decided that the toolkit would build on top of the Django framework. This provides several advantages: -1. The toolkit does not need to re-implement generic functionality that Django -already provides such as template rendering, routing, object relational mapping, -or even higher-level functionality like user accounts and database migrations. +#. The toolkit does not need to re-implement generic functionality that Django already provides such as template rendering, routing, object relational mapping, or even higher-level functionality like user accounts and database migrations. -2. TOM developers get to take advantage of the massive amounts of existing -knowledge that already exists for Django projects. In fact, much of the extra -functionality that a TOM developer might want to implement need not be dependent -on the the toolkit at all, but can instead be developed by referring to the -[excellent documentation](https://docs.djangoproject.com/en/2.2/) Django -provides. +#. TOM developers get to take advantage of the massive amounts of existing knowledge that already exists for Django projects. In fact, much of the extra functionality that a TOM developer might want to implement need not be dependent on the the toolkit at all, but can instead be developed by referring to the `excellent documentation `_ Django provides. -3. There are [thousands of Django packages](https://djangopackages.org) already -written that can be used in any TOM project. If a TOM developer wants to be able -to generate dynamic plots, or allow their users to login with Google, or even -turn their TOM into a Slack bot, chances are there is already a package -available that might suit their needs. +#. There are `thousands of Django packages `_ already written that can be used in any TOM project. If a TOM developer wants to be able to generate dynamic plots, or allow their users to login with Google, or even turn their TOM into a Slack bot, chances are there is already a package available that might suit their needs. We **highly recommend** that developers interested in utilizing the TOM Toolkit familiarize themselves with the basics of Django, especially if they want to -customize the toolkit in any significant fashion. The majority of the [guides -found in the TOM toolkit documentation](/index) are simply Django concepts -rewritten in a TOM context. +customize the toolkit in any significant fashion. The majority of the :doc:`guides found in the TOM toolkit documentation ` are simply Django concepts rewritten in a TOM context. -### Extending and Customizing the TOM Toolkit +Extending and Customizing the TOM Toolkit +========================================= -As mentioned before, Django is well known for its extendibility and modularity. +As mentioned before, Django is well known for its extensibility and modularity. The toolkit takes advantage of these strengths heavily. In many ways, the TOM Toolkit is a framework within a framework. -After a TOM developer follows the [getting started guide](getting_started) +After a TOM developer follows the :doc:`getting started guide ` they are left with a functioning but generic TOM. It is then up to the developer to implement the specific features that their science case requires. The toolkit tries to facilitate this as efficiently as possible and provides -[documentation](/index) in areas of customization from [changing the HTML layout -of a page](/customization/customize_templates) to [altering how observations are -submitted](/customization/customize_observations) and even [creating a new alert -broker](/customization/create_broker). +:doc:`documentation ` in areas of customization from :doc:`changing the HTML layout of a page ` +to :doc:`altering how observations are submitted ` and even +:doc:`creating a new alert broker `. Django, and by extension the toolkit, rely heavily on object oriented programming, especially inheritance. Most customization in the TOM toolkit comes from subclassing classes that provide generic functionality and overriding or extending methods. An experienced Django developer would feel right at home. For example, the -[ObservationRecordDetailView](https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/views.py#L143) -in the `tom_observations` module of the toolkit inherits from Django's -[DetailView](https://docs.djangoproject.com/en/2.2/ref/class-based-views/generic-display/#detailview). +`ObservationRecordDetailView `_ +in the ``tom_observations`` module of the toolkit inherits from Django's +`DetailView `_. This means TOM developers are able to take full advantage of the power of Django while still benefiting from the basic functionality that the toolkit provides. @@ -82,36 +69,38 @@ This is why we recommend TOM developers familiarize themselves with Django; most TOM Toolkit features are actually extended Django features. -#### Plugin Architecture +Plugin Architecture +=================== + Some areas of the TOM implement a plugin based architecture to support multiple implementations of a similar functionality. An example would be the -`tom_observations` module in which every supported observatory is implemented -as its own plugin. The `tom_catalogs` and `tom_alerts` work in the same way: the +`tom_observations`` module in which every supported observatory is implemented +as its own plugin. The ``tom_catalogs`` and ``tom_alerts`` work in the same way: the module defines the interface and generic functionality and each implementation fills in its own logic. This structure makes it easy for developers to write their own plugins which can then be shared and installed by others or even contributed to the main codebase. -The [gemini.py -module](https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/gemini.py) +The `gemini.py module `_ is an observation module plugin contributed by Bryan Miller to enable the triggering of observation requests on the Gemini telescope via the TOM Toolkit. Thanks Bryan! -#### Template Engine -The toolkit is able to take advantage of Django's excellent [template -engine](https://docs.djangoproject.com/en/2.2/topics/templates/). Part of the +Template Engine +=============== + +The toolkit is able to take advantage of Django's excellent `template engine `_. Part of the engine's power comes form the ability of templates to extend and override each other. This means a TOM developer can easily change the layout and style of any page without modifying the underlying framework's code directly. Entire pages may be replaced, or only "blocks" within a template. -Compare these screenshots of the [standard target detail -page](../../../_static/architecture/standardlayout.png) and the [Global Supernova -Project's target detail page](../../../_static/architecture/snex2layout.png), the +Compare these screenshots of the `standard target detail page <../../../_static/architecture/snex2layout.png>`_ and the +`Global Supernova Project's target detail page <../../../_static/architecture/snex2layout.png>`_, the latter taking heavy advantage of template inheritance. -### Data Storage, Deployment and Tooling +Data Storage, Deployment and Tooling +==================================== The toolkit is implemented as a web application backed by a relational database, uses (mostly) server side rendering, and is deployed using wsgi. @@ -123,30 +112,28 @@ ones. By default SQLite is deployed because of its ease of use. For non-database storage (data products, fits files, etc) the toolkit can be configured to use a variety of cloud-based storage services via -[django-storages](https://django-storages.readthedocs.io). The documentation -provides a guide for [storing data on Amazon S3](/deployment/amazons3). By default, +`django-storages `_. The documentation +provides a guide for :doc:`storing data on Amazon S3 `. By default, data is stored on disk. Similarly, deployment works with a variety of servers, including uWsgi and -Gunicorn. The documentation provides a guide to [deploying to -Heroku](/deployment/deployment_heroku) for those who want to get up and running +Gunicorn. The documentation provides a guide to :doc:`deploying to Heroku ` for those who want to get up and running quickly. Another option is to use Docker: the demo instance of the toolkit is deployed to a Kubernetes cluster and the -[Dockerfile](https://github.com/TOMToolkit/tom_demo/blob/master/Dockerfile) is +`Dockerfile `_ is available on Github. -On the frontend, the toolkit utilizes the very popular [Bootstrap4 css -framework](https://getbootstrap.com) for its layout and general look, making it +On the frontend, the toolkit utilizes the very popular `Bootstrap4 css framework `_ for its layout and general look, making it easy to pickup for anyone with experience with CSS. Javascript is introduced sparingly (astronomers love Python!) but is used in various situations to enhance the user experience and enable functionality such as interactive plotting and sky maps. -### Django Reusable Apps +Django Reusable Apps +==================== As previously mentioned, one of the reasons for Django's popularity is its -modularity. Django has the concept of [reusable -apps](https://docs.djangoproject.com/en/2.2/intro/reusable-apps/) which are just +modularity. Django has the concept of `reusable apps `_ which are just python packages that are specifically meant to be used inside a Django project. The majority of the the toolkit's functionality is implemented in a series of Django apps. While most of the apps are required, some may be omitted entirely @@ -154,38 +141,39 @@ from a TOM if the functionality is not desired. The following describes each app that ships with the toolkit and its purpose. -#### TOM Targets +TOM Targets +----------- -The -[tom_targets](https://github.com/TOMToolkit/tom_base/tree/master/tom_targets) +The `tom_targets `_ app is central to the entire TOM Toolkit project. It provides the database definitions for the storage and retrieval of targets and target lists. It also provides the views (pages) for viewing, creating, modifying and visualizing these targets in several ways including the visibility and target distribution plots. -Nearly every app depends on the `tom_targets` module in some way. +Nearly every app depends on the ``tom_targets`` module in some way. -#### TOM Observations +TOM Observations +---------------- -The -[tom_observations](https://github.com/TOMToolkit/tom_base/tree/master/tom_observations) +The `tom_observations `_ app handles all the logic for submitting and querying observations of targets at observatories. It defines the database models for observation requests and provides some views for working with them. -[facility.py](https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facility.py) +`facility.py `_ defines an interface that external facilities (observatories) can implement in order to integrate with the toolkit: -[gemini.py](https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/gemini.py) +`gemini.py `_ and -[lco.py](https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/lco.py) +`lco.py `_ are two examples, and we expect more in the future. -#### TOM Data Products +TOM Data Products +----------------- -Straddling both the `tom_targets` and `tom_observations` packages is -[tom_dataproducts](https://github.com/TOMToolkit/tom_base/tree/master/tom_dataproducts). +Straddling both the ``tom_targets`` and ``tom_observations`` packages is +`tom_dataproducts `_. This package contains the logic required for storing data related to targets and observations within the toolkit. Some data products are fetched from on-line archives (handled by an observatory's observation module) but data can also be @@ -196,107 +184,118 @@ or in the cloud) as well as displaying certain kinds of data. It also provides code hooks where TOM developers can run their own functions on the data in case specialized data processing, analytics or pipelining is required. -#### TOM Alerts +TOM Alerts +---------- -The [tom_alerts](https://github.com/TOMToolkit/tom_base/tree/master/tom_alerts) +The `tom_alerts `_ app contains modules related to the functionality of ingesting targets from various external services. These services, usually called brokers, provide rapidly changing target lists that are of interest to time domain astronomers. The -[alerts.py](https://github.com/TOMToolkit/tom_base/blob/master/tom_alerts/alerts.py) +`alerts.py `_ module provides a generic interface that other modules can implement, giving them the ability to integrate these brokers with the toolkit. Currently, there are -modules available for [Lasair](https://lasair.roe.ac.uk), -[MARS](https://mars.lco.global) and -[SCOUT](https://cneos.jpl.nasa.gov/scout/intro.html) with more planned for the -future. +modules available for `Lasair `_, +`MARS `_, `SCOUT `_, and others, +with more planned for the future. -#### TOM Catalogs +TOM Catalogs +------------ The -[tom_catalogs](https://github.com/TOMToolkit/tom_base/tree/master/tom_catalogs) +`tom_catalogs `_ app contains functionality related to querying astronomical catalogs. These "harvester" modules enable the querying and translation of targets found in databases such as Simbad and JPL Horizons directly into targets within the toolkit. The -[harvester.py](https://github.com/TOMToolkit/tom_base/blob/master/tom_catalogs/harvester.py) +`harvester.py `_ module provides the basic interface, and there are several modules already written for Simbad, NED, the MPC, JPL Horizons and the Transient Name Server. -#### TOM Setup and TOM Common +TOM Setup and TOM Common +------------------------ -The [tom_setup](https://github.com/TOMToolkit/tom_base/tree/master/tom_setup) +The `tom_setup `_ package is special in that its sole purpose is to help TOM developers bootstrap -new TOMs. See the [getting started](getting_started) guide for an example. -The [tom_common](https://github.com/TOMToolkit/tom_base/tree/master/tom_common) +new TOMs. See the :doc:`getting started ` guide for an example. +The `tom_common `_ package contains logic and data that doesn't fit anywhere else. -### Database Layout +Database Layout +--------------- + The following diagram is an Entity-relationship Diagram (ERD). It is meant to display the relationship between tables in a database. In this case, it may help illustrate how the data from each of the toolkit's packages relate to each other. It is not exhaustive; many tables and rows have been omitted for brevity. -[![db layout](../_static/architecture/erd.png)](../_images/erd.png) +.. image:: /_static/architecture/erd.png + :alt: DB Layout + +Models +====== -### Models Django models are the classes that map to the database tables in your Django application. The TOM Toolkit models and the rationale behind them do are largely intuitive, but may require some explanation. -#### Target -The `Target` model is relatively self-evident--it stores the data that describes the +Target +------ + +The ``Target`` model is relatively self-evident--it stores the data that describes the targets in your TOM. By default, that includes things like name, type, coordinates, and ephemerides. -#### TargetName -The `TargetName` model stores extra names for a target, aka aliases. The corresponding target +TargetName +---------- + +The ``TargetName`` model stores extra names for a target, aka aliases. The corresponding target is stored as a foreign key. -#### ObservationRecord -The `ObservationRecord` model describes an individual observation request for a single target. +ObservationRecord +----------------- + +The ``ObservationRecord`` model describes an individual observation request for a single target. It stores the target as a foreign key, and can optionally store facility information and the parameters submitted for the observation. -#### DataProduct -The `DataProduct` model can refer to a number of different things, but generally refers to a -single file that is associated with a `Target` and optionally an `ObservationRecord`. A -`DataProduct` has one of a number of tags, which at present include the following: +DataProduct +----------- + +The ``DataProduct`` model can refer to a number of different things, but generally refers to a +single file that is associated with a ``Target`` and optionally an ``ObservationRecord``. A +`DataProduct`` has one of a number of tags, which at present include the following: - Photometry, a file containing photometric data - FITS, any FITS file not falling into the other categories - Spectroscopy, a file containing spectroscopic data - Image, a file containing image data, such as a JPEG or PNG -A `DataProduct` type is file format-agnostic and refers to the data contained in the file, +A ``DataProduct`` type is file format-agnostic and refers to the data contained in the file, rather than the format itself. The type is necessary for making decisions on which operations can be executed using the data in a file. -#### ReducedDatum -A `ReducedDatum` is a single point of data associated with a `Target` and optionally a -`DataProduct`. The single data point is typically a single point of photometry or an individual -spectrum. The `ReducedDatum` model has the following fields, in addition to its aforementioned +ReducedDatum +------------ + +A ``ReducedDatum`` is a single point of data associated with a ``Target`` and optionally a +``DataProduct``. The single data point is typically a single point of photometry or an individual +spectrum. The ``ReducedDatum`` model has the following fields, in addition to its aforementioned foreign key relationships: -- `data_type` is maintained on both the `ReducedDatum` and `DataProduct` for the -case when data is brought in from another source, such as a broker -- The `source_name` optionally refers to the original source of the data. The -intent of this field was to track data ingested from brokers, but could potentially be used for -other purposes. -- `source_location` optionally gives a hard location to the source--for a -broker, it would be a link to the original alert. -- The `timestamp` time at which the datum was produced. -- `value` is a `TextField` that can take any series of data. As implemented, photometry -is stored as JSON with keys for magnitude and error, but the `TextField` provides flexibility for -additional photometry values on the datum. Spectroscopy is also stored as JSON, with keys for -`magnitude` and `flux`. - -### Feedback and bug reporting +- ``data_type`` is maintained on both the ``ReducedDatum`` and ``DataProduct`` for the case when data is brought in from another source, such as a broker +- The ``source_name`` optionally refers to the original source of the data. The intent of this field was to track data ingested from brokers, but could potentially be used for other purposes. +- ``source_location`` optionally gives a hard location to the source--for a broker, it would be a link to the original alert. +- The ``timestamp`` time at which the datum was produced. +- ``value`` is a ``TextField`` that can take any series of data. As implemented, photometry is stored as JSON with keys for magnitude and error, but the ``TextField`` provides flexibility for additional photometry values on the datum. Spectroscopy is also stored as JSON, with keys for ``magnitude`` and ``flux``. + +Feedback and bug reporting +========================== We hope the TOM Toolkit is helpful to you and your project. If you have any concerns about implementation details, or questions about your own needs, please -don't hesitate to [reach out](mailto:ariba@lco.global). Issues and pull requests -are also welcome on the project's [GitHub page](https://github.com/TOMToolkit/). +don't hesitate to `reach out `_. Issues and pull requests +are also welcome on the project's `GitHub page `_. diff --git a/docs/introduction/troubleshooting.md b/docs/introduction/troubleshooting.md index 6c52b9844..ded36f432 100644 --- a/docs/introduction/troubleshooting.md +++ b/docs/introduction/troubleshooting.md @@ -1,4 +1,4 @@ -# Troubleshooting your TOM Toolkit +# Troubleshooting your TOM When first installing or later updating your TOM, you may run into a few common issues. Fortunately, you can stand on our shoulders and hopefully find a solution here! diff --git a/docs/customization/customizing_data_processing.md b/docs/managing_data/customizing_data_processing.md similarity index 100% rename from docs/customization/customizing_data_processing.md rename to docs/managing_data/customizing_data_processing.md diff --git a/docs/managing_data/index.rst b/docs/managing_data/index.rst new file mode 100644 index 000000000..86a7108d4 --- /dev/null +++ b/docs/managing_data/index.rst @@ -0,0 +1,18 @@ +Managing Data +============= + +.. toctree:: + :maxdepth: 2 + :hidden: + + + ../api/tom_dataproducts/views + plotting_data + customizing_data_processing + + +:doc:`Creating Plots from TOM Data ` - Learn how to create plots using plot.ly and your TOM +data to display anywhere in your TOM. + +:doc:`Adding Custom Data Processing ` - Learn how you can process data into your +TOM from uploaded data products. diff --git a/docs/customization/plotting_data.md b/docs/managing_data/plotting_data.md similarity index 100% rename from docs/customization/plotting_data.md rename to docs/managing_data/plotting_data.md diff --git a/docs/customization/customize_observations.md b/docs/observing/customize_observations.md similarity index 100% rename from docs/customization/customize_observations.md rename to docs/observing/customize_observations.md diff --git a/docs/observing/index.rst b/docs/observing/index.rst new file mode 100644 index 000000000..ffa879be8 --- /dev/null +++ b/docs/observing/index.rst @@ -0,0 +1,29 @@ +Observing Facilities and Observations +===================================== + +.. toctree:: + :maxdepth: 2 + :hidden: + + customize_observations + ../common/scripts + strategies + observation_module + ../api/tom_observations/facilities + ../api/tom_observations/views + + +:doc:`Changing Request Submission Behavior ` - Learn how to customize observation forms +in order to add additional parameters to observation requests. + +`Programmatically Submitting Observations <../common/scripts.html#creating-observations-programmatically>`__ + +:doc:`Cadence and Observing Strategies ` - Learn how to build cadence strategies that submit observations based on +the result of prior observations, as well as how to leverage observing templates to submit observations with fewer clicks. + +:doc:`Building a TOM Observation Facility Module ` - Learn to build a module which will +allow your TOM to submit observation requests to observatories. + +:doc:`Facility Modules <../api/tom_observations/facilities>` - Take a look at the supported facilities. + +:doc:`Observation Views <../api/tom_observations/views>` - Familiarize yourself with the available Observation Views. diff --git a/docs/advanced/observation_module.md b/docs/observing/observation_module.md similarity index 100% rename from docs/advanced/observation_module.md rename to docs/observing/observation_module.md diff --git a/docs/advanced/strategies.md b/docs/observing/strategies.md similarity index 100% rename from docs/advanced/strategies.md rename to docs/observing/strategies.md diff --git a/docs/support.md b/docs/support.md deleted file mode 100644 index a2fb3d5a1..000000000 --- a/docs/support.md +++ /dev/null @@ -1,21 +0,0 @@ -Getting Support ---- - -This page will go over the process for reporting issues, requesting features, and getting -support for the TOM Toolkit. - -### Reporting Issues - -Issue reporting can be done via the [Github issues page](https://github.com/TOMToolkit/tom_base/issues) -of the tom_base project. Reporting an issue requires a Github account, but provides an easy way for -developers to ask follow-up questions about an issue in order to resolve it. - -Please include as much detail as possible, as well as the steps taken that trigger the issue, and be sure to tag it with the "bug" tag! - -### Requesting Features - -Like issues, feature and enhancement requests should be done via the same [Github issues page](https://github.com/TOMToolkit/tom_base/issues) of the tom_base project. This is also a great place to see what's being worked on and what's already been requested, which will allow you to voice support for a backlogged feature to be reprioritized. - -### Support - -If you're looking for help with some aspect of your TOM, the [Github issues page](https://github.com/TOMToolkit/tom_base/issues) is once again the place to go. The "question" or "help wanted" tags should be very useful when looking for support, and the TOM Toolkit developers are more than happy to provide the help necessary to get your TOM running. You may also want to peruse the [Closed Issues](https://github.com/TOMToolkit/tom_base/issues?q=is%3Aissue+is%3Aclosed), where someone may have already had (and solved!) your problem. \ No newline at end of file diff --git a/docs/targets/index.rst b/docs/targets/index.rst new file mode 100644 index 000000000..5947b4b52 --- /dev/null +++ b/docs/targets/index.rst @@ -0,0 +1,23 @@ +Targets +======= + +.. toctree:: + :maxdepth: 2 + :hidden: + + target_fields + ../api/tom_targets/models + ../api/tom_targets/views + + +The ``Target``, along with the associated ``TargetList``, ``TargetExtra``, and ``TargetName``, are the core models of the +TOM Toolkit. The ``Target`` defines the concept of an astronomical target. + +:doc:`Adding Custom Target Fields ` - Learn how to add custom fields to your TOM Targets if the +defaults do not suffice. + +:doc:`Target API <../api/tom_targets/models>` - Take a look at the available properties for a ``Target`` and its associated models. + +:doc:`Target Views <../api/tom_targets/views>` - Familiarize yourself with the available Target Views. + +:doc:`Target Groups <../api/tom_targets/groups>` - Check out the functions for operating on Target Groups. diff --git a/docs/customization/target_fields.md b/docs/targets/target_fields.md similarity index 100% rename from docs/customization/target_fields.md rename to docs/targets/target_fields.md From 5612cdeef3d7499d4f7bf43a91fb89eedec33cee Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 8 Jun 2020 17:49:47 -0700 Subject: [PATCH 164/424] Converted create_broker into rst in order to add admonition note --- docs/brokers/create_broker.md | 219 ----------------------------- docs/brokers/create_broker.rst | 245 +++++++++++++++++++++++++++++++++ 2 files changed, 245 insertions(+), 219 deletions(-) delete mode 100644 docs/brokers/create_broker.md create mode 100644 docs/brokers/create_broker.rst diff --git a/docs/brokers/create_broker.md b/docs/brokers/create_broker.md deleted file mode 100644 index 0307f52ca..000000000 --- a/docs/brokers/create_broker.md +++ /dev/null @@ -1,219 +0,0 @@ -Creating an Alert Broker Module for the TOM Toolkit ---------------------------------------------------- - -This guide will walk you through how to create a custom alert broker module using the TOM toolkit. - -At the end of this tutorial we will have a very simple module that connects to -an "alert broker" (in this case a static json file) and allows us to ingest -targets into our TOM. - -You can follow this example to build an alert broker module to connect to a real -alert broker. - -Be sure you've followed the [Getting Started](/introduction/getting_started) guide before continuing onto this tutorial. - -#### TOM Alerts module -The TOM Alerts module is a Django app which provides the methods and -classes needed to create a custom TOM alert broker module. A module may be created to ingest -alerts of an arbitrary form from a remote source. The TOM Alerts module provides -tools to transform these alerts into TOM-specific alerts to be used in the creation of TOM Targets. - -### Project Structure -After following the [Getting Started](/introduction/getting_started) guide, you will have -a Django project directory of the form: - -``` -mytom -├── db.sqlite3 -├── manage.py -└── mytom - ├── __init__.py - ├── settings.py - ├── urls.py - └── wsgi.py -``` - -### Creating a Broker Module -In this example, we will create a broker named __MyBroker__. - -Begin by creating a file `my_broker.py`, and placing it in the inner `mytom/` directory -of the project (in the directory with settings.py). `my_broker.py` will contain the classes that define our custom -TOM Alert Broker Module. - -Our custom broker module relies on the TOM Toolkit modules that were installed in the -[Getting Started](/introduction/getting_started) guide. Begin by editing `my_broker.py` -to import the necessary modules. - -```python -from tom_alerts.alerts import GenericQueryForm, GenericAlert, GenericBroker -from tom_alerts.models import BrokerQuery -from tom_targets.models import Target -``` - -In order to add custom forms to our broker module, we will also need Django's `forms` module. - -```python -from django import forms -``` -See [Working with Django Forms](https://docs.djangoproject.com/en/2.1/topics/forms/) - -Finally, import `requests` so that we can fetch some remote broker test data. - -```python -import requests -``` -See [Requests Official API Docs](http://docs.python-requests.org/en/master/) - -#### Test Data - -In place of a remote broker, we've uploaded a [sample JSON file to GitHub Gist](https://gist.githubusercontent.com/mgdaily/f5dfb4047aaeb393bf1996f0823e1064/raw/5e6a6142ff77e7eb783892f1d1d01b13489032cc/example_broker_data.json). - -For our `my_broker.py` module to use this data, we will set `broker_url` to it. -``` -broker_url = 'https://gist.githubusercontent.com/mgdaily/f5dfb4047aaeb393bf1996f0823e1064/raw/5e6a6142ff77e7eb783892f1d1d01b13489032cc/example_broker_data.json' -``` - -#### Broker Forms -To define the query forms for our custom broker module, we'll begin by creating class -`MyBrokerForm` inside `my_broker.py`, which inherits the `tom_alert` module's -`GenericQueryForm`. - -This will define the list of forms to be presented within the broker query. For -our example, we'll be querying simply on target name. - -```python -class MyBrokerForm(GenericQueryForm): - target_name = forms.CharField(required=True) -``` - -#### Broker Class -To define our broker module, we'll create the class `MyBroker`, also inside of `my_broker.py`. -Our broker class will encapsulate the logic for making queries to a remote alert broker, -retrieving and sanitizing data, and creating TOM alerts from it. - -Begin by defining the class, its name and default form. In our case, the name -will simply be 'MyBroker', and the form will be `MyBrokerForm` - the form that we -just defined! - -```python -class MyBroker(GenericBroker): - name = 'MyBroker' - form = MyBrokerForm -``` - -#### Required Broker Class Methods -Each TOM alert broker module is required to have a base set of class methods. These -methods enable the conversion of remote alert data into TOM-specific -alerts and targets. - -##### `fetch_alerts` Class Method -`fetch_alerts` is used to query the remote broker, and return an iterator -of results depending on the parameters passed into the query, so that -these results may be displayed on the query results page. In our case, `fetch_alerts` -will only filter on name, but this can be easily extended to other query parameters. - -```python -@classmethod -def fetch_alerts(clazz, parameters): - response = requests.get(broker_url) - response.raise_for_status() - test_alerts = response.json() - return iter([alert for alert in test_alerts if alert['name'] == parameters['target_name']]) -``` -**Why an iterator?** Because some alert brokers work by sending streams, not fully -evaluated lists. This simple example broker could easily return a list (in fact we -are coercing the list into an iterator!) but that would not work in the model -where a broker is sending an unending stream of alerts. - -Our implementation will get a response from our test broker source, check that our -request was successful, and return a iterator of alerts whose name field matches the -name passed into the query. - -##### `to_generic_alert` Class Method -In order to standardize alerts and display them in a consistent manner, -the `GenericAlert` class has been defined within the `tom_alerts` library. -This broker method converts a remote alert into a TOM Toolkit `GenericAlert`. - -```python -@classmethod -def to_generic_alert(clazz, alert): - return GenericAlert( - timestamp=alert['timestamp'], - url=broker_url, - id=alert['id'], - name=alert['name'], - ra=alert['ra'], - dec=alert['dec'], - mag=alert['mag'], - score=alert['score'] - ) -``` -In our case, the `GenericAlert` attributes match up *almost* directly with our test -data. How convenient! We'll just go ahead and define the `GenericAlert`'s `url` -field as the `broker_url` we retrieved our test data from. - -```python -... -url=broker_url, -... -``` - -#### Other methods - -`fetch_alerts` and `to_generic_alert` are the only methods required for your -broker module to function. Of course you are free to add any number of additional -methods or attributes to the module that you deem necessary. - -### Using Our New Alert Broker -Now that we've created our TOM alert broker, let's hook it into our TOM -so that we can ingest alerts and create targets. - -The `tom_alerts` module will look in `settings.py` for a list of alert -broker classes, so we'll need to add `MyBroker` to that list. - -```python -TOM_ALERT_CLASSES = [ - ... - 'tom_alerts.brokers.mars.MARSBroker', - 'mytom.my_broker.MyBroker', - ... -] -``` -Now, navigate to the top-level directory of your Django project, -where `manage.py` resides and run - -```bash -./manage.py makemigrations -./manage.py migrate -./manage.py runserver -``` - -Navigate to [http://127.0.0.1:8000/alerts/query/list/](http://127.0.0.1:8000/alerts/query/list/) - -You should now see 'MyBroker' listed as a broker! Clicking the link will bring you -to the query page, where you can make a query to our sample dataset. - -![](/_static/create_broker_doc/success_broker_list.png) - -#### Making a Query - -Since we're only going to be filtering on the alert's 'target_name' field, we're only -presented with that option. Name the query whatever you'd like, and we'll check -our remote data source for a target named 'Tatooine' - -![](/_static/create_broker_doc/example_query.png) - -Going back to [http://127.0.0.1:8000/alerts/query/list/](http://127.0.0.1:8000/alerts/query/list/), -our new query will appear. Click the 'run' button to run the query. - -![](/_static/create_broker_doc/populated_query_list.png) - -The query result will be presented. - -![](/_static/create_broker_doc/query_result.png) - -To create a target from any query result, click the 'create target' button. To view the raw -alert data, click the 'view' link. - -[Click here](https://gist.github.com/mgdaily/19aefebd05da91fe6ebfe928b4862a51) to view -the full source code detailed in this example. diff --git a/docs/brokers/create_broker.rst b/docs/brokers/create_broker.rst new file mode 100644 index 000000000..2b01109a4 --- /dev/null +++ b/docs/brokers/create_broker.rst @@ -0,0 +1,245 @@ +Creating an Alert Broker Module for the TOM Toolkit +################################################### + +This guide will walk you through how to create a custom alert broker module using the TOM toolkit. + +At the end of this tutorial we will have a very simple module that connects to +an "alert broker" (in this case a static json file) and allows us to ingest +targets into our TOM. + +You can follow this example to build an alert broker module to connect to a real +alert broker. + +Be sure you've followed the :doc:`Getting Started ` guide before continuing onto this tutorial. + +.. note:: + The following Python/Django concepts are used in this tutorial. While this tutorial does not assume familiarity with the concepts, you will likely find the tutorial easier to understand if you read these in advance. + + - `Working with Django Forms `_ + - `Requests Official API Docs `_ + +TOM Alerts module +***************** + +The TOM Alerts module is a Django app which provides the methods and +classes needed to create a custom TOM alert broker module. A module may be created to ingest +alerts of an arbitrary form from a remote source. The TOM Alerts module provides +tools to transform these alerts into TOM-specific alerts to be used in the creation of TOM Targets. + +Project Structure +***************** + +After following the :doc:`Getting Started ` guide, you will have +a Django project directory of the form: + +.. code-block:: + + mytom + ├── db.sqlite3 + ├── manage.py + └── mytom + ├── __init__.py + ├── settings.py + ├── urls.py + └── wsgi.py + +Creating a Broker Module +************************ + +In this example, we will create a broker named __MyBroker__. + +Begin by creating a file ``my_broker.py``, and placing it in the inner ``mytom/`` directory +of the project (in the directory with settings.py). ``my_broker.py`` will contain the classes that define our custom +TOM Alert Broker Module. + +Our custom broker module relies on the TOM Toolkit modules that were installed in the +:doc:`Getting Started ` guide. Begin by editing ``my_broker.py`` +to import the necessary modules. + +.. code-block:: python + + from tom_alerts.alerts import GenericQueryForm, GenericAlert, GenericBroker + from tom_alerts.models import BrokerQuery + from tom_targets.models import Target + +In order to add custom forms to our broker module, we will also need Django's `forms` module, as well the Python module `requests`, which will allow us to fetch some remote broker test data. + +.. code-block:: python + + from django import forms + import requests + +See `Working with Django Forms `_ and the `Requests Official API Docs `_. + +Test Data +********* + +In place of a remote broker, we've uploaded a `sample JSON file to GitHub Gist `_. + +For our ``my_broker.py`` module to use this data, we will set ``broker_url`` to it. + +.. code-block:: python + + broker_url = 'https://gist.githubusercontent.com/mgdaily/f5dfb4047aaeb393bf1996f0823e1064/raw/5e6a6142ff77e7eb783892f1d1d01b13489032cc/example_broker_data.json' + +Broker Forms +************ + +To define the query forms for our custom broker module, we'll begin by creating class +``MyBrokerForm`` inside ``my_broker.py``, which inherits the ``tom_alert`` module's +``GenericQueryForm``. + +This will define the list of forms to be presented within the broker query. For +our example, we'll be querying simply on target name. + +.. code-block:: python + + class MyBrokerForm(GenericQueryForm): + target_name = forms.CharField(required=True) + +Broker Class +************ + +To define our broker module, we'll create the class ``MyBroker``, also inside of ``my_broker.py``. +Our broker class will encapsulate the logic for making queries to a remote alert broker, +retrieving and sanitizing data, and creating TOM alerts from it. + +Begin by defining the class, its name and default form. In our case, the name +will simply be 'MyBroker', and the form will be ``MyBrokerForm`` - the form that we +just defined! + +.. code-block:: python + + class MyBroker(GenericBroker): + name = 'MyBroker' + form = MyBrokerForm + +Required Broker Class Methods +============================= + +Each TOM alert broker module is required to have a base set of class methods. These +methods enable the conversion of remote alert data into TOM-specific +alerts and targets. + +``fetch_alerts`` Class Method +----------------------------- + +`fetch_alerts` is used to query the remote broker, and return an iterator +of results depending on the parameters passed into the query, so that +these results may be displayed on the query results page. In our case, `fetch_alerts` +will only filter on name, but this can be easily extended to other query parameters. + +.. code-block:: python + + @classmethod + def fetch_alerts(clazz, parameters): + response = requests.get(broker_url) + response.raise_for_status() + test_alerts = response.json() + return iter([alert for alert in test_alerts if alert['name'] == parameters['target_name']]) + +**Why an iterator?** Because some alert brokers work by sending streams, not fully +evaluated lists. This simple example broker could easily return a list (in fact we +are coercing the list into an iterator!) but that would not work in the model +where a broker is sending an unending stream of alerts. + +Our implementation will get a response from our test broker source, check that our +request was successful, and return a iterator of alerts whose name field matches the +name passed into the query. + +``to_generic_alert`` Class Method +--------------------------------- + +In order to standardize alerts and display them in a consistent manner, +the ``GenericAlert`` class has been defined within the ``tom_alerts`` library. +This broker method converts a remote alert into a TOM Toolkit ``GenericAlert``. + +.. code-block:: python + + @classmethod + def to_generic_alert(clazz, alert): + return GenericAlert( + timestamp=alert['timestamp'], + url=broker_url, + id=alert['id'], + name=alert['name'], + ra=alert['ra'], + dec=alert['dec'], + mag=alert['mag'], + score=alert['score'] + ) + +In our case, the ``GenericAlert`` attributes match up *almost* directly with our test +data. How convenient! We'll just go ahead and define the ``GenericAlert``'s ``url`` +field as the ``broker_url`` we retrieved our test data from. + +.. code-block:: python + + ... + url=broker_url, + ... + +Other methods +============= + +``fetch_alerts`` and ``to_generic_alert`` are the only methods required for your +broker module to function. Of course you are free to add any number of additional +methods or attributes to the module that you deem necessary. + +Using Our New Alert Broker +************************** + +Now that we've created our TOM alert broker, let's hook it into our TOM +so that we can ingest alerts and create targets. + +The ``tom_alerts`` module will look in ``settings.py`` for a list of alert +broker classes, so we'll need to add ``MyBroker`` to that list. + +.. code-block:: python + + TOM_ALERT_CLASSES = [ + ... + 'tom_alerts.brokers.mars.MARSBroker', + 'mytom.my_broker.MyBroker', + ... + ] + +Now, navigate to the top-level directory of your Django project, +where ``manage.py`` resides and run + +.. code-block:: bash + + ./manage.py makemigrations + ./manage.py migrate + ./manage.py runserver + +Navigate to `http://127.0.0.1:8000/alerts/query/list/ `_ + +You should now see 'MyBroker' listed as a broker! Clicking the link will bring you +to the query page, where you can make a query to our sample dataset. + +.. image:: /_static/create_broker_doc/success_broker_list.png + +Making a Query +============== + +Since we're only going to be filtering on the alert's 'target_name' field, we're only +presented with that option. Name the query whatever you'd like, and we'll check +our remote data source for a target named 'Tatooine' + +.. image:: /_static/create_broker_doc/example_query.png + +Going back to `http://127.0.0.1:8000/alerts/query/list/ `_, +our new query will appear. Click the 'run' button to run the query. + +.. image:: /_static/create_broker_doc/populated_query_list.png + +The query result will be presented. + +.. image:: /_static/create_broker_doc/query_result.png + +To create a target from any query result, click the 'create target' button. To view the raw +alert data, click the 'view' link. + +`Click here `_ to view +the full source code detailed in this example. From bdab53c35fb3cccafc8e27c57bcd22ca7d8cadaa Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 8 Jun 2020 18:06:39 -0700 Subject: [PATCH 165/424] Used pandoc to convert several .md files to .rst --- docs/brokers/create_broker.rst | 5 +- docs/common/customsettings.md | 206 -------------------------- docs/common/customsettings.rst | 239 +++++++++++++++++++++++++++++++ docs/common/latex_generation.md | 193 ------------------------- docs/common/latex_generation.rst | 215 +++++++++++++++++++++++++++ 5 files changed, 457 insertions(+), 401 deletions(-) delete mode 100644 docs/common/customsettings.md create mode 100644 docs/common/customsettings.rst delete mode 100644 docs/common/latex_generation.md create mode 100644 docs/common/latex_generation.rst diff --git a/docs/brokers/create_broker.rst b/docs/brokers/create_broker.rst index 2b01109a4..2fea30cd6 100644 --- a/docs/brokers/create_broker.rst +++ b/docs/brokers/create_broker.rst @@ -12,8 +12,9 @@ alert broker. Be sure you've followed the :doc:`Getting Started ` guide before continuing onto this tutorial. -.. note:: - The following Python/Django concepts are used in this tutorial. While this tutorial does not assume familiarity with the concepts, you will likely find the tutorial easier to understand if you read these in advance. +.. tip:: Read these first! + + The following Python/Django concepts are used in this tutorial. While this tutorial does not assume familiarity with the concepts, you will likely find the tutorial easier to understand and build upon if you read these in advance. - `Working with Django Forms `_ - `Requests Official API Docs `_ diff --git a/docs/common/customsettings.md b/docs/common/customsettings.md deleted file mode 100644 index 33b8d6f14..000000000 --- a/docs/common/customsettings.md +++ /dev/null @@ -1,206 +0,0 @@ -TOM Specific Settings ---------------------- - -The following is a list of TOM Specific settings to be added/edited in your -project's `settings.py`. For explanations of Django specific settings, see the -[official documentation](https://docs.djangoproject.com/en/2.1/ref/settings/). - - -### [ALERT_CREDENTIALS](#alert_credentials) - -Default: - - { - 'TNS': { - 'api_key': '' - } - } - -Credentials for any brokers that require them. At the moment, the only built-in TOM Toolkit broker module that -requires credentials is the TNS. - - -### [AUTH_STRATEGY](#auth_strategy) - -Default: 'READ_ONLY' - -Determines how your TOM treats unauthenticated users. A value of **READ_ONLY** -allows unauthenticated users to view most pages on your TOM, but not to change -anything. A value of **LOCKED** requires all users to login before viewing any -page. Use the [**OPEN_URLS**](#open_urls) setting for adding exemptions. - - -### [DATA_PROCESSORS](#data_processors) - -Default: - - { - 'photometry': 'tom_dataproducts.processors.photometry_processor.PhotometryProcessor', - 'spectroscopy': 'tom_dataproducts.processors.spectroscopy_processor.SpectroscopyProcessor', - } - -The `DATA_PROCESSORS` dict specifies the subclasses of `DataProcessor` that should be used for processing the -corresponding `data_type`s. - -### [DATA_PRODUCT_TYPES](#data_types) - -Default: - - { - 'spectroscopy': ('spectroscopy', 'Spectroscopy'), - 'photometry': ('photometry', 'Photometry'), - 'spectroscopy': ('spectroscopy', 'Spectroscopy'), - 'image_file': ('image_file', 'Image File') - } - -A list of machine readable, human readable tuples which determine the choices -available to categorize reduced data. - - -### [EXTRA_FIELDS](#extra_fields) - -Default: [] - -A list of extra fields to add to your targets. These can be used if the predefined -target fields do not match your needs. Please see the documentation on [Adding -Custom Fields to Targets](/targets/target_fields) for an explanation of how to use -this feature. - - -### [FACILITIES](#facilities) - -Default: - - { - 'LCO': { - 'portal_url': 'https://observe.lco.global', - 'api_key': os.getenv('LCO_API_KEY', ''), - } - } - -Observation facilities read their configuration values from this dictionary. -Although each facility is different, if you plan on using one you'll probably have -to configure it here first. For example the LCO facility requires you to provide a -value for the `api_key` configuration value. - - -### [HINTS](#hints) - -Default: - -HINTS_ENABLED = False -HINT_LEVEL = 20 - -A few messages are sprinkled throughout the TOM Toolkit that offer suggestions on -things you might want to change right out of the gate. These can be turned on and -off, and the level adjusted. For more information on Django message levels, see -the [Django messages framework documentation](https://docs.djangoproject.com/en/2.2/ref/contrib/messages/#message-levels). - - -### [HOOKS](#hooks) - -Default: - - { - 'target_post_save': 'tom_common.hooks.target_post_save', - 'observation_change_state': 'tom_common.hooks.observation_change_state', - 'data_product_post_upload': 'tom_dataproducts.hooks.data_product_post_upload', - } - -A dictionary of action, method code hooks to run. These hooks allow running -arbitrary python code when specific actions happen within a TOM, such as an -observation changing state. See the documentation on [Running Custom Code on -Actions in your TOM](/code/custom_code) for more details and available hooks. - - -### [OPEN_URLS](#open_urls) - -Default: [] - -With an [**AUTH_STRATEGY**](#auth_strategy) value of **LOCKED**, urls in this list will remain -visible to unauthenticated users. You might add the homepage ('/'), for example. - - -### [TARGET_PERMISSIONS_ONLY](#target_permissions_only) - -Default: True - -This settings determines the permissions strategy of the TOM. When set to True, authorization permissions will be set -on Targets and cascade from there--that is, a group that can see a Target can see all ObservationRecords and Data -associated with the Target. When set to False, permissions can be set for a group at the Target level, the -ObservationRecord level, or the DataProduct level. - - -### [TARGET_TYPE](#target_type) - -Default: No default - -Can be either **SIDEREAL** or **NON_SIDEREAL**. This setting determines the -default target type for your TOM. TOMs can still create and work with targets of -both types even after this option is set, but setting it to one of the values will -optimize the workflow for that target type. - - -### [TOM_ALERT_CLASSES](#tom_alert_classes) - -Default: - - [ - 'tom_alerts.brokers.mars.MARSBroker', - 'tom_alerts.brokers.lasair.LasairBroker', - 'tom_alerts.brokers.scout.ScoutBroker', - 'tom_alerts.brokers.tns.TNSBroker', - 'tom_alerts.brokers.antares.ANTARESBroker', - 'tom_alerts.brokers.gaia.GaiaBroker' - ] - -A list of tom alert classes to make available to your TOM. If you have written or -downloaded additional alert classes you would make them available here. If you'd -like to write your own alert module please see the documentation on [Creating an -Alert Module for the TOM Toolkit](/brokers/create_broker). - - -### [TOM_FACILITY_CLASSES](#tom_facility_classes) - -Default: - - [ - 'tom_observations.facilities.lco.LCOFacility', - 'tom_observations.facilities.gemini.GEMFacility', - 'tom_observations.facilities.soar.SOARFacility', - 'tom_observations.facilities.lt.LTFacility' - ] - -A list of observation facility classes to make available to your TOM. If you have -written or downloaded a custom observation facility you would add the class to -this list to make your TOM load it. - - -### [TOM_HARVESTER_CLASSES](#tom_harvester_classes) - -Default: - - [ - 'tom_catalogs.harvesters.simbad.SimbadHarvester', - 'tom_catalogs.harvesters.ned.NEDHarvester', - 'tom_catalogs.harvesters.jplhorizons.JPLHorizonsHarvester', - 'tom_catalogs.harvesters.mpc.MPCHarvester', - 'tom_catalogs.harvesters.tns.TNSHarvester', - ] - -A list of TOM harverster classes to make available to your TOM. If you have -written or downloaded additional harvester classes you would make them available -here. - - -### [TOM_LATEX_PROCESSORS](#tom_latex_processors) - -Default: - - { - 'ObservationGroup': 'tom_publications.processors.latex_processor.ObservationGroupLatexProcessor', - 'TargetList': 'tom_publications.processors.target_list_latex_processor.TargetListLatexProcessor' - } - -A dictionary with the keys being TOM models classes and the values being the modules that should be used to generate -latex tables for those models. diff --git a/docs/common/customsettings.rst b/docs/common/customsettings.rst new file mode 100644 index 000000000..f9d84412a --- /dev/null +++ b/docs/common/customsettings.rst @@ -0,0 +1,239 @@ +TOM Specific Settings +--------------------- + +The following is a list of TOM Specific settings to be added/edited in +your project’s ``settings.py``. For explanations of Django specific +settings, see the `official +documentation `__. + +`ALERT_CREDENTIALS <#alert_credentials>`__ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: + +.. code-block:: + + { + 'TNS': { + 'api_key': '' + } + } + +Credentials for any brokers that require them. At the moment, the only +built-in TOM Toolkit broker module that requires credentials is the TNS. + +`AUTH_STRATEGY <#auth_strategy>`__ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: ‘READ_ONLY’ + +Determines how your TOM treats unauthenticated users. A value of +**READ_ONLY** allows unauthenticated users to view most pages on your +TOM, but not to change anything. A value of **LOCKED** requires all +users to login before viewing any page. Use the +`OPEN_URLS <#open_urls>`__ setting for adding exemptions. + +`DATA_PROCESSORS <#data_processors>`__ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: + +.. code-block:: + + { + 'photometry': 'tom_dataproducts.processors.photometry_processor.PhotometryProcessor', + 'spectroscopy': 'tom_dataproducts.processors.spectroscopy_processor.SpectroscopyProcessor', + } + +The ``DATA_PROCESSORS`` dict specifies the subclasses of +``DataProcessor`` that should be used for processing the corresponding +``data_type``\ s. + +`DATA_PRODUCT_TYPES <#data_types>`__ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: + +.. code-block:: + + { + 'spectroscopy': ('spectroscopy', 'Spectroscopy'), + 'photometry': ('photometry', 'Photometry'), + 'spectroscopy': ('spectroscopy', 'Spectroscopy'), + 'image_file': ('image_file', 'Image File') + } + +A list of machine readable, human readable tuples which determine the +choices available to categorize reduced data. + +`EXTRA_FIELDS <#extra_fields>`__ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: [] + +A list of extra fields to add to your targets. These can be used if the +predefined target fields do not match your needs. Please see the +documentation on `Adding Custom Fields to +Targets `__ for an explanation of how to use +this feature. + +`FACILITIES <#facilities>`__ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: + +.. code-block:: + + { + 'LCO': { + 'portal_url': 'https://observe.lco.global', + 'api_key': os.getenv('LCO_API_KEY', ''), + } + } + +Observation facilities read their configuration values from this +dictionary. Although each facility is different, if you plan on using +one you’ll probably have to configure it here first. For example the LCO +facility requires you to provide a value for the ``api_key`` +configuration value. + +`HINTS <#hints>`__ +~~~~~~~~~~~~~~~~~~ + +Default: + +.. code-block:: + + HINTS_ENABLED = False + HINT_LEVEL = 20 + +A few messages are sprinkled throughout the TOM Toolkit that offer +suggestions on things you might want to change right out of the gate. +These can be turned on and off, and the level adjusted. For more +information on Django message levels, see the `Django messages framework +documentation `__. + +`HOOKS <#hooks>`__ +~~~~~~~~~~~~~~~~~~ + +Default: + +.. code-block:: + + { + 'target_post_save': 'tom_common.hooks.target_post_save', + 'observation_change_state': 'tom_common.hooks.observation_change_state', + 'data_product_post_upload': 'tom_dataproducts.hooks.data_product_post_upload', + } + +A dictionary of action, method code hooks to run. These hooks allow +running arbitrary python code when specific actions happen within a TOM, +such as an observation changing state. See the documentation on `Running +Custom Code on Actions in your TOM `__ for more +details and available hooks. + +`OPEN_URLS <#open_urls>`__ +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: [] + +With an `AUTH_STRATEGY <#auth_strategy>`__ value of **LOCKED**, urls in +this list will remain visible to unauthenticated users. You might add +the homepage (‘/’), for example. + +`TARGET_PERMISSIONS_ONLY <#target_permissions_only>`__ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: True + +This settings determines the permissions strategy of the TOM. When set +to True, authorization permissions will be set on Targets and cascade +from there–that is, a group that can see a Target can see all +ObservationRecords and Data associated with the Target. When set to +False, permissions can be set for a group at the Target level, the +ObservationRecord level, or the DataProduct level. + +`TARGET_TYPE <#target_type>`__ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: No default + +Can be either **SIDEREAL** or **NON_SIDEREAL**. This setting determines +the default target type for your TOM. TOMs can still create and work +with targets of both types even after this option is set, but setting it +to one of the values will optimize the workflow for that target type. + +`TOM_ALERT_CLASSES <#tom_alert_classes>`__ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: + +.. code-block:: + + [ + 'tom_alerts.brokers.mars.MARSBroker', + 'tom_alerts.brokers.lasair.LasairBroker', + 'tom_alerts.brokers.scout.ScoutBroker', + 'tom_alerts.brokers.tns.TNSBroker', + 'tom_alerts.brokers.antares.ANTARESBroker', + 'tom_alerts.brokers.gaia.GaiaBroker' + ] + +A list of tom alert classes to make available to your TOM. If you have +written or downloaded additional alert classes you would make them +available here. If you’d like to write your own alert module please see +the documentation on `Creating an Alert Module for the TOM +Toolkit `__. + +`TOM_FACILITY_CLASSES <#tom_facility_classes>`__ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: + +.. code-block + + [ + 'tom_observations.facilities.lco.LCOFacility', + 'tom_observations.facilities.gemini.GEMFacility', + 'tom_observations.facilities.soar.SOARFacility', + 'tom_observations.facilities.lt.LTFacility' + ] + +A list of observation facility classes to make available to your TOM. If +you have written or downloaded a custom observation facility you would +add the class to this list to make your TOM load it. + +`TOM_HARVESTER_CLASSES <#tom_harvester_classes>`__ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: + +.. code-block + + [ + 'tom_catalogs.harvesters.simbad.SimbadHarvester', + 'tom_catalogs.harvesters.ned.NEDHarvester', + 'tom_catalogs.harvesters.jplhorizons.JPLHorizonsHarvester', + 'tom_catalogs.harvesters.mpc.MPCHarvester', + 'tom_catalogs.harvesters.tns.TNSHarvester', + ] + +A list of TOM harverster classes to make available to your TOM. If you +have written or downloaded additional harvester classes you would make +them available here. + +`TOM_LATEX_PROCESSORS <#tom_latex_processors>`__ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: + +.. code-block + + { + 'ObservationGroup': 'tom_publications.processors.latex_processor.ObservationGroupLatexProcessor', + 'TargetList': 'tom_publications.processors.target_list_latex_processor.TargetListLatexProcessor' + } + +A dictionary with the keys being TOM models classes and the values being +the modules that should be used to generate latex tables for those +models. diff --git a/docs/common/latex_generation.md b/docs/common/latex_generation.md deleted file mode 100644 index c27ab50d7..000000000 --- a/docs/common/latex_generation.md +++ /dev/null @@ -1,193 +0,0 @@ -# LaTeX Generation - -One of the features the TOM Toolkit offers is automated generation of LaTeX-formatted data tables. The LaTeX table tool -allows the user to select the parameters for an entity in their TOM--for example, a Target--and generate a table of -those parameters for all targets within a list. At the moment, the Toolkit supports table generation for two built-in models--``ObservationGroup``s and ``TargetList``s. - -A LaTeX processor can be created for any model, or, with some additional modifications, any combination of models. The -supported LaTeX processors must be specified in ``settings.py`` in the ``TOM_LATEX_PROCESSORS`` as key/value pairs, -with the model being the key, and the processor class being the value. By default, the following processors are -automatically present in ``settings.py``: - -```python -TOM_LATEX_PROCESSORS = { - 'ObservationGroup': 'tom_publications.processors.latex_processor.ObservationGroupLatexProcessor', - 'TargetList': 'tom_publications.processors.target_list_latex_processor.TargetListLatexProcessor' -} -``` - -## Custom Processing - -The built-in LaTeX table generation is good, but it certainly has some shortcomings, and can't be expected to cover -every or even most use cases. As such, the implementation allows for smooth addition of any custom processing. - -In order to generate a LaTeX table for a unique use case, we'll need to write a custom LaTeX processor, which we'll -go through below. A LaTeX processor has a custom Form class and a Processor class, and the Processor class has a -function which takes data from your TOM DB and outputs it in the preferred LaTeX-formatted table. To begin, here's a -brief look at part of the structure of the tom_publications app in the TOM Toolkit: - -``` -tom_publications -├──latex.py -└──processors - ├──target_list_latex_processor.py - └──observation_group_latex_processor.py -``` - -Perhaps one wants a processor that generates a table simply for all the photometric or spectroscopic data for a given -target. The first thing to be done is to create a ``target_photometry_latex_processor.py``. We'll create a new file for -our processor, and then create a ``TargetListLatexProcessor`` class that inherits from ``GenericLatexProcessor``. -``GenericLatexProcessor`` has an abstract method that must be implemented called ``create_latex``, so we'll also add -that: - -```python -from tom_publications.latex import GenericLatexProcessor - -class TargetDataLatexProcessor(GenericLatexProcessor): - - def create_latex(self, cleaned_data): - pass -``` - -The ``GenericLatexProcessor`` also has a form class that renders the correct set of fields to be generated. In our case, -we'd like the user to be able to choose between spectroscopy or photometry. So let's create the form. We'll also create -one form field, and populate it with our two choices: - -```python -from django import forms - -from tom_publications.latex import GenericLatexProcessor, GenericLatexForm - - -class TargetDataLatexForm(GenericLatexForm): - data_type = forms.ChoiceField( - choices=[('spectroscopy', 'Spectroscopy'), ('photometry', 'Photometry')], - required=True, - widget=forms.RadioSelect() - ) - - -class TargetDataLatexProcessor(GenericLatexProcessor): -... -``` - - -With the form implemented, we can implement our ``create_latex`` method and add our ``TargetDataLatexForm`` as the -``form_class``. - -The base form class always includes ``model_pk``, which gives us a way to access the object for which we're generating -data. - -```python -import json - -from django import forms - -from tom_dataproducts.models import ReducedDatum -from tom_publications.latex import GenericLatexProcessor, GenericLatexForm -from tom_targets.models import Target - -... - -class TargetDataLatexProcessor(GenericLatexProcessor): - form_class = TargetDataLatexForm - - def create_latex_table_data(self, cleaned_data): - target = Target.objects.get(pk=cleaned_data.get('model_pk')) - data = ReducedDatum.objects.filter(target=target, data_type=cleaned_data.get('data_type')) - - table_data = {} - if cleaned_data.get('data_type') == 'photometry': - for datum in data: - for key, value in json.loads(datum.value).items(): - table_data.setdefault(key, []).append(value) - elif cleaned_data.get('data_type') == 'spectroscopy': - ... - - return table_data -``` - -The above example only shows the photometric table generation, but spectroscopic can be left as an exercise to the -reader. - -The last two steps are to link our new processor to our existing code. First, in our ``settings.py`` (making sure you -replace the displayed path with the correct one for your TOM): - -```python -... -TOM_LATEX_PROCESSORS = { - 'ObservationGroup': 'tom_publications.processors.latex_processor.ObservationGroupLatexProcessor', - 'TargetList': 'tom_publications.processors.target_list_latex_processor.TargetListLatexProcessor', - 'Target': 'tom_publications.processors.target_data_latex_processor.TargetDataLatexProcessor' -} -... -``` - -We add a ``Target`` processor. For the default implementation, all processors must be tied to a TOM model, but with a -custom templatetag (or enough requests to the developers), it can be expanded further. - -Then, in our overridden ``target_detail.html`` template (details on overriding templates -can be found [here](https://tom-toolkit.readthedocs.io/en/latest/customization/customize_templates.html)), we add a -button: - -```html -... -
    - {% target_feature object %} - {% latex_button object %} - {% if object.future_observations %} -... -``` - -For context, the template tag being referenced by ``{% latex_button object %}`` can be seen below. It accepts an -instance of a model from your TOM and generates a button with the correct query parameters to send to your form. - -```python -@register.inclusion_tag('tom_publications/partials/latex_button.html') -def latex_button(object): - """ - Renders a button that redirects to the LaTeX table generation page for the specified model instance. Requires an - object, which is generally the object in the context for the page on which the templatetag will be used. - """ - model_name = object._meta.label - return {'model_name': object._meta.label, 'model_pk': object.id} -``` - -With all that done, you will now be able to generate tables of photometric (and eventually spectroscopic) data of any -target in your TOM. Here's our final ``target_data_latex_processor.py``: - -```python -import json - -from django import forms - -from tom_dataproducts.models import ReducedDatum -from tom_publications.latex import GenericLatexProcessor, GenericLatexForm -from tom_targets.models import Target - - -class TargetDataLatexForm(GenericLatexForm): - data_type = forms.ChoiceField( - choices=[('spectroscopy', 'Spectroscopy'), ('photometry', 'Photometry')], - required=True, - widget=forms.RadioSelect() - ) - - -class TargetDataLatexProcessor(GenericLatexProcessor): - form_class = TargetDataLatexForm - - def create_latex_table_data(self, cleaned_data): - target = Target.objects.get(pk=cleaned_data.get('model_pk')) - data = ReducedDatum.objects.filter(target=target, data_type=cleaned_data.get('data_type')) - - table_data = {} - if cleaned_data.get('data_type') == 'photometry': - for datum in data: - for key, value in json.loads(datum.value).items(): - table_data.setdefault(key, []).append(value) - elif cleaned_data.get('data_type') == 'spectroscopy': - ... - - return table_data -``` \ No newline at end of file diff --git a/docs/common/latex_generation.rst b/docs/common/latex_generation.rst new file mode 100644 index 000000000..929722d32 --- /dev/null +++ b/docs/common/latex_generation.rst @@ -0,0 +1,215 @@ +LaTeX Generation +================ + +One of the features the TOM Toolkit offers is automated generation of +LaTeX-formatted data tables. The LaTeX table tool allows the user to +select the parameters for an entity in their TOM–for example, a +Target–and generate a table of those parameters for all targets within a +list. At the moment, the Toolkit supports table generation for two +built-in models–``ObservationGroup``\ s and ``TargetList``\ s. + +A LaTeX processor can be created for any model, or, with some additional +modifications, any combination of models. The supported LaTeX processors +must be specified in ``settings.py`` in the ``TOM_LATEX_PROCESSORS`` as +key/value pairs, with the model being the key, and the processor class +being the value. By default, the following processors are automatically +present in ``settings.py``: + +.. code-block:: python + + TOM_LATEX_PROCESSORS = { + 'ObservationGroup': 'tom_publications.processors.latex_processor.ObservationGroupLatexProcessor', + 'TargetList': 'tom_publications.processors.target_list_latex_processor.TargetListLatexProcessor' + } + +Custom Processing +----------------- + +The built-in LaTeX table generation is good, but it certainly has some +shortcomings, and can’t be expected to cover every or even most use +cases. As such, the implementation allows for smooth addition of any +custom processing. + +In order to generate a LaTeX table for a unique use case, we’ll need to +write a custom LaTeX processor, which we’ll go through below. A LaTeX +processor has a custom Form class and a Processor class, and the +Processor class has a function which takes data from your TOM DB and +outputs it in the preferred LaTeX-formatted table. To begin, here’s a +brief look at part of the structure of the tom_publications app in the +TOM Toolkit: + +.. code-block:: + + tom_publications + ├──latex.py + └──processors + ├──target_list_latex_processor.py + └──observation_group_latex_processor.py + +Perhaps one wants a processor that generates a table simply for all the +photometric or spectroscopic data for a given target. The first thing to +be done is to create a ``target_photometry_latex_processor.py``. We’ll +create a new file for our processor, and then create a +``TargetListLatexProcessor`` class that inherits from +``GenericLatexProcessor``. ``GenericLatexProcessor`` has an abstract +method that must be implemented called ``create_latex``, so we’ll also +add that: + +.. code-block:: python + + from tom_publications.latex import GenericLatexProcessor + + class TargetDataLatexProcessor(GenericLatexProcessor): + + def create_latex(self, cleaned_data): + pass + +The ``GenericLatexProcessor`` also has a form class that renders the +correct set of fields to be generated. In our case, we’d like the user +to be able to choose between spectroscopy or photometry. So let’s create +the form. We’ll also create one form field, and populate it with our two +choices: + +.. code-block:: python + + from django import forms + + from tom_publications.latex import GenericLatexProcessor, GenericLatexForm + + + class TargetDataLatexForm(GenericLatexForm): + data_type = forms.ChoiceField( + choices=[('spectroscopy', 'Spectroscopy'), ('photometry', 'Photometry')], + required=True, + widget=forms.RadioSelect() + ) + + + class TargetDataLatexProcessor(GenericLatexProcessor): + ... + +With the form implemented, we can implement our ``create_latex`` method +and add our ``TargetDataLatexForm`` as the ``form_class``. + +The base form class always includes ``model_pk``, which gives us a way +to access the object for which we’re generating data. + +.. code-block:: python + + import json + + from django import forms + + from tom_dataproducts.models import ReducedDatum + from tom_publications.latex import GenericLatexProcessor, GenericLatexForm + from tom_targets.models import Target + + ... + + class TargetDataLatexProcessor(GenericLatexProcessor): + form_class = TargetDataLatexForm + + def create_latex_table_data(self, cleaned_data): + target = Target.objects.get(pk=cleaned_data.get('model_pk')) + data = ReducedDatum.objects.filter(target=target, data_type=cleaned_data.get('data_type')) + + table_data = {} + if cleaned_data.get('data_type') == 'photometry': + for datum in data: + for key, value in json.loads(datum.value).items(): + table_data.setdefault(key, []).append(value) + elif cleaned_data.get('data_type') == 'spectroscopy': + ... + + return table_data + +The above example only shows the photometric table generation, but +spectroscopic can be left as an exercise to the reader. + +The last two steps are to link our new processor to our existing code. +First, in our ``settings.py`` (making sure you replace the displayed +path with the correct one for your TOM): + +.. code-block:: python + + ... + TOM_LATEX_PROCESSORS = { + 'ObservationGroup': 'tom_publications.processors.latex_processor.ObservationGroupLatexProcessor', + 'TargetList': 'tom_publications.processors.target_list_latex_processor.TargetListLatexProcessor', + 'Target': 'tom_publications.processors.target_data_latex_processor.TargetDataLatexProcessor' + } + ... + +We add a ``Target`` processor. For the default implementation, all +processors must be tied to a TOM model, but with a custom templatetag +(or enough requests to the developers), it can be expanded further. + +Then, in our overridden ``target_detail.html`` template (details on +overriding templates can be found +`here `__), +we add a button: + +.. code-block:: html + + ... +
    + {% target_feature object %} + {% latex_button object %} + {% if object.future_observations %} + ... + +For context, the template tag being referenced by +``{% latex_button object %}`` can be seen below. It accepts an instance +of a model from your TOM and generates a button with the correct query +parameters to send to your form. + +.. code-block:: python + + @register.inclusion_tag('tom_publications/partials/latex_button.html') + def latex_button(object): + """ + Renders a button that redirects to the LaTeX table generation page for the specified model instance. Requires an + object, which is generally the object in the context for the page on which the templatetag will be used. + """ + model_name = object._meta.label + return {'model_name': object._meta.label, 'model_pk': object.id} + +With all that done, you will now be able to generate tables of +photometric (and eventually spectroscopic) data of any target in your +TOM. Here’s our final ``target_data_latex_processor.py``: + +.. code-block:: python + + import json + + from django import forms + + from tom_dataproducts.models import ReducedDatum + from tom_publications.latex import GenericLatexProcessor, GenericLatexForm + from tom_targets.models import Target + + + class TargetDataLatexForm(GenericLatexForm): + data_type = forms.ChoiceField( + choices=[('spectroscopy', 'Spectroscopy'), ('photometry', 'Photometry')], + required=True, + widget=forms.RadioSelect() + ) + + + class TargetDataLatexProcessor(GenericLatexProcessor): + form_class = TargetDataLatexForm + + def create_latex_table_data(self, cleaned_data): + target = Target.objects.get(pk=cleaned_data.get('model_pk')) + data = ReducedDatum.objects.filter(target=target, data_type=cleaned_data.get('data_type')) + + table_data = {} + if cleaned_data.get('data_type') == 'photometry': + for datum in data: + for key, value in json.loads(datum.value).items(): + table_data.setdefault(key, []).append(value) + elif cleaned_data.get('data_type') == 'spectroscopy': + ... + + return table_data From 4b9d55c95f2058b72747a0d445a71b2757b32922 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 8 Jun 2020 18:14:40 -0700 Subject: [PATCH 166/424] Converted three more sections to rST --- docs/code/automation.md | 199 ----------- docs/code/automation.rst | 246 +++++++++++++ docs/code/backgroundtasks.md | 234 ------------- docs/code/backgroundtasks.rst | 276 +++++++++++++++ docs/code/custom_code.md | 113 ------ docs/code/custom_code.rst | 132 +++++++ docs/code/querying.md | 70 ---- docs/code/querying.rst | 81 +++++ docs/customization/adding_pages.md | 204 ----------- docs/customization/adding_pages.rst | 217 ++++++++++++ docs/customization/customize_template_tags.md | 266 --------------- .../customization/customize_template_tags.rst | 322 ++++++++++++++++++ docs/customization/customize_templates.md | 152 --------- docs/customization/customize_templates.rst | 170 +++++++++ docs/customization/index.rst | 2 +- docs/deployment/amazons3.md | 108 ------ docs/deployment/amazons3.rst | 126 +++++++ docs/deployment/deployment_heroku.md | 169 --------- docs/deployment/deployment_heroku.rst | 195 +++++++++++ docs/deployment/deployment_tips.md | 42 --- docs/deployment/deployment_tips.rst | 50 +++ 21 files changed, 1816 insertions(+), 1558 deletions(-) delete mode 100644 docs/code/automation.md create mode 100644 docs/code/automation.rst delete mode 100644 docs/code/backgroundtasks.md create mode 100644 docs/code/backgroundtasks.rst delete mode 100644 docs/code/custom_code.md create mode 100644 docs/code/custom_code.rst delete mode 100644 docs/code/querying.md create mode 100644 docs/code/querying.rst delete mode 100644 docs/customization/adding_pages.md create mode 100644 docs/customization/adding_pages.rst delete mode 100644 docs/customization/customize_template_tags.md create mode 100644 docs/customization/customize_template_tags.rst delete mode 100644 docs/customization/customize_templates.md create mode 100644 docs/customization/customize_templates.rst delete mode 100644 docs/deployment/amazons3.md create mode 100644 docs/deployment/amazons3.rst delete mode 100644 docs/deployment/deployment_heroku.md create mode 100644 docs/deployment/deployment_heroku.rst delete mode 100644 docs/deployment/deployment_tips.md create mode 100644 docs/deployment/deployment_tips.rst diff --git a/docs/code/automation.md b/docs/code/automation.md deleted file mode 100644 index 6dbb6ea61..000000000 --- a/docs/code/automation.md +++ /dev/null @@ -1,199 +0,0 @@ -Automating tasks for your TOM ---- - -Your TOM may have a need to run a task on a regular schedule without human intervention. With the help of a built-in Django feature and cron, this can be accomplished. Perhaps you want to check for and download data from your scheduled observations every hour, or see if any brokers have published new candidates that meet the criteria of a previous search--all that would be required is a bit of code to call those built-in functions, and a crontab update. - -### Create a management command - -Django provides the ability to register actions using [management commands](https://docs.djangoproject.com/en/2.2/howto/custom-management-commands/). These actions can then be called from the command line. - -#### Starting a new django "app" - -Django recommends creating separate "apps" to contain your management commands -(among other things, like custom models and views) so we'll start with creating a -new app called "myapp". You can read more about Django reusable apps -[in the official -documentation](https://docs.djangoproject.com/en/2.2/intro/tutorial01/#creating-the-polls-app). - - ./manage.py startapp myapp - -Now your tom should have a new folder in the root directory called "myapp". Next -we need to tell Django to use this new application. In your `settings.py` file -file the `INSTALLED_APPS` settings and add `myapp.apps.MyappConfig` to the array: - -```python -INSTALLED_APPS = [ - 'django.contrib.admin', - ... - 'myapp.apps.MyappConfig' -] -``` - -Now we are read to start writing our new commands. - -#### Writing the command - -Let's walk through a command to download observation data every hour. The first -thing to be done is to create a `management/commands` directory within your -application to house our script. We'll call it `save_data.py`. The structure should -look like this: - -``` -mytom/ -├── manage.py -└── myapp/ - ├── __init__.py - ├── models.py - ├── tests.py - ├── views.py - └── management/ - └── commands/ - └── save_data.py -``` - -A management command simply needs a class called `Command` that inherits from `BaseCommand`, and a `handle` class method that contains the logic for the command. - -```python -from django.core.management.base import BaseCommand -from tom_observations.models import ObservationRecord - - -class Command(BaseCommand): - - help = 'Downloads data for all completed observations' - - def handle(self, *args, **options): -``` - -Now, we need to add the logic to query the facilities for data. We'll iterate -over each incomplete `ObservationRecord`, and save the data products locally for -that ObservationRecord. - -```python -observation_records = ObservationRecord.objects.all() -for record in observation_records: - if record.terminal: - record.save_data() - -return 'Success!' -``` - -So our final management command should look like this: - -```python -from django.core.management.base import BaseCommand -from tom_observations.models import ObservationRecord - - -class Command(BaseCommand): - - help = 'Downloads data for all completed observations' - - def handle(self, *args, **options): - observation_records = ObservationRecord.objects.all() - for record in observation_records: - if record.terminal: - record.save_data() - - return 'Success!' -``` - -#### Adding parameters - -Management commands also provide the ability to accept parameters. Doing this is as simple as implementing `add_arguments` as a class method on your `Command` class. Let's say we want to ensure that our command can be run for a single target: - -```python - def add_arguments(self, parser): - parser.add_argument('--target_id', help='Download data for a single target') -``` - -That code will process any additional parameters, and we simply need to handle -them in our, `handle` class method. We'll attempt to fetch the supplied target -from the database and filter the ObservationRecords accordingly: - -```python - def handle(self, *args, **options): - if options['target_id']: - try: - target = Target.objects.get(pk=options['target_id']) - observation_records = ObservationRecord.objects.filter(target=target) - except ObjectDoesNotExist: - raise Exception('Invalid target id provided') - else: - observation_records = ObservationRecord.objects.all() - ... -``` - -Finally, we filter our initial set of observation records, so this line: - -```python - observation_records = ObservationRecord.objects.all() -``` - -will become this: - -```python - observation_records = ObservationRecord.objects.filter(target=target) -``` - -And our final finished command looks as follows: - -```python -from django.core.management.base import BaseCommand -from tom_observations.models import ObservationRecord -from tom_targets.models import Target - - -class Command(BaseCommand): - - help = 'Downloads data for all completed observations' - - def add_arguments(self, parser): - parser.add_argument('--target_id', help='Download data for a single target') - - def handle(self, *args, **options): - if options['target_id']: - try: - target = Target.objects.get(pk=options['target_id']) - observation_records = ObservationRecord.objects.filter(target=target) - except Target.DoesNotExist: - raise Exception('Invalid target id provided') - else: - observation_records = ObservationRecord.objects.all() - for record in observation_records: - if record.terminal: - record.save_data() - - return 'Success!' -``` - -### Automating a management command - -#### Using cron - -On Unix-based systems, [cron](https://linux.die.net/man/8/cron) can be used to automate running of a Django management command. The syntax is very simple, as commands look like this: - -`30 2 * 6 3 /path/to/command /path/to/parameters` - -In the above case, the first five values, which can either be numbers or asterisks, represent elements of time. From left to right, they are minutes, hours, day of the month, month of the year, and day of the week. Our example would run a command every Wednesday (fourth day of the week, starting from 0) in June (sixth month of the year, starting from 1) at 2:30 AM. - -Websites like [crontab.guru](https://crontab.guru/) make it easier to reason about -crontab expressions. - -Scheduling can be made more complex as well--values can be comma-separated or presented as a range. Refer to the abundance of cron documentation for more information. An excellent beginner's guide can be found [here](https://www.ostechnix.com/a-beginners-guide-to-cron-jobs/). - -Now, how is cron called? Well, cron jobs are run by the system, and it reads the commands that need to be called from a cron table, or crontab. To edit this file, simple call `crontab -e`. - -#### Using cron with a management command - -To make this more specific to our example, let's say we want to update the observation data every hour. The command we would normally run in our project directory would be the following: - -`python manage.py save_data` - -However, cron is a system-level operation, so the command needs to be directory-agnostic, and we need to ensure we're using the right Python version. If you have a virtualenv, the command should be the absolute path to the Python interpreter in the virtualenv. If your TOM is in a Docker container, it should be the version of Python running in the container. Otherwise, just ensure that it's at least version 3.6 or higher. - -So, the line in our crontab should be as follows: - -`0 * * * * /path/to/virtualenv/bin/python /path/to/project/manage.py save_data` - -This will run every day on the hour. And that's it! Just exit the crontab and it will automatically restart cron, then your command will run on the next hour. diff --git a/docs/code/automation.rst b/docs/code/automation.rst new file mode 100644 index 000000000..04ae196e4 --- /dev/null +++ b/docs/code/automation.rst @@ -0,0 +1,246 @@ +Automating tasks for your TOM +----------------------------- + +Your TOM may have a need to run a task on a regular schedule without +human intervention. With the help of a built-in Django feature and cron, +this can be accomplished. Perhaps you want to check for and download +data from your scheduled observations every hour, or see if any brokers +have published new candidates that meet the criteria of a previous +search–all that would be required is a bit of code to call those +built-in functions, and a crontab update. + +Create a management command +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Django provides the ability to register actions using `management +commands `__. +These actions can then be called from the command line. + +Starting a new django “app” +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Django recommends creating separate “apps” to contain your management +commands (among other things, like custom models and views) so we’ll +start with creating a new app called “myapp”. You can read more about +Django reusable apps `in the official +documentation `__. + +:: + + ./manage.py startapp myapp + +Now your tom should have a new folder in the root directory called +“myapp”. Next we need to tell Django to use this new application. In +your ``settings.py`` file file the ``INSTALLED_APPS`` settings and add +``myapp.apps.MyappConfig`` to the array: + +.. code:: python + + INSTALLED_APPS = [ + 'django.contrib.admin', + ... + 'myapp.apps.MyappConfig' + ] + +Now we are read to start writing our new commands. + +Writing the command +^^^^^^^^^^^^^^^^^^^ + +Let’s walk through a command to download observation data every hour. +The first thing to be done is to create a ``management/commands`` +directory within your application to house our script. We’ll call it +``save_data.py``. The structure should look like this: + +:: + + mytom/ + ├── manage.py + └── myapp/ + ├── __init__.py + ├── models.py + ├── tests.py + ├── views.py + └── management/ + └── commands/ + └── save_data.py + +A management command simply needs a class called ``Command`` that +inherits from ``BaseCommand``, and a ``handle`` class method that +contains the logic for the command. + +.. code:: python + + from django.core.management.base import BaseCommand + from tom_observations.models import ObservationRecord + + + class Command(BaseCommand): + + help = 'Downloads data for all completed observations' + + def handle(self, *args, **options): + +Now, we need to add the logic to query the facilities for data. We’ll +iterate over each incomplete ``ObservationRecord``, and save the data +products locally for that ObservationRecord. + +.. code:: python + + observation_records = ObservationRecord.objects.all() + for record in observation_records: + if record.terminal: + record.save_data() + + return 'Success!' + +So our final management command should look like this: + +.. code:: python + + from django.core.management.base import BaseCommand + from tom_observations.models import ObservationRecord + + + class Command(BaseCommand): + + help = 'Downloads data for all completed observations' + + def handle(self, *args, **options): + observation_records = ObservationRecord.objects.all() + for record in observation_records: + if record.terminal: + record.save_data() + + return 'Success!' + +Adding parameters +^^^^^^^^^^^^^^^^^ + +Management commands also provide the ability to accept parameters. Doing +this is as simple as implementing ``add_arguments`` as a class method on +your ``Command`` class. Let’s say we want to ensure that our command can +be run for a single target: + +.. code:: python + + def add_arguments(self, parser): + parser.add_argument('--target_id', help='Download data for a single target') + +That code will process any additional parameters, and we simply need to +handle them in our, ``handle`` class method. We’ll attempt to fetch the +supplied target from the database and filter the ObservationRecords +accordingly: + +.. code:: python + + def handle(self, *args, **options): + if options['target_id']: + try: + target = Target.objects.get(pk=options['target_id']) + observation_records = ObservationRecord.objects.filter(target=target) + except ObjectDoesNotExist: + raise Exception('Invalid target id provided') + else: + observation_records = ObservationRecord.objects.all() + ... + +Finally, we filter our initial set of observation records, so this line: + +.. code:: python + + observation_records = ObservationRecord.objects.all() + +will become this: + +.. code:: python + + observation_records = ObservationRecord.objects.filter(target=target) + +And our final finished command looks as follows: + +.. code:: python + + from django.core.management.base import BaseCommand + from tom_observations.models import ObservationRecord + from tom_targets.models import Target + + + class Command(BaseCommand): + + help = 'Downloads data for all completed observations' + + def add_arguments(self, parser): + parser.add_argument('--target_id', help='Download data for a single target') + + def handle(self, *args, **options): + if options['target_id']: + try: + target = Target.objects.get(pk=options['target_id']) + observation_records = ObservationRecord.objects.filter(target=target) + except Target.DoesNotExist: + raise Exception('Invalid target id provided') + else: + observation_records = ObservationRecord.objects.all() + for record in observation_records: + if record.terminal: + record.save_data() + + return 'Success!' + +Automating a management command +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Using cron +^^^^^^^^^^ + +On Unix-based systems, `cron `__ can +be used to automate running of a Django management command. The syntax +is very simple, as commands look like this: + +``30 2 * 6 3 /path/to/command /path/to/parameters`` + +In the above case, the first five values, which can either be numbers or +asterisks, represent elements of time. From left to right, they are +minutes, hours, day of the month, month of the year, and day of the +week. Our example would run a command every Wednesday (fourth day of the +week, starting from 0) in June (sixth month of the year, starting from +1) at 2:30 AM. + +Websites like `crontab.guru `__ make it easier to +reason about crontab expressions. + +Scheduling can be made more complex as well–values can be +comma-separated or presented as a range. Refer to the abundance of cron +documentation for more information. An excellent beginner’s guide can be +found +`here `__. + +Now, how is cron called? Well, cron jobs are run by the system, and it +reads the commands that need to be called from a cron table, or crontab. +To edit this file, simple call ``crontab -e``. + +Using cron with a management command +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To make this more specific to our example, let’s say we want to update +the observation data every hour. The command we would normally run in +our project directory would be the following: + +``python manage.py save_data`` + +However, cron is a system-level operation, so the command needs to be +directory-agnostic, and we need to ensure we’re using the right Python +version. If you have a virtualenv, the command should be the absolute +path to the Python interpreter in the virtualenv. If your TOM is in a +Docker container, it should be the version of Python running in the +container. Otherwise, just ensure that it’s at least version 3.6 or +higher. + +So, the line in our crontab should be as follows: + +``0 * * * * /path/to/virtualenv/bin/python /path/to/project/manage.py save_data`` + +This will run every day on the hour. And that’s it! Just exit the +crontab and it will automatically restart cron, then your command will +run on the next hour. diff --git a/docs/code/backgroundtasks.md b/docs/code/backgroundtasks.md deleted file mode 100644 index 3bc693fba..000000000 --- a/docs/code/backgroundtasks.md +++ /dev/null @@ -1,234 +0,0 @@ -Running asynchronous background tasks ---- - -When you are using your TOM via the web interface, the code that is running in the -background is tied to the request/response cycle. What this means is that when -you click a button or link in the TOM, your browser constructs a web request, which -is then sent to the web server running your TOM. The TOM receives this request and -then runs a bunch of code, ultimately to generate a response that gets sent back -to the browser. This response is what you see when the next page loads. For the -purposes of this explanation, this all happens _synchronously_, meaning that your -browser has to wait for your TOM to respond before displaying the next page. - - ---------- request ---------- - | | -------------> | | - | browser | response | TOM | - | | <------------- | | - ---------- ---------- - -But what happens if your TOM performs some compute or IO heavy task while -constructing the response? One example would be running a source extraction on a data -product after a user uploads it to your TOM. Normally, the browser will just wait -for the response. This results in an agonizing wait time for the user as they -watch the browser's loading spinner slowly rotate. Eventually they will give up -and either reload the page or close it completely. In fact, according to a study -by Akamai, 50% of web users will not wait longer than 10-15 seconds for a page to -load before giving up. - -The way we avoid these wait times is to run our slow code _asynchronously_ in the -background, in a separate thread or process. In this model the TOM responds to the -browser with a response immediately, before the slow code has even finished. - - - ---------- request ---------- task ----------- - | | -------------> | | --------> | | - | browser | response | TOM | result | worker | - | | <------------- | | <-------- | | - ---------- ---------- ----------- - -A very common scenario is sending email. Many web applications require the -functionality of sending mail at some point. Let's say the PI of a project has the -option to mass notify their CIs that observations have been taken. Usually, sending -email takes a very short amount of time, but it is still good practice to remove -it from the request/response cycle, just in case it takes longer than usual or -errors in some way. - -In this tutorial, we will go over how to run tasks asynchronously in your TOM if -you have the need to do so. - -### Running tasks with Dramatiq - -[Dramatiq](https://dramatiq.io/) is a task processing library for python. Simply -put: it allows you to define functions as _actors_ and then execute those function -using _workers_. None of this can happen without a _broker_, though, which is the -piece that is responsible for passing messages from the _web process_ to the -_workers_. - - -#### Installing Redis - -Unfortunately, the broker is a separate piece of software outside of the task -library. Dramatiq supports using either RabbitMQ or Redis. We'll use Redis because -of its versatility: not only can it be used as a message broker but it can also -be used in your TOM as a cache (though not covered in this tutorial). - -Depending on your OS, there are a few ways to [install -Redis](https://redis.io/download). - -##### Using Docker - -One of the easiest ways to install Redis is to use Docker: - - docker run --name tom-redis -d -p6379:6379 redis - -##### Building from source - -You can also download Redis directly from the website and compile it: - - $ wget http://download.redis.io/releases/redis-5.0.5.tar.gz - $ tar xzf redis-5.0.5.tar.gz - $ cd redis-5.0.5 - $ make - -You can now run the server with: - - ./src/redis-server - -##### Using a package manager - -If you are running Linux, most likely Redis is included with your distribution via -its package manager. For example: - - apt install Redis - -Whichever way works for you, we should now have a Redis server up and running and -listening on port 6379. - - -#### Installing Dramatiq - -Now that we have our broker running, we can install and configure our TOM to run -Dramatiq. Start by installing the required dependencies into your virtualenv: - - pip install 'dramatiq[watch, redis]' django-dramatiq - -[django-dramatiq](https://github.com/Bogdanp/django_dramatiq) -will offer us some conveniences while working with tasks in our TOM. - -Install django-dramatiq to your `INSTALLED_APPS` setting, above the tom\_\* apps: - -```python -INSTALLED_APPS = [ - ... - 'django_gravatar', - 'django_dramatiq', - 'tom_targets', - ... -] -``` - -Add a section for dramatiq in `settings.py`: - -```python -DRAMATIQ_BROKER = { - "BROKER": "dramatiq.brokers.redis.RedisBroker", - "OPTIONS": { - "url": "redis://localhost:6379", - }, - "MIDDLEWARE": [ - "dramatiq.middleware.AgeLimit", - "dramatiq.middleware.TimeLimit", - "dramatiq.middleware.Callbacks", - "dramatiq.middleware.Retries", - "django_dramatiq.middleware.AdminMiddleware", - "django_dramatiq.middleware.DbConnectionsMiddleware", - ] -} -``` - -If you want to store the results of your tasks add a section in `settings.py` for -that as well: - -```python -DRAMATIQ_RESULT_BACKEND = { - "BACKEND": "dramatiq.results.backends.redis.RedisBackend", - "BACKEND_OPTIONS": { - "url": "redis://localhost:6379", - }, - "MIDDLEWARE_OPTIONS": { - "result_ttl": 60000 - } -} -``` - -Now that all the settings are in place, we need to run a `manage.py migrate` in order to create the `django_dramatiq` table. Then, we can test installation by starting up some workers: - - ./manage.py rundramatiq - -If all goes well you will see output that looks like this: - - % ./manage.py rundramatiq - * Discovered tasks module: 'django_dramatiq.tasks' - * Running dramatiq: "dramatiq --path . --processes 8 --threads 8 --watch . django_dramatiq.setup django_dramatiq.tasks" - - [2019-08-21 17:52:30,216] [PID 27267] [MainThread] [dramatiq.MainProcess] [INFO] Dramatiq '1.6.1' is booting up. - Worker process is ready for action. - -Your task workers are up and running! - -#### Writing a task -Now that we have some workers, lets put them to work. In order to do that we'll -write a task. - - -Create a file `mytom/myapp/tasks.py` where `myapp` is a django app you've -installed into `INSTALLED_APPS`. If you haven't started one, you can do so with: - - ./manage.py startapp myapp - -In `tasks.py`: - -```python -import dramatiq -import time -import logging - -logger = logging.getLogger(__name__) - - -@dramatiq.actor -def super_complicated_task(): - logger.info('starting task...') - time.sleep(2) - logger.info('still running...') - time.sleep(2) - logger.info('done!') -``` - -This task will emulate a function that blocks for 4 seconds, in practice this would -be a network call or some kind of heavy processing task. - -Now open up a Django shell: - - ./manage.py shell_plus - -And import and call the task: - - In [1]: from myapp.tasks import super_complicated_task - - In [2]: super_complicated_task.send() - Out[2]: Message(queue_name='default', actor_name='super_complicated_task', args=(), kwargs={}, options={'redis_message_id': '667821da-f236-4c4e-969a-9d1f1ff54be2'}, message_id='2c8893d8-4211-4cac-b0b9-0f2e9672d0ae', message_timestamp=1566416600481) - -In the terminal where you started the dramatiq workers (not the django shell!) you -should see the following output: - - starting task... - still running... - done! - - -Notice how calling the task returned immediately in the shell, but the task took a -few seconds to complete. This is how it would work in practice in your django app: -Somewhere in your code, for example in your app's `views.py`, you would import the -task just like we did in the terminal. Now when the view gets called, the task -will be queued for execution and the response can be sent back to the user's -browser right away. The task will finish in the background. - - -#### Conclusion -In this tutorial we went over the need for asynchronous tasks, the -installation of Dramatiq and the broker, and finally writing a running a task. - -We recommend reading the [Dramatiq](https://dramatiq.io/guide.html) documentation -for full details on what the library is capable of, as well as additional usage -examples. diff --git a/docs/code/backgroundtasks.rst b/docs/code/backgroundtasks.rst new file mode 100644 index 000000000..433a13972 --- /dev/null +++ b/docs/code/backgroundtasks.rst @@ -0,0 +1,276 @@ +Running asynchronous background tasks +------------------------------------- + +When you are using your TOM via the web interface, the code that is +running in the background is tied to the request/response cycle. What +this means is that when you click a button or link in the TOM, your +browser constructs a web request, which is then sent to the web server +running your TOM. The TOM receives this request and then runs a bunch of +code, ultimately to generate a response that gets sent back to the +browser. This response is what you see when the next page loads. For the +purposes of this explanation, this all happens *synchronously*, meaning +that your browser has to wait for your TOM to respond before displaying +the next page. + +:: + + ---------- request ---------- + | | -------------> | | + | browser | response | TOM | + | | <------------- | | + ---------- ---------- + +But what happens if your TOM performs some compute or IO heavy task +while constructing the response? One example would be running a source +extraction on a data product after a user uploads it to your TOM. +Normally, the browser will just wait for the response. This results in +an agonizing wait time for the user as they watch the browser’s loading +spinner slowly rotate. Eventually they will give up and either reload +the page or close it completely. In fact, according to a study by +Akamai, 50% of web users will not wait longer than 10-15 seconds for a +page to load before giving up. + +The way we avoid these wait times is to run our slow code +*asynchronously* in the background, in a separate thread or process. In +this model the TOM responds to the browser with a response immediately, +before the slow code has even finished. + +:: + + ---------- request ---------- task ----------- + | | -------------> | | --------> | | + | browser | response | TOM | result | worker | + | | <------------- | | <-------- | | + ---------- ---------- ----------- + +A very common scenario is sending email. Many web applications require +the functionality of sending mail at some point. Let’s say the PI of a +project has the option to mass notify their CIs that observations have +been taken. Usually, sending email takes a very short amount of time, +but it is still good practice to remove it from the request/response +cycle, just in case it takes longer than usual or errors in some way. + +In this tutorial, we will go over how to run tasks asynchronously in +your TOM if you have the need to do so. + +Running tasks with Dramatiq +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +`Dramatiq `__ is a task processing library for +python. Simply put: it allows you to define functions as *actors* and +then execute those function using *workers*. None of this can happen +without a *broker*, though, which is the piece that is responsible for +passing messages from the *web process* to the *workers*. + +Installing Redis +^^^^^^^^^^^^^^^^ + +Unfortunately, the broker is a separate piece of software outside of the +task library. Dramatiq supports using either RabbitMQ or Redis. We’ll +use Redis because of its versatility: not only can it be used as a +message broker but it can also be used in your TOM as a cache (though +not covered in this tutorial). + +Depending on your OS, there are a few ways to `install +Redis `__. + +Using Docker +'''''''''''' + +One of the easiest ways to install Redis is to use Docker: + +:: + + docker run --name tom-redis -d -p6379:6379 redis + +Building from source +'''''''''''''''''''' + +You can also download Redis directly from the website and compile it: + +:: + + $ wget http://download.redis.io/releases/redis-5.0.5.tar.gz + $ tar xzf redis-5.0.5.tar.gz + $ cd redis-5.0.5 + $ make + +You can now run the server with: + +:: + + ./src/redis-server + +Using a package manager +''''''''''''''''''''''' + +If you are running Linux, most likely Redis is included with your +distribution via its package manager. For example: + +:: + + apt install Redis + +Whichever way works for you, we should now have a Redis server up and +running and listening on port 6379. + +Installing Dramatiq +^^^^^^^^^^^^^^^^^^^ + +Now that we have our broker running, we can install and configure our +TOM to run Dramatiq. Start by installing the required dependencies into +your virtualenv: + +:: + + pip install 'dramatiq[watch, redis]' django-dramatiq + +`django-dramatiq `__ will +offer us some conveniences while working with tasks in our TOM. + +Install django-dramatiq to your ``INSTALLED_APPS`` setting, above the +tom_\* apps: + +.. code:: python + + INSTALLED_APPS = [ + ... + 'django_gravatar', + 'django_dramatiq', + 'tom_targets', + ... + ] + +Add a section for dramatiq in ``settings.py``: + +.. code:: python + + DRAMATIQ_BROKER = { + "BROKER": "dramatiq.brokers.redis.RedisBroker", + "OPTIONS": { + "url": "redis://localhost:6379", + }, + "MIDDLEWARE": [ + "dramatiq.middleware.AgeLimit", + "dramatiq.middleware.TimeLimit", + "dramatiq.middleware.Callbacks", + "dramatiq.middleware.Retries", + "django_dramatiq.middleware.AdminMiddleware", + "django_dramatiq.middleware.DbConnectionsMiddleware", + ] + } + +If you want to store the results of your tasks add a section in +``settings.py`` for that as well: + +.. code:: python + + DRAMATIQ_RESULT_BACKEND = { + "BACKEND": "dramatiq.results.backends.redis.RedisBackend", + "BACKEND_OPTIONS": { + "url": "redis://localhost:6379", + }, + "MIDDLEWARE_OPTIONS": { + "result_ttl": 60000 + } + } + +Now that all the settings are in place, we need to run a +``manage.py migrate`` in order to create the ``django_dramatiq`` table. +Then, we can test installation by starting up some workers: + +:: + + ./manage.py rundramatiq + +If all goes well you will see output that looks like this: + +:: + + % ./manage.py rundramatiq + * Discovered tasks module: 'django_dramatiq.tasks' + * Running dramatiq: "dramatiq --path . --processes 8 --threads 8 --watch . django_dramatiq.setup django_dramatiq.tasks" + + [2019-08-21 17:52:30,216] [PID 27267] [MainThread] [dramatiq.MainProcess] [INFO] Dramatiq '1.6.1' is booting up. + Worker process is ready for action. + +Your task workers are up and running! + +Writing a task +^^^^^^^^^^^^^^ + +Now that we have some workers, lets put them to work. In order to do +that we’ll write a task. + +Create a file ``mytom/myapp/tasks.py`` where ``myapp`` is a django app +you’ve installed into ``INSTALLED_APPS``. If you haven’t started one, +you can do so with: + +:: + + ./manage.py startapp myapp + +In ``tasks.py``: + +.. code:: python + + import dramatiq + import time + import logging + + logger = logging.getLogger(__name__) + + + @dramatiq.actor + def super_complicated_task(): + logger.info('starting task...') + time.sleep(2) + logger.info('still running...') + time.sleep(2) + logger.info('done!') + +This task will emulate a function that blocks for 4 seconds, in practice +this would be a network call or some kind of heavy processing task. + +Now open up a Django shell: + +:: + + ./manage.py shell_plus + +And import and call the task: + +:: + + In [1]: from myapp.tasks import super_complicated_task + + In [2]: super_complicated_task.send() + Out[2]: Message(queue_name='default', actor_name='super_complicated_task', args=(), kwargs={}, options={'redis_message_id': '667821da-f236-4c4e-969a-9d1f1ff54be2'}, message_id='2c8893d8-4211-4cac-b0b9-0f2e9672d0ae', message_timestamp=1566416600481) + +In the terminal where you started the dramatiq workers (not the django +shell!) you should see the following output: + +:: + + starting task... + still running... + done! + +Notice how calling the task returned immediately in the shell, but the +task took a few seconds to complete. This is how it would work in +practice in your django app: Somewhere in your code, for example in your +app’s ``views.py``, you would import the task just like we did in the +terminal. Now when the view gets called, the task will be queued for +execution and the response can be sent back to the user’s browser right +away. The task will finish in the background. + +Conclusion +^^^^^^^^^^ + +In this tutorial we went over the need for asynchronous tasks, the +installation of Dramatiq and the broker, and finally writing a running a +task. + +We recommend reading the `Dramatiq `__ +documentation for full details on what the library is capable of, as +well as additional usage examples. diff --git a/docs/code/custom_code.md b/docs/code/custom_code.md deleted file mode 100644 index 86471d60b..000000000 --- a/docs/code/custom_code.md +++ /dev/null @@ -1,113 +0,0 @@ -Running Custom Code on Actions in your TOM ------------------------------------------- - -Sometimes it would be desirable for your TOM to run custom code when certain -actions happen. For example: when an observation is completed you'd like to submit -your data to an outside service. Or when you add a new target you'd like to -automatically search a remote catalog for matches. You could even make your TOM -automatically tweet new observations! We can achieve these tasks -using code hooks. - -### An example code hook: send an email when observation completes. - -In this example, we'll write a little bit of code to send an email when an -observation record changes it's state to 'COMPLETED'. We'll assume you have gone -through the [getting started](/introduction/getting_started) guide, and that you have -working TOM up and running called mytom. - -First, let's create a python module where the entry point to our custom code will -live: - - touch mytom/hooks.py - -Inside this module, let's stub out a method to call when our observation changes -status: - -```python -import logging - -logger = logging.getLogger(__name__) - - -def observation_change_state(observation, previous_status): - logger.info( - 'Sending email, observation %s changed state from %s to %s', - observation, previous_status, observation.status - ) -``` - -This method, for now, will simply log the fact that we will send out an email. -Note that the method takes the observation and it's previous status as parameters. - -Next, we'll tell our TOM to execute this method when an observation changes state. -This is done via the `HOOKS` configuration parameter in your project's -`settings.py`: - -```python -HOOKS = { - 'target_post_save': 'tom_common.hooks.target_post_save', - 'observation_change_state': 'mytom.hooks.observation_change_state', - 'data_product_post_upload': 'tom_dataproducts.hooks.data_product_post_upload', -} -``` - -We changed the path for the `observation_change_state` method from it's default to -the module path for our custom method, `mytom.hooks.observation_change_state`. - -Now, when an observation changes state, you should see the following in your logs: - - Sending email, observation M42 @ LCO changed state from PENDING to COMPLETED - -You can test this by manually changing an observation via the -[django admin -page](http://127.0.0.1:8000/admin/tom_observations/observationrecord/). - -If you only wanted to know how to run code via a hook, you can stop here and -implement your own code hooks. If you'd like to learn how to send an email, read -on. - -### Sending email - -Django has [good support](https://docs.djangoproject.com/en/2.1/topics/email/) -for sending emails, and you'll need to read the documentation to get the basic -setup right. Once you have the proper settings configured, sending an email in -your hook becomes as simple as this: - -```python -import logging -from django.core.mail import send_mail - -logger = logging.getLogger(__name__) - - -def observation_change_state(observation, previous_status): - if observation.status == 'COMPLETED': - logger.info( - 'Sending email, observation %s changed state from %s to %s', - observation, previous_status, observation.status - ) - send_mail( - 'Observation complete', - 'The observation {} has completed'.format(observation), - 'from@mytom.com', - ['to@example.com'], - fail_silently=False, - ) -``` - -That is all that is necessary for sending an email, though you might want to look -into using asynchronous task runners such as [dramatiq](https://dramatiq.io/) or -[celery](http://www.celeryproject.org/). - -### Available code hooks - -At present, there are three available code hooks. - -* target_post_save: Runs after a target is created or updated. -* observation_change_state: Runs whenever an observation's state is updated. -* data_product_post_upload: Runs after a data product is successfully uploaded to the TOM. - -> **NOTE**: `target_post_save` does not run automatically following a programmatic create statement, such as: -> ```python -> Target.objects.create(name='m51') -> ``` diff --git a/docs/code/custom_code.rst b/docs/code/custom_code.rst new file mode 100644 index 000000000..92be0c32f --- /dev/null +++ b/docs/code/custom_code.rst @@ -0,0 +1,132 @@ +Running Custom Code on Actions in your TOM +------------------------------------------ + +Sometimes it would be desirable for your TOM to run custom code when +certain actions happen. For example: when an observation is completed +you’d like to submit your data to an outside service. Or when you add a +new target you’d like to automatically search a remote catalog for +matches. You could even make your TOM automatically tweet new +observations! We can achieve these tasks using code hooks. + +An example code hook: send an email when observation completes. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In this example, we’ll write a little bit of code to send an email when +an observation record changes it’s state to ‘COMPLETED’. We’ll assume +you have gone through the `getting +started `__ guide, and that you have +working TOM up and running called mytom. + +First, let’s create a python module where the entry point to our custom +code will live: + +:: + + touch mytom/hooks.py + +Inside this module, let’s stub out a method to call when our observation +changes status: + +.. code:: python + + import logging + + logger = logging.getLogger(__name__) + + + def observation_change_state(observation, previous_status): + logger.info( + 'Sending email, observation %s changed state from %s to %s', + observation, previous_status, observation.status + ) + +This method, for now, will simply log the fact that we will send out an +email. Note that the method takes the observation and it’s previous +status as parameters. + +Next, we’ll tell our TOM to execute this method when an observation +changes state. This is done via the ``HOOKS`` configuration parameter in +your project’s ``settings.py``: + +.. code:: python + + HOOKS = { + 'target_post_save': 'tom_common.hooks.target_post_save', + 'observation_change_state': 'mytom.hooks.observation_change_state', + 'data_product_post_upload': 'tom_dataproducts.hooks.data_product_post_upload', + } + +We changed the path for the ``observation_change_state`` method from +it’s default to the module path for our custom method, +``mytom.hooks.observation_change_state``. + +Now, when an observation changes state, you should see the following in +your logs: + +:: + + Sending email, observation M42 @ LCO changed state from PENDING to COMPLETED + +You can test this by manually changing an observation via the `django +admin +page `__. + +If you only wanted to know how to run code via a hook, you can stop here +and implement your own code hooks. If you’d like to learn how to send an +email, read on. + +Sending email +~~~~~~~~~~~~~ + +Django has `good +support `__ for +sending emails, and you’ll need to read the documentation to get the +basic setup right. Once you have the proper settings configured, sending +an email in your hook becomes as simple as this: + +.. code:: python + + import logging + from django.core.mail import send_mail + + logger = logging.getLogger(__name__) + + + def observation_change_state(observation, previous_status): + if observation.status == 'COMPLETED': + logger.info( + 'Sending email, observation %s changed state from %s to %s', + observation, previous_status, observation.status + ) + send_mail( + 'Observation complete', + 'The observation {} has completed'.format(observation), + 'from@mytom.com', + ['to@example.com'], + fail_silently=False, + ) + +That is all that is necessary for sending an email, though you might +want to look into using asynchronous task runners such as +`dramatiq `__ or +`celery `__. + +Available code hooks +~~~~~~~~~~~~~~~~~~~~ + +At present, there are three available code hooks. + +- target_post_save: Runs after a target is created or updated. +- observation_change_state: Runs whenever an observation’s state is + updated. +- data_product_post_upload: Runs after a data product is successfully + uploaded to the TOM. + +.. + + **NOTE**: ``target_post_save`` does not run automatically following a + programmatic create statement, such as: + + .. code:: python + + Target.objects.create(name='m51') diff --git a/docs/code/querying.md b/docs/code/querying.md deleted file mode 100644 index 26585e309..000000000 --- a/docs/code/querying.md +++ /dev/null @@ -1,70 +0,0 @@ -# Querying on related objects - -An aspect of programmatic TOM Toolkit access that is often desired is filtering by related objects. While this is -extensively documented in the Django documentation, it's certainly helpful to see a couple of examples in action. - -## Identifying Targets by TargetExtra values - -There may be times that you want to find Targets by specific parameters. It's fairly trivial to, for example, find a set -of Targets by RA: - -```python ->>> from tom_targets.models import Target ->>> Target.objects.filter(ra=356.58) -``` - -However, this isn't terribly helpful, as you need to know the exact value of RA that you're looking for to find your -Target. Fortunately, the Django QuerySet API offers a number of additional functions, including -[Field lookups](https://docs.djangoproject.com/en/3.0/ref/models/querysets/#field-lookups): - -```python ->>> Target.objects.filter(ra__lte=357, ra__gte=356) -``` - -The above query will look for Targets with RAs between 356 and 357. Field lookups can be used for more granular queries, -and it's encouraged to reference the Django Queryset API Docs to familiarize yourself. - -While the previous query is very useful for searching in a range, what about when you aren't filtering on base Target -fields? A common use case of the TOM `TargetExtra` model is for fields that aren't on Targets by default. Let's take the -example of supernovae. Let's say that you have a TOM for tracking supernovae, and you've added redshift as a TargetExtra. -How does one find Targets with the appropriate redshift? - -```python ->>> Target.objects.filter(targetextra__key='redshift', targetextra__value__gt=0.5) -``` - -That query will first find Targets with a TargetExtra of `redshift`, and will filter those particular TargetExtras for a -value of greater than 0.5. - - -## Adding Targets to Groups programmatically - -Another operation that one might desire to do programmatically is adding Targets to Groups. This can be done in a -relatively straightforward manner as well: - -```python -from tom_targets.models import TargetList ->>> Target.objects.all() -, ]> ->>> TargetList.objects.all() - ->>> tl = TargetList(name='My Target List') ->>> tl.save() ->>> tl.refresh_from_db() ->>> tl.id -1 ->>> tl.targets.all() - ->>> tl.targets.add(Target.objects.first()) ->>> tl.targets.all() -]> -``` - -Related objects can be obtained in either direction: - -```python ->>> t.targetlist_set.all() -]> ->>> tl.targets.all() -]> -``` diff --git a/docs/code/querying.rst b/docs/code/querying.rst new file mode 100644 index 000000000..f22674a9a --- /dev/null +++ b/docs/code/querying.rst @@ -0,0 +1,81 @@ +Querying on related objects +=========================== + +An aspect of programmatic TOM Toolkit access that is often desired is +filtering by related objects. While this is extensively documented in +the Django documentation, it’s certainly helpful to see a couple of +examples in action. + +Identifying Targets by TargetExtra values +----------------------------------------- + +There may be times that you want to find Targets by specific parameters. +It’s fairly trivial to, for example, find a set of Targets by RA: + +.. code:: python + + >>> from tom_targets.models import Target + >>> Target.objects.filter(ra=356.58) + +However, this isn’t terribly helpful, as you need to know the exact +value of RA that you’re looking for to find your Target. Fortunately, +the Django QuerySet API offers a number of additional functions, +including `Field +lookups `__: + +.. code:: python + + >>> Target.objects.filter(ra__lte=357, ra__gte=356) + +The above query will look for Targets with RAs between 356 and 357. +Field lookups can be used for more granular queries, and it’s encouraged +to reference the Django Queryset API Docs to familiarize yourself. + +While the previous query is very useful for searching in a range, what +about when you aren’t filtering on base Target fields? A common use case +of the TOM ``TargetExtra`` model is for fields that aren’t on Targets by +default. Let’s take the example of supernovae. Let’s say that you have a +TOM for tracking supernovae, and you’ve added redshift as a TargetExtra. +How does one find Targets with the appropriate redshift? + +.. code:: python + + >>> Target.objects.filter(targetextra__key='redshift', targetextra__value__gt=0.5) + +That query will first find Targets with a TargetExtra of ``redshift``, +and will filter those particular TargetExtras for a value of greater +than 0.5. + +Adding Targets to Groups programmatically +----------------------------------------- + +Another operation that one might desire to do programmatically is adding +Targets to Groups. This can be done in a relatively straightforward +manner as well: + +.. code:: python + + from tom_targets.models import TargetList + >>> Target.objects.all() + , ]> + >>> TargetList.objects.all() + + >>> tl = TargetList(name='My Target List') + >>> tl.save() + >>> tl.refresh_from_db() + >>> tl.id + 1 + >>> tl.targets.all() + + >>> tl.targets.add(Target.objects.first()) + >>> tl.targets.all() + ]> + +Related objects can be obtained in either direction: + +.. code:: python + + >>> t.targetlist_set.all() + ]> + >>> tl.targets.all() + ]> diff --git a/docs/customization/adding_pages.md b/docs/customization/adding_pages.md deleted file mode 100644 index d2de9d8c9..000000000 --- a/docs/customization/adding_pages.md +++ /dev/null @@ -1,204 +0,0 @@ -Adding pages to your TOM ------------------------- - -The TOM Toolkit provides many views (pages) by default, but at some point you may -want to add pages of your own. These could be simple static pages like project or -grant information. Or they can be fully dynamic, displaying data from the database -and containing forms of their own. - -In this tutorial we'll start out by adding a simple "About" page to our TOM. Then -to spice it up a little we'll add some dynamic info to the page (a list of -targets). Finally we'll learn how Django can help us create even more interactive -pages. - -### A simple template page - -Let's get started with some code and we'll explain it piece by piece afterwards. - -First, let's create a new file `about.html` and place it in the `templates/` -directory at the root of our TOM. This file will contain the content of our new -page. - -```html -

    -To know that we know what we know, and to know that we do not know -what we do not know, that is true knowledge.
    -Nicolaus Copernicus -

    -``` - -Next we need to tell Django about this new page and what url to serve it from. -Open the `urls.py` file (next to `settings.py`) and modify it so that it looks -something like this (you may have additional urls already, the important part is -the one relevant to `about.html`): - -```python -from django.urls import path, include -from django.views.generic import TemplateView - -urlpatterns = [ - path('', include('tom_common.urls')), - path('about/', TemplateView.as_view(template_name='about.html'), name='about') -] -``` - -Notice the `path` function we use here. It takes three arguments. Argument one is -the path in which this page should be made available in our TOM. In this case, -we used the sensible path "about/". The second argument is the view function. -In this case we passed in a -[TemplateView](https://docs.djangoproject.com/en/2.2/ref/class-based-views/base/#templateview) -. We'll talk about view functions a bit later, but just know that this class -simply takes the template it should render and renders it. The last argument is -the name of the url. This is so we can refer to this path elsewhere in the -application without the need to hardcode urls. - -Enough techno blabber. Launch your TOM and navigate to -[/about/](http://127.0.0.1:8000/about/). You should see something like this: - -![](/_static/adding_pages_doc/quote.png) - -That's progress, but our new page is pretty ugly. The navigation bar is missing -and we don't have any of the nice CSS that makes the rest of the TOM pages look -good! But wait, before you start copying in lines of HTML, know that all we need -to do is extend -[tom\_common/base.html](https://github.com/TOMToolkit/tom_base/blob/master/tom_common/templates/tom_common/base.html) - to get all that back. You can read more about extending templates from the guide - on [Customizing TOM Templates](/customization/customize_templates). Let's modify - `about.html` to extend the base template: - -```html -{% extends 'tom_common/base.html' %} -{% block content %} -

    -To know that we know what we know, and to know that we do not know -what we do not know, that is true knowledge.
    -Nicolaus Copernicus -

    -{% endblock %} -``` - -Now when you reload the page you should see this: - -![](/_static/adding_pages_doc/base.png) - -Much better! By extending a template and providing a `content` block, we are able -to make consistent looking pages without copying and pasting any code. - -You can read more about template inheritance in [Django's official -docs](https://docs.djangoproject.com/en/2.2/ref/templates/language/#template-inheritance) - - -### Adding in dynamic data - -We now know how to add basic static pages. But what if we want to show data from -our database? Let's try adding a list of all the targets in our TOM to the about -page. This is slightly more complex, so we're going to create a new file, -`views.py` alongside our `urls.py` file. Add the following content: - -```python -from django.views.generic import TemplateView -from tom_observations.models import Target - - -class AboutView(TemplateView): - template_name = 'about.html' - - def get_context_data(self, **kwargs): - return {'targets': Target.objects.all()} -``` - -Notice we are still using the `TemplateView` here. The only addition is that we -are implementing `get_context_data` which returns a dictionary of data that should -be available to our template. In this case, we are returning all the targets in -our TOM. - -Let's modify our `urls.py` to use our new view: - -```python -from django.urls import path, include -from .views import AboutView - -urlpatterns = [ - path('', include('tom_common.urls')), - path('about/', AboutView.as_view(), name='about') -] -``` - -We've replaced the import of `TemplateView` with an import of the view class we -just wrote, and modified the call to `path()` accordingly. - -Lastly let's update our `about.html` template to actually show the list of -targets: - -```html -{% extends 'tom_common/base.html' %} -{% block content %} -

    -To know that we know what we know, and to know that we do not know -what we do not know, that is true knowledge.
    -Nicolaus Copernicus -

    -
      - {% for target in targets %} -
    • {{ target.name }}
      • - {% endfor %} -
    -{% endblock %} - -``` - -`targets` in this template refers to the key in the dictionary we returned in the -`get_context_data` method in our view. We can add anything to the context -dictionary and have access to it in our templates. In this particular example, we're -iterating over all of the targets in our TOM and displaying all of their names. If you -don't see anything, make sure you have targets in your TOM! - -Reloading your about page, you should now see something like this: - -![](/_static/adding_pages_doc/targets.png) - -If the page looks exactly the same as last time, you might need to add some -targets. Navigate to -[http://localhost:8000/targets/](http://cygnus.lco.gtn:8000/targets/) to do so. -### Class based views -Django has the concept of [class based -views](https://docs.djangoproject.com/en/2.2/topics/class-based-views/intro/). -These classes do one job: they take in an HTTP request and return a response. In -this tutorial we took advantage of Django's -[TemplateView](https://docs.djangoproject.com/en/2.2/ref/class-based-views/base/#templateview) -which does a simple job of rendering templates. Django has [many more built in -class based -views](https://docs.djangoproject.com/en/2.2/topics/class-based-views/generic-display/) -that can be taken advantage of. For example, instead of using the `TemplateView` -for rendering a list of Targets, we could have used the -[ListView](https://docs.djangoproject.com/en/2.2/topics/class-based-views/generic-display/#generic-views-of-objects) -which provides additional functionality, such as pagination and filtering. - -When working with class based views, you'll almost always subclass them. We did -this with our `AboutView` earlier, and changed the `TemplateView`'s behavior to include a -list of our targets. Herein lies the power of class based views. You can even subclass -the views that ship with the TOM Toolkit itself. So for example, if you don't like -how the -[TargetListView](https://github.com/TOMToolkit/tom_base/blob/15870172e842bcbac17bd4a4b71c9e016b270cf9/tom_targets/views.py#L29) -in the base TOM Toolkit behaves, you could subclass it in your TOM: - -```python -from tom_targets.views import TargetListView - -class MyCustomTargetListView(TargetListView): - template_name = 'mysupertargetlist.html' - paginate_by = 100 -``` - -### Wrapping it all up - -In this tutorial we learned how to not only add static pages to our TOM, but also -how to display some information from our database. Along the way we learned about -Django's [class based -views](https://docs.djangoproject.com/en/2.2/topics/class-based-views/intro/) as -well as some of the things we could use them for. - -We didn't get into how to display forms or receive other parameters in our views, -but some [light reading the Django docs](https://docs.djangoproject.com/en/2.2/intro/tutorial04/#write-a-simple-form) -could familiarize one with those concepts. - diff --git a/docs/customization/adding_pages.rst b/docs/customization/adding_pages.rst new file mode 100644 index 000000000..63597c897 --- /dev/null +++ b/docs/customization/adding_pages.rst @@ -0,0 +1,217 @@ +Adding pages to your TOM +------------------------ + +The TOM Toolkit provides many views (pages) by default, but at some +point you may want to add pages of your own. These could be simple +static pages like project or grant information. Or they can be fully +dynamic, displaying data from the database and containing forms of their +own. + +In this tutorial we’ll start out by adding a simple “About” page to our +TOM. Then to spice it up a little we’ll add some dynamic info to the +page (a list of targets). Finally we’ll learn how Django can help us +create even more interactive pages. + +A simple template page +~~~~~~~~~~~~~~~~~~~~~~ + +Let’s get started with some code and we’ll explain it piece by piece +afterwards. + +First, let’s create a new file ``about.html`` and place it in the +``templates/`` directory at the root of our TOM. This file will contain +the content of our new page. + +.. code:: html + +

    + To know that we know what we know, and to know that we do not know + what we do not know, that is true knowledge.
    + Nicolaus Copernicus +

    + +Next we need to tell Django about this new page and what url to serve it +from. Open the ``urls.py`` file (next to ``settings.py``) and modify it +so that it looks something like this (you may have additional urls +already, the important part is the one relevant to ``about.html``): + +.. code:: python + + from django.urls import path, include + from django.views.generic import TemplateView + + urlpatterns = [ + path('', include('tom_common.urls')), + path('about/', TemplateView.as_view(template_name='about.html'), name='about') + ] + +Notice the ``path`` function we use here. It takes three arguments. +Argument one is the path in which this page should be made available in +our TOM. In this case, we used the sensible path “about/”. The second +argument is the view function. In this case we passed in a +`TemplateView `__ +. We’ll talk about view functions a bit later, but just know that this +class simply takes the template it should render and renders it. The +last argument is the name of the url. This is so we can refer to this +path elsewhere in the application without the need to hardcode urls. + +Enough techno blabber. Launch your TOM and navigate to +`/about/ `__. You should see something +like this: + +|image0| + +That’s progress, but our new page is pretty ugly. The navigation bar is +missing and we don’t have any of the nice CSS that makes the rest of the +TOM pages look good! But wait, before you start copying in lines of +HTML, know that all we need to do is extend +`tom_common/base.html `__ +to get all that back. You can read more about extending templates from +the guide on `Customizing TOM +Templates `__. Let’s modify +``about.html`` to extend the base template: + +.. code:: html + + {% extends 'tom_common/base.html' %} + {% block content %} +

    + To know that we know what we know, and to know that we do not know + what we do not know, that is true knowledge.
    + Nicolaus Copernicus +

    + {% endblock %} + +Now when you reload the page you should see this: + +|image1| + +Much better! By extending a template and providing a ``content`` block, +we are able to make consistent looking pages without copying and pasting +any code. + +You can read more about template inheritance in `Django’s official +docs `__ + +Adding in dynamic data +~~~~~~~~~~~~~~~~~~~~~~ + +We now know how to add basic static pages. But what if we want to show +data from our database? Let’s try adding a list of all the targets in +our TOM to the about page. This is slightly more complex, so we’re going +to create a new file, ``views.py`` alongside our ``urls.py`` file. Add +the following content: + +.. code:: python + + from django.views.generic import TemplateView + from tom_observations.models import Target + + + class AboutView(TemplateView): + template_name = 'about.html' + + def get_context_data(self, **kwargs): + return {'targets': Target.objects.all()} + +Notice we are still using the ``TemplateView`` here. The only addition +is that we are implementing ``get_context_data`` which returns a +dictionary of data that should be available to our template. In this +case, we are returning all the targets in our TOM. + +Let’s modify our ``urls.py`` to use our new view: + +.. code:: python + + from django.urls import path, include + from .views import AboutView + + urlpatterns = [ + path('', include('tom_common.urls')), + path('about/', AboutView.as_view(), name='about') + ] + +We’ve replaced the import of ``TemplateView`` with an import of the view +class we just wrote, and modified the call to ``path()`` accordingly. + +Lastly let’s update our ``about.html`` template to actually show the +list of targets: + +.. code:: html + + {% extends 'tom_common/base.html' %} + {% block content %} +

    + To know that we know what we know, and to know that we do not know + what we do not know, that is true knowledge.
    + Nicolaus Copernicus +

    +
      + {% for target in targets %} +
    • {{ target.name }}
      • + {% endfor %} +
    + {% endblock %} + +``targets`` in this template refers to the key in the dictionary we +returned in the ``get_context_data`` method in our view. We can add +anything to the context dictionary and have access to it in our +templates. In this particular example, we’re iterating over all of the +targets in our TOM and displaying all of their names. If you don’t see +anything, make sure you have targets in your TOM! + +Reloading your about page, you should now see something like this: + +|image2| + +If the page looks exactly the same as last time, you might need to add +some targets. Navigate to +`http://localhost:8000/targets/ `__ +to do so. ### Class based views Django has the concept of `class based +views `__. +These classes do one job: they take in an HTTP request and return a +response. In this tutorial we took advantage of Django’s +`TemplateView `__ +which does a simple job of rendering templates. Django has `many more +built in class based +views `__ +that can be taken advantage of. For example, instead of using the +``TemplateView`` for rendering a list of Targets, we could have used the +`ListView `__ +which provides additional functionality, such as pagination and +filtering. + +When working with class based views, you’ll almost always subclass them. +We did this with our ``AboutView`` earlier, and changed the +``TemplateView``\ ’s behavior to include a list of our targets. Herein +lies the power of class based views. You can even subclass the views +that ship with the TOM Toolkit itself. So for example, if you don’t like +how the +`TargetListView `__ +in the base TOM Toolkit behaves, you could subclass it in your TOM: + +.. code:: python + + from tom_targets.views import TargetListView + + class MyCustomTargetListView(TargetListView): + template_name = 'mysupertargetlist.html' + paginate_by = 100 + +Wrapping it all up +~~~~~~~~~~~~~~~~~~ + +In this tutorial we learned how to not only add static pages to our TOM, +but also how to display some information from our database. Along the +way we learned about Django’s `class based +views `__ +as well as some of the things we could use them for. + +We didn’t get into how to display forms or receive other parameters in +our views, but some `light reading the Django +docs `__ +could familiarize one with those concepts. + +.. |image0| image:: /_static/adding_pages_doc/quote.png +.. |image1| image:: /_static/adding_pages_doc/base.png +.. |image2| image:: /_static/adding_pages_doc/targets.png diff --git a/docs/customization/customize_template_tags.md b/docs/customization/customize_template_tags.md deleted file mode 100644 index e6a2fd7b0..000000000 --- a/docs/customization/customize_template_tags.md +++ /dev/null @@ -1,266 +0,0 @@ -# Customizing Template Tags - -The TOM Toolkit is designed to be as customizable as possible. A number of UI objects are rendered as Django templatetags. -Django has quite a few [built-in template tags](https://docs.djangoproject.com/en/3.0/ref/templates/builtins/), but also -allows the creation of [custom template tags](https://docs.djangoproject.com/en/3.0/howto/custom-template-tags/), which -the TOM Toolkit leverages heavily. - -However, it's possible that a TOM Toolkit template tag doesn't quite meet your needs. Maybe the axis labels for photometry -plotting aren't quite what you're looking for, or the target data isn't formatted the way you'd like. This tutorial will -show you how to write your own template tag to suit your own program better. - - -## Preparing your project for custom template tags - -The first thing your project will need is a custom app. You can read about custom apps in the Django tutorial -[here](https://docs.djangoproject.com/en/dev/intro/tutorial01/), but to quickly get started, the command to create a new -app is as follows: - -```python -./manage.py startapp custom_code -``` - -Where `custom_code` is the name of your app. You will also need to ensure that `custom_code` is in your `settings.py`. -Append it to the end of `INSTALLED_APPS`: - -```python -... -INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - ... - 'tom_dataproducts', - 'custom_code', -] -... -``` - -You should now have a directory within your TOM called `custom_code`, which looks like this: - - ├── custom_code - | ├── __init__.py - │ ├── admin.py - │ ├── apps.py - │ ├── models.py - │ ├── tests.py - │ └── views.py - ├── data - ├── db.sqlite3 - ├── manage.py - ├── mytom - │ ├── __init__.py - │ ├── settings.py - │ ├── urls.py - │ └── wsgi.py - ├── static - ├── templates - └── tmp - -Next, you'll need to add a `templatetags` directory within `custom_code`. Create an empty file called `__init__.py` within -that directory. Finally, we need a file to put the code for our custom template tags. Add a file in `custom_code` called -`custom_extras`. It's convention to use `_extras` within your template tag module name. - -Your `custom_code` directory should look like this: - - └── custom_code - ├── __init__.py - ├── admin.py - ├── apps.py - ├── models.py - ├── templatetags - | ├── __init__.py - | └── custom_extras.py - ├── tests.py - └── views.py - - -## Writing a custom template tag - -For our template tag, we're going to write a tag that displays the timestamp and magnitude for the most recent photometry -point available for a target. There are three aspects to a template tag: - -* The code in `custom_extras` to run the logic to get the data we'll be displaying -* The partial template to render the data -* Putting the custom tag somewhere we'd like it displayed - -### The Python code - -We're going to write a `recent_photometry` function in our `custom_extras` first. Step one is the necessary import and -initialization of the template library: - -```python -from django import template - - -register = template.Library() -``` - -Now, to the `recent_photometry` function. A couple notes about the approach here: - -* The function will have the decorator `@register.inclusion_tag()`. There are a couple of different types of template tags, -but we're using the `inclusion_tag` because it renders a template, allowing us to customize how it looks. The `simple_tag` -is a different type of template tag that simply modifies data, so that won't work for us. -* Within the decorator is a path to the partial template that will render the data--this doesn't exist yet, but remember the file name we're using! -* We'd like to get the latest photometry values for a specific target, so we'll need to pass a `Target` as a parameter. -* We'd also like to be able to specify how many photometry points we care about, so let's also include a keyword argument that defaults to just 1. - -```python -from django import template - - -register = template.Library() - - -@register.inclusion_tag('custom_code/partials/recent_photometry.html') -def recent_photometry(target, num_points=1): - return {} -``` - -You can see that we'll eventually be returning a dictionary, but first we need to add our logic. We'll need to use the -`Target` passed in to get all `ReducedDatum` objects for that `Target` with a `data_type` of `photometry`. Then we'll -need to order by `timestamp` descending, and slice just the first few. Make sure to take note of the imports in this step! - -```python -import json - -from django import template - -from tom_dataproducts.models import ReducedDatum - - -register = template.Library() - - -@register.inclusion_tag('custom_code/partials/recent_photometry.html') -def recent_photometry(target, num_points=1): - photometry = ReducedDatum.objects.filter(data_type='photometry').order_by('-timestamp')[:num_points] - return {'recent_photometry': [(datum.timestamp, json.loads(datum.value)['magnitude']) for datum in photometry]} -``` - -It's only a couple of lines, but there's a lot going on here. The first line does the aforemention database query and -slices the first point of the `QuerySet`. The second line constructs a dictionary--the only key is `recent_photometry`, -and the corresponding value is a list of tuples. Each tuple has the timestamp as the first item, and the magnitude as -the second item. - -Ultimately, this template tag will, when included, return the most recent photometry points for a `Target`. But it -can't display anything! - -### The partial template - -So now we need to create `custom_code/templates/custom_code/partials/recent_photometry.html`. We'll need to add yet -another series of directories and files. Your directory structure should now look like this: - -Let's start with the partial template. We'll need to add yet another series of directories and files. Add the following -to your directory structure: - - └── custom_code - └── templates - └── custom_code - └── partials - └── recent_photometry.html - -Your complete directory structure should look like this: - - └── custom_code - ├── __init__.py - ├── admin.py - ├── apps.py - ├── models.py - ├── templates - | └── custom_code - | └── partials - | └── recent_photometry.html - ├── templatetags - | ├── __init__.py - | └── custom_extras.py - ├── tests.py - └── views.py - -And let's open up `recent_photometry.html` and get to work. - - -```html -
    -
    - Recent Photometry -
    - - - - {% for datum in recent_photometry %} - - - - - {% empty %} - - - - {% endfor %} - -
    TimestampMagnitude
    {{ datum.0 }}{{ datum.1 }}
    No recent photometry.
    -
    -``` - -This template looks suspiciously like a few others in the TOM Toolkit, but that's okay! It will just render a two-column -table with columns for timestamp and magnitude. The dictionary we returned is accessible to the template, which is why -this line works: - -```html -{% for datum in recent_photometry %} -``` -It iterates over the value referred to by `recent_photometry`, which, if you recall, is a list of tuples. Then it -renders each element of the tuple in a `` element. - -So we have a partial template and a template tag that can be used anywhere, but we have to put it somewhere! - -### Using the template tag - -The target detail page seems like a logical place for this, so let's go there. First, we need to override our `target_detail.html` -template. If you haven't read the tutorial on template overriding, you can do so [here](customize_templates)-- -in the meantime, you'll need to add `target_detail.html` to `templates/tom_targets/` in the top level of your project. -Your project directory should look like this: - - ├── custom_code - ├── data - ├── db.sqlite3 - ├── manage.py - ├── mytom - ├── static - ├── templates - │ └── tom_targets - │ └── target_detail.html - └── tmp - -Then, you'll need to copy the contents of `target_detail.html` in the base TOM Toolkit to your `target_detail.html`. You -can find that file on [Github](https://github.com/TOMToolkit/tom_base/blob/master/tom_targets/templates/tom_targets/target_detail.html). - -Near the top of the file, there's a series of template tags that are loaded in. Add `custom_extras` to that list: - -```html -{% extends 'tom_common/base.html' %} -{% load comments bootstrap4 tom_common_extras targets_extras observation_extras dataproduct_extras publication_extras custom_extras static cache %} -... -``` - -Then, put your templatetag in the HTML somewhere, passing in `object` (which refers to the object value of the current -template context) and the desired number of photometry points: - -```html -... -{% endif %} -{% target_buttons object %} -{% target_data object %} -{% if object.type == 'SIDEREAL' %} -{% aladin object %} -{% endif %} -{% recent_photometry object num_points=3 %} -... -``` - -The new table should be displayed on your target detail page! Not only that, but you'll now be able to include that -template tag on other pages, too. And if it doesn't quite meet your needs--perhaps you want the most recent photometry -points for all targets, for example--it can be easily modified. - -As far as this template tag goes, as of this tutorial, it's now a part of the base TOM Toolkit, but all of the information -here should provide you with the ability to write your own. diff --git a/docs/customization/customize_template_tags.rst b/docs/customization/customize_template_tags.rst new file mode 100644 index 000000000..4d871e7ee --- /dev/null +++ b/docs/customization/customize_template_tags.rst @@ -0,0 +1,322 @@ +Customizing Template Tags +========================= + +The TOM Toolkit is designed to be as customizable as possible. A number +of UI objects are rendered as Django templatetags. Django has quite a +few `built-in template +tags `__, +but also allows the creation of `custom template +tags `__, +which the TOM Toolkit leverages heavily. + +However, it’s possible that a TOM Toolkit template tag doesn’t quite +meet your needs. Maybe the axis labels for photometry plotting aren’t +quite what you’re looking for, or the target data isn’t formatted the +way you’d like. This tutorial will show you how to write your own +template tag to suit your own program better. + +Preparing your project for custom template tags +----------------------------------------------- + +The first thing your project will need is a custom app. You can read +about custom apps in the Django tutorial +`here `__, but +to quickly get started, the command to create a new app is as follows: + +.. code:: python + + ./manage.py startapp custom_code + +Where ``custom_code`` is the name of your app. You will also need to +ensure that ``custom_code`` is in your ``settings.py``. Append it to the +end of ``INSTALLED_APPS``: + +.. code:: python + + ... + INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + ... + 'tom_dataproducts', + 'custom_code', + ] + ... + +You should now have a directory within your TOM called ``custom_code``, +which looks like this: + +:: + + ├── custom_code + | ├── __init__.py + │ ├── admin.py + │ ├── apps.py + │ ├── models.py + │ ├── tests.py + │ └── views.py + ├── data + ├── db.sqlite3 + ├── manage.py + ├── mytom + │ ├── __init__.py + │ ├── settings.py + │ ├── urls.py + │ └── wsgi.py + ├── static + ├── templates + └── tmp + +Next, you’ll need to add a ``templatetags`` directory within +``custom_code``. Create an empty file called ``__init__.py`` within that +directory. Finally, we need a file to put the code for our custom +template tags. Add a file in ``custom_code`` called ``custom_extras``. +It’s convention to use ``_extras`` within your template tag module name. + +Your ``custom_code`` directory should look like this: + +:: + + └── custom_code + ├── __init__.py + ├── admin.py + ├── apps.py + ├── models.py + ├── templatetags + | ├── __init__.py + | └── custom_extras.py + ├── tests.py + └── views.py + +Writing a custom template tag +----------------------------- + +For our template tag, we’re going to write a tag that displays the +timestamp and magnitude for the most recent photometry point available +for a target. There are three aspects to a template tag: + +- The code in ``custom_extras`` to run the logic to get the data we’ll + be displaying +- The partial template to render the data +- Putting the custom tag somewhere we’d like it displayed + +The Python code +~~~~~~~~~~~~~~~ + +We’re going to write a ``recent_photometry`` function in our +``custom_extras`` first. Step one is the necessary import and +initialization of the template library: + +.. code:: python + + from django import template + + + register = template.Library() + +Now, to the ``recent_photometry`` function. A couple notes about the +approach here: + +- The function will have the decorator ``@register.inclusion_tag()``. + There are a couple of different types of template tags, but we’re + using the ``inclusion_tag`` because it renders a template, allowing + us to customize how it looks. The ``simple_tag`` is a different type + of template tag that simply modifies data, so that won’t work for us. +- Within the decorator is a path to the partial template that will + render the data–this doesn’t exist yet, but remember the file name + we’re using! +- We’d like to get the latest photometry values for a specific target, + so we’ll need to pass a ``Target`` as a parameter. +- We’d also like to be able to specify how many photometry points we + care about, so let’s also include a keyword argument that defaults to + just 1. + +.. code:: python + + from django import template + + + register = template.Library() + + + @register.inclusion_tag('custom_code/partials/recent_photometry.html') + def recent_photometry(target, num_points=1): + return {} + +You can see that we’ll eventually be returning a dictionary, but first +we need to add our logic. We’ll need to use the ``Target`` passed in to +get all ``ReducedDatum`` objects for that ``Target`` with a +``data_type`` of ``photometry``. Then we’ll need to order by +``timestamp`` descending, and slice just the first few. Make sure to +take note of the imports in this step! + +.. code:: python + + import json + + from django import template + + from tom_dataproducts.models import ReducedDatum + + + register = template.Library() + + + @register.inclusion_tag('custom_code/partials/recent_photometry.html') + def recent_photometry(target, num_points=1): + photometry = ReducedDatum.objects.filter(data_type='photometry').order_by('-timestamp')[:num_points] + return {'recent_photometry': [(datum.timestamp, json.loads(datum.value)['magnitude']) for datum in photometry]} + +It’s only a couple of lines, but there’s a lot going on here. The first +line does the aforemention database query and slices the first point of +the ``QuerySet``. The second line constructs a dictionary–the only key +is ``recent_photometry``, and the corresponding value is a list of +tuples. Each tuple has the timestamp as the first item, and the +magnitude as the second item. + +Ultimately, this template tag will, when included, return the most +recent photometry points for a ``Target``. But it can’t display +anything! + +The partial template +~~~~~~~~~~~~~~~~~~~~ + +So now we need to create +``custom_code/templates/custom_code/partials/recent_photometry.html``. +We’ll need to add yet another series of directories and files. Your +directory structure should now look like this: + +Let’s start with the partial template. We’ll need to add yet another +series of directories and files. Add the following to your directory +structure: + +:: + + └── custom_code + └── templates + └── custom_code + └── partials + └── recent_photometry.html + +Your complete directory structure should look like this: + +:: + + └── custom_code + ├── __init__.py + ├── admin.py + ├── apps.py + ├── models.py + ├── templates + | └── custom_code + | └── partials + | └── recent_photometry.html + ├── templatetags + | ├── __init__.py + | └── custom_extras.py + ├── tests.py + └── views.py + +And let’s open up ``recent_photometry.html`` and get to work. + +.. code:: html + +
    +
    + Recent Photometry +
    + + + + {% for datum in recent_photometry %} + + + + + {% empty %} + + + + {% endfor %} + +
    TimestampMagnitude
    {{ datum.0 }}{{ datum.1 }}
    No recent photometry.
    +
    + +This template looks suspiciously like a few others in the TOM Toolkit, +but that’s okay! It will just render a two-column table with columns for +timestamp and magnitude. The dictionary we returned is accessible to the +template, which is why this line works: + +.. code:: html + + {% for datum in recent_photometry %} + +It iterates over the value referred to by ``recent_photometry``, which, +if you recall, is a list of tuples. Then it renders each element of the +tuple in a ```` element. + +So we have a partial template and a template tag that can be used +anywhere, but we have to put it somewhere! + +Using the template tag +~~~~~~~~~~~~~~~~~~~~~~ + +The target detail page seems like a logical place for this, so let’s go +there. First, we need to override our ``target_detail.html`` template. +If you haven’t read the tutorial on template overriding, you can do so +`here `__– in the meantime, you’ll need to add +``target_detail.html`` to ``templates/tom_targets/`` in the top level of +your project. Your project directory should look like this: + +:: + + ├── custom_code + ├── data + ├── db.sqlite3 + ├── manage.py + ├── mytom + ├── static + ├── templates + │ └── tom_targets + │ └── target_detail.html + └── tmp + +Then, you’ll need to copy the contents of ``target_detail.html`` in the +base TOM Toolkit to your ``target_detail.html``. You can find that file +on +`Github `__. + +Near the top of the file, there’s a series of template tags that are +loaded in. Add ``custom_extras`` to that list: + +.. code:: html + + {% extends 'tom_common/base.html' %} + {% load comments bootstrap4 tom_common_extras targets_extras observation_extras dataproduct_extras publication_extras custom_extras static cache %} + ... + +Then, put your templatetag in the HTML somewhere, passing in ``object`` +(which refers to the object value of the current template context) and +the desired number of photometry points: + +.. code:: html + + ... + {% endif %} + {% target_buttons object %} + {% target_data object %} + {% if object.type == 'SIDEREAL' %} + {% aladin object %} + {% endif %} + {% recent_photometry object num_points=3 %} + ... + +The new table should be displayed on your target detail page! Not only +that, but you’ll now be able to include that template tag on other +pages, too. And if it doesn’t quite meet your needs–perhaps you want the +most recent photometry points for all targets, for example–it can be +easily modified. + +As far as this template tag goes, as of this tutorial, it’s now a part +of the base TOM Toolkit, but all of the information here should provide +you with the ability to write your own. diff --git a/docs/customization/customize_templates.md b/docs/customization/customize_templates.md deleted file mode 100644 index b01c36b6f..000000000 --- a/docs/customization/customize_templates.md +++ /dev/null @@ -1,152 +0,0 @@ -Customizing TOM Templates -------------------------- - -So you've got a TOM up and running, and your homepage looks something like this: - -![](/_static/customize_templates_doc/tomhomepagenew.png) - -This is fine for starting out, but since you're running a TOM for a specific -project, the homepage ought to reflect that. - -If you haven't already, please read through the [Getting Started](/introduction/getting_started) -docs and return here when you have a project layout that looks something like this: - - -``` -mytom -├── db.sqlite3 -├── manage.py -└── mytom - ├── __init__.py - ├── settings.py - ├── urls.py - └── wsgi.py -``` - -We are going to override the html template included with the TOM Toolkit, `tom_common/index.html`, -so that we can edit some text and change the image. Overriding and extending templates is -[documented extensively](https://docs.djangoproject.com/en/2.1/howto/overriding-templates/) on -Django's website and we highly recommend reading these docs if you plan on customizing your -TOM further. - -Since the template we want to override is already part of the TOM Toolkit source -code, we can use it as a starting point for our customized template. In fact, -we'll copy and paste the entire thing from the [source code of TOM Toolkit](https://github.com/TOMToolkit/tom_base/blob/master/tom_common/templates/tom_common/index.html). -and place it in our project. The template we are looking for is `tom_common/index.html` - -Let's download and copy that template into our `templates` folder -(including the `tom_common` sub-directory) so that our directory structure now -looks like this: - -``` -├── db.sqlite3 -├── manage.py -├── templates -│ └── tom_common -│ └── index.html -└── mytom - ├── __init__.py - ├── settings.py - ├── urls.py - └── wsgi.py -``` - -Now let's make a few changes to the `templates/tom_common/index.html` template: - -```html -{% extends 'tom_common/base.html' %} -{% load static targets_extras observation_extras dataproduct_extras tom_common_extras %} -{% block title %}Home{% endblock %} -{% block content %} -
    -
    -

    Project LEO

    - - - -

    -

    Project LEO is a very serious survey of the most important constellation.

    - - - -

    Next steps

    - -

    Other Resources

    - -
    -
    -
    -
    - Latest Comments -
    - {% recent_comments %} -
    -
    -
    -
    - Latest Targets -
    - {% recent_targets %} -
    -
    -{% endblock %} -``` -Look for the block of HTML we changed between the <\!-- BEGIN MODIFIED CONTENT --> -and <\!-- END MODIFIED CONTENT --> comments. Everything else is the same as the -base template. - -We've just changed a few lines of HTML, but basically left the template alone. Reload your homepage, -and you should see something like this: - -![](/_static/customize_templates_doc/tomhomepagemod.png) - -Thats it! You've just customized your TOM homepage. - -### Using static files - -Instead of linking to an image hosted online already, we can display static files -in our project directly. For this we will use [Django's static -files](https://docs.djangoproject.com/en/2.1/howto/static-files/) capabilities. - -If you ran the tom_setup script, you should have a directory `static` at the top -level of your project. Within this folder, make a directory `img`. In this folder, -place an image you'd like to display on your homepage. For example, `mytom.jpg`. - - cp mytom.jpg static/img/ - -Now let's edit our template to use Django's `static` template tag to display the -image: - -```html -{% raw %} -

    -{% endraw %} -``` - -After reloading the page, you should now see `mytom.jpg` displayed instead of the -remote cat image. - -### Further Reading - -Any template included in the TOM Toolkit (or any other Django app) can be customized. Please -see the [official Django docs](https://docs.djangoproject.com/en/2.1/howto/overriding-templates/) -for more details. diff --git a/docs/customization/customize_templates.rst b/docs/customization/customize_templates.rst new file mode 100644 index 000000000..a5f91441d --- /dev/null +++ b/docs/customization/customize_templates.rst @@ -0,0 +1,170 @@ +Customizing TOM Templates +------------------------- + +So you’ve got a TOM up and running, and your homepage looks something +like this: + +|image0| + +This is fine for starting out, but since you’re running a TOM for a +specific project, the homepage ought to reflect that. + +If you haven’t already, please read through the `Getting +Started `__ docs and return here when you +have a project layout that looks something like this: + +:: + + mytom + ├── db.sqlite3 + ├── manage.py + └── mytom + ├── __init__.py + ├── settings.py + ├── urls.py + └── wsgi.py + +We are going to override the html template included with the TOM +Toolkit, ``tom_common/index.html``, so that we can edit some text and +change the image. Overriding and extending templates is `documented +extensively `__ +on Django’s website and we highly recommend reading these docs if you +plan on customizing your TOM further. + +Since the template we want to override is already part of the TOM +Toolkit source code, we can use it as a starting point for our +customized template. In fact, we’ll copy and paste the entire thing from +the `source code of TOM +Toolkit `__. +and place it in our project. The template we are looking for is +``tom_common/index.html`` + +Let’s download and copy that template into our ``templates`` folder +(including the ``tom_common`` sub-directory) so that our directory +structure now looks like this: + +:: + + ├── db.sqlite3 + ├── manage.py + ├── templates + │ └── tom_common + │ └── index.html + └── mytom + ├── __init__.py + ├── settings.py + ├── urls.py + └── wsgi.py + +Now let’s make a few changes to the ``templates/tom_common/index.html`` +template: + +.. code:: html + + {% extends 'tom_common/base.html' %} + {% load static targets_extras observation_extras dataproduct_extras tom_common_extras %} + {% block title %}Home{% endblock %} + {% block content %} +
    +
    +

    Project LEO

    + + + +

    +

    Project LEO is a very serious survey of the most important constellation.

    + + + +

    Next steps

    + +

    Other Resources

    + +
    +
    +
    +
    + Latest Comments +
    + {% recent_comments %} +
    +
    +
    +
    + Latest Targets +
    + {% recent_targets %} +
    +
    + {% endblock %} + +Look for the block of HTML we changed between the and comments. Everything else is +the same as the base template. + +We’ve just changed a few lines of HTML, but basically left the template +alone. Reload your homepage, and you should see something like this: + +|image1| + +Thats it! You’ve just customized your TOM homepage. + +Using static files +~~~~~~~~~~~~~~~~~~ + +Instead of linking to an image hosted online already, we can display +static files in our project directly. For this we will use `Django’s +static +files `__ +capabilities. + +If you ran the tom_setup script, you should have a directory ``static`` +at the top level of your project. Within this folder, make a directory +``img``. In this folder, place an image you’d like to display on your +homepage. For example, ``mytom.jpg``. + +:: + + cp mytom.jpg static/img/ + +Now let’s edit our template to use Django’s ``static`` template tag to +display the image: + +.. code:: html + + {% raw %} +

    + {% endraw %} + +After reloading the page, you should now see ``mytom.jpg`` displayed +instead of the remote cat image. + +Further Reading +~~~~~~~~~~~~~~~ + +Any template included in the TOM Toolkit (or any other Django app) can +be customized. Please see the `official Django +docs `__ +for more details. + +.. |image0| image:: /_static/customize_templates_doc/tomhomepagenew.png +.. |image1| image:: /_static/customize_templates_doc/tomhomepagemod.png diff --git a/docs/customization/index.rst b/docs/customization/index.rst index 6244fea60..87882b1f7 100644 --- a/docs/customization/index.rst +++ b/docs/customization/index.rst @@ -18,5 +18,5 @@ change the look and feel of your TOM. :doc:`Adding new Pages to your TOM ` - Learn how to add entirely new pages to your TOM, displaying static html pages or dynamic database-driven content. -:doc:`Customizing Template Tag ` - Learn how to write your own template tags to display +:doc:`Customizing Template Tags ` - Learn how to write your own template tags to display the data you need. diff --git a/docs/deployment/amazons3.md b/docs/deployment/amazons3.md deleted file mode 100644 index 8203c965f..000000000 --- a/docs/deployment/amazons3.md +++ /dev/null @@ -1,108 +0,0 @@ -Using Amazon S3 to Store Data for a TOM ---- - -If a TOM needs to store a large amount of data, like images or spectra, it may -eventually become impractical to do so on a local hard drive or network share. -This is where cloud storage services like [Amazon S3](https://aws.amazon.com/s3/) -come in handy. These services allow you to store large quantities of data at a low -cost, while providing high reliability and feature rich services. In most cases -using a cloud storage system also provides performance and speed increases to your -application. - -Configuring the TOM toolkit to store data on Amazon S3 is fairly straightforward. -Once enabled, data product downloads, uploads, and static assets (images, -stylesheets, etc) will be stored in Amazon S3 instead of the local filesystem -where your TOM is run. - -### Sign up for an AWS Account - -To use S3, you'll first need to sign up for an [Amazon Web -Services](https://portal.aws.amazon.com/billing/signup#/start) account. New -accounts get access to one year of free tier access which includes a year of S3 at -a max of 5GB. If you're interested in the cost beyond 5Gb, try out the [Amazon -cost calculator](https://calculator.s3.amazonaws.com/index.html). - -Once you have created an account, you'll need your access key id and secret access -key. These can be found under your profile settings -> "My security credentials". -Make sure you save these in a safe place, you'll need them later. - -### Create a bucket - -A bucket is like the highest level folder you can store data in S3. You should -create one for your TOM. Name it whatever you'd like. Most of the default settings -should be fine. - -**We need to enable CORS** for JS9 (or any other javascript code that wants to -access our data directly) to work. Under the "Permissions" tab for your bucket, -find the section for "CORS configuration". In the editor, paste the following policy: - -```xml - - - - * - GET - * - - -``` - -This policy allows GET requests from anywhere. Feel free to edit it to match your -particular use case specifically. - - -### Configure S3 Storage backend for your TOM - -Now that we have a bucket set up, let's configure our TOM to use it. First we need -to install two additional python packages. You should add these to your project's -`requirements.txt`.: - -* django-storages -* boto3 - -Next, we'll edit our TOM's `settings.py` to use S3 instead of local storage. Place -the following lines somewhere around the existing static files configuration -settings: - -```python -DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' -STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' - -AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID', '') -AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRECT_ACCESS_KEY', '') -AWS_STORAGE_BUCKET_NAME = os.getenv('AWS_STORAGE_BUCKET_NAME', '') -AWS_S3_REGION_NAME = os.getenv('AWS_S3_REGION_NAME', '') -AWS_DEFAULT_ACL = None -``` - -Notice that these settings get their values via environmental variables. Depending -on how you deploy your TOM, you can set these in a variety of ways. For example: -export AWS_ACCESS_KEY_ID=MyAccessKey would be one way to set them using Bash. - -* AWS_ACCESS_KEY_ID is your access key id from your security credentials. -* AWS_SECRECT_ACCESS_KEY is your secret access key from your security credentials. -* AWS_STORAGE_BUCKET_NAME is the name you gave to the bucket you created. -* AWS_S3_REGION_NAME is the name of th region you created your bucket in. - -Once these settings are filled out, your TOM should store all future data in S3. -If you had existing data in your TOM, you should copy it over to your bucket in -the exact same way it was stored locally. - -### For Heroku Users - -If you are using Heroku (perhaps by following the [Heroku deployment -guide](https://tomtoolkit.github.io/docs/deployment_heroku)) there is one more -additional step. At the very bottom of `settings.py` change the line: - - django_heroku.settings(locals()) - -to: - - django_heroku.settings(locals(), staticfiles=False) - -This instructs the `django-heroku` package to not automatically configure static -files for your TOM (since we are explicitly using S3 now). - -Additionally, Heroku makes it easy to set environmental variables. -See [Configuration and Config Vars]( -https://devcenter.heroku.com/articles/config-vars). diff --git a/docs/deployment/amazons3.rst b/docs/deployment/amazons3.rst new file mode 100644 index 000000000..2b90012a8 --- /dev/null +++ b/docs/deployment/amazons3.rst @@ -0,0 +1,126 @@ +Using Amazon S3 to Store Data for a TOM +--------------------------------------- + +If a TOM needs to store a large amount of data, like images or spectra, +it may eventually become impractical to do so on a local hard drive or +network share. This is where cloud storage services like `Amazon +S3 `__ come in handy. These services allow +you to store large quantities of data at a low cost, while providing +high reliability and feature rich services. In most cases using a cloud +storage system also provides performance and speed increases to your +application. + +Configuring the TOM toolkit to store data on Amazon S3 is fairly +straightforward. Once enabled, data product downloads, uploads, and +static assets (images, stylesheets, etc) will be stored in Amazon S3 +instead of the local filesystem where your TOM is run. + +Sign up for an AWS Account +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To use S3, you’ll first need to sign up for an `Amazon Web +Services `__ +account. New accounts get access to one year of free tier access which +includes a year of S3 at a max of 5GB. If you’re interested in the cost +beyond 5Gb, try out the `Amazon cost +calculator `__. + +Once you have created an account, you’ll need your access key id and +secret access key. These can be found under your profile settings -> “My +security credentials”. Make sure you save these in a safe place, you’ll +need them later. + +Create a bucket +~~~~~~~~~~~~~~~ + +A bucket is like the highest level folder you can store data in S3. You +should create one for your TOM. Name it whatever you’d like. Most of the +default settings should be fine. + +**We need to enable CORS** for JS9 (or any other javascript code that +wants to access our data directly) to work. Under the “Permissions” tab +for your bucket, find the section for “CORS configuration”. In the +editor, paste the following policy: + +.. code:: xml + + + + + * + GET + * + + + +This policy allows GET requests from anywhere. Feel free to edit it to +match your particular use case specifically. + +Configure S3 Storage backend for your TOM +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Now that we have a bucket set up, let’s configure our TOM to use it. +First we need to install two additional python packages. You should add +these to your project’s ``requirements.txt``.: + +- django-storages +- boto3 + +Next, we’ll edit our TOM’s ``settings.py`` to use S3 instead of local +storage. Place the following lines somewhere around the existing static +files configuration settings: + +.. code:: python + + DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' + STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' + + AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID', '') + AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRECT_ACCESS_KEY', '') + AWS_STORAGE_BUCKET_NAME = os.getenv('AWS_STORAGE_BUCKET_NAME', '') + AWS_S3_REGION_NAME = os.getenv('AWS_S3_REGION_NAME', '') + AWS_DEFAULT_ACL = None + +Notice that these settings get their values via environmental variables. +Depending on how you deploy your TOM, you can set these in a variety of +ways. For example: export AWS_ACCESS_KEY_ID=MyAccessKey would be one way +to set them using Bash. + +- AWS_ACCESS_KEY_ID is your access key id from your security + credentials. +- AWS_SECRECT_ACCESS_KEY is your secret access key from your security + credentials. +- AWS_STORAGE_BUCKET_NAME is the name you gave to the bucket you + created. +- AWS_S3_REGION_NAME is the name of th region you created your bucket + in. + +Once these settings are filled out, your TOM should store all future +data in S3. If you had existing data in your TOM, you should copy it +over to your bucket in the exact same way it was stored locally. + +For Heroku Users +~~~~~~~~~~~~~~~~ + +If you are using Heroku (perhaps by following the `Heroku deployment +guide `__) there is +one more additional step. At the very bottom of ``settings.py`` change +the line: + +:: + + django_heroku.settings(locals()) + +to: + +:: + + django_heroku.settings(locals(), staticfiles=False) + +This instructs the ``django-heroku`` package to not automatically +configure static files for your TOM (since we are explicitly using S3 +now). + +Additionally, Heroku makes it easy to set environmental variables. See +`Configuration and Config +Vars `__. diff --git a/docs/deployment/deployment_heroku.md b/docs/deployment/deployment_heroku.md deleted file mode 100644 index e97163459..000000000 --- a/docs/deployment/deployment_heroku.md +++ /dev/null @@ -1,169 +0,0 @@ -Deploy a TOM to Heroku ----------------------- - -[Heroku](https://heroku.com) is a -[PaaS](https://en.wikipedia.org/wiki/Platform_as_a_service) which allows you to -easily deploy web applications (like a TOM) to public servers without the need for -managing any of the underlying infrastructure yourself. - -Put simply: Heroku lets you make your TOM publicly available without needing to -run servers, open firewall rules, manage domain names, etc. You simply push your -code and Heroku will run your website for you. - -The service has a free tier that should be more than adequate for TOMs handling -10s of users. However note that the free tier processing power is limited, so if -you plan on doing lots of expensive processing on your data, you might want to -look into alternatives. - - -### Example code repository - -There is an example code repository, -[TOMToolkit/herokutom](https://github.com/TOMToolkit/herokutom), which contains the -minimal setup required to run a TOM in Heroku. It is running at -[https://herokutom.herokuapp.com/](https://herokutom.herokuapp.com/). - - -### Prerequisites - -1. You should have a local TOM up and running following the instructions in the -[getting started](/introduction/getting_started) guide. -2. You should be familiar with basic git commands. - -### Push your code to Github. -This guide will use the -[Github integration](https://devcenter.heroku.com/articles/github-integration) -method for deploying to Heroku. This way, we can tell Heroku to redeploy your TOM -each time we push changes to Github. Note: It's possible to [deploy to -Heroku](https://devcenter.heroku.com/articles/git) without using Github, but -you'll still need git. - -If you haven't already, push your TOM's code to Github. If you are unfamiliar with -Git and Github, [there are many tutorials -online](https://guides.github.com/activities/hello-world/) for getting started, -though the specifics are beyond the scope of this tutorial. - -Once you have your code up on Github, you're ready to move on to the next step. - -### Sign up for Heroku and create an app - -First, start off by [signing up for a Heroku account](https://signup.heroku.com/). - -Once you have logged in to your account, Heroku will ask you to start a new -project. Give it a name, but leave the pipeline stuff alone for now. - -After creating an app you'll be presented with a choice of Deployment methods. -Choose Github and click the "Connect to Github" button. - -![](/_static/heroku_deploy_doc/githubintegration.png) - -Once you have given Heroku access to your Github account and found your repo, your -app should successfully be connected and your deployment dashboard should look -like this: - -![](/_static/heroku_deploy_doc/githubconnected.png) - - -That's it for now, we'll return to this page after we've made some modifications -to our TOM to make it work with Heroku. - -### Make your TOM Heroku ready - -There are a few additions we'll need to make to our TOM before it can run in -Heroku. If you'd like to follow Heroku's guide directly, you can find it -[here](https://devcenter.heroku.com/articles/django-app-configuration). - -#### Defining project dependencies with requirements.txt - -If you haven't already, define a `requirements.txt` file. This is a file which is -used to list dependencies of your project. Heroku expects it so it knows which -python packages it needs to install to run your app. It should look something like -this: - - dataclasses - django - tomtoolkit - -Let's add 2 more lines: one for [gunicorn](https://gunicorn.org/), a high -performance http server and -[django-heroku](https://github.com/heroku/django-heroku) a utility that helps -autoconfigure Django projects for Heroku. Our `requirements.txt` file should now -look something like this: - - dataclasses - django - tomtoolkit - gunicorn - django-heroku - -You can make sure it works locally by installing your `requirements.txt` -dependencies with `pip`: - - pip install -r requirements.txt - -#### Settings.py changes - -Now we need to edit our projects `settings.py` file to make it work with Heroku. -At the top of the file, we should import django_heroku: - -```python -import django_heroku -``` - -At the bottom of the file, we'll call a method to autoconfigure our project: - -```python -django_heroku.settings(locals()) -``` - -#### Adding a Procfile - -Heroku requires the presence of a `Procfile` in your project. This file tells -Heroku how it is supposed to launch your app. Create a file `Procfile` in the root -of your project and add these contents: - - release: python manage.py migrate --noinput - web: gunicorn mytom.wsgi - -**Make sure to change mytom.wsgi above to the actual name of your project!** - -Note on the `release` command: you might want to remove this line if you'd like to -have manual control over when your migrations are run in the future. This is -simply a convenience for now. - - -#### Push to Github and deploy - -Once you have made the necessary modifications to `settings.py` above, you should -make a commit and push your code to Github. - -Now, navigate back to your app's dashboard on Heroku. Under the deploy tab, you -should see a section for Manual deploy, at the bottom, with a button "Deploy -Branch". - - -![](/_static/heroku_deploy_doc/herokudeploybranch.png) - -Select the branch to deploy (usually "master") and click the "Deploy Branch" -button. Heroku will begin launching your app. If all goes well, you should see -something like this: - -![](/_static/heroku_deploy_doc/branchdeployed.png) - -Your TOM should now be running at https://<>.herokuapp.com. -Congratulations! - -### Next steps - -You should spend some time familiarizing yourself with how Heroku works. As you -may have noticed, there are many configuration options and workflows available. -For example, just above the "Manual Deploy" section we used, there is a setting -that allows Heroku to automatically deploy your app when you push code to Github. - -Also note that Heroku has limitations, especially around storing data on disk. By -default, **Heroku only keeps files on disk for a maximum of 24 hours**. If you -plan on storing data (such as fits files or other supplementary data) you will -have to use an external stoage service. In this case, you might want to read ahead -on how to [Use Amazon S3 to Store Data for a -TOM](https://tomtoolkit.github.io/docs/amazons3). - diff --git a/docs/deployment/deployment_heroku.rst b/docs/deployment/deployment_heroku.rst new file mode 100644 index 000000000..ed5e4f748 --- /dev/null +++ b/docs/deployment/deployment_heroku.rst @@ -0,0 +1,195 @@ +Deploy a TOM to Heroku +---------------------- + +`Heroku `__ is a +`PaaS `__ which +allows you to easily deploy web applications (like a TOM) to public +servers without the need for managing any of the underlying +infrastructure yourself. + +Put simply: Heroku lets you make your TOM publicly available without +needing to run servers, open firewall rules, manage domain names, etc. +You simply push your code and Heroku will run your website for you. + +The service has a free tier that should be more than adequate for TOMs +handling 10s of users. However note that the free tier processing power +is limited, so if you plan on doing lots of expensive processing on your +data, you might want to look into alternatives. + +Example code repository +~~~~~~~~~~~~~~~~~~~~~~~ + +There is an example code repository, +`TOMToolkit/herokutom `__, +which contains the minimal setup required to run a TOM in Heroku. It is +running at https://herokutom.herokuapp.com/. + +Prerequisites +~~~~~~~~~~~~~ + +1. You should have a local TOM up and running following the instructions + in the `getting started `__ guide. +2. You should be familiar with basic git commands. + +Push your code to Github. +~~~~~~~~~~~~~~~~~~~~~~~~~ + +This guide will use the `Github +integration `__ +method for deploying to Heroku. This way, we can tell Heroku to redeploy +your TOM each time we push changes to Github. Note: It’s possible to +`deploy to Heroku `__ without +using Github, but you’ll still need git. + +If you haven’t already, push your TOM’s code to Github. If you are +unfamiliar with Git and Github, `there are many tutorials +online `__ for +getting started, though the specifics are beyond the scope of this +tutorial. + +Once you have your code up on Github, you’re ready to move on to the +next step. + +Sign up for Heroku and create an app +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +First, start off by `signing up for a Heroku +account `__. + +Once you have logged in to your account, Heroku will ask you to start a +new project. Give it a name, but leave the pipeline stuff alone for now. + +After creating an app you’ll be presented with a choice of Deployment +methods. Choose Github and click the “Connect to Github” button. + +|image0| + +Once you have given Heroku access to your Github account and found your +repo, your app should successfully be connected and your deployment +dashboard should look like this: + +|image1| + +That’s it for now, we’ll return to this page after we’ve made some +modifications to our TOM to make it work with Heroku. + +Make your TOM Heroku ready +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +There are a few additions we’ll need to make to our TOM before it can +run in Heroku. If you’d like to follow Heroku’s guide directly, you can +find it +`here `__. + +Defining project dependencies with requirements.txt +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you haven’t already, define a ``requirements.txt`` file. This is a +file which is used to list dependencies of your project. Heroku expects +it so it knows which python packages it needs to install to run your +app. It should look something like this: + +:: + + dataclasses + django + tomtoolkit + +Let’s add 2 more lines: one for `gunicorn `__, a +high performance http server and +`django-heroku `__ a utility +that helps autoconfigure Django projects for Heroku. Our +``requirements.txt`` file should now look something like this: + +:: + + dataclasses + django + tomtoolkit + gunicorn + django-heroku + +You can make sure it works locally by installing your +``requirements.txt`` dependencies with ``pip``: + +:: + + pip install -r requirements.txt + +Settings.py changes +^^^^^^^^^^^^^^^^^^^ + +Now we need to edit our projects ``settings.py`` file to make it work +with Heroku. At the top of the file, we should import django_heroku: + +.. code:: python + + import django_heroku + +At the bottom of the file, we’ll call a method to autoconfigure our +project: + +.. code:: python + + django_heroku.settings(locals()) + +Adding a Procfile +^^^^^^^^^^^^^^^^^ + +Heroku requires the presence of a ``Procfile`` in your project. This +file tells Heroku how it is supposed to launch your app. Create a file +``Procfile`` in the root of your project and add these contents: + +:: + + release: python manage.py migrate --noinput + web: gunicorn mytom.wsgi + +**Make sure to change mytom.wsgi above to the actual name of your +project!** + +Note on the ``release`` command: you might want to remove this line if +you’d like to have manual control over when your migrations are run in +the future. This is simply a convenience for now. + +Push to Github and deploy +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Once you have made the necessary modifications to ``settings.py`` above, +you should make a commit and push your code to Github. + +Now, navigate back to your app’s dashboard on Heroku. Under the deploy +tab, you should see a section for Manual deploy, at the bottom, with a +button “Deploy Branch”. + +|image2| + +Select the branch to deploy (usually “master”) and click the “Deploy +Branch” button. Heroku will begin launching your app. If all goes well, +you should see something like this: + +|image3| + +Your TOM should now be running at https://<>.herokuapp.com. +Congratulations! + +Next steps +~~~~~~~~~~ + +You should spend some time familiarizing yourself with how Heroku works. +As you may have noticed, there are many configuration options and +workflows available. For example, just above the “Manual Deploy” section +we used, there is a setting that allows Heroku to automatically deploy +your app when you push code to Github. + +Also note that Heroku has limitations, especially around storing data on +disk. By default, **Heroku only keeps files on disk for a maximum of 24 +hours**. If you plan on storing data (such as fits files or other +supplementary data) you will have to use an external stoage service. In +this case, you might want to read ahead on how to `Use Amazon S3 to +Store Data for a TOM `__. + +.. |image0| image:: /_static/heroku_deploy_doc/githubintegration.png +.. |image1| image:: /_static/heroku_deploy_doc/githubconnected.png +.. |image2| image:: /_static/heroku_deploy_doc/herokudeploybranch.png +.. |image3| image:: /_static/heroku_deploy_doc/branchdeployed.png diff --git a/docs/deployment/deployment_tips.md b/docs/deployment/deployment_tips.md deleted file mode 100644 index a5030e1bc..000000000 --- a/docs/deployment/deployment_tips.md +++ /dev/null @@ -1,42 +0,0 @@ -General Deployment Tips ---- - -When it comes to deploying your tom for general use, there are a few things you -might want to consider. - -### Choosing a database -By default Django (and thus TOMs) use Sqlite as their database backend. Sqlite is -sufficient for the majority of use cases and can scale up to the millions if not -billions of rows, as long as you have the disk space. - -The one place where Sqlite falls behind other databases is it's performance under -heavy concurrent writes. So if you are writing a TOM that, for example, listens -to the ZTF, LSST, and SCOUT alert streams and creates targets from each alert -you might want to look into Postgresql or MySQL. - - -### Set your TOM's hostname in the default site -In your TOM's admin area (/admin/) on your production TOM -you will notice a section called "Sites." There -should be one site object, you should edit it so that its hostname is accurate for -production. Some functionalities of the TOM rely on this value to properly set up -redirects, etc. - - -### Basic security -If you are exposing your TOM to the internet you should make sure that basic -security precautions have been taken. Make sure that any views which expose -sensitive data, perform any kind of modification to the database or cause large -amounts of server load are properly protected and require authentication. - -If you plan on making your TOM open source, take care not to check in any secrets, -passwords, or credentials. This includes database settings, API keys, or a -multitude of other things that you wouldn't want to throw out on the internet for -everyone to see. Note that if you are using git, removing a secret from a file and -then committing it **does not remove the secret** it will still exist in the -repo's history and be trivially accessible. You will need to clean your repo's -history if you commit and sensitive data. - -Enforce basic password requirements (TOMs by default will do this) and encourage -your users to exercise basic security measures, like using a password manager and -not reusing passwords. diff --git a/docs/deployment/deployment_tips.rst b/docs/deployment/deployment_tips.rst new file mode 100644 index 000000000..0632adf0d --- /dev/null +++ b/docs/deployment/deployment_tips.rst @@ -0,0 +1,50 @@ +General Deployment Tips +----------------------- + +When it comes to deploying your tom for general use, there are a few +things you might want to consider. + +Choosing a database +~~~~~~~~~~~~~~~~~~~ + +By default Django (and thus TOMs) use Sqlite as their database backend. +Sqlite is sufficient for the majority of use cases and can scale up to +the millions if not billions of rows, as long as you have the disk +space. + +The one place where Sqlite falls behind other databases is it’s +performance under heavy concurrent writes. So if you are writing a TOM +that, for example, listens to the ZTF, LSST, and SCOUT alert streams and +creates targets from each alert you might want to look into Postgresql +or MySQL. + +Set your TOM’s hostname in the default site +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In your TOM’s admin area (/admin/) on your production TOM you will +notice a section called “Sites.” There should be one site object, you +should edit it so that its hostname is accurate for production. Some +functionalities of the TOM rely on this value to properly set up +redirects, etc. + +Basic security +~~~~~~~~~~~~~~ + +If you are exposing your TOM to the internet you should make sure that +basic security precautions have been taken. Make sure that any views +which expose sensitive data, perform any kind of modification to the +database or cause large amounts of server load are properly protected +and require authentication. + +If you plan on making your TOM open source, take care not to check in +any secrets, passwords, or credentials. This includes database settings, +API keys, or a multitude of other things that you wouldn’t want to throw +out on the internet for everyone to see. Note that if you are using git, +removing a secret from a file and then committing it **does not remove +the secret** it will still exist in the repo’s history and be trivially +accessible. You will need to clean your repo’s history if you commit and +sensitive data. + +Enforce basic password requirements (TOMs by default will do this) and +encourage your users to exercise basic security measures, like using a +password manager and not reusing passwords. From ea7c111f4193e176d5748ca2151a0f5c29f8191b Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 8 Jun 2020 18:22:00 -0700 Subject: [PATCH 167/424] Finished converting .md to .rst --- docs/examples.md | 26 -- docs/examples.rst | 51 +++ docs/introduction/about.md | 45 --- docs/introduction/about.rst | 72 ++++ docs/introduction/contributing.md | 73 ---- docs/introduction/contributing.rst | 116 +++++++ docs/introduction/faqs.md | 97 ------ docs/introduction/faqs.rst | 121 +++++++ docs/introduction/getting_started.md | 91 ----- docs/introduction/getting_started.rst | 116 +++++++ docs/introduction/troubleshooting.md | 43 --- docs/introduction/troubleshooting.rst | 52 +++ docs/introduction/workflow.md | 86 ----- docs/introduction/workflow.rst | 93 +++++ .../customizing_data_processing.md | 157 --------- .../customizing_data_processing.rst | 177 ++++++++++ docs/managing_data/plotting_data.md | 179 ---------- docs/managing_data/plotting_data.rst | 206 +++++++++++ docs/observing/customize_observations.md | 297 ---------------- docs/observing/customize_observations.rst | 323 ++++++++++++++++++ docs/observing/observation_module.md | 299 ---------------- docs/observing/observation_module.rst | 321 +++++++++++++++++ docs/observing/strategies.md | 218 ------------ docs/observing/strategies.rst | 258 ++++++++++++++ docs/targets/target_fields.md | 134 -------- docs/targets/target_fields.rst | 149 ++++++++ 26 files changed, 2055 insertions(+), 1745 deletions(-) delete mode 100644 docs/examples.md create mode 100644 docs/examples.rst delete mode 100644 docs/introduction/about.md create mode 100644 docs/introduction/about.rst delete mode 100644 docs/introduction/contributing.md create mode 100644 docs/introduction/contributing.rst delete mode 100644 docs/introduction/faqs.md create mode 100644 docs/introduction/faqs.rst delete mode 100644 docs/introduction/getting_started.md create mode 100644 docs/introduction/getting_started.rst delete mode 100644 docs/introduction/troubleshooting.md create mode 100644 docs/introduction/troubleshooting.rst delete mode 100644 docs/introduction/workflow.md create mode 100644 docs/introduction/workflow.rst delete mode 100644 docs/managing_data/customizing_data_processing.md create mode 100644 docs/managing_data/customizing_data_processing.rst delete mode 100644 docs/managing_data/plotting_data.md create mode 100644 docs/managing_data/plotting_data.rst delete mode 100644 docs/observing/customize_observations.md create mode 100644 docs/observing/customize_observations.rst delete mode 100644 docs/observing/observation_module.md create mode 100644 docs/observing/observation_module.rst delete mode 100644 docs/observing/strategies.md create mode 100644 docs/observing/strategies.rst delete mode 100644 docs/targets/target_fields.md create mode 100644 docs/targets/target_fields.rst diff --git a/docs/examples.md b/docs/examples.md deleted file mode 100644 index 4eb7e1997..000000000 --- a/docs/examples.md +++ /dev/null @@ -1,26 +0,0 @@ -Example TOMs ---- - -### SNEx - -The [Supernova Exchange](https://supernova.exchange/public/) is an interface for viewing and sharing observational data of supernovae, and for requesting and managing observations with the Las Cumbres Observatory network. In order to make it more maintainable, it is being rewritten from scratch using the TOM Toolkit, which has already resulted in orders of magnitude fewer lines of code. The code can be found and referenced on [Github](https://github.com/jfrostburke/snex2/). - -### Asteroid Tracker - -[Asteroid Tracker](https://asteroidtracker.lco.global/) is an educational TOM built for Asteroid Day. It allows students and teachers to submit one-click observations of specific asteroids and see the resulting images. Originally built from scratch, it's being rewritten using the TOM Toolkit, which will allow the underlying TOM to be used with multiple front-ends for completely different educational purposes. - -### Microlensing TOM - -The [Microlensing TOM](https://github.com/KSNikolaus/ZTF_TOM) is being written in order to identify microlensing events from ZTF and conduct follow-up observations. - -### PhotTOM - -The [ROME/REA TOM](https://github.com/rachel3834/romerea_phot_tom) is being built to manage ROME/REA photometry for the [LCO key project of the same name](https://robonet.lco.global/). - -### Calibration TOM - -LCO is rewriting an existing piece of software that automatically schedules nightly telescope calibrations using the TOM Toolkit called the [Calibration TOM](https://github.com/LCOGT/calibration-tom/). - -### Others - -There are a few other TOMs in development that we're aware of, but if you're developing a TOM, feel free to contribute to this page, or let us know and we'll take care of it for you. \ No newline at end of file diff --git a/docs/examples.rst b/docs/examples.rst new file mode 100644 index 000000000..ebc8f4a05 --- /dev/null +++ b/docs/examples.rst @@ -0,0 +1,51 @@ +Example TOMs +------------ + +SNEx +~~~~ + +The `Supernova Exchange `__ is an +interface for viewing and sharing observational data of supernovae, and +for requesting and managing observations with the Las Cumbres +Observatory network. In order to make it more maintainable, it is being +rewritten from scratch using the TOM Toolkit, which has already resulted +in orders of magnitude fewer lines of code. The code can be found and +referenced on `Github `__. + +Asteroid Tracker +~~~~~~~~~~~~~~~~ + +`Asteroid Tracker `__ is an +educational TOM built for Asteroid Day. It allows students and teachers +to submit one-click observations of specific asteroids and see the +resulting images. Originally built from scratch, it’s being rewritten +using the TOM Toolkit, which will allow the underlying TOM to be used +with multiple front-ends for completely different educational purposes. + +Microlensing TOM +~~~~~~~~~~~~~~~~ + +The `Microlensing TOM `__ is +being written in order to identify microlensing events from ZTF and +conduct follow-up observations. + +PhotTOM +~~~~~~~ + +The `ROME/REA TOM `__ is +being built to manage ROME/REA photometry for the `LCO key project of +the same name `__. + +Calibration TOM +~~~~~~~~~~~~~~~ + +LCO is rewriting an existing piece of software that automatically +schedules nightly telescope calibrations using the TOM Toolkit called +the `Calibration TOM `__. + +Others +~~~~~~ + +There are a few other TOMs in development that we’re aware of, but if +you’re developing a TOM, feel free to contribute to this page, or let us +know and we’ll take care of it for you. diff --git a/docs/introduction/about.md b/docs/introduction/about.md deleted file mode 100644 index ff41afaa0..000000000 --- a/docs/introduction/about.md +++ /dev/null @@ -1,45 +0,0 @@ -About the TOM Toolkit ---------------------- - -### What’s a TOM? - -It stands for Target and Observation Manager, and its a software package designed to facilitate astronomical observing projects and collaborations. - -Though useful for a wide range of projects, TOM systems are particularly important for programs with a large number of potential targets and/or observations. - -TOM systems perform some or all of these functions: - -* Harvest target alerts, or upload catalogs of targets of interest to the project science goals. -* Search and cross-match additional information on targets from catalogs and archives. -* Store information from the project’s own analysis of the targets, related data and observations. -* Provide informative displays of the targets, data and observing program. -* Provide flexible search capabilities on parameters that are relevant to the science. -* Provide tools to plan appropriate observations. -* Enable observations to be requested from telescope facilities. -* Receive information about the status of observation requests. -* Harvest data obtained as a result of their observing requests. -* Facilitate the sharing of information and data. - -### Motivation for a TOM Toolkit - -Many projects, from several branches of astronomy, have found it necessary to develop TOM systems. Current examples include the PTF Marshall and NASA’s ExoFOP, as well as those customized for the LCO Network: SNEx, NEO Exchange and RoboNet. -These tools provide capabilities which enable the projects to identify and evaluate high priority targets in good time to plan and conduct suitable observations, and to analyze the results. These capabilities have proven to be essential for existing projects to keep track of their observing program and to achieve their scientific goals. They are likely to become increasingly vital as next generation surveys produce ever-larger and more rapidly-evolving target lists. - -However, designing the existing TOM systems required high levels of expertise in database and software development that are not common among astronomers. - -No two TOM systems are identical, as astronomers strongly prefer to directly control the science-specific aspects of their projects such as target selection, observing strategy and analysis techniques. At the same time, while all of these systems are customized for the science goals of the projects they support, much of their underlying infrastructure and functions are very similar. - -What’s needed is a software package that lets astronomers easily build a TOM, customized to suit the needs of their project, without becoming an IT expert or software engineer. - -### Financial Support - -The TOM Toolkit has been made possible through generous financial support from the [Heising-Simons Foundation](https://hsfoundation.org) and the [Zegar Family Foundation](https://sites.google.com/zegarff.org/site). - -
    - - Heising-Simons Foundation - - - Zegar Family Foundation - -
    \ No newline at end of file diff --git a/docs/introduction/about.rst b/docs/introduction/about.rst new file mode 100644 index 000000000..b43163d2e --- /dev/null +++ b/docs/introduction/about.rst @@ -0,0 +1,72 @@ +About the TOM Toolkit +--------------------- + +What’s a TOM? +~~~~~~~~~~~~~ + +It stands for Target and Observation Manager, and its a software package +designed to facilitate astronomical observing projects and +collaborations. + +Though useful for a wide range of projects, TOM systems are particularly +important for programs with a large number of potential targets and/or +observations. + +TOM systems perform some or all of these functions: + +- Harvest target alerts, or upload catalogs of targets of interest to + the project science goals. +- Search and cross-match additional information on targets from + catalogs and archives. +- Store information from the project’s own analysis of the targets, + related data and observations. +- Provide informative displays of the targets, data and observing + program. +- Provide flexible search capabilities on parameters that are relevant + to the science. +- Provide tools to plan appropriate observations. +- Enable observations to be requested from telescope facilities. +- Receive information about the status of observation requests. +- Harvest data obtained as a result of their observing requests. +- Facilitate the sharing of information and data. + +Motivation for a TOM Toolkit +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Many projects, from several branches of astronomy, have found it +necessary to develop TOM systems. Current examples include the PTF +Marshall and NASA’s ExoFOP, as well as those customized for the LCO +Network: SNEx, NEO Exchange and RoboNet. These tools provide +capabilities which enable the projects to identify and evaluate high +priority targets in good time to plan and conduct suitable observations, +and to analyze the results. These capabilities have proven to be +essential for existing projects to keep track of their observing program +and to achieve their scientific goals. They are likely to become +increasingly vital as next generation surveys produce ever-larger and +more rapidly-evolving target lists. + +However, designing the existing TOM systems required high levels of +expertise in database and software development that are not common among +astronomers. + +No two TOM systems are identical, as astronomers strongly prefer to +directly control the science-specific aspects of their projects such as +target selection, observing strategy and analysis techniques. At the +same time, while all of these systems are customized for the science +goals of the projects they support, much of their underlying +infrastructure and functions are very similar. + +What’s needed is a software package that lets astronomers easily build a +TOM, customized to suit the needs of their project, without becoming an +IT expert or software engineer. + +Financial Support +~~~~~~~~~~~~~~~~~ + +The TOM Toolkit has been made possible through generous financial +support from the `Heising-Simons +Foundation `_ and the `Zegar Family +Foundation `_. + +.. container:: partners + diff --git a/docs/introduction/contributing.md b/docs/introduction/contributing.md deleted file mode 100644 index 2e13a4984..000000000 --- a/docs/introduction/contributing.md +++ /dev/null @@ -1,73 +0,0 @@ -Contributing ------------- - -This page will go over the process for contributing to the TOM Toolkit. - -### Contributing Code/Documentation - -If you're interested in contributing code to the project, thank you! For those unfamiliar with the process of contributing to an open-source project, you may want to read through Github's own short informational section on [how to submit a contribution](https://opensource.guide/how-to-contribute/#how-to-submit-a-contribution). - -### Identifying a starting point - -The best place to begin contributing is by first looking at the [Github issues page](https://github.com/TOMToolkit/tom_base/issues), to see what's currently needed. Issues that don't require much familiarity with the TOM Toolkit will be tagged appropriately. - -### Familiarizing yourself with Git - -If you are not familiar with git, we encourage you to briefly look at the [Git Basics](https://git-scm.com/book/en/v2/Getting-Started-Git-Basics) page. - -### Git Workflow - -The workflow for submitting a code change is, more or less, the following: - -1. Fork the TOM Toolkit repository to your own Github account. -![](/_static/fork.png) -2. Clone the forked repository to your local working machine. - ``` - git clone git@github.com:/tom_base.git - ``` -3. Add the original "upstream" repository as a remote. - ``` - git remote add upstream https://github.com/TOMToolkit/tom_base.git - ``` -4. Ensure that you're synchronizing your repository with the "upstream" one relatively frequently. - ``` - git fetch upstream - git merge upstream/master - ``` -5. Create and checkout a branch for your changes (see [Branch Naming](#branch-naming)). - ``` - git checkout -b - ``` -6. Commit frequently, and push your changes to Github. Be sure to merge master in before submitting your pull request. - ``` - git push origin - ``` -7. When your code is complete and tested, create a pull request from the upstream TOM Toolkit repository. -![](/_static/pull-request.png) - -8. Be sure to click "compare across forks" in order to see your branch! -![](/_static/compare-across-forks.png) - -9. We may ask for some updates to your pull request, so revise as necessary and push when revisions are complete. This will automatically update your pull request. - -### Branch Naming - -Branch names should be prefixed with the purpose of the branch, be it a bugfix or an enhancement, along with a descriptive title for the branch. - -``` - bugfix/fix-typo-target-detail - feature/reticulating-splines - enhancement/refactor-planning-tool -``` - -### Code Style - -We recommend that you use a linter, as all pull requests must pass a `pycodestyle` check. We also recommend configuring your editor to automatically remove trailing whitespace, add newlines on save, and other such helpful style corrections. You can check if your styling will meet standards before submitting a pull request by doing a `pip install pycodestyle` and running the same command our Travis build does: - -``` -pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 -``` - -### Documentation - -We require any new features to diff --git a/docs/introduction/contributing.rst b/docs/introduction/contributing.rst new file mode 100644 index 000000000..a1a78ece8 --- /dev/null +++ b/docs/introduction/contributing.rst @@ -0,0 +1,116 @@ +Contributing +------------ + +This page will go over the process for contributing to the TOM Toolkit. + +Contributing Code/Documentation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you’re interested in contributing code to the project, thank you! For +those unfamiliar with the process of contributing to an open-source +project, you may want to read through Github’s own short informational +section on `how to submit a +contribution `__. + +Identifying a starting point +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The best place to begin contributing is by first looking at the `Github +issues page `__, to see +what’s currently needed. Issues that don’t require much familiarity with +the TOM Toolkit will be tagged appropriately. + +Familiarizing yourself with Git +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you are not familiar with git, we encourage you to briefly look at +the `Git +Basics `__ +page. + +Git Workflow +~~~~~~~~~~~~ + +The workflow for submitting a code change is, more or less, the +following: + +1. Fork the TOM Toolkit repository to your own Github account. |image0| +2. Clone the forked repository to your local working machine. + +:: + + git clone git@github.com:/tom_base.git + +3. Add the original “upstream” repository as a remote. + +:: + + git remote add upstream https://github.com/TOMToolkit/tom_base.git + +4. Ensure that you’re synchronizing your repository with the “upstream” + one relatively frequently. + +:: + + git fetch upstream + git merge upstream/master + +5. Create and checkout a branch for your changes (see `Branch + Naming <#branch-naming>`__). + +:: + + git checkout -b + +6. Commit frequently, and push your changes to Github. Be sure to merge + master in before submitting your pull request. + +:: + + git push origin + +7. When your code is complete and tested, create a pull request from the + upstream TOM Toolkit repository. |image1| + +8. Be sure to click “compare across forks” in order to see your branch! + |image2| + +9. We may ask for some updates to your pull request, so revise as + necessary and push when revisions are complete. This will + automatically update your pull request. + +Branch Naming +~~~~~~~~~~~~~ + +Branch names should be prefixed with the purpose of the branch, be it a +bugfix or an enhancement, along with a descriptive title for the branch. + +:: + + bugfix/fix-typo-target-detail + feature/reticulating-splines + enhancement/refactor-planning-tool + +Code Style +~~~~~~~~~~ + +We recommend that you use a linter, as all pull requests must pass a +``pycodestyle`` check. We also recommend configuring your editor to +automatically remove trailing whitespace, add newlines on save, and +other such helpful style corrections. You can check if your styling will +meet standards before submitting a pull request by doing a +``pip install pycodestyle`` and running the same command our Travis +build does: + +:: + + pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 + +Documentation +~~~~~~~~~~~~~ + +We require any new features to + +.. |image0| image:: /_static/fork.png +.. |image1| image:: /_static/pull-request.png +.. |image2| image:: /_static/compare-across-forks.png diff --git a/docs/introduction/faqs.md b/docs/introduction/faqs.md deleted file mode 100644 index fb1931b1e..000000000 --- a/docs/introduction/faqs.md +++ /dev/null @@ -1,97 +0,0 @@ -FAQ ---- - -### Can I use Jupyter Notebooks with my TOM? - -Yes. First install jupyterlab into your TOM virtualenv: - - pip install jupyterlab - -Inside your TOM directory, use the following management command to launch the -notebook server: - - ./manage.py shell_plus --notebook - -Under the new notebook menu, choose "Django Shell-Plus". This will create a new -notebook in the correct TOM context. - -There is also a [tutorial](../common/scripts) on interacting with your TOM using -Jupyter notebooks. - -### What are tags on the Target form? -You can add tags to targets via the target create/update forms or -programmatically. These are meant to be arbitrary data associated with a target. -You can then search for targets via tags on the target list page, by entering the -"key" and/or "value" fields in the filter list. They will also be displayed on the -target detail pages. - -If you'd like to have more control over extra target data, see the documentation -on [Adding Custom Target Fields](../targets/target_fields). - -### I try to observe a target with LCO but get an error. - -You might not have added your LCO api key to your settings file under the -`FACILITIES` settings. See [Custom Settings](../uncategorized/customsettings#facilities) for -more details. - -### How do I create a super user (PI)? -You can create a new superuser using the built in management command: - - ./manage.py createsuperuser - -The `manage.py` file can be found in the root of your project. - -Alternatively, you can give a user superuser status if you are already logged -in as a superuser by visiting the admin page for users: -[http://127.0.0.1/admin/auth/user/](http://127.0.0.1/admin/auth/user/) - - -### My science requires more parameters than are provided by the TOM Toolkit. -It is possible to add additional parameters to your targets within the TOM. See -the documentation on [Adding Custom Target Fields](../targets/target_fields). - - -### Yuck! My TOM is ugly. How do I change how it looks? -You have a few options. If you'd like to rearrange the layout or information on -the page, you can follow the tutorial on -[Customizing your TOM](../customization/customize_templates). If you'd like to modify colors, -typography, etc you'll want to use CSS. -[W3Schools](https://www.w3schools.com/Css/) is a good resource if you are -unfamiliar with Cascading Style Sheets. - - -### How do I add a new page to my TOM? -We would recommend you read the [Django tutorial](https://docs.djangoproject.com/en/2.2/contents/) -🙂. But if you want the quick and dirty, edit the `urls.py` (located next to -`settings.py`): - -```python -from django.urls import path, include -from django.views.generic import TemplateView - -urlpatterns = [ - path('', include('tom_common.urls')), - path('newpage/', TemplateView.as_view(template_name='newpage.html'), name='newpage') -] -``` - -And make sure `newpage.html` is located within the `templates/` directory in your -project. - -This will make the contents of `newpage.html` available under the path -[/newpage/](http://127.0.0.1/newpage/). - - -### Who is AnonymousUser? - -AnonymousUser is a special profile that django-guardian, our permissions library, creates automatically. AnonymousUser -represents an unauthenticated user. The user has no first name, last name, or password, and allows unauthenticated -users to view unprotected pages within your TOM. You can choose to delete the user if you don't want any pages to be -visible without logging in. - -### How can I display an error message when authentication to an external facility fails? - -For any modules exposing external services, such as brokers, harvesters, or facilities, a failed authentication should -raise an `ImproperCredentialsException`. Exceptions of this type are caught by the TOM Toolkit's built-in -`ExternalServiceMiddleware`. This middleware will display an error at the top of the page and redirect the user to the -home page. diff --git a/docs/introduction/faqs.rst b/docs/introduction/faqs.rst new file mode 100644 index 000000000..0485a788d --- /dev/null +++ b/docs/introduction/faqs.rst @@ -0,0 +1,121 @@ +FAQ +### + +Can I use Jupyter Notebooks with my TOM? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Yes. First install jupyterlab into your TOM virtualenv: + +:: + + pip install jupyterlab + +Inside your TOM directory, use the following management command to +launch the notebook server: + +:: + + ./manage.py shell_plus --notebook + +Under the new notebook menu, choose “Django Shell-Plus”. This will +create a new notebook in the correct TOM context. + +There is also a `tutorial <../common/scripts>`__ on interacting with +your TOM using Jupyter notebooks. + +What are tags on the Target form? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can add tags to targets via the target create/update forms or +programmatically. These are meant to be arbitrary data associated with a +target. You can then search for targets via tags on the target list +page, by entering the “key” and/or “value” fields in the filter list. +They will also be displayed on the target detail pages. + +If you’d like to have more control over extra target data, see the +documentation on `Adding Custom Target +Fields <../targets/target_fields>`__. + +I try to observe a target with LCO but get an error. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You might not have added your LCO api key to your settings file under +the ``FACILITIES`` settings. See `Custom +Settings <../uncategorized/customsettings#facilities>`__ for more +details. + +How do I create a super user (PI)? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can create a new superuser using the built in management command: + +:: + + ./manage.py createsuperuser + +The ``manage.py`` file can be found in the root of your project. + +Alternatively, you can give a user superuser status if you are already +logged in as a superuser by visiting the admin page for users: +http://127.0.0.1/admin/auth/user/ + +My science requires more parameters than are provided by the TOM Toolkit. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It is possible to add additional parameters to your targets within the +TOM. See the documentation on `Adding Custom Target +Fields <../targets/target_fields>`__. + +Yuck! My TOM is ugly. How do I change how it looks? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You have a few options. If you’d like to rearrange the layout or +information on the page, you can follow the tutorial on `Customizing +your TOM <../customization/customize_templates>`__. If you’d like to +modify colors, typography, etc you’ll want to use CSS. +`W3Schools `__ is a good resource if you +are unfamiliar with Cascading Style Sheets. + +How do I add a new page to my TOM? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We would recommend you read the `Django +tutorial `__ 🙂. But if +you want the quick and dirty, edit the ``urls.py`` (located next to +``settings.py``): + +.. code:: python + + from django.urls import path, include + from django.views.generic import TemplateView + + urlpatterns = [ + path('', include('tom_common.urls')), + path('newpage/', TemplateView.as_view(template_name='newpage.html'), name='newpage') + ] + +And make sure ``newpage.html`` is located within the ``templates/`` +directory in your project. + +This will make the contents of ``newpage.html`` available under the path +`/newpage/ `__. + +Who is AnonymousUser? +~~~~~~~~~~~~~~~~~~~~~ + +AnonymousUser is a special profile that django-guardian, our permissions +library, creates automatically. AnonymousUser represents an +unauthenticated user. The user has no first name, last name, or +password, and allows unauthenticated users to view unprotected pages +within your TOM. You can choose to delete the user if you don’t want any +pages to be visible without logging in. + +How can I display an error message when authentication to an external facility fails? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For any modules exposing external services, such as brokers, harvesters, +or facilities, a failed authentication should raise an +``ImproperCredentialsException``. Exceptions of this type are caught by +the TOM Toolkit’s built-in ``ExternalServiceMiddleware``. This +middleware will display an error at the top of the page and redirect the +user to the home page. diff --git a/docs/introduction/getting_started.md b/docs/introduction/getting_started.md deleted file mode 100644 index 593563c5b..000000000 --- a/docs/introduction/getting_started.md +++ /dev/null @@ -1,91 +0,0 @@ -Getting Started with the TOM Toolkit ------------------------------------- - -So you've decided to run a Target Observation Manager. This article will help you get started. - -The TOM Toolkit is a [Django](https://www.djangoproject.com/) project. This means you'll be running -an application based on the Django framework when you run a TOM. If you decide to customize -your TOM, you'll be working in Django. You'll likely need some basic understanding of python -and we recommend all users work their way through the -[Django tutorial](https://docs.djangoproject.com/en/2.1/contents/) first before starting with -the TOM Toolkit. It doesn't take long, and you most likely won't need to utilize any advanced -features. - -Ready to go? Let's get started. - -### Prerequisites - -The TOM toolkit requires Python >= 3.7 - -If you are using Python 3.6 and cannot upgrade to 3.7, install the `dataclasses` -backport: - - pip install dataclasses - -### Installing the TOM Toolkit and Django - -First, we recommend using a -[virtual environment](https://docs.python.org/3/tutorial/venv.html) for your -project. -This will keep your TOM python packages seperate from your system python packages. - - python3 -m venv tom_env/ - -Now that we have created the virtual environment, we can activate it: - - source tom_env/bin/activate - -You should now see `(tom_env)` prepended to your terminal prompt. - -Now, install the TOM Toolkit: - - pip install tomtoolkit - -...and create a new project, just like in the tutorial: - - django-admin startproject mytom - -You should now have a fully functional standard Django installation inside the -`mytom` folder, with the TOM dependencies installed as well. - -### Getting started with the `tom_setup` script. - -We need to add the `tom_setup` app to our project's `INSTALLED_APPS`. Locate the -`settings.py` file inside your project directory (usually in a subdirectory of the -main folder, i.e. mytom/mytom/settings.py) and edit it so that it looks like this: - -```python -INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'tom_setup', -] -``` - -### Run the setup script - -The `tom_setup` app contains a script that will bootstrap a new TOM in your -current project. Run it: - - ./manage.py tom_setup - -The install script will ask you a few questions and then install your TOM. - -### Running the dev server - -Now that the toolkit is installed, you're ready to try it out! - -First, run the necessary migrations: - - ./manage.py migrate - -Now, start the dev server: - - ./manage.py runserver - -Your new TOM should now be running on [http://127.0.0.1:8000](http://127.0.0.1:8000)! - diff --git a/docs/introduction/getting_started.rst b/docs/introduction/getting_started.rst new file mode 100644 index 000000000..3f3a8349a --- /dev/null +++ b/docs/introduction/getting_started.rst @@ -0,0 +1,116 @@ +Getting Started with the TOM Toolkit +------------------------------------ + +So you’ve decided to run a Target Observation Manager. This article will +help you get started. + +The TOM Toolkit is a `Django `__ +project. This means you’ll be running an application based on the Django +framework when you run a TOM. If you decide to customize your TOM, +you’ll be working in Django. You’ll likely need some basic understanding +of python and we recommend all users work their way through the `Django +tutorial `__ first +before starting with the TOM Toolkit. It doesn’t take long, and you most +likely won’t need to utilize any advanced features. + +Ready to go? Let’s get started. + +Prerequisites +~~~~~~~~~~~~~ + +The TOM toolkit requires Python >= 3.7 + +If you are using Python 3.6 and cannot upgrade to 3.7, install the +``dataclasses`` backport: + +:: + + pip install dataclasses + +Installing the TOM Toolkit and Django +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +First, we recommend using a `virtual +environment `__ for your +project. This will keep your TOM python packages seperate from your +system python packages. + +:: + + python3 -m venv tom_env/ + +Now that we have created the virtual environment, we can activate it: + +:: + + source tom_env/bin/activate + +You should now see ``(tom_env)`` prepended to your terminal prompt. + +Now, install the TOM Toolkit: + +:: + + pip install tomtoolkit + +…and create a new project, just like in the tutorial: + +:: + + django-admin startproject mytom + +You should now have a fully functional standard Django installation +inside the ``mytom`` folder, with the TOM dependencies installed as +well. + +Getting started with the ``tom_setup`` script. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We need to add the ``tom_setup`` app to our project’s +``INSTALLED_APPS``. Locate the ``settings.py`` file inside your project +directory (usually in a subdirectory of the main folder, +i.e. mytom/mytom/settings.py) and edit it so that it looks like this: + +.. code:: python + + INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'tom_setup', + ] + +Run the setup script +~~~~~~~~~~~~~~~~~~~~ + +The ``tom_setup`` app contains a script that will bootstrap a new TOM in +your current project. Run it: + +:: + + ./manage.py tom_setup + +The install script will ask you a few questions and then install your +TOM. + +Running the dev server +~~~~~~~~~~~~~~~~~~~~~~ + +Now that the toolkit is installed, you’re ready to try it out! + +First, run the necessary migrations: + +:: + + ./manage.py migrate + +Now, start the dev server: + +:: + + ./manage.py runserver + +Your new TOM should now be running on http://127.0.0.1:8000! diff --git a/docs/introduction/troubleshooting.md b/docs/introduction/troubleshooting.md deleted file mode 100644 index ded36f432..000000000 --- a/docs/introduction/troubleshooting.md +++ /dev/null @@ -1,43 +0,0 @@ -# Troubleshooting your TOM - -When first installing or later updating your TOM, you may run into a few common issues. Fortunately, you can stand on our -shoulders and hopefully find a solution here! - - -## Check that you've migrated - -Oftentimes, updating the TOM Toolkit requires running migrations. Usually, a directive to do so will be included in the -release notes, or Django will remind you that `You have unapplied migrations`. If you don't happen to see those, you may -also see a ` does not exist` when you load a page, or an error about an `applabel`. Those are generally -indicators that you need to run a database migration. - -You can confirm that you are missing a migration by running: - -``` -./manage.py showmigrations --list -``` - -Migrations that have been applied will have a `[X]` next to them, so make sure they all have one. If any are missing: - -``` -./manage.py migrate -``` - - -## Make sure you're in a virtual environment - -Everyone forgets to activate their virtualenv from time to time. If you get a missing package or some such, ensure that -you've activated your virtualenv: - -``` -source env/bin/activate -``` - -You may need to adapt the above for your particular shell. Also be sure that the virtualenv was created with a compatible -version of Python, and that you installed your dependencies into that virtualenv. - - -## Check your shell - -It's a small development team, and we all use bash. We've seen some issues with people running zsh, fish, and even csh. -You may need to adapt the commands given in the setup guide. \ No newline at end of file diff --git a/docs/introduction/troubleshooting.rst b/docs/introduction/troubleshooting.rst new file mode 100644 index 000000000..b08af9dc9 --- /dev/null +++ b/docs/introduction/troubleshooting.rst @@ -0,0 +1,52 @@ +Troubleshooting your TOM +======================== + +When first installing or later updating your TOM, you may run into a few +common issues. Fortunately, you can stand on our shoulders and hopefully +find a solution here! + +Check that you’ve migrated +-------------------------- + +Oftentimes, updating the TOM Toolkit requires running migrations. +Usually, a directive to do so will be included in the release notes, or +Django will remind you that ``You have unapplied migrations``. If you +don’t happen to see those, you may also see a +`` does not exist`` when you load a page, or an error +about an ``applabel``. Those are generally indicators that you need to +run a database migration. + +You can confirm that you are missing a migration by running: + +:: + + ./manage.py showmigrations --list + +Migrations that have been applied will have a ``[X]`` next to them, so +make sure they all have one. If any are missing: + +:: + + ./manage.py migrate + +Make sure you’re in a virtual environment +----------------------------------------- + +Everyone forgets to activate their virtualenv from time to time. If you +get a missing package or some such, ensure that you’ve activated your +virtualenv: + +:: + + source env/bin/activate + +You may need to adapt the above for your particular shell. Also be sure +that the virtualenv was created with a compatible version of Python, and +that you installed your dependencies into that virtualenv. + +Check your shell +---------------- + +It’s a small development team, and we all use bash. We’ve seen some +issues with people running zsh, fish, and even csh. You may need to +adapt the commands given in the setup guide. diff --git a/docs/introduction/workflow.md b/docs/introduction/workflow.md deleted file mode 100644 index 59ed01b2b..000000000 --- a/docs/introduction/workflow.md +++ /dev/null @@ -1,86 +0,0 @@ -TOM Workflow ---- - -### Targets - -Targets are the central entity of the TOM Toolkit. Most functionality in the -toolkit requires a target as they are the object of study. A target represents an -astronomical object (star, galaxy, asteroid, etc) and is usually represented using -coordinates on the sky along with other meta data. - -#### Creating Targets - -The TOM Toolkit provides a variety of methods for importing astronomical targets -into the TOM: - -![](/_static/target_sources.png) - - -* The Alert Module provides the functionality to create targets from alert brokers -such as [MARS](https://mars.lco.global) and [ANTARES](https://antares.noao.edu/). -These brokers generally provide alerts from transient phenomena as soon as they -happen, and a scientist who is interested in studying these phenomena can import -these alerts as targets into their TOM to study in real time. - -* Online catalogs such as SIMBAD and the JPL Horizons contain information on - millions of existing astronomical objects. If a scientist wishes to study one of - these existing objects, they can query these catalogs directly from the TOM and - use the returned data to create TOM Targets. - -* Manual entry/bulk upload allows a scientist to create targets that aren't known - by any of the existing catalogs or use more precise information that they know - of. - - -### Observations - -After creating targets, the scientist needs to collect data on these targets. The -TOM Observing module provides an interface to several observatories for which -observations can be requested. - -#### Requesting Observations - -Using the TOM Observation module, scientists can request observations of their -targets to one or many different observatories. Since the observing module has -access to targets stored in the TOM database it can automatically fill in many of -the observing parameters required by observing facilities, greatly reducing the -workload of the scientist. The observing module also provides a common interface, -removing the need of the scientist to navigate many different online systems to -request observations. - -Observations can also be requested in a completely automated manner, which is -particularly useful for rapid response time domain follow-up programs. - - -![](/_static/common_interface.png) - -#### Observation Status - -Once an observation for a target is created it's status is kept up to date within -the TOM. When the status of an observation request at an observatory changes -(failed, completed, postponed, etc) the scientist may be notified by the TOM. - -### Data - -The ultimate goal of the TOM toolkit is to collect and organize data. The TOM data -module provides several methods for obtaining data, the most obvious being from -completed observations. Scientists can also upload any data they'd like to -associate with their targets as well. - -#### Data Processing - -The TOM toolkit provides a framework to write custom code to -interact with the data the TOM obtains (among other things). These are called -"hooks" and they can be used by scientists to write custom image pipelines, data -quality checks, or to hook into entirely different systems. For example: if a -scientist has existing code that checks images of microlensed stars for -exoplanets, they may hook the code into the TOM toolkit directly to run whenever -new data is acquired. - -#### Downloading Data - -Data is stored in the TOM toolkit by default, but many scientists may want to -download the data somewhere else to do offline processing. Scientists can easily -download data to their local machines, and the data module by default stores all -it's data on a local file system. However, it can be customized to store data on -cloud services, like Amazon S3, when desired. diff --git a/docs/introduction/workflow.rst b/docs/introduction/workflow.rst new file mode 100644 index 000000000..d54291a11 --- /dev/null +++ b/docs/introduction/workflow.rst @@ -0,0 +1,93 @@ +TOM Workflow +------------ + +Targets ~~~~~~~ + +Targets are the central entity of the TOM Toolkit. Most functionality in +the toolkit requires a target as they are the object of study. A target +represents an astronomical object (star, galaxy, asteroid, etc) and is +usually represented using coordinates on the sky along with other meta +data. + +Creating Targets ^^^^^^^^^^^^^^^^ + +The TOM Toolkit provides a variety of methods for importing astronomical +targets into the TOM: + +\|image0\| + +- The Alert Module provides the functionality to create targets from + alert brokers such as ``MARS ``\ \_\_ and + ``ANTARES ``\ \__. These brokers generally + provide alerts from transient phenomena as soon as they happen, and a + scientist who is interested in studying these phenomena can import + these alerts as targets into their TOM to study in real time. + +- Online catalogs such as SIMBAD and the JPL Horizons contain + information on millions of existing astronomical objects. If a + scientist wishes to study one of these existing objects, they can + query these catalogs directly from the TOM and use the returned data + to create TOM Targets. + +- Manual entry/bulk upload allows a scientist to create targets that + aren’t known by any of the existing catalogs or use more precise + information that they know of. + +Observations ~~~~~~~~~~~~ + +After creating targets, the scientist needs to collect data on these +targets. The TOM Observing module provides an interface to several +observatories for which observations can be requested. + +Requesting Observations ^^^^^^^^^^^^^^^^^^^^^^^ + +Using the TOM Observation module, scientists can request observations of +their targets to one or many different observatories. Since the +observing module has access to targets stored in the TOM database it can +automatically fill in many of the observing parameters required by +observing facilities, greatly reducing the workload of the scientist. +The observing module also provides a common interface, removing the need +of the scientist to navigate many different online systems to request +observations. + +Observations can also be requested in a completely automated manner, +which is particularly useful for rapid response time domain follow-up +programs. + +\|image1\| + +Observation Status ^^^^^^^^^^^^^^^^^^ + +Once an observation for a target is created it’s status is kept up to +date within the TOM. When the status of an observation request at an +observatory changes (failed, completed, postponed, etc) the scientist +may be notified by the TOM. + +Data ~~~~ + +The ultimate goal of the TOM toolkit is to collect and organize data. +The TOM data module provides several methods for obtaining data, the +most obvious being from completed observations. Scientists can also +upload any data they’d like to associate with their targets as well. + +Data Processing ^^^^^^^^^^^^^^^ + +The TOM toolkit provides a framework to write custom code to interact +with the data the TOM obtains (among other things). These are called +“hooks” and they can be used by scientists to write custom image +pipelines, data quality checks, or to hook into entirely different +systems. For example: if a scientist has existing code that checks +images of microlensed stars for exoplanets, they may hook the code into +the TOM toolkit directly to run whenever new data is acquired. + +Downloading Data ^^^^^^^^^^^^^^^^ + +Data is stored in the TOM toolkit by default, but many scientists may +want to download the data somewhere else to do offline processing. +Scientists can easily download data to their local machines, and the +data module by default stores all it’s data on a local file system. +However, it can be customized to store data on cloud services, like +Amazon S3, when desired. + +.. \|image0\| image:: /_static/target_sources.png .. \|image1\| image:: +/_static/common_interface.png diff --git a/docs/managing_data/customizing_data_processing.md b/docs/managing_data/customizing_data_processing.md deleted file mode 100644 index fea329b95..000000000 --- a/docs/managing_data/customizing_data_processing.md +++ /dev/null @@ -1,157 +0,0 @@ -Customizing Data Processing ---------------------------- - -One of the many goals of the TOM Toolkit is to enable the simplification of the flow of your data from observations. To -that end, there's some built-in functionality that can be overridden to allow your TOM to work for your use case. - -To begin, here's a brief look at part of the structure of the tom_dataproducts app in the TOM Toolkit: - -``` -tom_dataproducts -├──hooks.py -├──models.py -└──processors - ├──data_serializers.py - ├──photometry_processor.py - └──spectroscopy_processor.py -``` - -Let's start with a quick overview of `models.py`. The file contains the Django models for the dataproducts app--in our -case, `DataProduct` and `ReducedDatum`. The `DataProduct` contains information about uploaded or saved `DataProducts`, -such as the file name, file path, and what kind of file it is. The `ReducedDatum` contains individual science data -points that are taken from the `DataProduct` files. Examples of `ReducedDatum` points would be individual photometry -points or individual spectra. - -Each `DataProduct` also has a `data_product_type`. The `data_product_type` is simply a description of what the file is, -more or less, and is customizable. The list of supported `data_product_type`s is maintained in `settings.py`: - -```python -# Define the valid data product types for your TOM. Be careful when removing items, as previously valid types will no -# longer be valid, and may cause issues unless the offending records are modified. -DATA_PRODUCT_TYPES = { - 'photometry': ('photometry', 'Photometry'), - 'fits_file': ('fits_file', 'FITS File'), - 'spectroscopy': ('spectroscopy', 'Spectroscopy'), - 'image_file': ('image_file', 'Image File') -} -``` - -In order to add new data product types, simply add a new key/value pair, with the value being a 2-tuple. The first -tuple item is the database value, and the second is the display value. - -All data products are automatically "processed" on upload, as well. Of course, that can mean different things to -different TOMs! The TOM has two built-in data processors, both of which simply ingest the data into the database, -and those are also specified in `settings.py`: - -```python -DATA_PROCESSORS = { - 'photometry': 'tom_dataproducts.processors.photometry_processor.PhotometryProcessor', - 'spectroscopy': 'tom_dataproducts.processors.spectroscopy_processor.SpectroscopyProcessor', -} -``` - -When a user either uploads a `DataProduct` to their TOM, the TOM runs `process_data()` from the corresponding -`DataProcessor` subclass specified in `DATA_PROCESSORS` seen above. To illustrate, this is the base `DataProcessor` -class: - -```python -import mimetypes - -... - -class DataProcessor(): - - FITS_MIMETYPES = ['image/fits', 'application/fits'] - PLAINTEXT_MIMETYPES = ['text/plain', 'text/csv'] - - mimetypes.add_type('image/fits', '.fits') - mimetypes.add_type('image/fits', '.fz') - mimetypes.add_type('application/fits', '.fits') - mimetypes.add_type('application/fits', '.fz') - - def process_data(self, data_product): - pass - -``` - -Now let's look at the built-in data processors. First, let's check out the `PhotometryProcessor`, which inherits from -`DataProcessor`: - -```python -class PhotometryProcessor(DataProcessor): - - def process_data(self, data_product): - mimetype = mimetypes.guess_type(data_product.data.path)[0] - if mimetype in self.PLAINTEXT_MIMETYPES: - photometry = self._process_photometry_from_plaintext(data_product) - return [(datum.pop('timestamp'), json.dumps(datum)) for datum in photometry] - else: - raise InvalidFileFormatException('Unsupported file type') -``` - -This class has an implementation of `process_data()` from the superclass `DataProcessor`. The implementation calls an -internal method `_process_photometry_from_plaintext()`, which return a `list` of `dict`s. Each dict contains the values -for the timestamp, magnitude, filter, and error for that photometry point. The list is then transformed into a list of -2-tuples, with the first value being the photometry timestamp, and the second being the JSON-ified remaining values. - -Next, let's look at the `SpectroscopyProcessor`: - -```python -class SpectroscopyProcessor(DataProcessor): - - DEFAULT_WAVELENGTH_UNITS = units.angstrom - DEFAULT_FLUX_CONSTANT = units.erg / units.cm ** 2 / units.second / units.angstrom - - def process_data(self, data_product): - - mimetype = mimetypes.guess_type(data_product.data.path)[0] - if mimetype in self.FITS_MIMETYPES: - spectrum, obs_date = self._process_spectrum_from_fits(data_product) - elif mimetype in self.PLAINTEXT_MIMETYPES: - spectrum, obs_date = self._process_spectrum_from_plaintext(data_product) - else: - raise InvalidFileFormatException('Unsupported file type') - - serialized_spectrum = SpectrumSerializer().serialize(spectrum) - - return [(obs_date, serialized_spectrum)] -``` - -Just like the `PhotometryProcessor`, this class inherits from `DataProcessor` and implements `process_data()`. This is a -requirement for a custom DataProcessor! This `process_data()` method handles two file types, unlike the previous -example, each of which calls an internal method that returns a `Spectrum1D` object. Again, like the -`PhotometryProcessor`, a list of 2-tuples is created, with the first value being the timestamp, and the second being -the JSON spectrum. - -You may be wondering why these two methods return lists of 2-tuples, especially when the `SpectroscopyProcessor` only -returns a list of length one. The rationale is to ensure that you, the TOM user, shouldn't have to worry about the -database insertion, so the internal logic handles that aspect, and it can do so whether you return one data point or -many data points. - -For a custom `DataProcessor`, there are just a few required steps. The first is to create a class that implements -`DataProcessor`, like so: - -```python -from tom_dataproducts.data_processor import DataProcessor - - -class MyDataProcessor(DataProcessor): - - def process_data(self, data_product): - # custom data processing here - - return [(timestamp1, json1), (timestamp2, json2), ..., (timestampN, dictN)] -``` - -Let's say that this file lives at `mytom/my_data_processor.py`. Now the processor needs to be added to -`DATA_PROCESSORS`, and it can either process a new data product type, or replace an existing one. Let's replace -spectroscopy: - -```python -DATA_PROCESSORS = { - 'photometry': 'tom_dataproducts.processors.photometry_processor.PhotometryProcessor', - 'spectroscopy': 'mytom.my_data_processor.MyDataProcessor', -} -``` - -And that's it! Now your TOM will run the data processing specific to your case instead of the default one. diff --git a/docs/managing_data/customizing_data_processing.rst b/docs/managing_data/customizing_data_processing.rst new file mode 100644 index 000000000..62b295535 --- /dev/null +++ b/docs/managing_data/customizing_data_processing.rst @@ -0,0 +1,177 @@ +Customizing Data Processing +--------------------------- + +One of the many goals of the TOM Toolkit is to enable the simplification +of the flow of your data from observations. To that end, there’s some +built-in functionality that can be overridden to allow your TOM to work +for your use case. + +To begin, here’s a brief look at part of the structure of the +tom_dataproducts app in the TOM Toolkit: + +:: + + tom_dataproducts + ├──hooks.py + ├──models.py + └──processors + ├──data_serializers.py + ├──photometry_processor.py + └──spectroscopy_processor.py + +Let’s start with a quick overview of ``models.py``. The file contains +the Django models for the dataproducts app–in our case, ``DataProduct`` +and ``ReducedDatum``. The ``DataProduct`` contains information about +uploaded or saved ``DataProducts``, such as the file name, file path, +and what kind of file it is. The ``ReducedDatum`` contains individual +science data points that are taken from the ``DataProduct`` files. +Examples of ``ReducedDatum`` points would be individual photometry +points or individual spectra. + +Each ``DataProduct`` also has a ``data_product_type``. The +``data_product_type`` is simply a description of what the file is, more +or less, and is customizable. The list of supported +``data_product_type``\ s is maintained in ``settings.py``: + +.. code:: python + + # Define the valid data product types for your TOM. Be careful when removing items, as previously valid types will no + # longer be valid, and may cause issues unless the offending records are modified. + DATA_PRODUCT_TYPES = { + 'photometry': ('photometry', 'Photometry'), + 'fits_file': ('fits_file', 'FITS File'), + 'spectroscopy': ('spectroscopy', 'Spectroscopy'), + 'image_file': ('image_file', 'Image File') + } + +In order to add new data product types, simply add a new key/value pair, +with the value being a 2-tuple. The first tuple item is the database +value, and the second is the display value. + +All data products are automatically “processed” on upload, as well. Of +course, that can mean different things to different TOMs! The TOM has +two built-in data processors, both of which simply ingest the data into +the database, and those are also specified in ``settings.py``: + +.. code:: python + + DATA_PROCESSORS = { + 'photometry': 'tom_dataproducts.processors.photometry_processor.PhotometryProcessor', + 'spectroscopy': 'tom_dataproducts.processors.spectroscopy_processor.SpectroscopyProcessor', + } + +When a user either uploads a ``DataProduct`` to their TOM, the TOM runs +``process_data()`` from the corresponding ``DataProcessor`` subclass +specified in ``DATA_PROCESSORS`` seen above. To illustrate, this is the +base ``DataProcessor`` class: + +.. code:: python + + import mimetypes + + ... + + class DataProcessor(): + + FITS_MIMETYPES = ['image/fits', 'application/fits'] + PLAINTEXT_MIMETYPES = ['text/plain', 'text/csv'] + + mimetypes.add_type('image/fits', '.fits') + mimetypes.add_type('image/fits', '.fz') + mimetypes.add_type('application/fits', '.fits') + mimetypes.add_type('application/fits', '.fz') + + def process_data(self, data_product): + pass + +Now let’s look at the built-in data processors. First, let’s check out +the ``PhotometryProcessor``, which inherits from ``DataProcessor``: + +.. code:: python + + class PhotometryProcessor(DataProcessor): + + def process_data(self, data_product): + mimetype = mimetypes.guess_type(data_product.data.path)[0] + if mimetype in self.PLAINTEXT_MIMETYPES: + photometry = self._process_photometry_from_plaintext(data_product) + return [(datum.pop('timestamp'), json.dumps(datum)) for datum in photometry] + else: + raise InvalidFileFormatException('Unsupported file type') + +This class has an implementation of ``process_data()`` from the +superclass ``DataProcessor``. The implementation calls an internal +method ``_process_photometry_from_plaintext()``, which return a ``list`` +of ``dict``\ s. Each dict contains the values for the timestamp, +magnitude, filter, and error for that photometry point. The list is then +transformed into a list of 2-tuples, with the first value being the +photometry timestamp, and the second being the JSON-ified remaining +values. + +Next, let’s look at the ``SpectroscopyProcessor``: + +.. code:: python + + class SpectroscopyProcessor(DataProcessor): + + DEFAULT_WAVELENGTH_UNITS = units.angstrom + DEFAULT_FLUX_CONSTANT = units.erg / units.cm ** 2 / units.second / units.angstrom + + def process_data(self, data_product): + + mimetype = mimetypes.guess_type(data_product.data.path)[0] + if mimetype in self.FITS_MIMETYPES: + spectrum, obs_date = self._process_spectrum_from_fits(data_product) + elif mimetype in self.PLAINTEXT_MIMETYPES: + spectrum, obs_date = self._process_spectrum_from_plaintext(data_product) + else: + raise InvalidFileFormatException('Unsupported file type') + + serialized_spectrum = SpectrumSerializer().serialize(spectrum) + + return [(obs_date, serialized_spectrum)] + +Just like the ``PhotometryProcessor``, this class inherits from +``DataProcessor`` and implements ``process_data()``. This is a +requirement for a custom DataProcessor! This ``process_data()`` method +handles two file types, unlike the previous example, each of which calls +an internal method that returns a ``Spectrum1D`` object. Again, like the +``PhotometryProcessor``, a list of 2-tuples is created, with the first +value being the timestamp, and the second being the JSON spectrum. + +You may be wondering why these two methods return lists of 2-tuples, +especially when the ``SpectroscopyProcessor`` only returns a list of +length one. The rationale is to ensure that you, the TOM user, shouldn’t +have to worry about the database insertion, so the internal logic +handles that aspect, and it can do so whether you return one data point +or many data points. + +For a custom ``DataProcessor``, there are just a few required steps. The +first is to create a class that implements ``DataProcessor``, like so: + +.. code:: python + + from tom_dataproducts.data_processor import DataProcessor + + + class MyDataProcessor(DataProcessor): + + def process_data(self, data_product): + # custom data processing here + + return [(timestamp1, json1), (timestamp2, json2), ..., (timestampN, dictN)] + +Let’s say that this file lives at ``mytom/my_data_processor.py``. Now +the processor needs to be added to ``DATA_PROCESSORS``, and it can +either process a new data product type, or replace an existing one. +Let’s replace spectroscopy: + +.. code:: python + + DATA_PROCESSORS = { + 'photometry': 'tom_dataproducts.processors.photometry_processor.PhotometryProcessor', + 'spectroscopy': 'mytom.my_data_processor.MyDataProcessor', + } + +And that’s it! Now your TOM will run the data processing specific to +your case instead of the default one. diff --git a/docs/managing_data/plotting_data.md b/docs/managing_data/plotting_data.md deleted file mode 100644 index 83f9cb235..000000000 --- a/docs/managing_data/plotting_data.md +++ /dev/null @@ -1,179 +0,0 @@ -Plotting Data -------------- - -The TOM Toolkit provides a few basic plots, such as photometry, spectroscopy and -target distribution. Sometimes it would be useful to visualize data in a different -way. - -In this tutorial you will learn how to build and display a very simple plot -in our TOM: number of reduced data per target. The end result will demonstrate -how to create a [plot.ly](https://plot.ly) plot with data from our TOM. You will -even package the code in it's own app so we can share it with other TOM users that -might find it useful. - -If you haven't already read the documentation on [customizing -templates](/customization/customize_templates) you should read it first. You'll need to -edit a template in order to view your new plot somewhere. - -First, start a new app in our project to house the new plot (and perhaps -other additions!): - - ./manage.py startapp myplots - -This will create a new [Django -app](https://docs.djangoproject.com/en/2.1/intro/tutorial01/#creating-the-polls-app) in your project -named myplots: - - myplots - ├── admin.py - ├── apps.py - ├── __init__.py - ├── migrations - │ └── __init__.py - ├── models.py - ├── tests.py - └── views.py - - 1 directory, 7 files - -Note you don't necessarily have to start a new app. If you've already started an -app that you'd like to reuse, that works too. - -Now install the new app into your project's settings.py file: - -```python -INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - ... - 'myplots', -] -``` - -Now that the `myplots` app is installed, create the directories necessary to -contain your new plot: - - mkdir -p myplots/templates/myplots - mkdir myplots/templatetags - -The templates directory will contain the html template you can include in other -templates to display your plot. The templatetags directory will contain the python -code to construct the plot.ly plot. - -Start by creating the -[templatetags](https://docs.djangoproject.com/en/2.1/howto/custom-template-tags/) -file: - - touch myplots/templatetags/myplots_tags.py - -Edit this file, starting with the necessary imports: - -```python -from plotly import offline -import plotly.graph_objs as go -from django import template - -from tom_targets.models import Target -``` - -The `plotly` imports are needed for building an offline plot. The django -`template` import gives access to the template library, which will allow for -registering the template tag. Finally, the TOM Toolkit `Target` class will allow -access to the `Target` model (for querying). - -Next, add the boiler plate code for a template tag: - -```python -register = template.Library() - - -@register.inclusion_tag('myplots/targets_reduceddata.html') -def targets_reduceddata(targets=Target.objects.all()): -``` - -First we instantiate the `register` decorator. You don't need to know much about -this other that it allows us to register functions as templatetags. The function -`targets_reduceddata` is decorated with the `register` decorator, which takes as -an argument the template to render. The function definition takes in a queryset of -`Target`s as a keyword argument, but if none are supplied, defaults to all `Target`s -in the database. - -Next, add the function body: - -```python - # order targets by creation date - targets = targets.order_by('-created') - # x axis: target names. y axis: datum count - data = [go.Bar( - x=[target.name for target in targets], - y=[target.reduceddatum_set.count() for target in targets] - )] - # Create the plot - figure = offline.plot(go.Figure(data=data), output_type='div', show_link=False) - # Add plot to the template context - return {'figure': figure} -``` - -As the comments describe, the function code iterates over each `Target` in the -`targets` queryset adding the target name and datum count as x/y values to the -`Bar` data structure. Check out the [plot.ly bar chart -documentation](https://plot.ly/python/bar-charts/) for more information about the -options available to you. As an exercise, try changing the values in the y axis. -Or you could use a different chart type. - -Finally, the code adds the plot.ly plot to the template rendering context. Next we -will create this template where this context will be rendered. - -Create the file, making sure it matches the template name specified in the -template tag definition beforehand: - - touch myplots/templates/myplots/targets_reduceddata.html - -This file contains the simple contents: - - {% raw %} - {{ figure|safe }} - {% endraw %} - -All this template does is output the `figure` variable, which is the html -generated from plotly in the templatetag. We also tell django that the output is -safe, so that it doesn't escape the html. That's it. - -**Note:** If you're running the development server, restart it now. Django doesn't -automatically pick up new templatetags. - -Now that the templatetag and template are complete, we can use it in any template. -You might have your own templates which you'd like to add the plot to, or perhaps -you've customized one of the TOM supplied templates as per the [customizing -templates](/customization/customize_templates) documentation. Either way, including the -templatetag works the same way. At the top of the template (after any 'extends') -load the new tag library: - - {% raw %} - {% load myplots_tags %} - {% endraw %} - -Now insert the templatetag somewhere in the template where you'd like it to -appear: - - {% raw %} - {% targets_reduceddata %} - {% endraw %} - -If your parent template already has a queryset of targets available in the context -(for example, a target list page) you can pass it in to be used in your plot: - - {% raw %} - {% targets_reduceddata targets %} - {% endraw %} - -Otherwise the plot will simply use all targets in your database. Either way, you -should end up with something like this: - -![](/_static/plotting_data_doc/plot.png) - -That's it! Plot.ly provides a wide range of plotting capabilities, you should -reference [the documentation](https://plot.ly/python/) for more information. It -would also be helpful to read [Django's -ORM](https://docs.djangoproject.com/en/2.1/topics/db/) to become familiarized with -wide range of methods of querying data. diff --git a/docs/managing_data/plotting_data.rst b/docs/managing_data/plotting_data.rst new file mode 100644 index 000000000..57ae08537 --- /dev/null +++ b/docs/managing_data/plotting_data.rst @@ -0,0 +1,206 @@ +Plotting Data +------------- + +The TOM Toolkit provides a few basic plots, such as photometry, +spectroscopy and target distribution. Sometimes it would be useful to +visualize data in a different way. + +In this tutorial you will learn how to build and display a very simple +plot in our TOM: number of reduced data per target. The end result will +demonstrate how to create a `plot.ly `__ plot with data +from our TOM. You will even package the code in it’s own app so we can +share it with other TOM users that might find it useful. + +If you haven’t already read the documentation on `customizing +templates `__ you should read it +first. You’ll need to edit a template in order to view your new plot +somewhere. + +First, start a new app in our project to house the new plot (and perhaps +other additions!): + +:: + + ./manage.py startapp myplots + +This will create a new `Django +app `__ +in your project named myplots: + +:: + + myplots + ├── admin.py + ├── apps.py + ├── __init__.py + ├── migrations + │ └── __init__.py + ├── models.py + ├── tests.py + └── views.py + + 1 directory, 7 files + +Note you don’t necessarily have to start a new app. If you’ve already +started an app that you’d like to reuse, that works too. + +Now install the new app into your project’s settings.py file: + +.. code:: python + + INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + ... + 'myplots', + ] + +Now that the ``myplots`` app is installed, create the directories +necessary to contain your new plot: + +:: + + mkdir -p myplots/templates/myplots + mkdir myplots/templatetags + +The templates directory will contain the html template you can include +in other templates to display your plot. The templatetags directory will +contain the python code to construct the plot.ly plot. + +Start by creating the +`templatetags `__ +file: + +:: + + touch myplots/templatetags/myplots_tags.py + +Edit this file, starting with the necessary imports: + +.. code:: python + + from plotly import offline + import plotly.graph_objs as go + from django import template + + from tom_targets.models import Target + +The ``plotly`` imports are needed for building an offline plot. The +django ``template`` import gives access to the template library, which +will allow for registering the template tag. Finally, the TOM Toolkit +``Target`` class will allow access to the ``Target`` model (for +querying). + +Next, add the boiler plate code for a template tag: + +.. code:: python + + register = template.Library() + + + @register.inclusion_tag('myplots/targets_reduceddata.html') + def targets_reduceddata(targets=Target.objects.all()): + +First we instantiate the ``register`` decorator. You don’t need to know +much about this other that it allows us to register functions as +templatetags. The function ``targets_reduceddata`` is decorated with the +``register`` decorator, which takes as an argument the template to +render. The function definition takes in a queryset of ``Target``\ s as +a keyword argument, but if none are supplied, defaults to all +``Target``\ s in the database. + +Next, add the function body: + +.. code:: python + + # order targets by creation date + targets = targets.order_by('-created') + # x axis: target names. y axis: datum count + data = [go.Bar( + x=[target.name for target in targets], + y=[target.reduceddatum_set.count() for target in targets] + )] + # Create the plot + figure = offline.plot(go.Figure(data=data), output_type='div', show_link=False) + # Add plot to the template context + return {'figure': figure} + +As the comments describe, the function code iterates over each +``Target`` in the ``targets`` queryset adding the target name and datum +count as x/y values to the ``Bar`` data structure. Check out the +`plot.ly bar chart documentation `__ +for more information about the options available to you. As an exercise, +try changing the values in the y axis. Or you could use a different +chart type. + +Finally, the code adds the plot.ly plot to the template rendering +context. Next we will create this template where this context will be +rendered. + +Create the file, making sure it matches the template name specified in +the template tag definition beforehand: + +:: + + touch myplots/templates/myplots/targets_reduceddata.html + +This file contains the simple contents: + +:: + + {% raw %} + {{ figure|safe }} + {% endraw %} + +All this template does is output the ``figure`` variable, which is the +html generated from plotly in the templatetag. We also tell django that +the output is safe, so that it doesn’t escape the html. That’s it. + +**Note:** If you’re running the development server, restart it now. +Django doesn’t automatically pick up new templatetags. + +Now that the templatetag and template are complete, we can use it in any +template. You might have your own templates which you’d like to add the +plot to, or perhaps you’ve customized one of the TOM supplied templates +as per the `customizing +templates `__ documentation. Either +way, including the templatetag works the same way. At the top of the +template (after any ‘extends’) load the new tag library: + +:: + + {% raw %} + {% load myplots_tags %} + {% endraw %} + +Now insert the templatetag somewhere in the template where you’d like it +to appear: + +:: + + {% raw %} + {% targets_reduceddata %} + {% endraw %} + +If your parent template already has a queryset of targets available in +the context (for example, a target list page) you can pass it in to be +used in your plot: + +:: + + {% raw %} + {% targets_reduceddata targets %} + {% endraw %} + +Otherwise the plot will simply use all targets in your database. Either +way, you should end up with something like this: + +|image0| + +That’s it! Plot.ly provides a wide range of plotting capabilities, you +should reference `the documentation `__ for +more information. It would also be helpful to read `Django’s +ORM `__ to become +familiarized with wide range of methods of querying data. + +.. |image0| image:: /_static/plotting_data_doc/plot.png diff --git a/docs/observing/customize_observations.md b/docs/observing/customize_observations.md deleted file mode 100644 index 2c58ff0a0..000000000 --- a/docs/observing/customize_observations.md +++ /dev/null @@ -1,297 +0,0 @@ -Changing How Observations are Submitted ---------------------------------------- - -The LCO Observation module for the TOM Toolkit ships with a default HTML form that -facilitates submitting basic observations to the LCO network. It may sometimes be -desirable to customize the form to show or hide fields, add new parameters, or -change the submission logic itself, depending on the needs of the project. In this -tutorial we will customize our LCO module to submit multiple observations with -different filters at the same time. - -This guide assumes you have followed the [getting -started](/introduction/getting_started) guide and have a working TOM up and running. - -### Create a new Observation Module - -Many methods of customizing the TOM Toolkit involve inheriting/extending existing -functionality. This time will be no different: we'll crate a new observation -module that inherits the existing functionality from -`tom_observations.facilities.LCOFacility`. - -First, create a python file somewhere in your project to house your new module. -For example it could live next to your `settings.py`, or if you've started a new -app, it could live there. It doesn't really matter, as -long as it's located somewhere in your project: - - touch mytom/mytom/lcomultifilter.py - -Now add some code to this file to create a new observation module: - -```python -# lcomultifilter.py -from tom_observations.facilities.lco import LCOFacility - - -class LCOMultiFilterFacility(LCOFacility): - name = 'LCOMultiFilter' -``` -So what does the above code do? - -1. Line 1 imports the LCOFacility that is already shipped with the TOM Toolkit. We -want this class because it contains functionality we will re-use in our own -implementation. -2. Line 4 defines a new class named `LCOMultiFilterFacility` that inherits from -`LCOFacility`. -3. Line 5 sets the name attribute of this class to `LCOMultiFilter`. - -What you have done is created a new observation module that is functionally -identical to the existing LCO module, but has a different name: `LCOMultiFilter`. -A good start! - -Now we need to tell our TOM where to find our new module so we can use it to -submit observations. Add (or edit) the following lines to your `settings.py`: - -```python -# settings.py -TOM_FACILITY_CLASSES = [ - 'tom_observations.facilities.lco.LCOFacility', - 'tom_observations.facilities.gemini.GEMFacility', - 'mytom.lcomultifilter.LCOMultiFilterFacility', -] -``` -This code lists all of the observation modules that should be available to our -TOM. - -With that done, go to any target in your TOM and you should see your new module in -the list: - -![](/_static/customize_observations/observebutton.png) - -You could now use the new module now to make an observation, and it would work the -same as the old LCO module. - -Note that if you see an error like: "There was a problem authenticating with LCO" -then you need to [add your LCO api key](/docs/customsettings#facilities) to your -`settings.py` file. - -### Adding additional fields - -Now that you've created a new observation module that's functionally the same as -the old LCO module, how do we change it? One thing that might be useful is to add some extra -fields to the form: two more choices of filters and exposure times. Back in the -`lcomultifilter.py` file add a new import and create a new class that will become -the new form: - -```python -# lcomultifilter.py -from tom_observations.facilities.lco import LCOFacility, LCOObservationForm, filter_choices -from django import forms - - -class LCOMultiFilterForm(LCOObservationForm): - filter2 = forms.ChoiceField(choices=filter_choices) - exposure_time2 = forms.FloatField(min_value=0.1) - filter3 = forms.ChoiceField(choices=filter_choices) - exposure_time3 = forms.FloatField(min_value=0.1) - - -class LCOMultiFilterFacility(LCOFacility): - name = 'LCOMultiFilter' - form = LCOMultiFilterForm -``` - -There is now a new class, `LCOMultiFilterForm` which inherits from -`LCOObservationForm`, the form for the default interface. Additionally there are -definitions for 4 fields: `fiter2`, `exposure_time2`, `filter3`, and -`exposure_time3`. - -A `form` attribute has been added on the `LCOMultiFilterFacility` -class, this tells our observation module to use the new `LCOMultiFilterForm` -instead of the default LCO observation form. - - -### Modifying the form layout - -Now that the desired fields have been added to the `LCOMultiFilterForm`, the -form's layout needs to be modified in order to actually display them. In this -example we'll split the form into two rows: one row for the three filter choices -and exposure times, and another row for everything else. Note that the default -form already has fields for `filter` and `exposure_time`, so we'll overwrite the -entire layout so that they appear next to the new fields we added. - -The `LCOObservationForm` has a method `layout()` that returns the desired layout -using the [crispy forms Layout](https://django-crispy-forms.readthedocs.io/en/d-0/layouts.html) -class. Familiarizing yourself with the basic functionality of crispy forms would -be a good idea if you wish to deeply customize your observation module's form. - -With our modified layout added, the `lcomultifilter.py` file now looks like this: - -```python -# lcomultifilter.py -from tom_observations.facilities.lco import LCOFacility, LCOObservationForm, filter_choices -from django import forms -from crispy_forms.layout import Div - - -class LCOMultiFilterForm(LCOObservationForm): - filter2 = forms.ChoiceField(choices=filter_choices) - exposure_time2 = forms.FloatField(min_value=0.1) - filter3 = forms.ChoiceField(choices=filter_choices) - exposure_time3 = forms.FloatField(min_value=0.1) - - def layout(self): - return Div( - Div( - Div( - 'name', 'proposal', 'ipp_value', 'observation_type', 'start', 'end', - css_class='col' - ), - Div( - 'instrument_name', 'exposure_count', 'max_airmass', - css_class='col' - ), - css_class='form-row' - ), - Div( - Div( - 'filter', 'exposure_time', - css_class='col' - ), - Div( - 'filter2', 'exposure_time2', - css_class='col' - ), - Div( - 'filter3', 'exposure_time3', - css_class='col' - ), - css_class='form-row' - ) - ) - - -class LCOMultiFilterFacility(LCOFacility): - name = 'LCOMultiFilter' - form = LCOMultiFilterForm -``` - -Take a look at the layout and compare it to the [existing lco layout](https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/lco.py#L169). A second -row has been added that includes all the filter choices. Note that the original -`filter` and `exposure_time` have been moved from their original location to the -new row. - -Now if you select "LCOMultiFilter" from the list of observation facilities on a -target you should see your new form: - -![](/_static/customize_observations/newform.png) - -Is the form still too ugly for you? Trying playing with the layout definition to -suit your needs. - -### Changing the form submission behavior - -If you are not familiar with the [LCO submission -API](https://developers.lco.global/#observations) now might be a good time to take -a look. The LCO Observation module uses this API to submit observations using the -data provided in the form, so we need to modify how this happens. More -specifically, we'd like to add two additional `Configuration` to our observation -request, one for each of our additional filters and exposure times. - -Using the `observation_payload()` method, we can use `super()` to get the -original LCO module's observation request, then modify it to suit the needs of our -`LCOMultiFilter` class: - -```python -#lcomultifilter.py -from tom_observations.facilities.lco import LCOFacility, LCOObservationForm, filter_choices -from django import forms -from crispy_forms.layout import Div -from copy import deepcopy - -class LCOMultiFilterForm(LCOObservationForm): - filter2 = forms.ChoiceField(choices=filter_choices) - exposure_time2 = forms.FloatField(min_value=0.1) - filter3 = forms.ChoiceField(choices=filter_choices) - exposure_time3 = forms.FloatField(min_value=0.1) - - def layout(self): - return Div( - Div( - Div( - 'name', 'proposal', 'ipp_value', 'observation_type', 'start', 'end', - css_class='col' - ), - Div( - 'instrument_type', 'exposure_count', 'max_airmass', - css_class='col' - ), - css_class='form-row' - ), - Div( - Div( - 'filter', 'exposure_time', - css_class='col' - ), - Div( - 'filter2', 'exposure_time2', - css_class='col' - ), - Div( - 'filter3', 'exposure_time3', - css_class='col' - ), - css_class='form-row' - ) - ) - - def observation_payload(self): - payload = super().observation_payload() - configuration2 = deepcopy(payload['requests'][0]['configurations'][0]) - configuration3 = deepcopy(payload['requests'][0]['configurations'][0]) - configuration2['instrument_configs'][0]['optical_elements']['filter'] = self.cleaned_data['filter2'] - configuration2['instrument_configs'][0]['exposure_time'] = self.cleaned_data['exposure_time2'] - configuration3['instrument_configs'][0]['optical_elements']['filter'] = self.cleaned_data['filter3'] - configuration3['instrument_configs'][0]['exposure_time'] = self.cleaned_data['exposure_time3'] - payload['requests'][0]['configurations'].extend([configuration2, configuration3]) - return payload - - -class LCOMultiFilterFacility(LCOFacility): - name = 'LCOMultiFilter' - form = LCOMultiFilterForm -``` - -Let's go over what we did in this new `observation_payload()` method: - -1. Line 1: We call `super().observation_payload()` to get the observation request -which the parent class (LCOFacility) would have called. -2. Line 2-3 We copy the Request's Configuration into two new Configurations: `configuration2` and -`configuration3`. These will be the additional Configuration we send to LCO. -3. Lines 5-8: We set the value of these new Configuration `filter` and -`exposure_time` to the values we collected from our custom form. -4. lines 10-11: Finally, we extend the original Request's Configuration array to -include the 2 new Configuration we built. Return it and we're done! - -If you submit an observation request with the `LCOMultiFilter` observation module -now you should see that it creates an observation request with LCO with three -Configuration! - -### Summary - -Our original requirement was to be able to submit observations to LCO with some -additional filters and exposure times. We accomplished this by: - -1. Creating a new observation module: a `LCOMultiFilterFacility` class and a -`LCOMultiFilterForm`, both of which were child classes of the original -`LCOFacility` class (since we wanted to keep most of the functionality intact) and -then added this new class to our `TOM_FACILITY_CLASSES` setting. - -2. We added a few fields to `LCOMultiFilterForm` and modified it's layout to -include these new fields using `layout()`. - -3. We implemented the `LCOMultiFilterForm` `observation_payload()` which used the -parent's class return value and then modified it to suit our needs. - -This is a good example of Object Oriented Programming in Python. If you are -curious about how this all works, we recommend reading up on OOP in general, as -well as how objects in Python 3 work. diff --git a/docs/observing/customize_observations.rst b/docs/observing/customize_observations.rst new file mode 100644 index 000000000..7d0db1bbf --- /dev/null +++ b/docs/observing/customize_observations.rst @@ -0,0 +1,323 @@ +Changing How Observations are Submitted +--------------------------------------- + +The LCO Observation module for the TOM Toolkit ships with a default HTML +form that facilitates submitting basic observations to the LCO network. +It may sometimes be desirable to customize the form to show or hide +fields, add new parameters, or change the submission logic itself, +depending on the needs of the project. In this tutorial we will +customize our LCO module to submit multiple observations with different +filters at the same time. + +This guide assumes you have followed the `getting +started `__ guide and have a working TOM +up and running. + +Create a new Observation Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Many methods of customizing the TOM Toolkit involve inheriting/extending +existing functionality. This time will be no different: we’ll crate a +new observation module that inherits the existing functionality from +``tom_observations.facilities.LCOFacility``. + +First, create a python file somewhere in your project to house your new +module. For example it could live next to your ``settings.py``, or if +you’ve started a new app, it could live there. It doesn’t really matter, +as long as it’s located somewhere in your project: + +:: + + touch mytom/mytom/lcomultifilter.py + +Now add some code to this file to create a new observation module: + +.. code:: python + + # lcomultifilter.py + from tom_observations.facilities.lco import LCOFacility + + + class LCOMultiFilterFacility(LCOFacility): + name = 'LCOMultiFilter' + +So what does the above code do? + +1. Line 1 imports the LCOFacility that is already shipped with the TOM + Toolkit. We want this class because it contains functionality we will + re-use in our own implementation. +2. Line 4 defines a new class named ``LCOMultiFilterFacility`` that + inherits from ``LCOFacility``. +3. Line 5 sets the name attribute of this class to ``LCOMultiFilter``. + +What you have done is created a new observation module that is +functionally identical to the existing LCO module, but has a different +name: ``LCOMultiFilter``. A good start! + +Now we need to tell our TOM where to find our new module so we can use +it to submit observations. Add (or edit) the following lines to your +``settings.py``: + +.. code:: python + + # settings.py + TOM_FACILITY_CLASSES = [ + 'tom_observations.facilities.lco.LCOFacility', + 'tom_observations.facilities.gemini.GEMFacility', + 'mytom.lcomultifilter.LCOMultiFilterFacility', + ] + +This code lists all of the observation modules that should be available +to our TOM. + +With that done, go to any target in your TOM and you should see your new +module in the list: + +|image0| + +You could now use the new module now to make an observation, and it +would work the same as the old LCO module. + +Note that if you see an error like: “There was a problem authenticating +with LCO” then you need to `add your LCO api +key `__ to your ``settings.py`` file. + +Adding additional fields +~~~~~~~~~~~~~~~~~~~~~~~~ + +Now that you’ve created a new observation module that’s functionally the +same as the old LCO module, how do we change it? One thing that might be +useful is to add some extra fields to the form: two more choices of +filters and exposure times. Back in the ``lcomultifilter.py`` file add a +new import and create a new class that will become the new form: + +.. code:: python + + # lcomultifilter.py + from tom_observations.facilities.lco import LCOFacility, LCOObservationForm, filter_choices + from django import forms + + + class LCOMultiFilterForm(LCOObservationForm): + filter2 = forms.ChoiceField(choices=filter_choices) + exposure_time2 = forms.FloatField(min_value=0.1) + filter3 = forms.ChoiceField(choices=filter_choices) + exposure_time3 = forms.FloatField(min_value=0.1) + + + class LCOMultiFilterFacility(LCOFacility): + name = 'LCOMultiFilter' + form = LCOMultiFilterForm + +There is now a new class, ``LCOMultiFilterForm`` which inherits from +``LCOObservationForm``, the form for the default interface. Additionally +there are definitions for 4 fields: ``fiter2``, ``exposure_time2``, +``filter3``, and ``exposure_time3``. + +A ``form`` attribute has been added on the ``LCOMultiFilterFacility`` +class, this tells our observation module to use the new +``LCOMultiFilterForm`` instead of the default LCO observation form. + +Modifying the form layout +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Now that the desired fields have been added to the +``LCOMultiFilterForm``, the form’s layout needs to be modified in order +to actually display them. In this example we’ll split the form into two +rows: one row for the three filter choices and exposure times, and +another row for everything else. Note that the default form already has +fields for ``filter`` and ``exposure_time``, so we’ll overwrite the +entire layout so that they appear next to the new fields we added. + +The ``LCOObservationForm`` has a method ``layout()`` that returns the +desired layout using the `crispy forms +Layout `__ +class. Familiarizing yourself with the basic functionality of crispy +forms would be a good idea if you wish to deeply customize your +observation module’s form. + +With our modified layout added, the ``lcomultifilter.py`` file now looks +like this: + +.. code:: python + + # lcomultifilter.py + from tom_observations.facilities.lco import LCOFacility, LCOObservationForm, filter_choices + from django import forms + from crispy_forms.layout import Div + + + class LCOMultiFilterForm(LCOObservationForm): + filter2 = forms.ChoiceField(choices=filter_choices) + exposure_time2 = forms.FloatField(min_value=0.1) + filter3 = forms.ChoiceField(choices=filter_choices) + exposure_time3 = forms.FloatField(min_value=0.1) + + def layout(self): + return Div( + Div( + Div( + 'name', 'proposal', 'ipp_value', 'observation_type', 'start', 'end', + css_class='col' + ), + Div( + 'instrument_name', 'exposure_count', 'max_airmass', + css_class='col' + ), + css_class='form-row' + ), + Div( + Div( + 'filter', 'exposure_time', + css_class='col' + ), + Div( + 'filter2', 'exposure_time2', + css_class='col' + ), + Div( + 'filter3', 'exposure_time3', + css_class='col' + ), + css_class='form-row' + ) + ) + + + class LCOMultiFilterFacility(LCOFacility): + name = 'LCOMultiFilter' + form = LCOMultiFilterForm + +Take a look at the layout and compare it to the `existing lco +layout `__. +A second row has been added that includes all the filter choices. Note +that the original ``filter`` and ``exposure_time`` have been moved from +their original location to the new row. + +Now if you select “LCOMultiFilter” from the list of observation +facilities on a target you should see your new form: + +|image1| + +Is the form still too ugly for you? Trying playing with the layout +definition to suit your needs. + +Changing the form submission behavior +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you are not familiar with the `LCO submission +API `__ now might be a good +time to take a look. The LCO Observation module uses this API to submit +observations using the data provided in the form, so we need to modify +how this happens. More specifically, we’d like to add two additional +``Configuration`` to our observation request, one for each of our +additional filters and exposure times. + +Using the ``observation_payload()`` method, we can use ``super()`` to +get the original LCO module’s observation request, then modify it to +suit the needs of our ``LCOMultiFilter`` class: + +.. code:: python + + #lcomultifilter.py + from tom_observations.facilities.lco import LCOFacility, LCOObservationForm, filter_choices + from django import forms + from crispy_forms.layout import Div + from copy import deepcopy + + class LCOMultiFilterForm(LCOObservationForm): + filter2 = forms.ChoiceField(choices=filter_choices) + exposure_time2 = forms.FloatField(min_value=0.1) + filter3 = forms.ChoiceField(choices=filter_choices) + exposure_time3 = forms.FloatField(min_value=0.1) + + def layout(self): + return Div( + Div( + Div( + 'name', 'proposal', 'ipp_value', 'observation_type', 'start', 'end', + css_class='col' + ), + Div( + 'instrument_type', 'exposure_count', 'max_airmass', + css_class='col' + ), + css_class='form-row' + ), + Div( + Div( + 'filter', 'exposure_time', + css_class='col' + ), + Div( + 'filter2', 'exposure_time2', + css_class='col' + ), + Div( + 'filter3', 'exposure_time3', + css_class='col' + ), + css_class='form-row' + ) + ) + + def observation_payload(self): + payload = super().observation_payload() + configuration2 = deepcopy(payload['requests'][0]['configurations'][0]) + configuration3 = deepcopy(payload['requests'][0]['configurations'][0]) + configuration2['instrument_configs'][0]['optical_elements']['filter'] = self.cleaned_data['filter2'] + configuration2['instrument_configs'][0]['exposure_time'] = self.cleaned_data['exposure_time2'] + configuration3['instrument_configs'][0]['optical_elements']['filter'] = self.cleaned_data['filter3'] + configuration3['instrument_configs'][0]['exposure_time'] = self.cleaned_data['exposure_time3'] + payload['requests'][0]['configurations'].extend([configuration2, configuration3]) + return payload + + + class LCOMultiFilterFacility(LCOFacility): + name = 'LCOMultiFilter' + form = LCOMultiFilterForm + +Let’s go over what we did in this new ``observation_payload()`` method: + +1. Line 1: We call ``super().observation_payload()`` to get the + observation request which the parent class (LCOFacility) would have + called. +2. Line 2-3 We copy the Request’s Configuration into two new + Configurations: ``configuration2`` and ``configuration3``. These will + be the additional Configuration we send to LCO. +3. Lines 5-8: We set the value of these new Configuration ``filter`` and + ``exposure_time`` to the values we collected from our custom form. +4. lines 10-11: Finally, we extend the original Request’s Configuration + array to include the 2 new Configuration we built. Return it and + we’re done! + +If you submit an observation request with the ``LCOMultiFilter`` +observation module now you should see that it creates an observation +request with LCO with three Configuration! + +Summary +~~~~~~~ + +Our original requirement was to be able to submit observations to LCO +with some additional filters and exposure times. We accomplished this +by: + +1. Creating a new observation module: a ``LCOMultiFilterFacility`` class + and a ``LCOMultiFilterForm``, both of which were child classes of the + original ``LCOFacility`` class (since we wanted to keep most of the + functionality intact) and then added this new class to our + ``TOM_FACILITY_CLASSES`` setting. + +2. We added a few fields to ``LCOMultiFilterForm`` and modified it’s + layout to include these new fields using ``layout()``. + +3. We implemented the ``LCOMultiFilterForm`` ``observation_payload()`` + which used the parent’s class return value and then modified it to + suit our needs. + +This is a good example of Object Oriented Programming in Python. If you +are curious about how this all works, we recommend reading up on OOP in +general, as well as how objects in Python 3 work. + +.. |image0| image:: /_static/customize_observations/observebutton.png +.. |image1| image:: /_static/customize_observations/newform.png diff --git a/docs/observing/observation_module.md b/docs/observing/observation_module.md deleted file mode 100644 index bce4bcb64..000000000 --- a/docs/observing/observation_module.md +++ /dev/null @@ -1,299 +0,0 @@ -# Writing an observation module to interface with observatories - -This guide will walk you through how to create a custom observation facility -module using some mocked up endpoints to simulate a real observatory interface. It will also -provide information on creating a custom manual observation facility for tracking observations that -were not created through an API. - -You can use this example as the foundation to build an observing facility module -to connect to a real observatory or track observations on non-API supported facilities. - -Be sure you've followed the [Getting Started](/introduction/getting_started) guide before continuing onto this tutorial. - -### What is a observing facility module? - -A TOM Toolkit observing facility module is a python module which contains the code -necessary to provide an interface to an observing facility in a TOM. Some examples -of existing modules are the [Las Cumbres -Observatory](https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/lco.py) -and the -[Gemini](https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/gemini.py) -modules. Both allow the submission of observation requests to their respective -observatories through a TOM. - -### Prerequisites - -You should have a working TOM already. You can start where the [Getting -Started](/introduction/getting_started) guide leaves off. You should also be familiar with -the observing facility's API that you would like to work with. - - -## Creating a custom robotic facility - -### Defining the minimal implementation - -Within any existing module in your TOM you should create a new python module -(file) named `myfacility.py`. For example, if you have a fresh TOM installation -you'll have a directory structure that looks something like this: - - ├── data - ├── db.sqlite3 - ├── manage.py - ├── mytom - │ ├── __init__.py - │ ├── settings.py - │ ├── urls.py - │ └── wsgi.py - ├── static - ├── templates - └── tmp - -We'll place our `myfacility.py` file inside the `mytom` directory, next to -`settings.py`. For now, copy the following lines into `myfacility.py`: - -```python -from tom_observations.facility import BaseRoboticObservationFacility, BaseRoboticObservationForm - - -class MyObservationFacilityForm(BaseRoboticObservationForm): - pass - - -class MyObservationFacility(BaseRoboticObservationFacility): - name = 'MyFacility' - observation_types = [('OBSERVATION', 'Custom Observation')] -``` - -We'll go over what these lines mean soon. First, we'll add a setting to our -project's `settings.py` to tell the TOM Toolkit to use our new class: - -```python -TOM_FACILITY_CLASSES = [ - 'tom_observations.facilities.lco.LCOFacility', - 'tom_observations.facilities.gemini.GEMFacility', - 'mytom.myfacility.MyObservationFacility' -] -``` - -Now go ahead and view a target in your TOM, you should see something like this: - -![](/_static/observation_module/myfacility.png) - -This means our new observation facility module has been successfully loaded. - - -### BaseRoboticObservationFacility and BaseRoboticObservationForm - -You will have noticed our module consists of two classes that inherit from two -other classes. - -`MyObservationFacility` is the class that will contain the "business logic" -for interacting with the remote observatory. This includes methods to submit -observations, check observation status, etc. It inherits from -`BaseRoboticObservationFacility`, which contains some functionality that all -observation facility classes will want. - -`MyObservationFacilityForm` is the class that will display a GUI form for our -users to create an observation. We can submit observations programmatically, but it -is also nice to have a GUI for our users to use. The `BaseRoboticObservationForm` -class, just like the previous super class, contains logic and layout that all -observation facility form classes should contain. - -### Implementing observation submission - -Try to click on the button for `MyFacility`. -It should return an error that says everything it's missing: - -``` -Can't instantiate abstract class MyObservationFacility with abstract methods -data_products, get_form, get_observation_status, get_observation_url, get_observing_sites, -get_terminal_observing_states, submit_observation, validate_observation -``` - -To start, let's define new functions in `MyObservationFacility` -for each missing function like so: - -```python -class MyObservationFacility(BaseRoboticObservationFacility): - name = 'MyFacility' - observation_types = [('OBSERVATION', 'Custom Observation')] - - def data_products(self): - return - - def get_form(self): - return - ... -``` - -Reload the server, click the `MyFacility` button, and you should get . . . -a different error! Progress! - -``` -get_form() takes 1 positional argument but 2 were given -``` - -To fix up `get_form`, adjust it to: - -```python - def get_form(self, observation_type): - return MyObservationFacilityForm -``` - -Reload the page and now it should look something like this: - -![](/_static/observation_module/empty_form.png) - -Some notes: -1. The form is empty, but we'll fix that next. -2. The `name` variable of `MyObservationFacility` determines what the top of -the page says (`Submit an observation to MyFacility`). -It also determines the name of the button under "Observe" on the target's page. -3. You should see a tab for `Custom Observation` as the only option on the page. - This is read from the `observation_types` variable in `MyObservationFacility`. -That variable is a list of 2-tuples. -The second value of each tuple is what will be displayed on the webpage, -as different tabs of observation types to submit. -The first value of each tuple is what should be used to distinguish -different observation types in your code. -To see a demonstration of this, check out the -[Las Cumbres Observatory](https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/lco.py) -facility's `observation_types` and `get_form`. - -Now let's populate the form. -Let's assume our observatory only requires us to send 2 parameters -(besides the target data): exposure\_time and exposure\_count. Let's start by -adding them to our form class: - -```python -from django import forms -from tom_observations.facility import GenericObservationFacility, GenericObservationForm - - -class MyObservationFacilityForm(GenericObservationForm): - exposure_time = forms.IntegerField() - exposure_count = forms.IntegerField() -``` - -Notice that we've added the two field definitions on our form. We've also imported -the django form module with `from django import forms`. - -Now if we reload the page, we should see something like this: - -![](/_static/observation_module/fields.png) - - -This is progress, but remember that most of the functions in `MyObservationFacility` -have blank return statements. -Next we'll implement the methods that perform actions with our form when we -submit the observation request: - -```python -from django import forms -from tom_observations.facility import BaseRoboticObservationFacility, BaseRoboticObservationForm - -class MyObservationFacilityForm(BaseRoboticObservationForm): - exposure_time = forms.IntegerField() - exposure_count = forms.IntegerField() - -class MyObservationFacility(BaseRoboticObservationFacility): - name = 'MyFacility' - observation_types = [('OBSERVATION', 'Custom Observation')] - - def data_products(self, observation_id, product_id=None): - return [] - - def get_form(self, observation_type): - return MyObservationFacilityForm - - def get_observation_status(self, observation_id): - return ['IN_PROGRESS'] - - def get_observation_url(self, observation_id): - return '' - - def get_observing_sites(self): - return {} - - def get_terminal_observing_states(self): - return ['IN_PROGRESS', 'COMPLETED'] - - def submit_observation(self, observation_payload): - print(observation_payload) - return [1] - - def validate_observation(self, observation_payload): - pass - -``` - -The important method here is `submit_observation`. This method, when implemented -fully, will send the observation payload to the remote observatory and then return -a list of observation ids. Those ids will be stored in the database to be used -later, in methods like `get_observation_status(self, observation_id)`. In our -dummy implementation, we simply print out the observation payload and return a -single fake id with `return [1]`. - -If you now "submit" an observation using the MyFacility module, you should see -this in the server console: - - {'target_id': 1, 'params': '{"facility": "MyFacility", "target_id": 1, "observation_type": "(\'OBSERVATION\', \'Custom Observation\')", "exposure_time": 100, "exposure_count": 2}'} - -That was our print statement! -Additionally, you should see `1 upcoming observation` on the target's page, -and if you navigate to its "Observations" tab you can see the parameters of the - observation you just submitted in more detail. - -### Filling in the rest of the functionality -You'll notice we added many more methods other than `submit_observation` to our -Facility class. For now they return dummy data, but when you adapt it to work with -a real observatory you should fill them in with the correct logic so that the -whole module works correctly with the TOM. You can view explanations of each -method [in the source -code](https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facility.py#L142) - -###Airmass plotting for new facilities -The last step in adding a new facility is to get it to appear on airmass plots. -If you input two dates into the "Plan" form under the "Observe" tab -on a target's page, you'll see the target's visibility. -By default, the plot shows you the airmass at LCO and Gemini sites. - -In our `MyObservationFacility` class, let's define a new variable called `SITES`. -Modeling our `SITES` on the one defined for -[Las Cumbres Observatory](https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/lco.py), -we can easily put new sites into the airmass plots: - -```python -class MyObservationFacility(BaseRoboticObservationFacility): - name = 'MyFacility' - observation_types = [('OBSERVATION', 'Custom Observation')] - - SITES = { - 'Itagaki': { - 'latitude': 38.188020, - 'longitude': 140.335113, - 'elevation': 350 - } - } - - ... - - def get_observing_sites(self): - return self.SITES - -``` - -(Koichi Itagaki is an "amateur" astronomer in Japan who has discovered -many extremely interesting supernovae.) - -Now the new observatory site should show up when you generate airmass plots. -Even if the facilities you observe at are not -API-accessible, you can still add them to your TOM's airmass plots -to judge what targets to observe when. - -Happy developing! - - -## Creating a custom manual facility - - diff --git a/docs/observing/observation_module.rst b/docs/observing/observation_module.rst new file mode 100644 index 000000000..6dca67ae1 --- /dev/null +++ b/docs/observing/observation_module.rst @@ -0,0 +1,321 @@ +Writing an observation module to interface with observatories +============================================================= + +This guide will walk you through how to create a custom observation +facility module using some mocked up endpoints to simulate a real +observatory interface. It will also provide information on creating a +custom manual observation facility for tracking observations that were +not created through an API. + +You can use this example as the foundation to build an observing +facility module to connect to a real observatory or track observations +on non-API supported facilities. + +Be sure you’ve followed the `Getting +Started `__ guide before continuing onto +this tutorial. + +What is a observing facility module? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A TOM Toolkit observing facility module is a python module which +contains the code necessary to provide an interface to an observing +facility in a TOM. Some examples of existing modules are the `Las +Cumbres +Observatory `__ +and the +`Gemini `__ +modules. Both allow the submission of observation requests to their +respective observatories through a TOM. + +Prerequisites +~~~~~~~~~~~~~ + +You should have a working TOM already. You can start where the `Getting +Started `__ guide leaves off. You should +also be familiar with the observing facility’s API that you would like +to work with. + +Creating a custom robotic facility +---------------------------------- + +Defining the minimal implementation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Within any existing module in your TOM you should create a new python +module (file) named ``myfacility.py``. For example, if you have a fresh +TOM installation you’ll have a directory structure that looks something +like this: + +:: + + ├── data + ├── db.sqlite3 + ├── manage.py + ├── mytom + │ ├── __init__.py + │ ├── settings.py + │ ├── urls.py + │ └── wsgi.py + ├── static + ├── templates + └── tmp + +We’ll place our ``myfacility.py`` file inside the ``mytom`` directory, +next to ``settings.py``. For now, copy the following lines into +``myfacility.py``: + +.. code:: python + + from tom_observations.facility import BaseRoboticObservationFacility, BaseRoboticObservationForm + + + class MyObservationFacilityForm(BaseRoboticObservationForm): + pass + + + class MyObservationFacility(BaseRoboticObservationFacility): + name = 'MyFacility' + observation_types = [('OBSERVATION', 'Custom Observation')] + +We’ll go over what these lines mean soon. First, we’ll add a setting to +our project’s ``settings.py`` to tell the TOM Toolkit to use our new +class: + +.. code:: python + + TOM_FACILITY_CLASSES = [ + 'tom_observations.facilities.lco.LCOFacility', + 'tom_observations.facilities.gemini.GEMFacility', + 'mytom.myfacility.MyObservationFacility' + ] + +Now go ahead and view a target in your TOM, you should see something +like this: + +|image0| + +This means our new observation facility module has been successfully +loaded. + +BaseRoboticObservationFacility and BaseRoboticObservationForm +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You will have noticed our module consists of two classes that inherit +from two other classes. + +``MyObservationFacility`` is the class that will contain the “business +logic” for interacting with the remote observatory. This includes +methods to submit observations, check observation status, etc. It +inherits from ``BaseRoboticObservationFacility``, which contains some +functionality that all observation facility classes will want. + +``MyObservationFacilityForm`` is the class that will display a GUI form +for our users to create an observation. We can submit observations +programmatically, but it is also nice to have a GUI for our users to +use. The ``BaseRoboticObservationForm`` class, just like the previous +super class, contains logic and layout that all observation facility +form classes should contain. + +Implementing observation submission +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Try to click on the button for ``MyFacility``. It should return an error +that says everything it’s missing: + +:: + + Can't instantiate abstract class MyObservationFacility with abstract methods + data_products, get_form, get_observation_status, get_observation_url, get_observing_sites, + get_terminal_observing_states, submit_observation, validate_observation + +To start, let’s define new functions in ``MyObservationFacility`` for +each missing function like so: + +.. code:: python + + class MyObservationFacility(BaseRoboticObservationFacility): + name = 'MyFacility' + observation_types = [('OBSERVATION', 'Custom Observation')] + + def data_products(self): + return + + def get_form(self): + return + ... + +Reload the server, click the ``MyFacility`` button, and you should get . +. . a different error! Progress! + +:: + + get_form() takes 1 positional argument but 2 were given + +To fix up ``get_form``, adjust it to: + +.. code:: python + + def get_form(self, observation_type): + return MyObservationFacilityForm + +Reload the page and now it should look something like this: + +|image1| + +Some notes: 1. The form is empty, but we’ll fix that next. 2. The +``name`` variable of ``MyObservationFacility`` determines what the top +of the page says (``Submit an observation to MyFacility``). It also +determines the name of the button under “Observe” on the target’s page. +3. You should see a tab for ``Custom Observation`` as the only option on +the page. This is read from the ``observation_types`` variable in +``MyObservationFacility``. That variable is a list of 2-tuples. The +second value of each tuple is what will be displayed on the webpage, as +different tabs of observation types to submit. The first value of each +tuple is what should be used to distinguish different observation types +in your code. To see a demonstration of this, check out the `Las Cumbres +Observatory `__ +facility’s ``observation_types`` and ``get_form``. + +Now let’s populate the form. Let’s assume our observatory only requires +us to send 2 parameters (besides the target data): exposure_time and +exposure_count. Let’s start by adding them to our form class: + +.. code:: python + + from django import forms + from tom_observations.facility import GenericObservationFacility, GenericObservationForm + + + class MyObservationFacilityForm(GenericObservationForm): + exposure_time = forms.IntegerField() + exposure_count = forms.IntegerField() + +Notice that we’ve added the two field definitions on our form. We’ve +also imported the django form module with ``from django import forms``. + +Now if we reload the page, we should see something like this: + +|image2| + +This is progress, but remember that most of the functions in +``MyObservationFacility`` have blank return statements. Next we’ll +implement the methods that perform actions with our form when we submit +the observation request: + +.. code:: python + + from django import forms + from tom_observations.facility import BaseRoboticObservationFacility, BaseRoboticObservationForm + + class MyObservationFacilityForm(BaseRoboticObservationForm): + exposure_time = forms.IntegerField() + exposure_count = forms.IntegerField() + + class MyObservationFacility(BaseRoboticObservationFacility): + name = 'MyFacility' + observation_types = [('OBSERVATION', 'Custom Observation')] + + def data_products(self, observation_id, product_id=None): + return [] + + def get_form(self, observation_type): + return MyObservationFacilityForm + + def get_observation_status(self, observation_id): + return ['IN_PROGRESS'] + + def get_observation_url(self, observation_id): + return '' + + def get_observing_sites(self): + return {} + + def get_terminal_observing_states(self): + return ['IN_PROGRESS', 'COMPLETED'] + + def submit_observation(self, observation_payload): + print(observation_payload) + return [1] + + def validate_observation(self, observation_payload): + pass + +The important method here is ``submit_observation``. This method, when +implemented fully, will send the observation payload to the remote +observatory and then return a list of observation ids. Those ids will be +stored in the database to be used later, in methods like +``get_observation_status(self, observation_id)``. In our dummy +implementation, we simply print out the observation payload and return a +single fake id with ``return [1]``. + +If you now “submit” an observation using the MyFacility module, you +should see this in the server console: + +:: + + {'target_id': 1, 'params': '{"facility": "MyFacility", "target_id": 1, "observation_type": "(\'OBSERVATION\', \'Custom Observation\')", "exposure_time": 100, "exposure_count": 2}'} + +That was our print statement! Additionally, you should see +``1 upcoming observation`` on the target’s page, and if you navigate to +its “Observations” tab you can see the parameters of the observation you +just submitted in more detail. + +Filling in the rest of the functionality +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You’ll notice we added many more methods other than +``submit_observation`` to our Facility class. For now they return dummy +data, but when you adapt it to work with a real observatory you should +fill them in with the correct logic so that the whole module works +correctly with the TOM. You can view explanations of each method `in the +source +code `__ + +###Airmass plotting for new facilities The last step in adding a new +facility is to get it to appear on airmass plots. If you input two dates +into the “Plan” form under the “Observe” tab on a target’s page, you’ll +see the target’s visibility. By default, the plot shows you the airmass +at LCO and Gemini sites. + +In our ``MyObservationFacility`` class, let’s define a new variable +called ``SITES``. Modeling our ``SITES`` on the one defined for `Las +Cumbres +Observatory `__, +we can easily put new sites into the airmass plots: + +.. code:: python + + class MyObservationFacility(BaseRoboticObservationFacility): + name = 'MyFacility' + observation_types = [('OBSERVATION', 'Custom Observation')] + + SITES = { + 'Itagaki': { + 'latitude': 38.188020, + 'longitude': 140.335113, + 'elevation': 350 + } + } + + ... + + def get_observing_sites(self): + return self.SITES + +(Koichi Itagaki is an “amateur” astronomer in Japan who has discovered +many extremely interesting supernovae.) + +Now the new observatory site should show up when you generate airmass +plots. Even if the facilities you observe at are not API-accessible, you +can still add them to your TOM’s airmass plots to judge what targets to +observe when. + +Happy developing! + +Creating a custom manual facility +--------------------------------- + +.. |image0| image:: /_static/observation_module/myfacility.png +.. |image1| image:: /_static/observation_module/empty_form.png +.. |image2| image:: /_static/observation_module/fields.png diff --git a/docs/observing/strategies.md b/docs/observing/strategies.md deleted file mode 100644 index b5d68e4da..000000000 --- a/docs/observing/strategies.md +++ /dev/null @@ -1,218 +0,0 @@ -# Cadence and Observing Strategies - -The TOM has a couple of unique concepts that may be unfamiliar to some at first, that will be describe here before going -into detail. - -The first concept is that of an observing strategy. An observing strategy is something of a template. If an observer is -consistently submitting observations with a lot of similar parameters, it may be useful to save those as a kind of -template, which can just be loaded later. The TOM Toolkit offers an interface that allows facilities to define a -strategy form, that will be saved as an observing strategy. The strategy can then be applied to an observation, with the -remaining parameters filled in or changed. An observing strategy can also be creating from a past observation, with a -button to do so that's available on any ObservationRecord detail page. - -The second concept referred to is a cadence strategy. A cadence is as it sounds--a series of observations that are -performed at regular intervals. However, most observatories don't have built-in support for cadences, and, if they do, -they may be limited to a predetermined cadence. The TOM Toolkit, on the other hand, allows for a *reactive* cadence. -Because data is collected programmatically, and observations are submitted programmatically, a user can write their own -cadence to submit observations depending on the success of a prior observation or the data collected from a prior -observation. - - -## Writing a custom cadence strategy - -Many of the TOM modules leverage a plugin architecture that enables you to write your own implementation, and the -cadence strategy plugin is no different. If you're familiar with the other modules, you've already seen examples of this -in the :doc:`Writing an alert broker <../customization/create_broker>`, :doc:`Writing an observation module -`, and :doc:`Customizing data processing <../customization/customizing_data_processing>` tutorials. - - -Create a cadence strategy file ------------------------------- - -First, you'll need a file where you'll put your custom cadence strategy. If you have a fresh TOM installation, you'll -have a directory structure that looks something like this: - - ├── data - ├── db.sqlite3 - ├── manage.py - ├── mytom - │ ├── __init__.py - │ ├── settings.py - │ ├── urls.py - │ └── wsgi.py - ├── static - ├── templates - └── tmp - -We'll create a new file called ``mycadence.py`` and place it next to ``settings.py``. To get started, we'll just put a -small skeleton into our new file, so to begin with, it should look like this: - -```python -from tom_observations.cadence import CadenceStrategy - -class MyCadenceStrategy(CadenceStrategy): - pass -``` - -We also need to add the cadence strategy to ``settings.py`` so that our TOM knows that it exists: - -```python -TOM_CADENCE_STRATEGIES = [ - 'tom_observations.cadence.RetryFailedObservationsStrategy', - 'tom_observations.cadence.ResumeCadenceAfterFailureStrategy', - 'mytom.mycadence.MyCadenceStrategy' -] -``` - -Add logic to the new cadence strategy -------------------------------------- - -You may have noticed that our ``MyCadenceStrategy`` class inherits from ``CadenceStrategy``. The ``CadenceStrategy`` -interface only has one method, which is ``run()``. All of the logic for a ``CadenceStrategy`` lives in the ``run()`` -method. Rather than demonstrating the implementation of a new cadence strategy, this tutorial is going to walk through -the business logic of a built-in cadence strategy. We're going to review the ``ResumeCadenceAfterFailureStrategy``. - -It should also be worth mentioning at this point that the ``CadenceStrategy`` constructor takes an ``observation group``. -The ``observation_group`` is the set of observations that make up the cadence, and is created in the ``ObservationCreateView`` -when the first observation of a cadence is submitted. - -The ``ResumeCadenceAfterFailureStrategy`` is designed to ensure that, even after an observation fails, the cadence remains -consistent. If, for example, you submit an observation with a cadence of three days, and the observation fails, the cadence -should attempt to get the observation as soon as possible, and then resume observing once every three days. - -Let's look at the strategy piece by piece. - -```python -last_obs = self.observation_group.observation_records.order_by('-created').first() -facility = get_service_class(last_obs.facility)() -facility.update_observation_status(last_obs.observation_id) -last_obs.refresh_from_db() -``` - -The first thing this strategy does is get a couple of pieces of information. First, from the observation group that the -cadence consists of, the most recent observation is selected. The facility class for the facility that the cadence is -submitting observations to is also instantiated. With these values, the status of the most recent cadence observation is -updated, and the ``ObservationRecord`` object is refreshed. - -```python -start_keyword, end_keyword = facility.get_start_end_keywords() -observation_payload = last_obs.parameters_as_dict -new_observations = [] -``` - -These lines are, again, just more setup. Each facility has its own unique keywords representing the start and the end of -the observation window, so we get those from the facility class. Then, we get the original observation parameters that -were submitted to the facility, and we initialize a list for any new observations that will be submitted when the cadence -is updated. - -```python -if not last_obs.terminal: - return -elif last_obs.failed: - # Submit next observation to be taken as soon as possible - window_length = parse(observation_payload[end_keyword]) - parse(observation_payload[start_keyword]) - observation_payload[start_keyword] = datetime.now().isoformat() - observation_payload[end_keyword] = (parse(observation_payload[start_keyword]) + window_length).isoformat() -else: - # Advance window normally according to cadence parameters - observation_payload = self.advance_window( - observation_payload, start_keyword=start_keyword, end_keyword=end_keyword - ) -``` - -Here we have some logic for the three cases--either the most recent observation hasn't happened yet, it failed, or it succeeded. -If it hasn't happened, then there's nothing to do--we'll check again later. If if failed, we want to submit it again to be taken -immediately, so we get the original length of the observation window, and set our new observation payload to start immediately, -and end such that the new window length is the same. Finally, if our observation succeeded, we update our new observation -parameters to start 72 hours after the last observation, using a utility method that's part of the -``ResumeCadenceAfterFailureStrategy`` called ``advance_window``. - -```python -obs_type = last_obs.parameters_as_dict.get('observation_type') -form = facility.get_form(obs_type)(observation_payload) -form.is_valid() -observation_ids = facility.submit_observation(form.observation_payload()) - -for observation_id in observation_ids: - # Create Observation record - record = ObservationRecord.objects.create( - target=last_obs.target, - facility=facility.name, - parameters=json.dumps(observation_payload), - observation_id=observation_id - ) - self.observation_group.observation_records.add(record) - self.observation_group.save() - new_observations.append(record) - - for obsr in new_observations: - facility = get_service_class(obsr.facility)() - facility.update_observation_status(obsr.observation_id) - - return new_observations -``` - -The last part of our strategy is when we submit our new observations. Regardless of how we modified the observing window, -we initialize our observation form, validate it, and submit the observation to our facility. The rest of the code is -saving any resulting observations to the database, getting their new status from the facility, and returning them. - -Just to review, here is the strategy's ``run()`` in its entirety: - -```python -def run(self): - last_obs = self.observation_group.observation_records.order_by('-created').first() - facility = get_service_class(last_obs.facility)() - facility.update_observation_status(last_obs.observation_id) - last_obs.refresh_from_db() - start_keyword, end_keyword = facility.get_start_end_keywords() - observation_payload = last_obs.parameters_as_dict - new_observations = [] - if not last_obs.terminal: - return - elif last_obs.failed: - # Submit next observation to be taken as soon as possible - window_length = parse(observation_payload[end_keyword]) - parse(observation_payload[start_keyword]) - observation_payload[start_keyword] = datetime.now().isoformat() - observation_payload[end_keyword] = (parse(observation_payload[start_keyword]) + window_length).isoformat() - else: - # Advance window normally according to cadence parameters - observation_payload = self.advance_window( - observation_payload, start_keyword=start_keyword, end_keyword=end_keyword - ) - - obs_type = last_obs.parameters_as_dict.get('observation_type') - form = facility.get_form(obs_type)(observation_payload) - form.is_valid() - observation_ids = facility.submit_observation(form.observation_payload()) - - for observation_id in observation_ids: - # Create Observation record - record = ObservationRecord.objects.create( - target=last_obs.target, - facility=facility.name, - parameters=json.dumps(observation_payload), - observation_id=observation_id - ) - self.observation_group.observation_records.add(record) - self.observation_group.save() - new_observations.append(record) - - for obsr in new_observations: - facility = get_service_class(obsr.facility)() - facility.update_observation_status(obsr.observation_id) - - return new_observations -``` - - -## Configuring the cadence strategy to run automatically - -As you may have noticed, the cadence strategies act on updates to the status of an ``ObservationRecord``. Ideally, we want -the cadence strategies to run as soon as an observation status changes--so, we need to automate that and have it run -periodically. - -Fortunately, the TOM Toolkit comes with a built-in management command to update all cadences in the TOM. If you've perused -the TOM Toolkit documentation previously, you may have noticed a section about automation of tasks, and, more specifically, -a subsection about :doc:`Using cron with a management command <../customization/automation>`. You can simply apply the -instructions here, but use the management command ``runcadencestrategies.py`` in place of the example. If you set your cron -to run every few minutes or so, you'll ensure that your cadences are kept up to date! \ No newline at end of file diff --git a/docs/observing/strategies.rst b/docs/observing/strategies.rst new file mode 100644 index 000000000..87dd6c2d9 --- /dev/null +++ b/docs/observing/strategies.rst @@ -0,0 +1,258 @@ +Cadence and Observing Strategies +================================ + +The TOM has a couple of unique concepts that may be unfamiliar to some +at first, that will be describe here before going into detail. + +The first concept is that of an observing strategy. An observing +strategy is something of a template. If an observer is consistently +submitting observations with a lot of similar parameters, it may be +useful to save those as a kind of template, which can just be loaded +later. The TOM Toolkit offers an interface that allows facilities to +define a strategy form, that will be saved as an observing strategy. The +strategy can then be applied to an observation, with the remaining +parameters filled in or changed. An observing strategy can also be +creating from a past observation, with a button to do so that’s +available on any ObservationRecord detail page. + +The second concept referred to is a cadence strategy. A cadence is as it +sounds–a series of observations that are performed at regular intervals. +However, most observatories don’t have built-in support for cadences, +and, if they do, they may be limited to a predetermined cadence. The TOM +Toolkit, on the other hand, allows for a *reactive* cadence. Because +data is collected programmatically, and observations are submitted +programmatically, a user can write their own cadence to submit +observations depending on the success of a prior observation or the data +collected from a prior observation. + +Writing a custom cadence strategy +--------------------------------- + +Many of the TOM modules leverage a plugin architecture that enables you +to write your own implementation, and the cadence strategy plugin is no +different. If you’re familiar with the other modules, you’ve already +seen examples of this in the +:doc:``Writing an alert broker <../customization/create_broker>``, +:doc:``Writing an observation module ``, and +:doc:``Customizing data processing <../customization/customizing_data_processing>`` +tutorials. + +Create a cadence strategy file +------------------------------ + +First, you’ll need a file where you’ll put your custom cadence strategy. +If you have a fresh TOM installation, you’ll have a directory structure +that looks something like this: + +:: + + ├── data + ├── db.sqlite3 + ├── manage.py + ├── mytom + │ ├── __init__.py + │ ├── settings.py + │ ├── urls.py + │ └── wsgi.py + ├── static + ├── templates + └── tmp + +We’ll create a new file called ``mycadence.py`` and place it next to +``settings.py``. To get started, we’ll just put a small skeleton into +our new file, so to begin with, it should look like this: + +.. code:: python + + from tom_observations.cadence import CadenceStrategy + + class MyCadenceStrategy(CadenceStrategy): + pass + +We also need to add the cadence strategy to ``settings.py`` so that our +TOM knows that it exists: + +.. code:: python + + TOM_CADENCE_STRATEGIES = [ + 'tom_observations.cadence.RetryFailedObservationsStrategy', + 'tom_observations.cadence.ResumeCadenceAfterFailureStrategy', + 'mytom.mycadence.MyCadenceStrategy' + ] + +Add logic to the new cadence strategy +------------------------------------- + +You may have noticed that our ``MyCadenceStrategy`` class inherits from +``CadenceStrategy``. The ``CadenceStrategy`` interface only has one +method, which is ``run()``. All of the logic for a ``CadenceStrategy`` +lives in the ``run()`` method. Rather than demonstrating the +implementation of a new cadence strategy, this tutorial is going to walk +through the business logic of a built-in cadence strategy. We’re going +to review the ``ResumeCadenceAfterFailureStrategy``. + +It should also be worth mentioning at this point that the +``CadenceStrategy`` constructor takes an ``observation group``. The +``observation_group`` is the set of observations that make up the +cadence, and is created in the ``ObservationCreateView`` when the first +observation of a cadence is submitted. + +The ``ResumeCadenceAfterFailureStrategy`` is designed to ensure that, +even after an observation fails, the cadence remains consistent. If, for +example, you submit an observation with a cadence of three days, and the +observation fails, the cadence should attempt to get the observation as +soon as possible, and then resume observing once every three days. + +Let’s look at the strategy piece by piece. + +.. code:: python + + last_obs = self.observation_group.observation_records.order_by('-created').first() + facility = get_service_class(last_obs.facility)() + facility.update_observation_status(last_obs.observation_id) + last_obs.refresh_from_db() + +The first thing this strategy does is get a couple of pieces of +information. First, from the observation group that the cadence consists +of, the most recent observation is selected. The facility class for the +facility that the cadence is submitting observations to is also +instantiated. With these values, the status of the most recent cadence +observation is updated, and the ``ObservationRecord`` object is +refreshed. + +.. code:: python + + start_keyword, end_keyword = facility.get_start_end_keywords() + observation_payload = last_obs.parameters_as_dict + new_observations = [] + +These lines are, again, just more setup. Each facility has its own +unique keywords representing the start and the end of the observation +window, so we get those from the facility class. Then, we get the +original observation parameters that were submitted to the facility, and +we initialize a list for any new observations that will be submitted +when the cadence is updated. + +.. code:: python + + if not last_obs.terminal: + return + elif last_obs.failed: + # Submit next observation to be taken as soon as possible + window_length = parse(observation_payload[end_keyword]) - parse(observation_payload[start_keyword]) + observation_payload[start_keyword] = datetime.now().isoformat() + observation_payload[end_keyword] = (parse(observation_payload[start_keyword]) + window_length).isoformat() + else: + # Advance window normally according to cadence parameters + observation_payload = self.advance_window( + observation_payload, start_keyword=start_keyword, end_keyword=end_keyword + ) + +Here we have some logic for the three cases–either the most recent +observation hasn’t happened yet, it failed, or it succeeded. If it +hasn’t happened, then there’s nothing to do–we’ll check again later. If +if failed, we want to submit it again to be taken immediately, so we get +the original length of the observation window, and set our new +observation payload to start immediately, and end such that the new +window length is the same. Finally, if our observation succeeded, we +update our new observation parameters to start 72 hours after the last +observation, using a utility method that’s part of the +``ResumeCadenceAfterFailureStrategy`` called ``advance_window``. + +.. code:: python + + obs_type = last_obs.parameters_as_dict.get('observation_type') + form = facility.get_form(obs_type)(observation_payload) + form.is_valid() + observation_ids = facility.submit_observation(form.observation_payload()) + + for observation_id in observation_ids: + # Create Observation record + record = ObservationRecord.objects.create( + target=last_obs.target, + facility=facility.name, + parameters=json.dumps(observation_payload), + observation_id=observation_id + ) + self.observation_group.observation_records.add(record) + self.observation_group.save() + new_observations.append(record) + + for obsr in new_observations: + facility = get_service_class(obsr.facility)() + facility.update_observation_status(obsr.observation_id) + + return new_observations + +The last part of our strategy is when we submit our new observations. +Regardless of how we modified the observing window, we initialize our +observation form, validate it, and submit the observation to our +facility. The rest of the code is saving any resulting observations to +the database, getting their new status from the facility, and returning +them. + +Just to review, here is the strategy’s ``run()`` in its entirety: + +.. code:: python + + def run(self): + last_obs = self.observation_group.observation_records.order_by('-created').first() + facility = get_service_class(last_obs.facility)() + facility.update_observation_status(last_obs.observation_id) + last_obs.refresh_from_db() + start_keyword, end_keyword = facility.get_start_end_keywords() + observation_payload = last_obs.parameters_as_dict + new_observations = [] + if not last_obs.terminal: + return + elif last_obs.failed: + # Submit next observation to be taken as soon as possible + window_length = parse(observation_payload[end_keyword]) - parse(observation_payload[start_keyword]) + observation_payload[start_keyword] = datetime.now().isoformat() + observation_payload[end_keyword] = (parse(observation_payload[start_keyword]) + window_length).isoformat() + else: + # Advance window normally according to cadence parameters + observation_payload = self.advance_window( + observation_payload, start_keyword=start_keyword, end_keyword=end_keyword + ) + + obs_type = last_obs.parameters_as_dict.get('observation_type') + form = facility.get_form(obs_type)(observation_payload) + form.is_valid() + observation_ids = facility.submit_observation(form.observation_payload()) + + for observation_id in observation_ids: + # Create Observation record + record = ObservationRecord.objects.create( + target=last_obs.target, + facility=facility.name, + parameters=json.dumps(observation_payload), + observation_id=observation_id + ) + self.observation_group.observation_records.add(record) + self.observation_group.save() + new_observations.append(record) + + for obsr in new_observations: + facility = get_service_class(obsr.facility)() + facility.update_observation_status(obsr.observation_id) + + return new_observations + +Configuring the cadence strategy to run automatically +----------------------------------------------------- + +As you may have noticed, the cadence strategies act on updates to the +status of an ``ObservationRecord``. Ideally, we want the cadence +strategies to run as soon as an observation status changes–so, we need +to automate that and have it run periodically. + +Fortunately, the TOM Toolkit comes with a built-in management command to +update all cadences in the TOM. If you’ve perused the TOM Toolkit +documentation previously, you may have noticed a section about +automation of tasks, and, more specifically, a subsection about +:doc:``Using cron with a management command <../customization/automation>``. +You can simply apply the instructions here, but use the management +command ``runcadencestrategies.py`` in place of the example. If you set +your cron to run every few minutes or so, you’ll ensure that your +cadences are kept up to date! diff --git a/docs/targets/target_fields.md b/docs/targets/target_fields.md deleted file mode 100644 index dbf3838e7..000000000 --- a/docs/targets/target_fields.md +++ /dev/null @@ -1,134 +0,0 @@ -Adding Custom Fields to Targets ---- - - -Sometimes you'd like to store data for targets but the predefined fields that the -TOM Toolkit provides aren't enough. The TOM Toolkit allows you to define extra -fields for your targets so you can associate different kinds of data with them. -For example, you might be studying high redshift galaxies. In this case, it would -make sense to be able to store the redshift of your targets. You could then do a -search for targets with a redshift less than or greater than a particular value, -or use the redshift value to make decisions in your science code. - -**Note**: There is a performance hit when using extra fields. Try to use the -built in fields whenever possible. - -### Enabling extra fields - -To start, find the `EXTRA_FIELDS` definition in your `settings.py`: - -```python -# Define extra target fields here. Types can be any of "number", "string", "boolean" or "datetime" -# For example: -# EXTRA_FIELDS = [ -# {'name': 'redshift', 'type': 'number'}, -# {'name': 'discoverer', 'type': 'string'} -# {'name': 'eligible', 'type': 'boolean'}, -# {'name': 'dicovery_date', 'type': 'datetime'} -# ] -EXTRA_FIELDS = [] -``` - -We can define any number of extra fields in the array. Each item in the array -is a dictionary with two values: name and type. Name is simply what you would like -to name your field. Type is the datatype of the field and can be one of: `number`, -`string`, `boolean` or `datetime`. These types allow the TOM Toolkit to properly -store, filter and display these values elsewhere. - -As an example, let's change the setting to look like this: - -```python - EXTRA_FIELDS = [ - {'name': 'redshift', 'type': 'number'}, - ] -``` - -This will make an extra field with the name "redshift" and a type of "number" -available to add to our targets. - -### Using extra fields - -Now if you go to the target creation page, you should see the new field available: - -![](/_static/target_fields_doc/redshift.png) - -And if we go to our list of targets, we should see redshift as a field available -to filter on: - -![](/_static/target_fields_doc/redshift_filter.png) - -Extra fields with the `number` type allow filtering on range of values. The same -goes for fields with the `datetime` type. `string` types to a case insensitive -inclusive search, and `boolean` fields to a simple matching comparison. - -Of course, redshift does appear on our target's display page as well: - -![](/_static/target_fields_doc/redshift_display.png) - -To hide extra fields from the target page, we can set the "hidden" key (this -doesn't affect filtering and searching): - -```python - EXTRA_FIELDS = [ - {'name': 'redshift', 'type': 'number', 'hidden': True}, - ] -``` - -And we can set a default value for an extra field by including a default key/value pair: - -```python - EXTRA_FIELDS = [ - {'name': 'redshift', 'type': 'number', 'default': 0}, - ] -``` - -### Displaying extra fields in templates - -If we want to display the redshift in other places, we can use a template filter to -do that. For example, we might want to display the redshift value in the target -list table. - -At the top of our template make sure to load `targets_extras`: - -``` -{% raw %} - {% load targets_extras %} -{% endraw %} -``` - -Now we can use the `target_extra_field` filter wherever a target object is -available in the template context: - -``` -{% raw %} - {{ target|target_extra_field:"redshift" }} -{% endraw %} -``` - -The result is the redshift value being printed on the template: - -![](/_static/target_fields_doc/redshift_tag.png) - -### Working with extra fields programatically - -If you'd like to update or save extra fields to your targets in code, there are a -few methods you can use. The simplest is to simply pass in a dictionary of extra data to your -target's `save()` method using the `extras` keyword argument: - -```python -target = Target.objects.get(name='example') -target.save(extras={'foo': 42}) -``` - -The example target above will now have an extra field "foo" with the value 42. - -For more precise control, you can access `TargetExtra` models directly. To remove -an extra, for example: - -```python -target = Target.objects.get(name='example') -target_extra = target.targetextra_set.get(key='foo') -target_extra.delete() -``` - -The above deleted the target extra on a target with the key of "foo". diff --git a/docs/targets/target_fields.rst b/docs/targets/target_fields.rst new file mode 100644 index 000000000..59c416194 --- /dev/null +++ b/docs/targets/target_fields.rst @@ -0,0 +1,149 @@ +Adding Custom Fields to Targets +------------------------------- + +Sometimes you’d like to store data for targets but the predefined fields +that the TOM Toolkit provides aren’t enough. The TOM Toolkit allows you +to define extra fields for your targets so you can associate different +kinds of data with them. For example, you might be studying high +redshift galaxies. In this case, it would make sense to be able to store +the redshift of your targets. You could then do a search for targets +with a redshift less than or greater than a particular value, or use the +redshift value to make decisions in your science code. + +**Note**: There is a performance hit when using extra fields. Try to use +the built in fields whenever possible. + +Enabling extra fields +~~~~~~~~~~~~~~~~~~~~~ + +To start, find the ``EXTRA_FIELDS`` definition in your ``settings.py``: + +.. code:: python + + # Define extra target fields here. Types can be any of "number", "string", "boolean" or "datetime" + # For example: + # EXTRA_FIELDS = [ + # {'name': 'redshift', 'type': 'number'}, + # {'name': 'discoverer', 'type': 'string'} + # {'name': 'eligible', 'type': 'boolean'}, + # {'name': 'dicovery_date', 'type': 'datetime'} + # ] + EXTRA_FIELDS = [] + +We can define any number of extra fields in the array. Each item in the +array is a dictionary with two values: name and type. Name is simply +what you would like to name your field. Type is the datatype of the +field and can be one of: ``number``, ``string``, ``boolean`` or +``datetime``. These types allow the TOM Toolkit to properly store, +filter and display these values elsewhere. + +As an example, let’s change the setting to look like this: + +.. code:: python + + EXTRA_FIELDS = [ + {'name': 'redshift', 'type': 'number'}, + ] + +This will make an extra field with the name “redshift” and a type of +“number” available to add to our targets. + +Using extra fields +~~~~~~~~~~~~~~~~~~ + +Now if you go to the target creation page, you should see the new field +available: + +|image0| + +And if we go to our list of targets, we should see redshift as a field +available to filter on: + +|image1| + +Extra fields with the ``number`` type allow filtering on range of +values. The same goes for fields with the ``datetime`` type. ``string`` +types to a case insensitive inclusive search, and ``boolean`` fields to +a simple matching comparison. + +Of course, redshift does appear on our target’s display page as well: + +|image2| + +To hide extra fields from the target page, we can set the “hidden” key +(this doesn’t affect filtering and searching): + +.. code:: python + + EXTRA_FIELDS = [ + {'name': 'redshift', 'type': 'number', 'hidden': True}, + ] + +And we can set a default value for an extra field by including a default +key/value pair: + +.. code:: python + + EXTRA_FIELDS = [ + {'name': 'redshift', 'type': 'number', 'default': 0}, + ] + +Displaying extra fields in templates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If we want to display the redshift in other places, we can use a +template filter to do that. For example, we might want to display the +redshift value in the target list table. + +At the top of our template make sure to load ``targets_extras``: + +:: + + {% raw %} + {% load targets_extras %} + {% endraw %} + +Now we can use the ``target_extra_field`` filter wherever a target +object is available in the template context: + +:: + + {% raw %} + {{ target|target_extra_field:"redshift" }} + {% endraw %} + +The result is the redshift value being printed on the template: + +|image3| + +Working with extra fields programatically +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you’d like to update or save extra fields to your targets in code, +there are a few methods you can use. The simplest is to simply pass in a +dictionary of extra data to your target’s ``save()`` method using the +``extras`` keyword argument: + +.. code:: python + + target = Target.objects.get(name='example') + target.save(extras={'foo': 42}) + +The example target above will now have an extra field “foo” with the +value 42. + +For more precise control, you can access ``TargetExtra`` models +directly. To remove an extra, for example: + +.. code:: python + + target = Target.objects.get(name='example') + target_extra = target.targetextra_set.get(key='foo') + target_extra.delete() + +The above deleted the target extra on a target with the key of “foo”. + +.. |image0| image:: /_static/target_fields_doc/redshift.png +.. |image1| image:: /_static/target_fields_doc/redshift_filter.png +.. |image2| image:: /_static/target_fields_doc/redshift_display.png +.. |image3| image:: /_static/target_fields_doc/redshift_tag.png From 6412fc29e21921d87203fc0d95849233a59904ff Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 8 Jun 2020 18:27:33 -0700 Subject: [PATCH 168/424] fixed a couple errors from migration --- docs/code/backgroundtasks.rst | 2 +- docs/introduction/about.rst | 4 ++-- docs/introduction/workflow.rst | 31 ++++++++++++++++++------------- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/docs/code/backgroundtasks.rst b/docs/code/backgroundtasks.rst index 433a13972..788577198 100644 --- a/docs/code/backgroundtasks.rst +++ b/docs/code/backgroundtasks.rst @@ -129,7 +129,7 @@ your virtualenv: offer us some conveniences while working with tasks in our TOM. Install django-dramatiq to your ``INSTALLED_APPS`` setting, above the -tom_\* apps: +tom_* apps: .. code:: python diff --git a/docs/introduction/about.rst b/docs/introduction/about.rst index b43163d2e..993652591 100644 --- a/docs/introduction/about.rst +++ b/docs/introduction/about.rst @@ -68,5 +68,5 @@ support from the `Heising-Simons Foundation `_ and the `Zegar Family Foundation `_. -.. container:: partners - +.. image:: /_static/hs.jpg +.. image:: /_static/zff.png diff --git a/docs/introduction/workflow.rst b/docs/introduction/workflow.rst index d54291a11..ba40683ab 100644 --- a/docs/introduction/workflow.rst +++ b/docs/introduction/workflow.rst @@ -1,7 +1,8 @@ TOM Workflow ------------ -Targets ~~~~~~~ +Targets +~~~~~~~ Targets are the central entity of the TOM Toolkit. Most functionality in the toolkit requires a target as they are the object of study. A target @@ -9,12 +10,13 @@ represents an astronomical object (star, galaxy, asteroid, etc) and is usually represented using coordinates on the sky along with other meta data. -Creating Targets ^^^^^^^^^^^^^^^^ +Creating Targets +^^^^^^^^^^^^^^^^ The TOM Toolkit provides a variety of methods for importing astronomical targets into the TOM: -\|image0\| +.. image:: /_static/target_sources.png - The Alert Module provides the functionality to create targets from alert brokers such as ``MARS ``\ \_\_ and @@ -33,13 +35,15 @@ targets into the TOM: aren’t known by any of the existing catalogs or use more precise information that they know of. -Observations ~~~~~~~~~~~~ +Observations +~~~~~~~~~~~~ After creating targets, the scientist needs to collect data on these targets. The TOM Observing module provides an interface to several observatories for which observations can be requested. -Requesting Observations ^^^^^^^^^^^^^^^^^^^^^^^ +Requesting Observations +^^^^^^^^^^^^^^^^^^^^^^^ Using the TOM Observation module, scientists can request observations of their targets to one or many different observatories. Since the @@ -54,23 +58,26 @@ Observations can also be requested in a completely automated manner, which is particularly useful for rapid response time domain follow-up programs. -\|image1\| +.. image:: /_static/common_interface.png -Observation Status ^^^^^^^^^^^^^^^^^^ +Observation Status +^^^^^^^^^^^^^^^^^^ Once an observation for a target is created it’s status is kept up to date within the TOM. When the status of an observation request at an observatory changes (failed, completed, postponed, etc) the scientist may be notified by the TOM. -Data ~~~~ +Data +~~~~ The ultimate goal of the TOM toolkit is to collect and organize data. The TOM data module provides several methods for obtaining data, the most obvious being from completed observations. Scientists can also upload any data they’d like to associate with their targets as well. -Data Processing ^^^^^^^^^^^^^^^ +Data Processing +^^^^^^^^^^^^^^^ The TOM toolkit provides a framework to write custom code to interact with the data the TOM obtains (among other things). These are called @@ -80,7 +87,8 @@ systems. For example: if a scientist has existing code that checks images of microlensed stars for exoplanets, they may hook the code into the TOM toolkit directly to run whenever new data is acquired. -Downloading Data ^^^^^^^^^^^^^^^^ +Downloading Data +^^^^^^^^^^^^^^^^ Data is stored in the TOM toolkit by default, but many scientists may want to download the data somewhere else to do offline processing. @@ -88,6 +96,3 @@ Scientists can easily download data to their local machines, and the data module by default stores all it’s data on a local file system. However, it can be customized to store data on cloud services, like Amazon S3, when desired. - -.. \|image0\| image:: /_static/target_sources.png .. \|image1\| image:: -/_static/common_interface.png From 85a9e898ac62e0b384b7e58fb3ba3cdccc5e150e Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 8 Jun 2020 21:21:29 -0700 Subject: [PATCH 169/424] Updated link to release notes and added to README-dev development and deployment workflows --- README-dev.md | 50 ++++++++++++++++++++++++++++++++++++++++---------- docs/index.rst | 2 +- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/README-dev.md b/README-dev.md index 98ed0726d..a808878c6 100644 --- a/README-dev.md +++ b/README-dev.md @@ -25,31 +25,47 @@ Following deployment of a release, a Github Release is created, and this should ## Deployment Workflow _**This section of this document is a work-in-progress**_ #### Pre-release deployment -* _meet pre-deployment criteria documented [here]()_. +* Meet pre-deployment criteria. + * Pass [Codacy code quality check](https://app.codacy.com/gh/TOMToolkit/tom_base/pullRequests). + * Doesn't decrease [Coveralls test coverage](https://coveralls.io/github/TOMToolkit/tom_base). + * Passes [Travis tests and code style check](https://travis-ci.com/github/TOMToolkit/tom_base/branches). + * Successfully builds [ReadTheDocs documentation](https://readthedocs.org/projects/tom-toolkit/builds/) (not an automated check) (TODO: fix webhook). + * One review approval by a repository owner. * merge to `development` -* `git tag -a x.y.z-alpha.w -m "x.y.z-aplha.w"` +* `git tag -a x.y.z-alpha.w -m "x.y.z-alpha.w"` -- must follow semantic versioning * `git push --tags` * This causes Travis to create a draft release in GitHub and push to PyPI -* Edit the release notes in GitHub; Update, edit; repeat until satisfied. Release notes should contain: +* deploy `tom-demo-dev` with new features demonstrated, pulling `tomtoolkit==x.y.z-alpha.w` from PyPI + Examples: + * Release of observing strategies should include saving an observing strategy and submitting an observation via the observing strategy + * Release of manual facility interface should include an implementation of the new interface + * Release of a new template tag should include that template tag in a template +* Edit the release notes in GitHub; Update, edit; repeat until satisfied. Release notes should contain (as needed): * Links to Read the Docs API (docstring) docs * Links to Read the Docs higher level docs * Link to Tom Demo feature demonstration - * what else? - - For example: TODO: _insert example here_ + * Links to issues that have been fixed * When satisfied, `Publish Release` Repo watchers are notified by email. -* deploy `tom-demo-dev` with new features demonstrated, pulling `tom_base-x.y.z-alpha.w` from PyPI #### Public release deployment * Create PR: `master <- development` +* Meet pre-deployment criteria. + * Include docstrings for any new or updated methods + * Include tutorial documentation for any new major features as needed + * Pass [Codacy code quality check](https://app.codacy.com/gh/TOMToolkit/tom_base/dashboard?bid=18204585). + * Doesn't decrease [Coveralls test coverage](https://coveralls.io/github/TOMToolkit/tom_base?branch=development). + * Passes [Travis tests and code style check](https://travis-ci.com/github/TOMToolkit/tom_base/branches). + * Successfully builds [ReadTheDocs documentation](https://readthedocs.org/projects/tom-toolkit/builds/) (not an automated check) (TODO: fix webhook). * Merge PR -* `git tag -a x.y.z -m "Release x.y.z"` + * Must be a repository owner to merge. +* `git tag -a x.y.z -m "Release x.y.z"` -- must follow semantic versioning * `git push --tags` Triggers Travis to: * build, build * push release to PyPI * create GitHub draft release +* deploy `tom-demo` with new features demonstrated, pulling `tomtoolkit==x.y.z` from PyPI * Update Release Notes in GitHub draft release. (This should be the accumulation of the all the development-release release notes: For example, release notes for releases x.y.z-alpha.1, x.y.z-alpha.2, etc. should be combined into release notes for release x.y.z. @@ -57,8 +73,22 @@ Following deployment of a release, a Github Release is created, and this should * Post notification to Slack, Tom Toolkit workspace, #general channel. (In the future, we hope to have automated release notification to a dedicated #releases slack channel). -### Preview Read the Docs doc strings + +### Development Notes - Doing checks locally + +#### Preview Read the Docs doc strings * `cd /path/to/tom_base/docs` * `pip install -r requirements.txt # make sure sphinx is installed to your venv` * `make html # make clean first, if things are weird` -* point a browser to the html files in `./_build/html/` to proof read before deployment \ No newline at end of file +* point a browser to the html files in `./_build/html/` to proof read before deployment + +#### Run code style checks +* `pip install pycodestyle` +* `pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120` + +#### Run tests +* `./manage.py test` +* Examples for running specific tests or test suites: + * `./manage.py test tom_targets.tests` + * `./manage.py test tom_targets.tests.tests.TestTargetDetail` + * `./manage.py test tom_targets.tests.tests.TestTargetDetail.test_sidereal_target_detail` \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 040c81e27..23920c7a5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -94,7 +94,7 @@ If you just need an idea, checkout out the :doc:`examples` of existing examples introduction/contributing - common/releasenotes + Release Notes Github API Documentation From 03a14e0b6d4ae613df30fea8aa2423f41941efd0 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 8 Jun 2020 21:29:42 -0700 Subject: [PATCH 170/424] Fixed some codacy stuff cause why not --- .github/ISSUE_TEMPLATE/bug_report.md | 8 ++++---- tom_alerts/brokers/gaia.py | 2 -- tom_common/models.py | 2 -- tom_common/static/tom_common/css/main.css | 6 ++++-- tom_observations/static/tom_observations/css/main.css | 2 +- tom_targets/models.py | 2 +- tom_targets/static/tom_targets/css/main.css | 2 +- 7 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 425469eb5..ecab41bbe 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -24,10 +24,10 @@ A clear and concise description of what you expected to happen. If applicable, add screenshots to help explain your problem. **Desktop (please complete the following information):** - - OS: [e.g. iOS] - - Browser [e.g. chrome, safari] - - Version [e.g. 22] - - Python Version [e.g. 3.7.2] +- OS: [e.g. iOS] +- Browser [e.g. chrome, safari] +- Version [e.g. 22] +- Python Version [e.g. 3.7.2] **Additional context** Add any other context about the problem here. diff --git a/tom_alerts/brokers/gaia.py b/tom_alerts/brokers/gaia.py index 5f038bbb7..1cb66c064 100644 --- a/tom_alerts/brokers/gaia.py +++ b/tom_alerts/brokers/gaia.py @@ -1,6 +1,4 @@ from tom_alerts.alerts import GenericQueryForm, GenericAlert, GenericBroker -from tom_alerts.models import BrokerQuery -from tom_targets.models import Target from tom_dataproducts.models import ReducedDatum from dateutil.parser import parse from django import forms diff --git a/tom_common/models.py b/tom_common/models.py index 71a836239..6b2021999 100644 --- a/tom_common/models.py +++ b/tom_common/models.py @@ -1,3 +1 @@ -from django.db import models - # Create your models here. diff --git a/tom_common/static/tom_common/css/main.css b/tom_common/static/tom_common/css/main.css index 91cf0ea4a..daf313be9 100644 --- a/tom_common/static/tom_common/css/main.css +++ b/tom_common/static/tom_common/css/main.css @@ -1,13 +1,15 @@ body { padding-top: 5rem; } + .content { padding: 3rem 1.5rem; } + .navbar-brand > img { - max-height: 25px; + max-height: 25px; } .input-group-text { - font-family: monospace; + font-family: monospace; } diff --git a/tom_observations/static/tom_observations/css/main.css b/tom_observations/static/tom_observations/css/main.css index 8fe55b6e6..492c50835 100644 --- a/tom_observations/static/tom_observations/css/main.css +++ b/tom_observations/static/tom_observations/css/main.css @@ -9,4 +9,4 @@ display: block; width: inherit; height: auto; -} \ No newline at end of file +} diff --git a/tom_targets/models.py b/tom_targets/models.py index e9619a0d8..d2d7b8b46 100644 --- a/tom_targets/models.py +++ b/tom_targets/models.py @@ -442,7 +442,7 @@ def save(self, *args, **kwargs): self.time_value = self.value else: self.time_value = parse(self.value) - except (TypeError, ValueError, OverflowError) as e: + except (TypeError, ValueError, OverflowError): self.time_value = None super().save(*args, **kwargs) diff --git a/tom_targets/static/tom_targets/css/main.css b/tom_targets/static/tom_targets/css/main.css index 3b127f7ec..b6eec2ab5 100644 --- a/tom_targets/static/tom_targets/css/main.css +++ b/tom_targets/static/tom_targets/css/main.css @@ -23,4 +23,4 @@ dl { span.featured { pointer-events: none; -} \ No newline at end of file +} From 3b57582033bcffc01a3a8f546dd1054f2bc05df2 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 8 Jun 2020 21:31:28 -0700 Subject: [PATCH 171/424] Fixing more codacy --- README-dev.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README-dev.md b/README-dev.md index a808878c6..ed15eb245 100644 --- a/README-dev.md +++ b/README-dev.md @@ -24,7 +24,8 @@ Following deployment of a release, a Github Release is created, and this should ## Deployment Workflow _**This section of this document is a work-in-progress**_ -#### Pre-release deployment + +### Pre-release deployment * Meet pre-deployment criteria. * Pass [Codacy code quality check](https://app.codacy.com/gh/TOMToolkit/tom_base/pullRequests). * Doesn't decrease [Coveralls test coverage](https://coveralls.io/github/TOMToolkit/tom_base). @@ -48,7 +49,7 @@ Following deployment of a release, a Github Release is created, and this should * When satisfied, `Publish Release` Repo watchers are notified by email. -#### Public release deployment +### Public release deployment * Create PR: `master <- development` * Meet pre-deployment criteria. @@ -74,19 +75,19 @@ Following deployment of a release, a Github Release is created, and this should have automated release notification to a dedicated #releases slack channel). -### Development Notes - Doing checks locally +## Development Notes - Doing checks locally -#### Preview Read the Docs doc strings +### Preview Read the Docs doc strings * `cd /path/to/tom_base/docs` * `pip install -r requirements.txt # make sure sphinx is installed to your venv` * `make html # make clean first, if things are weird` * point a browser to the html files in `./_build/html/` to proof read before deployment -#### Run code style checks +### Run code style checks * `pip install pycodestyle` * `pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120` -#### Run tests +### Run tests * `./manage.py test` * Examples for running specific tests or test suites: * `./manage.py test tom_targets.tests` From e50ce0041a1eff176344a54a9688af4ad7b334f1 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 9 Jun 2020 08:36:51 -0700 Subject: [PATCH 172/424] More codacy fixes --- README-dev.md | 33 ++++++++++++++------ README.md | 2 +- docs/api/plugins.md | 2 +- docs/api/tom_dataproducts/data_processing.md | 2 +- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/README-dev.md b/README-dev.md index ed15eb245..75f6e36f1 100644 --- a/README-dev.md +++ b/README-dev.md @@ -28,26 +28,41 @@ Following deployment of a release, a Github Release is created, and this should ### Pre-release deployment * Meet pre-deployment criteria. * Pass [Codacy code quality check](https://app.codacy.com/gh/TOMToolkit/tom_base/pullRequests). + * Doesn't decrease [Coveralls test coverage](https://coveralls.io/github/TOMToolkit/tom_base). + * Passes [Travis tests and code style check](https://travis-ci.com/github/TOMToolkit/tom_base/branches). + * Successfully builds [ReadTheDocs documentation](https://readthedocs.org/projects/tom-toolkit/builds/) (not an automated check) (TODO: fix webhook). + * One review approval by a repository owner. + * merge to `development` + * `git tag -a x.y.z-alpha.w -m "x.y.z-alpha.w"` -- must follow semantic versioning + * `git push --tags` + * This causes Travis to create a draft release in GitHub and push to PyPI + * deploy `tom-demo-dev` with new features demonstrated, pulling `tomtoolkit==x.y.z-alpha.w` from PyPI Examples: - * Release of observing strategies should include saving an observing strategy and submitting an observation via the observing strategy - * Release of manual facility interface should include an implementation of the new interface - * Release of a new template tag should include that template tag in a template + * Release of observing strategies should include saving an observing strategy and submitting an observation via the observing strategy + + * Release of manual facility interface should include an implementation of the new interface + + * Release of a new template tag should include that template tag in a template + * Edit the release notes in GitHub; Update, edit; repeat until satisfied. Release notes should contain (as needed): * Links to Read the Docs API (docstring) docs + * Links to Read the Docs higher level docs + * Link to Tom Demo feature demonstration + * Links to issues that have been fixed -* When satisfied, `Publish Release` Repo watchers are notified by email. +* When satisfied, `Publish Release` Repo watchers are notified by email. ### Public release deployment @@ -63,9 +78,9 @@ Following deployment of a release, a Github Release is created, and this should * Must be a repository owner to merge. * `git tag -a x.y.z -m "Release x.y.z"` -- must follow semantic versioning * `git push --tags` Triggers Travis to: - * build, build - * push release to PyPI - * create GitHub draft release + * build, build + * push release to PyPI + * create GitHub draft release * deploy `tom-demo` with new features demonstrated, pulling `tomtoolkit==x.y.z` from PyPI * Update Release Notes in GitHub draft release. (This should be the accumulation of the all the development-release release notes: For example, release notes for releases x.y.z-alpha.1, @@ -74,7 +89,6 @@ Following deployment of a release, a Github Release is created, and this should * Post notification to Slack, Tom Toolkit workspace, #general channel. (In the future, we hope to have automated release notification to a dedicated #releases slack channel). - ## Development Notes - Doing checks locally ### Preview Read the Docs doc strings @@ -92,4 +106,5 @@ have automated release notification to a dedicated #releases slack channel). * Examples for running specific tests or test suites: * `./manage.py test tom_targets.tests` * `./manage.py test tom_targets.tests.tests.TestTargetDetail` - * `./manage.py test tom_targets.tests.tests.TestTargetDetail.test_sidereal_target_detail` \ No newline at end of file + * `./manage.py test tom_targets.tests.tests.TestTargetDetail.test_sidereal_target_detail` + \ No newline at end of file diff --git a/README.md b/README.md index e7ef6aee6..bdf250c1e 100644 --- a/README.md +++ b/README.md @@ -51,4 +51,4 @@ This module provides the ability to submit observations to the Liverpool Telesco state, with little error handling and minimal instrument options, but can successfully submit well-formed observation requests. -[Github](https://github.com/TOMToolkit/tom_lt) \ No newline at end of file +[Github](https://github.com/TOMToolkit/tom_lt) diff --git a/docs/api/plugins.md b/docs/api/plugins.md index d3c62b27c..dcac0dd27 100644 --- a/docs/api/plugins.md +++ b/docs/api/plugins.md @@ -36,4 +36,4 @@ minimally supported while its successor is in development. The library used for This module provides the ability to submit observations to the Liverpool Telescope Phase 2 system. It is in a very alpha state, with little error handling and minimal instrument options, but can successfully submit well-formed -observation requests. \ No newline at end of file +observation requests. diff --git a/docs/api/tom_dataproducts/data_processing.md b/docs/api/tom_dataproducts/data_processing.md index e7fe6a10d..3fc9ebc55 100644 --- a/docs/api/tom_dataproducts/data_processing.md +++ b/docs/api/tom_dataproducts/data_processing.md @@ -32,4 +32,4 @@ Data Processors .. autoclass:: tom_dataproducts.data_processor.DataProcessor :members: :private-members: - :member-order: bysource \ No newline at end of file + :member-order: bysource From 22c0be1f227b2508a65c486c6e18a67927de509d Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 9 Jun 2020 08:40:45 -0700 Subject: [PATCH 173/424] Codacy is a fickle beast --- README-dev.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/README-dev.md b/README-dev.md index 75f6e36f1..2181107d1 100644 --- a/README-dev.md +++ b/README-dev.md @@ -67,25 +67,41 @@ Following deployment of a release, a Github Release is created, and this should ### Public release deployment * Create PR: `master <- development` + * Meet pre-deployment criteria. * Include docstrings for any new or updated methods + * Include tutorial documentation for any new major features as needed + * Pass [Codacy code quality check](https://app.codacy.com/gh/TOMToolkit/tom_base/dashboard?bid=18204585). + * Doesn't decrease [Coveralls test coverage](https://coveralls.io/github/TOMToolkit/tom_base?branch=development). + * Passes [Travis tests and code style check](https://travis-ci.com/github/TOMToolkit/tom_base/branches). + * Successfully builds [ReadTheDocs documentation](https://readthedocs.org/projects/tom-toolkit/builds/) (not an automated check) (TODO: fix webhook). + * Merge PR + * Must be a repository owner to merge. + * `git tag -a x.y.z -m "Release x.y.z"` -- must follow semantic versioning + * `git push --tags` Triggers Travis to: * build, build + * push release to PyPI + * create GitHub draft release + * deploy `tom-demo` with new features demonstrated, pulling `tomtoolkit==x.y.z` from PyPI + * Update Release Notes in GitHub draft release. (This should be the accumulation of the all the development-release release notes: For example, release notes for releases x.y.z-alpha.1, x.y.z-alpha.2, etc. should be combined into release notes for release x.y.z. + * Publish Release + * Post notification to Slack, Tom Toolkit workspace, #general channel. (In the future, we hope to have automated release notification to a dedicated #releases slack channel). @@ -93,18 +109,24 @@ have automated release notification to a dedicated #releases slack channel). ### Preview Read the Docs doc strings * `cd /path/to/tom_base/docs` + * `pip install -r requirements.txt # make sure sphinx is installed to your venv` + * `make html # make clean first, if things are weird` + * point a browser to the html files in `./_build/html/` to proof read before deployment ### Run code style checks * `pip install pycodestyle` + * `pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120` ### Run tests * `./manage.py test` + * Examples for running specific tests or test suites: * `./manage.py test tom_targets.tests` + * `./manage.py test tom_targets.tests.tests.TestTargetDetail` + * `./manage.py test tom_targets.tests.tests.TestTargetDetail.test_sidereal_target_detail` - \ No newline at end of file From d6e76ddd8d57640e2452c242a6c017fe038bc435 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 9 Jun 2020 09:12:49 -0700 Subject: [PATCH 174/424] Why did we introduce codacy again? --- README-dev.md | 40 +--------------------------------------- 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/README-dev.md b/README-dev.md index 2181107d1..eac5d6e95 100644 --- a/README-dev.md +++ b/README-dev.md @@ -28,80 +28,48 @@ Following deployment of a release, a Github Release is created, and this should ### Pre-release deployment * Meet pre-deployment criteria. * Pass [Codacy code quality check](https://app.codacy.com/gh/TOMToolkit/tom_base/pullRequests). - * Doesn't decrease [Coveralls test coverage](https://coveralls.io/github/TOMToolkit/tom_base). - * Passes [Travis tests and code style check](https://travis-ci.com/github/TOMToolkit/tom_base/branches). - * Successfully builds [ReadTheDocs documentation](https://readthedocs.org/projects/tom-toolkit/builds/) (not an automated check) (TODO: fix webhook). - * One review approval by a repository owner. - * merge to `development` - * `git tag -a x.y.z-alpha.w -m "x.y.z-alpha.w"` -- must follow semantic versioning - * `git push --tags` - * This causes Travis to create a draft release in GitHub and push to PyPI - * deploy `tom-demo-dev` with new features demonstrated, pulling `tomtoolkit==x.y.z-alpha.w` from PyPI Examples: * Release of observing strategies should include saving an observing strategy and submitting an observation via the observing strategy - * Release of manual facility interface should include an implementation of the new interface - * Release of a new template tag should include that template tag in a template - * Edit the release notes in GitHub; Update, edit; repeat until satisfied. Release notes should contain (as needed): * Links to Read the Docs API (docstring) docs - * Links to Read the Docs higher level docs - * Link to Tom Demo feature demonstration - * Links to issues that have been fixed - * When satisfied, `Publish Release` Repo watchers are notified by email. ### Public release deployment * Create PR: `master <- development` - * Meet pre-deployment criteria. * Include docstrings for any new or updated methods - * Include tutorial documentation for any new major features as needed - * Pass [Codacy code quality check](https://app.codacy.com/gh/TOMToolkit/tom_base/dashboard?bid=18204585). - * Doesn't decrease [Coveralls test coverage](https://coveralls.io/github/TOMToolkit/tom_base?branch=development). - * Passes [Travis tests and code style check](https://travis-ci.com/github/TOMToolkit/tom_base/branches). - * Successfully builds [ReadTheDocs documentation](https://readthedocs.org/projects/tom-toolkit/builds/) (not an automated check) (TODO: fix webhook). - * Merge PR - * Must be a repository owner to merge. - * `git tag -a x.y.z -m "Release x.y.z"` -- must follow semantic versioning - * `git push --tags` Triggers Travis to: * build, build - - * push release to PyPI - + * push release to PyPI * create GitHub draft release - * deploy `tom-demo` with new features demonstrated, pulling `tomtoolkit==x.y.z` from PyPI - * Update Release Notes in GitHub draft release. (This should be the accumulation of the all the development-release release notes: For example, release notes for releases x.y.z-alpha.1, x.y.z-alpha.2, etc. should be combined into release notes for release x.y.z. - * Publish Release - * Post notification to Slack, Tom Toolkit workspace, #general channel. (In the future, we hope to have automated release notification to a dedicated #releases slack channel). @@ -109,16 +77,12 @@ have automated release notification to a dedicated #releases slack channel). ### Preview Read the Docs doc strings * `cd /path/to/tom_base/docs` - * `pip install -r requirements.txt # make sure sphinx is installed to your venv` - * `make html # make clean first, if things are weird` - * point a browser to the html files in `./_build/html/` to proof read before deployment ### Run code style checks * `pip install pycodestyle` - * `pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120` ### Run tests @@ -126,7 +90,5 @@ have automated release notification to a dedicated #releases slack channel). * Examples for running specific tests or test suites: * `./manage.py test tom_targets.tests` - * `./manage.py test tom_targets.tests.tests.TestTargetDetail` - * `./manage.py test tom_targets.tests.tests.TestTargetDetail.test_sidereal_target_detail` From c0c7f0a7043cd54ebb36cdab12d9eb2fdebcdb3c Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 9 Jun 2020 17:43:46 +0000 Subject: [PATCH 175/424] incremental improvement to Dev Deployment section --- README-dev.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/README-dev.md b/README-dev.md index eac5d6e95..d24238bb0 100644 --- a/README-dev.md +++ b/README-dev.md @@ -50,18 +50,20 @@ Following deployment of a release, a Github Release is created, and this should ### Public release deployment -* Create PR: `master <- development` -* Meet pre-deployment criteria. - * Include docstrings for any new or updated methods - * Include tutorial documentation for any new major features as needed - * Pass [Codacy code quality check](https://app.codacy.com/gh/TOMToolkit/tom_base/dashboard?bid=18204585). - * Doesn't decrease [Coveralls test coverage](https://coveralls.io/github/TOMToolkit/tom_base?branch=development). - * Passes [Travis tests and code style check](https://travis-ci.com/github/TOMToolkit/tom_base/branches). - * Successfully builds [ReadTheDocs documentation](https://readthedocs.org/projects/tom-toolkit/builds/) (not an automated check) (TODO: fix webhook). -* Merge PR +1. Create PR: `master <- development` +2. Meet pre-deployment criteria. + * Include docstrings for any new or updated methods + * Include tutorial documentation for any new major features as needed + * Pass [Codacy code quality check](https://app.codacy.com/gh/TOMToolkit/tom_base/dashboard?bid=18204585). + * Doesn't decrease [Coveralls test coverage](https://coveralls.io/github/TOMToolkit/tom_base?branch=development). + * Passes [Travis tests and code style check](https://travis-ci.com/github/TOMToolkit/tom_base/branches). + * Successfully builds [ReadTheDocs documentation](https://readthedocs.org/projects/tom-toolkit/builds/) (not an automated check) (TODO: fix webhook). + +3. Merge PR * Must be a repository owner to merge. * `git tag -a x.y.z -m "Release x.y.z"` -- must follow semantic versioning * `git push --tags` Triggers Travis to: + * build, build * push release to PyPI * create GitHub draft release From 88f95bbf0f70154d20a68e01596b14dd09b219e3 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 9 Jun 2020 17:49:45 +0000 Subject: [PATCH 176/424] more README-dev updates --- README-dev.md | 83 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 32 deletions(-) diff --git a/README-dev.md b/README-dev.md index d24238bb0..797e12f77 100644 --- a/README-dev.md +++ b/README-dev.md @@ -26,44 +26,63 @@ Following deployment of a release, a Github Release is created, and this should _**This section of this document is a work-in-progress**_ ### Pre-release deployment -* Meet pre-deployment criteria. - * Pass [Codacy code quality check](https://app.codacy.com/gh/TOMToolkit/tom_base/pullRequests). - * Doesn't decrease [Coveralls test coverage](https://coveralls.io/github/TOMToolkit/tom_base). - * Passes [Travis tests and code style check](https://travis-ci.com/github/TOMToolkit/tom_base/branches). - * Successfully builds [ReadTheDocs documentation](https://readthedocs.org/projects/tom-toolkit/builds/) (not an automated check) (TODO: fix webhook). - * One review approval by a repository owner. -* merge to `development` -* `git tag -a x.y.z-alpha.w -m "x.y.z-alpha.w"` -- must follow semantic versioning -* `git push --tags` -* This causes Travis to create a draft release in GitHub and push to PyPI -* deploy `tom-demo-dev` with new features demonstrated, pulling `tomtoolkit==x.y.z-alpha.w` from PyPI - Examples: - * Release of observing strategies should include saving an observing strategy and submitting an observation via the observing strategy - * Release of manual facility interface should include an implementation of the new interface - * Release of a new template tag should include that template tag in a template -* Edit the release notes in GitHub; Update, edit; repeat until satisfied. Release notes should contain (as needed): - * Links to Read the Docs API (docstring) docs - * Links to Read the Docs higher level docs - * Link to Tom Demo feature demonstration - * Links to issues that have been fixed -* When satisfied, `Publish Release` Repo watchers are notified by email. +1. Meet pre-deployment criteria. + * Pass [Codacy code quality check](https://app.codacy.com/gh/TOMToolkit/tom_base/pullRequests). + * Doesn't decrease [Coveralls test coverage](https://coveralls.io/github/TOMToolkit/tom_base). + * Passes [Travis tests and code style check](https://travis-ci.com/github/TOMToolkit/tom_base/branches). + * Successfully builds [ReadTheDocs documentation](https://readthedocs.org/projects/tom-toolkit/builds/) (not an automated check) (TODO: fix webhook). + * One review approval by a repository owner. + +2. Merge your feature branch into the `development` branch + * `git checkout development` + * `git merge feature/your_feature_branch` + +3. Tag the release, triggering GitHub and PyPI actions: + + Release tags must follow [semantic versioning](https://semver.org) syntax. + * `git tag -a x.y.z-alpha.w -m "x.y.z-alpha.w"` + * `git push --tags` + * Pushing the tags causes Travis to create a draft release in GitHub and push to PyPI + +4. Deploy `tom-demo-dev` with new features demonstrated, pulling `tomtoolkit==x.y.z-alpha.w` from PyPI. + + Examples: + * Release of observing strategies should include saving an observing strategy and submitting an observation via the observing strategy + * Release of manual facility interface should include an implementation of the new interface + * Release of a new template tag should include that template tag in a template + +5. Edit the Release Notes in GitHub + + When the tags were pushed above, GitHub created draft Release Notes + which need to be filled out. (These can be found by following the `releases` link on the [front page](https://github.com/TOMToolkit/tom_base) of the repo. + Or, [here](https://github.com/TOMToolkit/tom_base/releases)). + + Edit, Update, and repeat until satisfied. + Release notes should contain (as needed): + * Links to Read the Docs API (docstring) docs + * Links to Read the Docs higher level docs + * Link to Tom Demo feature demonstration + * Links to issues that have been fixed + +6. Publish the Release + + When satisfied with the Release Notes, `Publish Release`. + Repo watchers are notified by email. ### Public release deployment -1. Create PR: `master <- development` -2. Meet pre-deployment criteria. - * Include docstrings for any new or updated methods - * Include tutorial documentation for any new major features as needed - * Pass [Codacy code quality check](https://app.codacy.com/gh/TOMToolkit/tom_base/dashboard?bid=18204585). - * Doesn't decrease [Coveralls test coverage](https://coveralls.io/github/TOMToolkit/tom_base?branch=development). - * Passes [Travis tests and code style check](https://travis-ci.com/github/TOMToolkit/tom_base/branches). - * Successfully builds [ReadTheDocs documentation](https://readthedocs.org/projects/tom-toolkit/builds/) (not an automated check) (TODO: fix webhook). - -3. Merge PR +* Create PR: `master <- development` +* Meet pre-deployment criteria. + * Include docstrings for any new or updated methods + * Include tutorial documentation for any new major features as needed + * Pass [Codacy code quality check](https://app.codacy.com/gh/TOMToolkit/tom_base/dashboard?bid=18204585). + * Doesn't decrease [Coveralls test coverage](https://coveralls.io/github/TOMToolkit/tom_base?branch=development). + * Passes [Travis tests and code style check](https://travis-ci.com/github/TOMToolkit/tom_base/branches). + * Successfully builds [ReadTheDocs documentation](https://readthedocs.org/projects/tom-toolkit/builds/) (not an automated check) (TODO: fix webhook). +* Merge PR * Must be a repository owner to merge. * `git tag -a x.y.z -m "Release x.y.z"` -- must follow semantic versioning * `git push --tags` Triggers Travis to: - * build, build * push release to PyPI * create GitHub draft release From 0904aa091b4205fc97566c604688c2f44a0cf76a Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 9 Jun 2020 17:55:24 +0000 Subject: [PATCH 177/424] incremental changes to Public Release section (still wip) --- README-dev.md | 53 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/README-dev.md b/README-dev.md index 797e12f77..03990fcf0 100644 --- a/README-dev.md +++ b/README-dev.md @@ -70,28 +70,39 @@ Following deployment of a release, a Github Release is created, and this should Repo watchers are notified by email. ### Public release deployment +The public release deployment workflow parallels the pre-release deployment work flow +and more details for a particular step may be found above. + +1. Create PR: `master <- development` +2. Meet pre-deployment criteria. + * Include docstrings for any new or updated methods + * Include tutorial documentation for any new major features as needed + * Pass [Codacy code quality check](https://app.codacy.com/gh/TOMToolkit/tom_base/dashboard?bid=18204585). + * Doesn't decrease [Coveralls test coverage](https://coveralls.io/github/TOMToolkit/tom_base?branch=development). + * Passes [Travis tests and code style check](https://travis-ci.com/github/TOMToolkit/tom_base/branches). + * Successfully builds [ReadTheDocs documentation](https://readthedocs.org/projects/tom-toolkit/builds/) (not an automated check) (TODO: fix webhook). + +3. Merge PR + * Must be a repository owner to merge. + +4. Tag the release, triggering GitHub and PyPI actions: + * `git tag -a x.y.z -m "Release x.y.z"` -- must follow semantic versioning + * `git push --tags` Triggers Travis to: + * build, build + * push release to PyPI + * create GitHub draft release + +5. deploy `tom-demo` with new features demonstrated, pulling `tomtoolkit==x.y.z` from PyPI + +6. Update Release Notes in GitHub draft release. + + This should be the accumulation of the all + the development-release release notes: For example, release notes for releases x.y.z-alpha.1, + x.y.z-alpha.2, etc. should be combined into release notes for release x.y.z. + +7. Publish Release -* Create PR: `master <- development` -* Meet pre-deployment criteria. - * Include docstrings for any new or updated methods - * Include tutorial documentation for any new major features as needed - * Pass [Codacy code quality check](https://app.codacy.com/gh/TOMToolkit/tom_base/dashboard?bid=18204585). - * Doesn't decrease [Coveralls test coverage](https://coveralls.io/github/TOMToolkit/tom_base?branch=development). - * Passes [Travis tests and code style check](https://travis-ci.com/github/TOMToolkit/tom_base/branches). - * Successfully builds [ReadTheDocs documentation](https://readthedocs.org/projects/tom-toolkit/builds/) (not an automated check) (TODO: fix webhook). -* Merge PR - * Must be a repository owner to merge. -* `git tag -a x.y.z -m "Release x.y.z"` -- must follow semantic versioning -* `git push --tags` Triggers Travis to: - * build, build - * push release to PyPI - * create GitHub draft release -* deploy `tom-demo` with new features demonstrated, pulling `tomtoolkit==x.y.z` from PyPI -* Update Release Notes in GitHub draft release. (This should be the accumulation of the all - the development-release release notes: For example, release notes for releases x.y.z-alpha.1, - x.y.z-alpha.2, etc. should be combined into release notes for release x.y.z. -* Publish Release -* Post notification to Slack, Tom Toolkit workspace, #general channel. (In the future, we hope to +8. Post notification to Slack, Tom Toolkit workspace, #general channel. (In the future, we hope to have automated release notification to a dedicated #releases slack channel). ## Development Notes - Doing checks locally From bab3914fc73bdb2738668900c64706a1ea1726d3 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 9 Jun 2020 13:35:21 -0700 Subject: [PATCH 178/424] A few doc updates --- docs/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 23920c7a5..445926f36 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -66,9 +66,9 @@ Topics :doc:`The Permissions System ` - Use the permissions system to limit access to targets in your TOM. -:doc:`LaTeX Generation ` +:doc:`LaTeX Generation ` - Generate data tables for your targets and observations -:doc:`Interacting with your TOM through code ` +:doc:`Interacting with your TOM through code ` - Learn how to programmatically interact with your TOM. :doc:`Deploying your TOM Online ` - Resources for deploying your TOM to a cloud provider From 1a9375e420ccdc27cb42128f6d17f08bc4a89ba2 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 30 Jun 2020 14:20:34 -0700 Subject: [PATCH 179/424] Removed non-create actions --- tom_targets/api_views.py | 15 ++++++++++----- tom_targets/serializers.py | 2 +- tom_targets/validators.py | 0 tom_targets/views.py | 10 ++++++---- 4 files changed, 17 insertions(+), 10 deletions(-) create mode 100644 tom_targets/validators.py diff --git a/tom_targets/api_views.py b/tom_targets/api_views.py index d8f1a8891..a83140df2 100644 --- a/tom_targets/api_views.py +++ b/tom_targets/api_views.py @@ -1,12 +1,17 @@ -from rest_framework import viewsets from guardian.mixins import PermissionListMixin +from rest_framework.mixins import CreateModelMixin +from rest_framework.viewsets import GenericViewSet -from .serializers import TargetSerializer -from .models import Target -from .filters import TargetFilter +from tom_targets.filters import TargetFilter +from tom_targets.models import Target +from tom_targets.serializers import TargetSerializer -class TargetViewSet(PermissionListMixin, viewsets.ModelViewSet): +# Until we have the bandwidth to add the appropriate validation and ensure that DRF will +# properly respect permissions, this class will inherit from GenericViewSet and the necessary +# mixins for the supported actions. Once we add the appropriate logic for all actions, we +# can update it to just inherit from ModelViewSet. +class TargetViewSet(CreateModelMixin, PermissionListMixin, GenericViewSet): """Viewset for Target objects. By default supports CRUD operations. See the docs on viewsets: https://www.django-rest-framework.org/api-guide/viewsets/ """ diff --git a/tom_targets/serializers.py b/tom_targets/serializers.py index c418b71c0..d6071eb68 100644 --- a/tom_targets/serializers.py +++ b/tom_targets/serializers.py @@ -28,7 +28,7 @@ class Meta: def create(self, validated_data): """DRF requires explicitly handling writeable nested serializers, - here we pop the alias/tag data and save it using thier respective + here we pop the alias/tag data and save it using their respective serializers """ aliases = validated_data.pop('aliases', []) diff --git a/tom_targets/validators.py b/tom_targets/validators.py new file mode 100644 index 000000000..e69de29bb diff --git a/tom_targets/views.py b/tom_targets/views.py index 5cf5b184c..4c0c97c14 100644 --- a/tom_targets/views.py +++ b/tom_targets/views.py @@ -21,6 +21,7 @@ from django.views.generic.list import ListView from django.views.generic import TemplateView, View from django_filters.views import FilterView +from rest_framework import viewsets from guardian.mixins import PermissionListMixin from guardian.shortcuts import get_objects_for_user, get_groups_with_perms, assign_perm @@ -30,14 +31,15 @@ from tom_common.mixins import Raise403PermissionRequiredMixin from tom_observations.observing_strategy import RunStrategyForm from tom_observations.models import ObservingStrategy -from tom_targets.models import Target, TargetList +from tom_targets.filters import TargetFilter from tom_targets.forms import ( SiderealTargetCreateForm, NonSiderealTargetCreateForm, TargetExtraFormset, TargetNamesFormset ) +from tom_targets.groups import ( + add_all_to_grouping, add_selected_to_grouping, remove_all_from_grouping, remove_selected_from_grouping +) +from tom_targets.models import Target, TargetList from tom_targets.utils import import_targets, export_targets -from tom_targets.filters import TargetFilter -from tom_targets.groups import add_all_to_grouping, add_selected_to_grouping -from tom_targets.groups import remove_all_from_grouping, remove_selected_from_grouping logger = logging.getLogger(__name__) From 3a9abf55722d10bf238a779a2f8e97f00c58f30b Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 30 Jun 2020 15:30:39 -0700 Subject: [PATCH 180/424] Fixing broken link --- docs/observing/strategies.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/observing/strategies.rst b/docs/observing/strategies.rst index 87dd6c2d9..98365530c 100644 --- a/docs/observing/strategies.rst +++ b/docs/observing/strategies.rst @@ -251,7 +251,7 @@ Fortunately, the TOM Toolkit comes with a built-in management command to update all cadences in the TOM. If you’ve perused the TOM Toolkit documentation previously, you may have noticed a section about automation of tasks, and, more specifically, a subsection about -:doc:``Using cron with a management command <../customization/automation>``. +:doc:`Using cron with a management command <../code/automation>`. You can simply apply the instructions here, but use the management command ``runcadencestrategies.py`` in place of the example. If you set your cron to run every few minutes or so, you’ll ensure that your From 2464f89380eb6edba85332879cb42283d117daf9 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 1 Jul 2020 10:14:35 -0700 Subject: [PATCH 181/424] Added cross-field validation to target serializers --- tom_targets/serializers.py | 14 ++++++++++++-- tom_targets/validators.py | 26 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/tom_targets/serializers.py b/tom_targets/serializers.py index d6071eb68..715981a73 100644 --- a/tom_targets/serializers.py +++ b/tom_targets/serializers.py @@ -1,5 +1,7 @@ from rest_framework import serializers -from .models import Target, TargetExtra, TargetName + +from tom_targets.models import Target, TargetExtra, TargetName +from tom_targets.validators import RequiredFieldsTogetherValidator class TargetNameSerializer(serializers.ModelSerializer): @@ -25,10 +27,18 @@ class TargetSerializer(serializers.ModelSerializer): class Meta: model = Target fields = '__all__' + validators = [RequiredFieldsTogetherValidator('type', 'SIDEREAL', 'ra', 'dec'), + RequiredFieldsTogetherValidator('type', 'NON_SIDEREAL', 'epoch_of_elements', 'inclination', + 'lng_asc_node', 'arg_of_perihelion', 'eccentricity'), + RequiredFieldsTogetherValidator('scheme', 'MPC_COMET', 'perihdist', 'epoch_of_perihelion'), + RequiredFieldsTogetherValidator('scheme', 'MPC_MINOR_PLANET', 'mean_anomaly', 'semimajor_axis'), + RequiredFieldsTogetherValidator('scheme', 'JPL_MAJOR_PLANET', 'mean_daily_motion', 'mean_anomaly', + 'semimajor_axis') + ] def create(self, validated_data): """DRF requires explicitly handling writeable nested serializers, - here we pop the alias/tag data and save it using their respective + here we pop the alias/tag data and save it using thier respective serializers """ aliases = validated_data.pop('aliases', []) diff --git a/tom_targets/validators.py b/tom_targets/validators.py index e69de29bb..155edc5f1 100644 --- a/tom_targets/validators.py +++ b/tom_targets/validators.py @@ -0,0 +1,26 @@ +from rest_framework.serializers import ValidationError + +class RequiredFieldsTogetherValidator(object): + + def __init__(self, type_name, type_value, *args): + self.type_name = type_name + self.type_value = type_value + self.required_fields = args + + def __call__(self, attrs): + print(attrs) + values = dict(attrs) + print(values) + if self.type_value != values.get(self.type_name): + return + + missing_fields = [] + + print(self.required_fields) + for field in self.required_fields: + print(field) + if not values.get(field): + missing_fields.append(field) + + if missing_fields: + raise ValidationError(f'The following fields are required for {self.type_value} targets: {missing_fields}') From a4f528f326205d36ddf15955d39f9aff465ca3d8 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 1 Jul 2020 10:22:16 -0700 Subject: [PATCH 182/424] Modified verbose name of TargetName name and updated respective test --- tom_targets/models.py | 2 +- tom_targets/tests/tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tom_targets/models.py b/tom_targets/models.py index d2d7b8b46..1509ebd48 100644 --- a/tom_targets/models.py +++ b/tom_targets/models.py @@ -369,7 +369,7 @@ class TargetName(models.Model): :type modified: datetime """ target = models.ForeignKey(Target, on_delete=models.CASCADE, related_name='aliases') - name = models.CharField(max_length=100, unique=True, verbose_name='Alias for target') + name = models.CharField(max_length=100, unique=True, verbose_name='Alias') created = models.DateTimeField( auto_now_add=True, help_text='The time which this target name was created.' ) diff --git a/tom_targets/tests/tests.py b/tom_targets/tests/tests.py index bea91f72f..a86c08e65 100644 --- a/tom_targets/tests/tests.py +++ b/tom_targets/tests/tests.py @@ -392,7 +392,7 @@ def test_create_targets_with_conflicting_aliases(self): self.client.post(reverse('targets:create'), data=target_data, follow=True) target_data['name'] = 'multiple_names_target2' second_response = self.client.post(reverse('targets:create'), data=target_data, follow=True) - self.assertContains(second_response, 'Target name with this Alias for target already exists.') + self.assertContains(second_response, 'Target name with this Alias already exists.') class TestTargetImport(TestCase): From daa928d59bbde6cd8c86cecb07ddc332b4d9abc1 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Wed, 1 Jul 2020 23:30:43 +0000 Subject: [PATCH 183/424] add basic seriaizers for data_products.models --- tom_dataproducts/serializers.py | 40 +++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 tom_dataproducts/serializers.py diff --git a/tom_dataproducts/serializers.py b/tom_dataproducts/serializers.py new file mode 100644 index 000000000..439caa71c --- /dev/null +++ b/tom_dataproducts/serializers.py @@ -0,0 +1,40 @@ +from rest_framework import serializers +from .models import DataProductGroup, DataProduct, ReducedDatum + + +class DataProductGroupSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = DataProductGroup + fields = ('name', 'created', 'modified') + + +class DataProductSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = DataProduct + fields = ( + 'product_id', + 'target', + 'observation_record', + 'data', + 'extra_data', + 'group', + 'created', + 'modified', + 'data_product_type', + 'featured', + 'thumbnail' + ) + + +class ReducedDatumSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = ReducedDatum + fields = ( + 'target', + 'data_product', + 'data_type', + 'source_name', + 'source_location', + 'timestamp', + 'value' + ) From f67297f39b81fb7b5b90fc6300e24f7243a8a1fb Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Thu, 2 Jul 2020 14:35:58 +0000 Subject: [PATCH 184/424] Add minimal ViewSets for dataproducts model classes --- tom_dataproducts/api_views.py | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 tom_dataproducts/api_views.py diff --git a/tom_dataproducts/api_views.py b/tom_dataproducts/api_views.py new file mode 100644 index 000000000..0edb55024 --- /dev/null +++ b/tom_dataproducts/api_views.py @@ -0,0 +1,38 @@ +from guardian.mixins import PermissionListMixin +from rest_framework.mixins import CreateModelMixin +from rest_framework.viewsets import GenericViewSet + +from tom_dataproducts.models import DataProductGroup, DataProduct, ReducedDatum +from tom_dataproducts.serializers import DataProductGroupSerializer, DataProductSerializer, ReducedDatumSerializer + +# TODO: see Davids comment in tom_targets/api_views.py + + +class DataProductGroupViewSet(CreateModelMixin, PermissionListMixin, GenericViewSet): + """Viewset for Target objects. By default supports CRUD operations. + See the docs on viewsets: https://www.django-rest-framework.org/api-guide/viewsets/ + """ + queryset = DataProductGroup + serializer_class = DataProductGroupSerializer + # TODO: define filterset_class + # TODO: define permission_required + + +class DataProductViewSet(CreateModelMixin, PermissionListMixin, GenericViewSet): + """Viewset for Target objects. By default supports CRUD operations. + See the docs on viewsets: https://www.django-rest-framework.org/api-guide/viewsets/ + """ + queryset = DataProduct + serializer_class = DataProductSerializer + # TODO: define filterset_class + # TODO: define permission_required + + +class ReducedDatumViewSet(CreateModelMixin, PermissionListMixin, GenericViewSet): + """Viewset for Target objects. By default supports CRUD operations. + See the docs on viewsets: https://www.django-rest-framework.org/api-guide/viewsets/ + """ + queryset = ReducedDatum + serializer_class = ReducedDatumSerializer + # TODO: define filterset_class + # TODO: define permission_required From 1e1887bc40e328360bcc4a6a4ed38cc1666a645d Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Thu, 2 Jul 2020 15:06:40 +0000 Subject: [PATCH 185/424] add url routes for dataproducts REST endpoints --- tom_dataproducts/api_urls.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 tom_dataproducts/api_urls.py diff --git a/tom_dataproducts/api_urls.py b/tom_dataproducts/api_urls.py new file mode 100644 index 000000000..d0a6fd8b3 --- /dev/null +++ b/tom_dataproducts/api_urls.py @@ -0,0 +1,18 @@ +from rest_framework.routers import DefaultRouter + +from .api_views import DataProductGroupViewSet, DataProductViewSet, ReducedDatumViewSet + +"""A url module specifically for api paths, separate from +the rest so it can be included in a modular fashion. +""" + +app_name = 'api' + +router = DefaultRouter() + +# prefix, ViewSet, basename +router.register(r'dataproductgroups', DataProductGroupViewSet) +router.register(r'dataproducts', DataProductViewSet) +router.register(r'reduceddatums', ReducedDatumViewSet) + +urlpatterns = router.urls From c61e1d861242ba0993413eccecbbab694d4fd442 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 7 Jul 2020 10:49:00 -0700 Subject: [PATCH 186/424] added some tests and TODOs --- tom_targets/api_views.py | 6 +++--- tom_targets/serializers.py | 2 ++ tom_targets/tests/test_api.py | 35 +++++++++++++++++++++++++++++++++++ tom_targets/validators.py | 4 ---- 4 files changed, 40 insertions(+), 7 deletions(-) diff --git a/tom_targets/api_views.py b/tom_targets/api_views.py index a83140df2..96aa6b553 100644 --- a/tom_targets/api_views.py +++ b/tom_targets/api_views.py @@ -1,6 +1,6 @@ from guardian.mixins import PermissionListMixin -from rest_framework.mixins import CreateModelMixin -from rest_framework.viewsets import GenericViewSet +from rest_framework.mixins import CreateModelMixin, DestroyModelMixin, UpdateModelMixin +from rest_framework.viewsets import GenericViewSet, ModelViewSet from tom_targets.filters import TargetFilter from tom_targets.models import Target @@ -11,7 +11,7 @@ # properly respect permissions, this class will inherit from GenericViewSet and the necessary # mixins for the supported actions. Once we add the appropriate logic for all actions, we # can update it to just inherit from ModelViewSet. -class TargetViewSet(CreateModelMixin, PermissionListMixin, GenericViewSet): +class TargetViewSet(PermissionListMixin, ModelViewSet): """Viewset for Target objects. By default supports CRUD operations. See the docs on viewsets: https://www.django-rest-framework.org/api-guide/viewsets/ """ diff --git a/tom_targets/serializers.py b/tom_targets/serializers.py index 715981a73..9415cfe66 100644 --- a/tom_targets/serializers.py +++ b/tom_targets/serializers.py @@ -27,6 +27,8 @@ class TargetSerializer(serializers.ModelSerializer): class Meta: model = Target fields = '__all__' + # TODO: We should investigate if this validator logic can be reused in the forms to reduce code duplication. + # TODO: Try to put validators in settings to allow user changes validators = [RequiredFieldsTogetherValidator('type', 'SIDEREAL', 'ra', 'dec'), RequiredFieldsTogetherValidator('type', 'NON_SIDEREAL', 'epoch_of_elements', 'inclination', 'lng_asc_node', 'arg_of_perihelion', 'eccentricity'), diff --git a/tom_targets/tests/test_api.py b/tom_targets/tests/test_api.py index 0304f08ae..0ebb1a29e 100644 --- a/tom_targets/tests/test_api.py +++ b/tom_targets/tests/test_api.py @@ -50,6 +50,41 @@ def test_target_create(self): self.assertEqual(response.json()['name'], target_data['name']) self.assertEqual(response.json()['aliases'][0]['name'], target_data['aliases'][0]['name']) + def test_target_create_sidereal_missing_parameters(self): + target_data = { + 'name': 'test_target_name_wtf', + 'type': Target.SIDEREAL, + 'ra': 123.456, + 'targetextra_set': [ + {'key': 'foo', 'value': 5} + ], + 'aliases': [ + {'name': 'alternative name'} + ] + } + response = self.client.post(reverse('api:targets-list'), data=target_data) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(response.json()['name'], target_data['name']) + self.assertEqual(response.json()['aliases'][0]['name'], target_data['aliases'][0]['name']) + + def test_target_create_non_sidereal_missing_parameters(self): + target_data = { + 'name': 'test_target_name_wtf', + 'type': Target.SIDEREAL, + 'ra': 123.456, + 'dec': -32.1, + 'targetextra_set': [ + {'key': 'foo', 'value': 5} + ], + 'aliases': [ + {'name': 'alternative name'} + ] + } + response = self.client.post(reverse('api:targets-list'), data=target_data) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(response.json()['name'], target_data['name']) + self.assertEqual(response.json()['aliases'][0]['name'], target_data['aliases'][0]['name']) + def test_target_update(self): updates = {'ra': 123.456} response = self.client.patch(reverse('api:targets-detail', args=(self.st.id,)), data=updates) diff --git a/tom_targets/validators.py b/tom_targets/validators.py index 155edc5f1..b785bed76 100644 --- a/tom_targets/validators.py +++ b/tom_targets/validators.py @@ -8,17 +8,13 @@ def __init__(self, type_name, type_value, *args): self.required_fields = args def __call__(self, attrs): - print(attrs) values = dict(attrs) - print(values) if self.type_value != values.get(self.type_name): return missing_fields = [] - print(self.required_fields) for field in self.required_fields: - print(field) if not values.get(field): missing_fields.append(field) From fd051d16b5f78ea768b4f13277dcd6e13d77728a Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 9 Jul 2020 11:12:21 -0700 Subject: [PATCH 187/424] Removed facility references in target tests to prevent external api calls during unit tests --- tom_targets/tests/tests.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tom_targets/tests/tests.py b/tom_targets/tests/tests.py index bea91f72f..20c5b78e7 100644 --- a/tom_targets/tests/tests.py +++ b/tom_targets/tests/tests.py @@ -65,6 +65,11 @@ def test_list_targets_limited_permissions(self): self.assertNotContains(response, self.st1.name) +# Because the target detail page has a templatetag that tries to get the facility status, these tests fail without +# network. While the preferred solution would be to create a mock facility class, in order to avoid any potential +# circular imports, we're simply disabling the facility classes for these tests. This can be revisited if need be at a +# future time, but currently the target tests don't do anything with ObservationRecords anyway. +@override_settings(TOM_FACILITY_CLASSES=[]) class TestTargetDetail(TestCase): def setUp(self): user = User.objects.create(username='testuser') @@ -100,6 +105,7 @@ def test_target_bad_permissions(self): self.assertContains(response, 'You do not have permission to access this page') +@override_settings(TOM_FACILITY_CLASSES=[]) class TestTargetCreate(TestCase): def setUp(self): user = User.objects.create(username='testuser') From 4132fef020ca7b2c5ef246ff246277afbd3c2ccc Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 9 Jul 2020 11:16:32 -0700 Subject: [PATCH 188/424] Fixing code style issues --- tom_targets/tests/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tom_targets/tests/tests.py b/tom_targets/tests/tests.py index 20c5b78e7..f3df7dde5 100644 --- a/tom_targets/tests/tests.py +++ b/tom_targets/tests/tests.py @@ -65,9 +65,9 @@ def test_list_targets_limited_permissions(self): self.assertNotContains(response, self.st1.name) -# Because the target detail page has a templatetag that tries to get the facility status, these tests fail without -# network. While the preferred solution would be to create a mock facility class, in order to avoid any potential -# circular imports, we're simply disabling the facility classes for these tests. This can be revisited if need be at a +# Because the target detail page has a templatetag that tries to get the facility status, these tests fail without +# network. While the preferred solution would be to create a mock facility class, in order to avoid any potential +# circular imports, we're simply disabling the facility classes for these tests. This can be revisited if need be at a # future time, but currently the target tests don't do anything with ObservationRecords anyway. @override_settings(TOM_FACILITY_CLASSES=[]) class TestTargetDetail(TestCase): From 2646be441e63a6e507a73eb1a7b318dd10f29083 Mon Sep 17 00:00:00 2001 From: fraserw Date: Thu, 9 Jul 2020 12:06:16 -0700 Subject: [PATCH 189/424] --setup for creation of unit tests --- tom_targets/tests/tests.py | 70 ++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/tom_targets/tests/tests.py b/tom_targets/tests/tests.py index bea91f72f..39f69c4f3 100644 --- a/tom_targets/tests/tests.py +++ b/tom_targets/tests/tests.py @@ -13,6 +13,29 @@ from guardian.shortcuts import assign_perm +base_data_Major_Planet = { + 'name': 'nonsidereal_target', + 'identifier': 'nonsidereal_identifier', + 'type': Target.NON_SIDEREAL, + 'epoch_of_elements': 100, + 'lng_asc_node': 100, + 'arg_of_perihelion': 100, + 'eccentricity': 100, + 'mean_anomaly': 100, + 'inclination': 100, + 'semimajor_axis': 100, + 'targetextra_set-TOTAL_FORMS': 1, + 'targetextra_set-INITIAL_FORMS': 0, + 'targetextra_set-MIN_NUM_FORMS': 0, + 'targetextra_set-MAX_NUM_FORMS': 1000, + 'targetextra_set-0-key': '', + 'targetextra_set-0-value': '', + 'aliases-TOTAL_FORMS': 1, + 'aliases-INITIAL_FORMS': 0, + 'aliases-MIN_NUM_FORMS': 0, + 'aliases-MAX_NUM_FORMS': 1000, +} + class TestTargetListUserPermissions(TestCase): def setUp(self): self.user = User.objects.create(username='testuser') @@ -74,6 +97,9 @@ def setUp(self): assign_perm('tom_targets.view_target', user, self.st) assign_perm('tom_targets.view_target', user, self.nst) + """ + these four below tests are the ones to comment out if you can't get to api/telescope_states + """ def test_sidereal_target_detail(self): response = self.client.get(reverse('targets:detail', kwargs={'pk': self.st.id})) self.assertContains(response, self.st.id) @@ -92,13 +118,17 @@ def test_extra_fields(self): self.assertContains(response, 'somevalue') self.assertNotContains(response, 'hiddenvalue') + ## Wes: probably want to test that an EPHEMERIS object with custom scheme actually + ## has the necessary ephemeris variables. Like test_extra_fields above + def test_target_bad_permissions(self): other_user = User.objects.create(username='otheruser') self.client.force_login(other_user) response = self.client.get(reverse('targets:detail', kwargs={'pk': self.st.id}), follow=True) self.assertRedirects(response, '{}?next=/targets/{}/'.format(reverse('login'), self.st.id)) self.assertContains(response, 'You do not have permission to access this page') - + """ + """ class TestTargetCreate(TestCase): def setUp(self): @@ -113,6 +143,8 @@ def test_target_create_form(self): self.assertContains(response, Target.SIDEREAL) self.assertContains(response, Target.NON_SIDEREAL) + ## Wes: add a test ephemeris target creation here with all four schemes like test_create_target below + ## Actually probably want to test the items as in test_non_sidereal_required_fields def test_create_target(self): target_data = { 'name': 'test_target_name', @@ -250,33 +282,13 @@ def test_target_save_programmatic_extras(self): target.save(extras={'foo': 5}) self.assertTrue(TargetExtra.objects.filter(target=target, key='foo', value='5').exists()) - def test_non_sidereal_required_fields(self): - base_data = { - 'name': 'nonsidereal_target', - 'identifier': 'nonsidereal_identifier', - 'type': Target.NON_SIDEREAL, - 'epoch_of_elements': 100, - 'lng_asc_node': 100, - 'arg_of_perihelion': 100, - 'eccentricity': 100, - 'mean_anomaly': 100, - 'inclination': 100, - 'semimajor_axis': 100, - 'targetextra_set-TOTAL_FORMS': 1, - 'targetextra_set-INITIAL_FORMS': 0, - 'targetextra_set-MIN_NUM_FORMS': 0, - 'targetextra_set-MAX_NUM_FORMS': 1000, - 'targetextra_set-0-key': '', - 'targetextra_set-0-value': '', - 'aliases-TOTAL_FORMS': 1, - 'aliases-INITIAL_FORMS': 0, - 'aliases-MIN_NUM_FORMS': 0, - 'aliases-MAX_NUM_FORMS': 1000, - } + def test_non_sidereal_required_fields_Major_Planet(self): + print('here Major') + create_url = reverse('targets:create') + '?type=NON_SIDEREAL' # Make data for a major planet scheme: missing 'mean_daily_motion' - maj_planet_data = dict(**base_data, scheme='JPL_MAJOR_PLANET') + maj_planet_data = dict(**base_data_Major_Planet, scheme='JPL_MAJOR_PLANET') response = self.client.post(create_url, data=maj_planet_data, follow=True) errors = response.context['form'].errors self.assertEqual(set(errors.keys()), {'__all__'}) @@ -285,8 +297,13 @@ def test_non_sidereal_required_fields(self): self.assertTrue(messages[0].startswith("Scheme 'JPL Major Planet' requires fields")) self.assertIn('Daily Motion', messages[0]) + def test_non_sidereal_required_fields_Minor_Planet(self): + print('here Minor') + + create_url = reverse('targets:create') + '?type=NON_SIDEREAL' + # Use the same data for minor planet: should be no errors - min_planet_data = dict(**base_data, scheme='MPC_MINOR_PLANET') + min_planet_data = dict(**base_data_Major_Planet, scheme='MPC_MINOR_PLANET') response = self.client.post(create_url, data=min_planet_data, follow=True) errors = response.context['form'].errors self.assertEqual(errors, {}) @@ -395,6 +412,7 @@ def test_create_targets_with_conflicting_aliases(self): self.assertContains(second_response, 'Target name with this Alias for target already exists.') +# Wes: probably want to test importing an ephemeris csv file class TestTargetImport(TestCase): def setUp(self): user = User.objects.create(username='testuser') From 3c8e5f333df42d5b0aa54cf3c9b670075589a27c Mon Sep 17 00:00:00 2001 From: fraserw Date: Thu, 9 Jul 2020 12:47:19 -0700 Subject: [PATCH 190/424] --added eph_json item to factory --- tom_targets/tests/factories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_targets/tests/factories.py b/tom_targets/tests/factories.py index 808d49fcd..1162fc0a3 100644 --- a/tom_targets/tests/factories.py +++ b/tom_targets/tests/factories.py @@ -32,7 +32,7 @@ class Meta: ephemeris_period_err = factory.Faker('pyfloat') ephemeris_epoch = factory.Faker('pyfloat') ephemeris_epoch_err = factory.Faker('pyfloat') - + eph_json = factory.Faker('pystr') class TargetNameFactory(factory.django.DjangoModelFactory): class Meta: From 9cbc4e17b91682ad40f786e8e78e8605fedf6911 Mon Sep 17 00:00:00 2001 From: fraserw Date: Thu, 9 Jul 2020 12:48:06 -0700 Subject: [PATCH 191/424] --clean up of a function doc --- tom_targets/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tom_targets/views.py b/tom_targets/views.py index 3910fc961..0d33a5e9c 100644 --- a/tom_targets/views.py +++ b/tom_targets/views.py @@ -393,7 +393,8 @@ def post(self, request): class TargetImportEphemerisView(LoginRequiredMixin, TemplateView): """ - View that handles the import of targets from a .eph . Requires authentication. + View that handles the import of targets from a .eph file for EPHEMERIS scheme. + Requires authentication. """ template_name = 'tom_targets/target_ephemeris_import.html' From 4fb4d442703f32ff1642ea141c3d84be561bb2f9 Mon Sep 17 00:00:00 2001 From: fraserw Date: Thu, 9 Jul 2020 12:48:35 -0700 Subject: [PATCH 192/424] --critical unit tests created for non-sidereal-EPHEMERIS target --- tom_targets/tests/tests.py | 81 +++++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/tom_targets/tests/tests.py b/tom_targets/tests/tests.py index 1526caeab..7fc97bb61 100644 --- a/tom_targets/tests/tests.py +++ b/tom_targets/tests/tests.py @@ -1,5 +1,6 @@ import pytz from datetime import datetime +from io import StringIO from django.contrib.auth.models import User, Group from django.contrib.messages import get_messages @@ -9,7 +10,8 @@ from .factories import SiderealTargetFactory, NonSiderealTargetFactory, TargetGroupingFactory, TargetNameFactory from tom_targets.models import Target, TargetExtra, TargetList, TargetName -from tom_targets.utils import import_targets +from tom_targets.utils import import_targets, import_ephemeris_target +import tom_targets from guardian.shortcuts import assign_perm @@ -36,6 +38,54 @@ 'aliases-MAX_NUM_FORMS': 1000, } +base_data_Comet = { + 'name': 'nonsidereal_target', + 'identifier': 'nonsidereal_identifier', + 'type': Target.NON_SIDEREAL, + 'epoch_of_elements': 100, + 'perihdist': 1.0, + 'epoch_of_perihelion': 100.0, + 'arg_of_perihelion': 100, + 'eccentricity': 100, + 'lng_asc_node': 100, + 'semimajor_axis': 100, + 'inclination': 100, + 'targetextra_set-TOTAL_FORMS': 1, + 'targetextra_set-INITIAL_FORMS': 0, + 'targetextra_set-MIN_NUM_FORMS': 0, + 'targetextra_set-MAX_NUM_FORMS': 1000, + 'targetextra_set-0-key': '', + 'targetextra_set-0-value': '', + 'aliases-TOTAL_FORMS': 1, + 'aliases-INITIAL_FORMS': 0, + 'aliases-MIN_NUM_FORMS': 0, + 'aliases-MAX_NUM_FORMS': 1000, +} + +base_data_EPH = { + 'name': 'nonsidereal_target', + 'identifier': 'nonsidereal_identifier', + 'type': Target.NON_SIDEREAL, + 'eph_json': {'568': [{'t': '58940.0', 'R': '196.9809167', 'D': ' 23.904639', 'dR': 0.0, 'dD': 0.0}, + {'t': '58940.1', 'R': '196.9806667', 'D': ' 23.904722', 'dR': 0.0, 'dD': 0.0}, + {'t': '58940.2', 'R': '196.9804167', 'D': ' 23.904833', 'dR': 0.0, 'dD': 0.0}] + }, + 'epoch_of_elements': 100, + 'targetextra_set-TOTAL_FORMS': 1, + 'targetextra_set-INITIAL_FORMS': 0, + 'targetextra_set-MIN_NUM_FORMS': 0, + 'targetextra_set-MAX_NUM_FORMS': 1000, + 'targetextra_set-0-key': '', + 'targetextra_set-0-value': '', + 'aliases-TOTAL_FORMS': 1, + 'aliases-INITIAL_FORMS': 0, + 'aliases-MIN_NUM_FORMS': 0, + 'aliases-MAX_NUM_FORMS': 1000, + +} + +#{'568': [{'t': '58940.0', 'R': '196.9809167', 'D': ' 23.904639', 'dR': 0.0, 'dD': 0.0}, {'t': '58940.01388888899', 'R': '196.9806667', 'D': ' 23.904722', 'dR': 0.0, 'dD': 0.0}, {'t': '58940.027777777985', 'R': '196.9804167', 'D': ' 23.904833', 'dR': 0.0, 'dD': 0.0} + class TestTargetListUserPermissions(TestCase): def setUp(self): self.user = User.objects.create(username='testuser') @@ -314,6 +364,28 @@ def test_non_sidereal_required_fields_Minor_Planet(self): errors = response.context['form'].errors self.assertEqual(errors, {}) + def test_non_sidereal_required_fields_Comet(self): + print('here Comet') + + create_url = reverse('targets:create') + '?type=NON_SIDEREAL' + + # Use the same data for minor planet: should be no errors + min_planet_data = dict(**base_data_Comet, scheme='MPC_COMET') + response = self.client.post(create_url, data=min_planet_data, follow=True) + errors = response.context['form'].errors + self.assertEqual(errors, {}) + + def test_non_sidereal_required_fields_EPHEMERIS(self): + print('here Ephemeris') + + create_url = reverse('targets:create') + '?type=NON_SIDEREAL' + + # Use the same data for minor planet: should be no errors + min_planet_data = dict(**base_data_EPH, scheme='EPHEMERIS') + response = self.client.post(create_url, data=min_planet_data, follow=True) + errors = response.context['form'].errors + self.assertEqual(errors, {}) + def test_create_form_failure(self): """ If a failure occurs when creating a non-sidereal target, make sure the @@ -459,6 +531,13 @@ def test_import_csv_with_multiple_names(self): for alias in aliases[target_name].split(','): self.assertTrue(TargetName.objects.filter(target=target, name=alias).exists()) + def test_import_ephemeris_csv(self): + root = tom_targets.__file__.split('__')[0] + eph_file = open(root + 'static/tom_targets/target_ephemeris_import.eph') + eph_stream = StringIO(eph_file.read(), newline='\n') + result = import_ephemeris_target(eph_stream) + eph_file.close() + self.assertEqual(len(result['targets']), 1) class TestTargetExport(TestCase): """ From 4da7c66df5508d34eec84ccdc65e9fa6c7c6bcd6 Mon Sep 17 00:00:00 2001 From: fraserw Date: Thu, 9 Jul 2020 12:50:45 -0700 Subject: [PATCH 193/424] --moved EPHEMERIS logic to LCOBaseObservationForm as per new refactoring --- tom_observations/facilities/lco.py | 46 +++++++++++++++++------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index e23946581..31947338c 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -110,20 +110,6 @@ class LCOBaseForm(forms.Form): exposure_time = forms.FloatField(min_value=0.1) max_airmass = forms.FloatField() - site = forms.ChoiceField( - choices=(('all', 'All Sites'), - ('coj', 'Siding Spring'), - ('cpt', 'Sutherland'), - ('tfn', 'Teide'), - ('tlv', 'Wise'), - ('lsc', 'Cerro Tololo'), - ('elp', 'McDonald'), - ('ogg', 'Haleakala')) - ) - - imaging_interval = forms.FloatField( - label='Interval (hrs). Will schedule exposure count per interval.' - ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -137,12 +123,6 @@ def __init__(self, *args, **kwargs): choices=self.instrument_choices() ) - self.eph_target = False - target = Target.objects.get(pk=kwargs['initial']['target_id']) - if target.type == Target.NON_SIDEREAL: - if target.scheme == 'EPHEMERIS': - self.eph_target = True - def _get_instruments(self): cached_instruments = cache.get('lco_instruments') @@ -203,7 +183,30 @@ class LCOBaseObservationForm(BaseRoboticObservationForm, LCOBaseForm, CadenceFor help_text=observation_mode_help ) + site = forms.ChoiceField( + choices=(('all', 'All Sites'), + ('coj', 'Siding Spring'), + ('cpt', 'Sutherland'), + ('tfn', 'Teide'), + ('tlv', 'Wise'), + ('lsc', 'Cerro Tololo'), + ('elp', 'McDonald'), + ('ogg', 'Haleakala')) + ) + + imaging_interval = forms.FloatField( + label='Interval (hrs). Will schedule exposure count per interval.' + ) + def __init__(self, *args, **kwargs): + # the ephemeris target stuff must come before super() + self.eph_target = False + if 'initial' in kwargs: + target = Target.objects.get(pk=kwargs['initial']['target_id']) + if target.type == Target.NON_SIDEREAL: + if target.scheme == 'EPHEMERIS': + self.eph_target = True + super().__init__(*args, **kwargs) self.helper.layout = Layout( self.common_layout, @@ -212,6 +215,8 @@ def __init__(self, *args, **kwargs): self.button_layout() ) + + def layout(self): return Div( Div( @@ -239,6 +244,7 @@ def layout(self): ), css_class='form-row' ), + self.extra_layout() ) def extra_layout(self): From 650477eb75d19a100b094f5be74f88ae2cb7324e Mon Sep 17 00:00:00 2001 From: fraserw Date: Thu, 9 Jul 2020 12:55:38 -0700 Subject: [PATCH 194/424] --somehow adding a pystr faker for eph_json causes a failure. Removed. --- tom_targets/tests/factories.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tom_targets/tests/factories.py b/tom_targets/tests/factories.py index 1162fc0a3..6048deb5e 100644 --- a/tom_targets/tests/factories.py +++ b/tom_targets/tests/factories.py @@ -32,7 +32,6 @@ class Meta: ephemeris_period_err = factory.Faker('pyfloat') ephemeris_epoch = factory.Faker('pyfloat') ephemeris_epoch_err = factory.Faker('pyfloat') - eph_json = factory.Faker('pystr') class TargetNameFactory(factory.django.DjangoModelFactory): class Meta: From e2abcf143f37ae0d3d25c1fe18356b59faabe0d2 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 14 Jul 2020 11:19:00 -0700 Subject: [PATCH 195/424] Added business logic to handle update/create of targetextras and targetnames --- tom_targets/serializers.py | 71 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 68 insertions(+), 3 deletions(-) diff --git a/tom_targets/serializers.py b/tom_targets/serializers.py index 9415cfe66..0a1fea33b 100644 --- a/tom_targets/serializers.py +++ b/tom_targets/serializers.py @@ -5,15 +5,19 @@ class TargetNameSerializer(serializers.ModelSerializer): + id = serializers.IntegerField(required=False) + class Meta: model = TargetName - fields = ('name',) + fields = ('id', 'name',) class TargetExtraSerializer(serializers.ModelSerializer): + id = serializers.IntegerField(required=False) + class Meta: model = TargetExtra - fields = ('key', 'value') + fields = ('id', 'key', 'value') class TargetSerializer(serializers.ModelSerializer): @@ -40,7 +44,7 @@ class Meta: def create(self, validated_data): """DRF requires explicitly handling writeable nested serializers, - here we pop the alias/tag data and save it using thier respective + here we pop the alias/tag data and save it using their respective serializers """ aliases = validated_data.pop('aliases', []) @@ -57,3 +61,64 @@ def create(self, validated_data): tes.save(target=target) return target + + + def update(self, instance, validated_data): + """ + For TargetExtra and TargetName objects, if the ID is present, it will update the corresponding row. If the ID is + not present, it will attempt to create a new TargetExtra or TargetName associated with this Target. + """ + aliases = validated_data.pop('aliases', []) + targetextras = validated_data.pop('targetextra_set', []) + + instance.name = validated_data.get('name', instance.name) + instance.type = validated_data.get('type', instance.type) + instance.ra = validated_data.get('ra', instance.ra) + instance.dec = validated_data.get('dec', instance.dec) + instance.epoch = validated_data.get('epoch', instance.epoch) + instance.parallax = validated_data.get('parallax', instance.parallax) + instance.pm_ra = validated_data.get('pm_ra', instance.pm_ra) + instance.pm_dec = validated_data.get('pm_dec', instance.pm_dec) + instance.galactic_lng = validated_data.get('galactic_lng', instance.galactic_lng) + instance.galactic_lat = validated_data.get('galactic_lat', instance.galactic_lat) + instance.distance = validated_data.get('distance', instance.distance) + instance.distance_err = validated_data.get('distance_err', instance.distance_err) + instance.scheme = validated_data.get('scheme', instance.scheme) + instance.epoch_of_elements = validated_data.get('epoch_of_elements', instance.epoch_of_elements) + instance.mean_anomaly = validated_data.get('mean_anomaly', instance.mean_anomaly) + instance.arg_of_perihelion = validated_data.get('arg_of_perihelion', instance.arg_of_perihelion) + instance.eccentricity = validated_data.get('eccentricity', instance.eccentricity) + instance.lng_asc_node = validated_data.get('lng_asc_node', instance.lng_asc_node) + instance.inclination = validated_data.get('inclination', instance.inclination) + instance.mean_daily_motion = validated_data.get('mean_daily_motion', instance.mean_daily_motion) + instance.semimajor_axis = validated_data.get('semimajor_axis', instance.semimajor_axis) + instance.epoch_of_perihelion = validated_data.get('epoch_of_perihelion', instance.epoch_of_perihelion) + instance.ephemeris_period = validated_data.get('ephemeris_period', instance.ephemeris_period) + instance.ephemeris_period_err = validated_data.get('ephemeris_period_err', instance.ephemeris_period_err) + instance.ephemeris_epoch = validated_data.get('ephemeris_epoch', instance.ephemeris_epoch) + instance.ephemeris_epoch_err = validated_data.get('ephemeris_epoch_err', instance.ephemeris_epoch_err) + instance.perihdist = validated_data.get('perihdist', instance.perihdist) + instance.save() + + # TODO: updating an existing TargetName with the same value results in an integrity error + for alias_data in aliases: + alias = dict(alias_data) + if alias.get('id'): + tn_instance = TargetName.objects.get(pk=alias['id']) + tns = TargetNameSerializer(tn_instance, data=alias_data) + else: + tns = TargetNameSerializer(data=alias_data) + if tns.is_valid(): + tns.save(target=instance) + + for te_data in targetextras: + te = dict(te_data) + if te_data.get('id'): + te_instance = TargetExtra.objects.get(pk=te['id']) + tes = TargetExtraSerializer(te_instance, data=te_data) + else: + tes = TargetExtraSerializer(data=te_data) + if tes.is_valid(): + tes.save(target=instance) + + return instance From 36b0d9cdbfd8ad4cc304b9622b03b30bd95bcd0b Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 15 Jul 2020 07:15:13 -0700 Subject: [PATCH 196/424] Added some notes and start targetname viewsets --- tom_base/settings.py | 7 ++++++- tom_setup/templates/tom_setup/settings.tmpl | 10 +++++++++- tom_targets/api_urls.py | 3 ++- tom_targets/api_views.py | 22 ++++++++++++++++----- tom_targets/serializers.py | 1 + 5 files changed, 35 insertions(+), 8 deletions(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index 1c73b1024..651f63e9b 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -26,7 +26,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [] +ALLOWED_HOSTS = [''] # Application definition @@ -123,6 +123,8 @@ }, ] +# TODO: Release notes MUST document this change!! +LOGIN_URL = '/accounts/login/' LOGIN_REDIRECT_URL = '/' LOGOUT_REDIRECT_URL = '/' @@ -263,7 +265,10 @@ HINTS_ENABLED = False HINT_LEVEL = 20 +# TODO: Release notes MUST document this change!! REST_FRAMEWORK = { + 'DEFAULT_PERMISSION_CLASSES': [ + ], 'TEST_REQUEST_DEFAULT_FORMAT': 'json', 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', 'PAGE_SIZE': 100 diff --git a/tom_setup/templates/tom_setup/settings.tmpl b/tom_setup/templates/tom_setup/settings.tmpl index 794594c32..c05526688 100644 --- a/tom_setup/templates/tom_setup/settings.tmpl +++ b/tom_setup/templates/tom_setup/settings.tmpl @@ -124,7 +124,7 @@ AUTH_PASSWORD_VALIDATORS = [ }, ] - +LOGIN_URL = '/accounts/login/' LOGIN_REDIRECT_URL = '/' LOGOUT_REDIRECT_URL = '/' @@ -297,6 +297,14 @@ THUMBNAIL_DEFAULT_SIZE = (200, 200) HINTS_ENABLED = {{ HINTS_ENABLED }} HINT_LEVEL = 20 +REST_FRAMEWORK = { + 'DEFAULT_PERMISSION_CLASSES': [ + ], + 'TEST_REQUEST_DEFAULT_FORMAT': 'json', + 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', + 'PAGE_SIZE': 100 +} + try: from local_settings import * # noqa except ImportError: diff --git a/tom_targets/api_urls.py b/tom_targets/api_urls.py index 0225feb0b..5df24497c 100644 --- a/tom_targets/api_urls.py +++ b/tom_targets/api_urls.py @@ -1,6 +1,6 @@ from rest_framework.routers import DefaultRouter -from .api_views import TargetViewSet +from .api_views import TargetViewSet, TargetNamesViewSet """A url module specifically for api paths, seperate from the rest so it can be included modularly. @@ -10,5 +10,6 @@ router = DefaultRouter() router.register(r'targets', TargetViewSet, 'targets') +router.register(r'targetnames', TargetNamesViewSet, 'targetnames') urlpatterns = router.urls diff --git a/tom_targets/api_views.py b/tom_targets/api_views.py index 96aa6b553..26991560f 100644 --- a/tom_targets/api_views.py +++ b/tom_targets/api_views.py @@ -1,21 +1,33 @@ from guardian.mixins import PermissionListMixin -from rest_framework.mixins import CreateModelMixin, DestroyModelMixin, UpdateModelMixin +from rest_framework.mixins import DestroyModelMixin, RetrieveModelMixin +from rest_framework.permissions import DjangoObjectPermissions, IsAuthenticated from rest_framework.viewsets import GenericViewSet, ModelViewSet from tom_targets.filters import TargetFilter -from tom_targets.models import Target -from tom_targets.serializers import TargetSerializer +from tom_targets.models import Target, TargetName +from tom_targets.serializers import TargetSerializer, TargetNameSerializer # Until we have the bandwidth to add the appropriate validation and ensure that DRF will # properly respect permissions, this class will inherit from GenericViewSet and the necessary # mixins for the supported actions. Once we add the appropriate logic for all actions, we # can update it to just inherit from ModelViewSet. -class TargetViewSet(PermissionListMixin, ModelViewSet): +class TargetViewSet(ModelViewSet): """Viewset for Target objects. By default supports CRUD operations. See the docs on viewsets: https://www.django-rest-framework.org/api-guide/viewsets/ """ queryset = Target.objects.all() serializer_class = TargetSerializer filterset_class = TargetFilter - permission_required = 'tom_targets.view_target' + permission_classes = [IsAuthenticated, DjangoObjectPermissions] + # permission_required = 'tom_targets.view_target' + + +class TargetNamesViewSet(DestroyModelMixin, PermissionListMixin, RetrieveModelMixin, GenericViewSet): + queryset = TargetName.objects.all() + serializer_class = TargetNameSerializer + permission_classes = [DjangoObjectPermissions] + permission_required = 'tom_targets.change_target' + +# def get_queryset(self): + \ No newline at end of file diff --git a/tom_targets/serializers.py b/tom_targets/serializers.py index 0a1fea33b..3e955dbad 100644 --- a/tom_targets/serializers.py +++ b/tom_targets/serializers.py @@ -101,6 +101,7 @@ def update(self, instance, validated_data): instance.save() # TODO: updating an existing TargetName with the same value results in an integrity error + # TODO: validate_unique is not called on TargetName for alias_data in aliases: alias = dict(alias_data) if alias.get('id'): From 349d1b62936e8690d552ea4a964e5a77dd88f961 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 17 Jul 2020 16:26:03 -0700 Subject: [PATCH 197/424] Added to tests and fixed them --- tom_targets/api_views.py | 12 ++--- .../migrations/0018_auto_20200714_1832.py | 18 +++++++ tom_targets/models.py | 3 +- tom_targets/tests/test_api.py | 54 +++++++++++++++---- 4 files changed, 70 insertions(+), 17 deletions(-) create mode 100644 tom_targets/migrations/0018_auto_20200714_1832.py diff --git a/tom_targets/api_views.py b/tom_targets/api_views.py index 26991560f..113cf8a84 100644 --- a/tom_targets/api_views.py +++ b/tom_targets/api_views.py @@ -1,4 +1,5 @@ from guardian.mixins import PermissionListMixin +from guardian.shortcuts import get_objects_for_user from rest_framework.mixins import DestroyModelMixin, RetrieveModelMixin from rest_framework.permissions import DjangoObjectPermissions, IsAuthenticated from rest_framework.viewsets import GenericViewSet, ModelViewSet @@ -8,19 +9,16 @@ from tom_targets.serializers import TargetSerializer, TargetNameSerializer -# Until we have the bandwidth to add the appropriate validation and ensure that DRF will -# properly respect permissions, this class will inherit from GenericViewSet and the necessary -# mixins for the supported actions. Once we add the appropriate logic for all actions, we -# can update it to just inherit from ModelViewSet. class TargetViewSet(ModelViewSet): """Viewset for Target objects. By default supports CRUD operations. See the docs on viewsets: https://www.django-rest-framework.org/api-guide/viewsets/ """ - queryset = Target.objects.all() serializer_class = TargetSerializer filterset_class = TargetFilter - permission_classes = [IsAuthenticated, DjangoObjectPermissions] - # permission_required = 'tom_targets.view_target' + permission_classes = [IsAuthenticated&DjangoObjectPermissions] + + def get_queryset(self): + return get_objects_for_user(self.request.user, 'tom_targets.view_target') class TargetNamesViewSet(DestroyModelMixin, PermissionListMixin, RetrieveModelMixin, GenericViewSet): diff --git a/tom_targets/migrations/0018_auto_20200714_1832.py b/tom_targets/migrations/0018_auto_20200714_1832.py new file mode 100644 index 000000000..d151fa41f --- /dev/null +++ b/tom_targets/migrations/0018_auto_20200714_1832.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.7 on 2020-07-14 18:32 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tom_targets', '0017_auto_20200130_2350'), + ] + + operations = [ + migrations.AlterField( + model_name='targetname', + name='name', + field=models.CharField(max_length=100, unique=True, verbose_name='Alias'), + ), + ] diff --git a/tom_targets/models.py b/tom_targets/models.py index 1509ebd48..f84d45e71 100644 --- a/tom_targets/models.py +++ b/tom_targets/models.py @@ -387,7 +387,8 @@ def validate_unique(self, *args, **kwargs): """ super().validate_unique(*args, **kwargs) if self.name == self.target.name: - raise ValidationError('Target name and target aliases must be unique') + raise ValidationError(f'''Alias {self.name} has a conflict with the primary name of the target + {self.target.name} (id={self.target.id})''') class TargetExtra(models.Model): diff --git a/tom_targets/tests/test_api.py b/tom_targets/tests/test_api.py index 0ebb1a29e..2528aa40f 100644 --- a/tom_targets/tests/test_api.py +++ b/tom_targets/tests/test_api.py @@ -10,25 +10,37 @@ class TestTargetViewset(APITestCase): def setUp(self): - user = User.objects.create(username='testuser') - self.client.force_login(user) + self.user = User.objects.create(username='testuser') + self.user2 = User.objects.create(username='testuser2') + # self.client.force_login(user) self.st = SiderealTargetFactory.create() self.nst = NonSiderealTargetFactory.create() - assign_perm('tom_targets.view_target', user, self.st) - assign_perm('tom_targets.view_target', user, self.nst) + assign_perm('tom_targets.view_target', self.user, self.st) + assign_perm('tom_targets.add_target', self.user, self.st) + assign_perm('tom_targets.change_target', self.user, self.st) + assign_perm('tom_targets.delete_target', self.user, self.st) + assign_perm('tom_targets.view_target', self.user, self.nst) + assign_perm('tom_targets.view_target', self.user2, self.st) def test_target_list(self): + self.client.force_login(self.user) response = self.client.get(reverse('api:targets-list')) self.assertEqual(response.json()['count'], 2) + # Ensure that a user without view_target permission on all targets can only retrieve the subset of targets for + # which they have permission + self.client.force_login(self.user2) + response = self.client.get(reverse('api:targets-list')) + self.assertEqual(response.json()['count'], 1) + def test_target_detail(self): + self.client.force_login(self.user) response = self.client.get(reverse('api:targets-detail', args=(self.st.id,))) self.assertEqual(response.json()['name'], self.st.name) - def test_target_detail_bad_permissions(self): - other_user = User.objects.create(username='otheruser') - self.client.force_login(other_user) - response = self.client.get(reverse('api:targets-detail', args=(self.st.id,)), follow=True) + # Ensure that a user without view_target permission cannot access the target + self.client.force_login(self.user2) + response = self.client.get(reverse('api:targets-detail', args=(self.nst.id,))) self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) self.assertEqual(response.json()['detail'], 'Not found.') @@ -45,11 +57,17 @@ def test_target_create(self): {'name': 'alternative name'} ] } + self.client.force_login(self.user) response = self.client.post(reverse('api:targets-list'), data=target_data) self.assertEqual(response.status_code, status.HTTP_201_CREATED) self.assertEqual(response.json()['name'], target_data['name']) self.assertEqual(response.json()['aliases'][0]['name'], target_data['aliases'][0]['name']) + # self.client.force_login(self.user2) + # target_data['name'] = 'test_target_create_bad_permissions' + # response = self.client.post(reverse('api:targets-list'), data=target_data) + # self.assertEqual(response.status_code, status.HTTP_302_REDIRECT) + def test_target_create_sidereal_missing_parameters(self): target_data = { 'name': 'test_target_name_wtf', @@ -86,13 +104,31 @@ def test_target_create_non_sidereal_missing_parameters(self): self.assertEqual(response.json()['aliases'][0]['name'], target_data['aliases'][0]['name']) def test_target_update(self): + self.client.force_login(self.user) updates = {'ra': 123.456} - response = self.client.patch(reverse('api:targets-detail', args=(self.st.id,)), data=updates) + response = self.client.patch(reverse('api:targets-detail', args=(self.st.id,)), data=updates, follow=True) + print(response.content) self.assertEqual(response.status_code, status.HTTP_200_OK) self.st.refresh_from_db() self.assertEqual(self.st.ra, updates['ra']) + # self.client.force_login(self.user2) + # updates = {'ra': 654.321} + # response = self.client.patch(reverse('api:targets-detail', args=(self.st.id,)), data=updates) + # self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + def test_target_delete(self): response = self.client.delete(reverse('api:targets-detail', args=(self.st.id,))) + print(response.content) self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) self.assertFalse(Target.objects.filter(pk=self.st.id).exists()) + + +class TestTargetDetailAPIView(APITestCase): + def setUp(self): + user = User.objects.create(username='testuser') + self.client.force_login(user) + self.st = SiderealTargetFactory.create() + self.nst = NonSiderealTargetFactory.create() + assign_perm('tom_targets.view_target', user, self.st) + assign_perm('tom_targets.view_target', user, self.nst) \ No newline at end of file From 650d40a5406d4810c114de4a2137bb7fef9c515c Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 20 Jul 2020 08:30:48 -0700 Subject: [PATCH 198/424] Added photometric sequence form with most fields --- tom_dataproducts/widgets.py | 23 ---------- tom_observations/cadence.py | 21 +++++++++- tom_observations/facilities/lco.py | 67 ++++++++++++++++++++++++++++-- tom_observations/widgets.py | 15 +++++++ 4 files changed, 98 insertions(+), 28 deletions(-) delete mode 100644 tom_dataproducts/widgets.py create mode 100644 tom_observations/widgets.py diff --git a/tom_dataproducts/widgets.py b/tom_dataproducts/widgets.py deleted file mode 100644 index d2679750f..000000000 --- a/tom_dataproducts/widgets.py +++ /dev/null @@ -1,23 +0,0 @@ -from django.forms import widgets - - -class ObservationDateTimeWidget(widgets.SplitDateTimeWidget): - def __init__(self, attrs=None): - date_attrs = attrs - time_attrs = attrs - date_attrs['label'] = attrs.get('date-label', 'Observation Date') - time_attrs['label'] = attrs.get('time-label', 'Observation Time') - _widgets = ( - widgets.DateInput(attrs=date_attrs), - widgets.TimeInput(attrs=time_attrs) - ) - super().__init__(_widgets, attrs) - - def decompress(self, value): - if value: - return [value.date, value.time] - return [None, None] - - def compress(self, data_list): - if data_list: - return diff --git a/tom_observations/cadence.py b/tom_observations/cadence.py index 68878d52a..8f17a7724 100644 --- a/tom_observations/cadence.py +++ b/tom_observations/cadence.py @@ -195,13 +195,15 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.fields['cadence_strategy'].widget.attrs['readonly'] = True self.fields['cadence_frequency'].widget.attrs['readonly'] = True + self.cadence_layout = self.cadence_layout() + def cadence_layout(self): # If cadence strategy or cadence frequency aren't set, this is a normal observation and the widgets shouldn't # be rendered if not (self.initial.get('cadence_strategy') or self.initial.get('cadence_frequency')): - self.cadence_layout = Layout() + return Layout() else: - self.cadence_layout = Layout( + return Layout( Div( HTML('

    Reactive cadencing parameters. Leave blank if no reactive cadencing is desired.

    '), ), @@ -217,3 +219,18 @@ def __init__(self, *args, **kwargs): css_class='form-row' ) ) + + +class DelayedCadenceForm(CadenceForm): + delay = forms.IntegerField(min_value=0) + cadence_type = forms.ChoiceField(choices=[('repeat', 'Repeating every'), ('once', 'Once in the next')]) + + def __init__(self, *args, **kwargs): + super().__init__(*args ,**kwargs) + self.fields['cadence_strategy'].widget.attrs['readonly'] = False + self.fields['cadence_frequency'].widget.attrs['readonly'] = False + + def cadence_layout(self): + return Layout( + 'delay', 'cadence_type' + ) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index b76b08546..e702d11ed 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -1,16 +1,17 @@ import requests from astropy import units as u -from crispy_forms.layout import Div, HTML, Layout +from crispy_forms.layout import Column, Div, HTML, Layout, Row from dateutil.parser import parse from django import forms from django.conf import settings from django.core.cache import cache from tom_common.exceptions import ImproperCredentialsException -from tom_observations.cadence import CadenceForm +from tom_observations.cadence import CadenceForm, DelayedCadenceForm from tom_observations.facility import BaseRoboticObservationFacility, BaseRoboticObservationForm, get_service_class from tom_observations.observing_strategy import GenericStrategyForm +from tom_observations.widgets import FilterMultiExposureWidget from tom_targets.models import Target, REQUIRED_NON_SIDEREAL_FIELDS, REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME # Determine settings for this module. @@ -417,6 +418,64 @@ def _build_instrument_config(self): return instrument_config +class LCOPhotometricSequenceForm(LCOBaseObservationForm, DelayedCadenceForm): + U_filter = forms.CharField(widget=FilterMultiExposureWidget()) + B_filter = forms.CharField(widget=FilterMultiExposureWidget()) + v_filter = forms.CharField(widget=FilterMultiExposureWidget()) + R_filter = forms.CharField(widget=FilterMultiExposureWidget()) + I_filter = forms.CharField(widget=FilterMultiExposureWidget()) + u_filter = forms.CharField(widget=FilterMultiExposureWidget()) + g_filter = forms.CharField(widget=FilterMultiExposureWidget()) + r_filter = forms.CharField(widget=FilterMultiExposureWidget()) + i_filter = forms.CharField(widget=FilterMultiExposureWidget()) + z_filter = forms.CharField(widget=FilterMultiExposureWidget()) + w_filter = forms.CharField(widget=FilterMultiExposureWidget()) + moon_distance = forms.IntegerField(min_value=0) + + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.helper.layout = Layout( + self.common_layout, + self.cadence_layout, + self.layout(), + self.button_layout() + ) + print(self.fields) + + def instrument_choices(self): + return [i for i in super().instrument_choices() if i[0] in ['1M0-SCICAM-SINISTRO', '0M4-SCICAM-SBIG', '2M0-SPECTRAL-AG']] + + def layout(self): + return Div( + Row(), + Row( + Column('U_filter'), Column('max_airmass') + ), + Row( + Column('B_filter'), Column('instrument_type') + ), + Row( + Column('v_filter'), Column('proposal') + ), + Row( + Column('R_filter'), Column('ipp_value') + ), + Row( + Column('I_filter'), Column('') + ), + Row('u_filter'), + Row('g_filter'), + Row('r_filter'), + Row('i_filter'), + Row('z_filter'), + Row('w_filter') + ) + + def observation_payload(self): + pass + + class LCOObservingStrategyForm(GenericStrategyForm, LCOBaseForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -448,7 +507,7 @@ class LCOFacility(BaseRoboticObservationFacility): """ name = 'LCO' - observation_types = [('IMAGING', 'Imaging'), ('SPECTRA', 'Spectroscopy')] + observation_types = [('IMAGING', 'Imaging'), ('SPECTRA', 'Spectroscopy'), ('SEQUENCE', 'Photometric Sequence')] # The SITES dictionary is used to calculate visibility intervals in the # planning tool. All entries should contain latitude, longitude, elevation # and a code. @@ -498,6 +557,8 @@ def get_form(self, observation_type): return LCOImagingObservationForm elif observation_type == 'SPECTRA': return LCOSpectroscopyObservationForm + elif observation_type == 'SEQUENCE': + return LCOPhotometricSequenceForm else: return LCOBaseObservationForm diff --git a/tom_observations/widgets.py b/tom_observations/widgets.py new file mode 100644 index 000000000..988f27e0a --- /dev/null +++ b/tom_observations/widgets.py @@ -0,0 +1,15 @@ +from django.forms.widgets import MultiWidget, NumberInput + + +class FilterMultiExposureWidget(MultiWidget): + def __init__(self, attrs={}): + _widgets = ( + NumberInput(attrs=attrs), + NumberInput(attrs=attrs), + NumberInput(attrs=attrs) + ) + + super().__init__(_widgets, attrs) + + def decompress(self, value): + return [value.exposure_time, value.exposure_count, value.block_num] if value else [None, None, None] \ No newline at end of file From 8d80d4422955c7f8bfe36c547c6466c87db77eb1 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 20 Jul 2020 23:42:00 -0700 Subject: [PATCH 199/424] Fixed build instrument configuration --- tom_observations/cadence.py | 8 +- tom_observations/facilities/lco.py | 135 ++++++++++++++++++++--------- tom_observations/widgets.py | 23 +++-- tom_targets/forms.py | 6 +- 4 files changed, 120 insertions(+), 52 deletions(-) diff --git a/tom_observations/cadence.py b/tom_observations/cadence.py index 8f17a7724..8dfcc3d33 100644 --- a/tom_observations/cadence.py +++ b/tom_observations/cadence.py @@ -4,7 +4,7 @@ from importlib import import_module import json -from crispy_forms.layout import Div, HTML, Layout +from crispy_forms.layout import Column, Div, HTML, Layout, Row from django import forms from django.conf import settings @@ -222,8 +222,8 @@ def cadence_layout(self): class DelayedCadenceForm(CadenceForm): - delay = forms.IntegerField(min_value=0) cadence_type = forms.ChoiceField(choices=[('repeat', 'Repeating every'), ('once', 'Once in the next')]) + delay = forms.IntegerField(min_value=0, help_text='Delay is in days') def __init__(self, *args, **kwargs): super().__init__(*args ,**kwargs) @@ -232,5 +232,7 @@ def __init__(self, *args, **kwargs): def cadence_layout(self): return Layout( - 'delay', 'cadence_type' + Row( + Column('cadence_type'), Column('delay') + ) ) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index e702d11ed..87cbb642b 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -1,6 +1,7 @@ import requests from astropy import units as u +from crispy_forms.bootstrap import PrependedText from crispy_forms.layout import Column, Div, HTML, Layout, Row from dateutil.parser import parse from django import forms @@ -11,7 +12,7 @@ from tom_observations.cadence import CadenceForm, DelayedCadenceForm from tom_observations.facility import BaseRoboticObservationFacility, BaseRoboticObservationForm, get_service_class from tom_observations.observing_strategy import GenericStrategyForm -from tom_observations.widgets import FilterMultiExposureWidget +from tom_observations.widgets import FilterField from tom_targets.models import Target, REQUIRED_NON_SIDEREAL_FIELDS, REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME # Determine settings for this module. @@ -151,6 +152,7 @@ class LCOBaseObservationForm(BaseRoboticObservationForm, LCOBaseForm, CadenceFor widget=forms.TextInput(attrs={'placeholder': 'Seconds'}), help_text=exposure_time_help) max_airmass = forms.FloatField(help_text=max_airmass_help) + min_lunar_distance = forms.IntegerField(min_value=0, label='Minimum Lunar Distance', required=False) period = forms.FloatField(required=False) jitter = forms.FloatField(required=False) observation_mode = forms.ChoiceField( @@ -175,7 +177,7 @@ def layout(self): css_class='col' ), Div( - 'filter', 'instrument_type', 'exposure_count', 'exposure_time', 'max_airmass', + 'filter', 'instrument_type', 'exposure_count', 'exposure_time', 'max_airmass', 'min_lunar_distance', css_class='col' ), css_class='form-row', @@ -287,14 +289,14 @@ def _build_instrument_config(self): } } - return instrument_config + return [instrument_config] def _build_configuration(self): return { 'type': self.instrument_to_type(self.cleaned_data['instrument_type']), 'instrument_type': self.cleaned_data['instrument_type'], 'target': self._build_target_fields(), - 'instrument_configs': [self._build_instrument_config()], + 'instrument_configs': self._build_instrument_config(), 'acquisition_config': { }, @@ -405,32 +407,44 @@ def filter_choices(self): def _build_instrument_config(self): instrument_config = super()._build_instrument_config() if self.cleaned_data['filter'] != 'None': - instrument_config['optical_elements'] = { + instrument_config[0]['optical_elements'] = { 'slit': self.cleaned_data['filter'] } else: - instrument_config.pop('optical_elements') - instrument_config['rotator_mode'] = 'VFLOAT' # TODO: Should be a distinct field, SKY & VFLOAT are both valid - instrument_config['extra_params'] = { + instrument_config[0].pop('optical_elements') + instrument_config[0]['rotator_mode'] = 'VFLOAT' # TODO: Should be a distinct field, SKY & VFLOAT are both valid + instrument_config[0]['extra_params'] = { 'rotator_angle': self.cleaned_data['rotator_angle'] } - return instrument_config + return [instrument_config] class LCOPhotometricSequenceForm(LCOBaseObservationForm, DelayedCadenceForm): - U_filter = forms.CharField(widget=FilterMultiExposureWidget()) - B_filter = forms.CharField(widget=FilterMultiExposureWidget()) - v_filter = forms.CharField(widget=FilterMultiExposureWidget()) - R_filter = forms.CharField(widget=FilterMultiExposureWidget()) - I_filter = forms.CharField(widget=FilterMultiExposureWidget()) - u_filter = forms.CharField(widget=FilterMultiExposureWidget()) - g_filter = forms.CharField(widget=FilterMultiExposureWidget()) - r_filter = forms.CharField(widget=FilterMultiExposureWidget()) - i_filter = forms.CharField(widget=FilterMultiExposureWidget()) - z_filter = forms.CharField(widget=FilterMultiExposureWidget()) - w_filter = forms.CharField(widget=FilterMultiExposureWidget()) - moon_distance = forms.IntegerField(min_value=0) + U_filter = FilterField(label='U', required=False) + B_filter = FilterField(label='B', required=False) + V_filter = FilterField(label='V', required=False) + R_filter = FilterField(label='R', required=False) + I_filter = FilterField(label='I', required=False) + u_filter = FilterField(label='u', required=False) + g_filter = FilterField(label='g', required=False) + r_filter = FilterField(label='r', required=False) + i_filter = FilterField(label='i', required=False) + z_filter = FilterField(label='z', required=False) + w_filter = FilterField(label='w', required=False) + filter_mapping = { + 'U_filter': 'U', + 'B_filter': 'B', + 'V_filter': 'v', + 'R_filter': 'R', + 'I_filter': 'I', + 'u_filter': 'up', + 'g_filter': 'gp', + 'r_filter': 'rp', + 'i_filter': 'ip', + 'z_filter': 'zs', + 'w_filter': 'w' + } def __init__(self, *args, **kwargs): @@ -441,39 +455,69 @@ def __init__(self, *args, **kwargs): self.layout(), self.button_layout() ) - print(self.fields) + self.fields['cadence_type'].required = False + self.fields['delay'].required = False def instrument_choices(self): return [i for i in super().instrument_choices() if i[0] in ['1M0-SCICAM-SINISTRO', '0M4-SCICAM-SBIG', '2M0-SPECTRAL-AG']] + def _build_instrument_configs(self): + instrument_config = [] + for label, filter in self.filter_mapping.items(): + filter_parameters = list(self.cleaned_data[label]) + if len(self.cleaned_data[label]) > 0: + instrument_config.append({ + 'exposure_count': self.cleaned_data[label][1], + 'exposure_time': self.cleaned_data[label][0], + 'optical_elements': { + self.filter_mapping[label] + } + }) + + return instrument_config + def layout(self): return Div( - Row(), - Row( - Column('U_filter'), Column('max_airmass') - ), - Row( - Column('B_filter'), Column('instrument_type') - ), - Row( - Column('v_filter'), Column('proposal') - ), - Row( - Column('R_filter'), Column('ipp_value') + Div( + Row( + Column(HTML('Exposure Time')), + Column(HTML('No. of Exposures')), + Column(HTML('Block No.')), + ), + Row('U_filter'), + Row('B_filter'), + Row('V_filter'), + Row('R_filter'), + Row('I_filter'), + Row('u_filter'), + Row('g_filter'), + Row('r_filter'), + Row('i_filter'), + Row('z_filter'), + Row('w_filter'), + css_class='col-md-6' ), - Row( - Column('I_filter'), Column('') + Div( + Row('max_airmass'), + Row( + PrependedText('min_lunar_distance', '>') + ), + Row('instrument_type'), + Row('proposal'), + Row('observation_mode'), + Row('ipp_value'), + css_class='col-md-6' ), - Row('u_filter'), - Row('g_filter'), - Row('r_filter'), - Row('i_filter'), - Row('z_filter'), - Row('w_filter') + css_class='form-row' ) def observation_payload(self): - pass + print(self.cleaned_data) + print(self.cleaned_data['U_filter']) + print(type(self.cleaned_data['U_filter'])) + print(self.cleaned_data['R_filter']) + print(self._build_instrument_configs()) + # return super().observation_payload() class LCOObservingStrategyForm(GenericStrategyForm, LCOBaseForm): @@ -553,6 +597,11 @@ class LCOFacility(BaseRoboticObservationFacility): } def get_form(self, observation_type): + # try: + # form_class = settings['LCO']['observation_types'][observation_type]['form_class'] + # return form_class + # except: + # return LCOBaseObservationForm if observation_type == 'IMAGING': return LCOImagingObservationForm elif observation_type == 'SPECTRA': diff --git a/tom_observations/widgets.py b/tom_observations/widgets.py index 988f27e0a..11b6e1ffd 100644 --- a/tom_observations/widgets.py +++ b/tom_observations/widgets.py @@ -1,15 +1,28 @@ +from django import forms from django.forms.widgets import MultiWidget, NumberInput -class FilterMultiExposureWidget(MultiWidget): +class FilterConfigurationWidget(forms.widgets.MultiWidget): + def __init__(self, attrs={}): _widgets = ( - NumberInput(attrs=attrs), - NumberInput(attrs=attrs), - NumberInput(attrs=attrs) + forms.widgets.NumberInput(attrs={'class': 'form-control col-md-3', 'style': 'margin-right: 10px; display: inline-block'}), + forms.widgets.NumberInput(attrs={'class': 'form-control col-md-3', 'style': 'margin-right: 10px; display: inline-block'}), + forms.widgets.NumberInput(attrs={'class': 'form-control col-md-3', 'style': 'margin-right: 10px; display: inline-block'}) ) super().__init__(_widgets, attrs) def decompress(self, value): - return [value.exposure_time, value.exposure_count, value.block_num] if value else [None, None, None] \ No newline at end of file + return [value.exposure_time, value.exposure_count, value.block_num] if value else [None, None, None] + + +class FilterField(forms.MultiValueField): + widget = FilterConfigurationWidget + + def __init__(self, *args, **kwargs): + fields = (forms.IntegerField(), forms.IntegerField(), forms.IntegerField()) + super().__init__(fields, *args, **kwargs) + + def compress(self, data_list): + return data_list \ No newline at end of file diff --git a/tom_targets/forms.py b/tom_targets/forms.py index 149ec5cdb..1590c950e 100644 --- a/tom_targets/forms.py +++ b/tom_targets/forms.py @@ -10,7 +10,8 @@ Target, TargetExtra, TargetName, SIDEREAL_FIELDS, NON_SIDEREAL_FIELDS, REQUIRED_SIDEREAL_FIELDS, REQUIRED_NON_SIDEREAL_FIELDS, REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME ) - +from tom_targets.serializers import TargetSerializer +from tom_targets.validators import RequiredFieldsTogetherValidator def extra_field_to_form_field(field_type): if field_type == 'number': @@ -127,6 +128,9 @@ def clean(self): specified field have been given """ cleaned_data = super().clean() + # serializer = TargetSerializer(data=self.cleaned_data) + # print(serializer) + # serializer.is_valid(raise_exception=True) scheme = cleaned_data['scheme'] # scheme is a required field, so this should be safe required_fields = REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME[scheme] From 930a603c483df951ac77b3f130aab2fc000907e0 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 21 Jul 2020 00:47:19 -0700 Subject: [PATCH 200/424] Finally got a working photseq submission --- docs/api/tom_observations/facilities.rst | 3 +- tom_observations/cadence.py | 9 ++-- tom_observations/facilities/lco.py | 64 +++++++++++++++++------- 3 files changed, 53 insertions(+), 23 deletions(-) diff --git a/docs/api/tom_observations/facilities.rst b/docs/api/tom_observations/facilities.rst index 7f3e0e5fa..c19ab1176 100644 --- a/docs/api/tom_observations/facilities.rst +++ b/docs/api/tom_observations/facilities.rst @@ -16,13 +16,14 @@ Gemini .. automodule:: tom_observations.facilities.gemini :members: + *********************** Las Cumbres Observatory *********************** .. automodule:: tom_observations.facilities.lco :members: - + **** SOAR diff --git a/tom_observations/cadence.py b/tom_observations/cadence.py index 8dfcc3d33..c96bc99b2 100644 --- a/tom_observations/cadence.py +++ b/tom_observations/cadence.py @@ -222,17 +222,16 @@ def cadence_layout(self): class DelayedCadenceForm(CadenceForm): - cadence_type = forms.ChoiceField(choices=[('repeat', 'Repeating every'), ('once', 'Once in the next')]) - delay = forms.IntegerField(min_value=0, help_text='Delay is in days') + cadence_type = forms.ChoiceField(choices=[('', ''), ('repeat', 'Repeating every')]) def __init__(self, *args, **kwargs): - super().__init__(*args ,**kwargs) - self.fields['cadence_strategy'].widget.attrs['readonly'] = False + super().__init__(*args, **kwargs) + self.fields['cadence_strategy'].widget = forms.HiddenInput() self.fields['cadence_frequency'].widget.attrs['readonly'] = False def cadence_layout(self): return Layout( Row( - Column('cadence_type'), Column('delay') + Column('cadence_type'), Column('cadence_frequency') ) ) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 87cbb642b..cd955414c 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -1,3 +1,5 @@ +from datetime import datetime, timedelta +import json import requests from astropy import units as u @@ -139,6 +141,12 @@ def proposal_choices(self): class LCOBaseObservationForm(BaseRoboticObservationForm, LCOBaseForm, CadenceForm): + """ + The LCOBaseObservationForm provides the base set of utilities to construct an observation at Las Cumbres + Observatory. While the forms that inherit from it provide a subset of instruments and filters, the + LCOBaseObservationForm presents the user with all of the instrument and filter options that the facility has to + offer. + """ name = forms.CharField() ipp_value = forms.FloatField(label='Intra Proposal Priority (IPP factor)', min_value=0.5, @@ -353,6 +361,10 @@ def observation_payload(self): class LCOImagingObservationForm(LCOBaseObservationForm): + """ + The LCOImagingObservationForm allows the selection of parameters for observing using LCO's Imagers. The list of + Imagers and their details can be found here: https://lco.global/observatory/instruments/ + """ def instrument_choices(self): return [(k, v['name']) for k, v in self._get_instruments().items() if 'IMAGE' in v['type']] @@ -364,6 +376,10 @@ def filter_choices(self): class LCOSpectroscopyObservationForm(LCOBaseObservationForm): + """ + The LCOSpectroscopyObservationForm allows the selection of parameters for observing using LCO's Spectrographs. The + list of spectrographs and their details can be found here: https://lco.global/observatory/instruments/ + """ rotator_angle = forms.FloatField(min_value=0.0, initial=0.0) def layout(self): @@ -421,6 +437,11 @@ def _build_instrument_config(self): class LCOPhotometricSequenceForm(LCOBaseObservationForm, DelayedCadenceForm): + """ + The LCOPhotometricSequenceForm provides a form offering a subset of the parameters in the LCOImagingObservationForm. + The form is modeled after the Supernova Exchange application's Photometric Sequence Request Form, and allows the + configuration of multiple filters, as well as a more intuitive proactive cadence form. + """ U_filter = FilterField(label='U', required=False) B_filter = FilterField(label='B', required=False) V_filter = FilterField(label='V', required=False) @@ -446,22 +467,24 @@ class LCOPhotometricSequenceForm(LCOBaseObservationForm, DelayedCadenceForm): 'w_filter': 'w' } - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.helper.layout = Layout( + Div( + Column('name'), + Column('cadence_type'), + Column('cadence_frequency'), + css_class='form-row' + ), self.common_layout, - self.cadence_layout, self.layout(), self.button_layout() ) self.fields['cadence_type'].required = False - self.fields['delay'].required = False + self.fields['cadence_strategy'].required = False + self.fields['cadence_frequency'].required = False - def instrument_choices(self): - return [i for i in super().instrument_choices() if i[0] in ['1M0-SCICAM-SINISTRO', '0M4-SCICAM-SBIG', '2M0-SPECTRAL-AG']] - - def _build_instrument_configs(self): + def _build_instrument_config(self): instrument_config = [] for label, filter in self.filter_mapping.items(): filter_parameters = list(self.cleaned_data[label]) @@ -470,12 +493,26 @@ def _build_instrument_configs(self): 'exposure_count': self.cleaned_data[label][1], 'exposure_time': self.cleaned_data[label][0], 'optical_elements': { - self.filter_mapping[label] + 'filter': self.filter_mapping[label] } }) return instrument_config + def clean(self): + cleaned_data = super().clean() + now = datetime.now() + cleaned_data['start'] = datetime.strftime(datetime.now(), '%Y-%m-%dT%H:%M:%S') + cadence_frequency = 24 if not cleaned_data.get('cadence_frequency') else cleaned_data.get('cadence_frequency') + cleaned_data['end'] = datetime.strftime(now + timedelta(hours=cadence_frequency), '%Y-%m-%dT%H:%M:%S') + if cleaned_data['cadence_type'] == 'repeat': + cleaned_data['cadence_strategy'] = 'Resume Cadence After Failure' + + return cleaned_data + + def instrument_choices(self): + return [i for i in super().instrument_choices() if i[0] in ['1M0-SCICAM-SINISTRO', '0M4-SCICAM-SBIG', '2M0-SPECTRAL-AG']] + def layout(self): return Div( Div( @@ -511,14 +548,6 @@ def layout(self): css_class='form-row' ) - def observation_payload(self): - print(self.cleaned_data) - print(self.cleaned_data['U_filter']) - print(type(self.cleaned_data['U_filter'])) - print(self.cleaned_data['R_filter']) - print(self._build_instrument_configs()) - # return super().observation_payload() - class LCOObservingStrategyForm(GenericStrategyForm, LCOBaseForm): def __init__(self, *args, **kwargs): @@ -697,7 +726,8 @@ def get_facility_weather_urls(self): return facility_weather_urls def get_facility_status(self): - """Get the telescope_states from the LCO API endpoint and simply + """ + Get the telescope_states from the LCO API endpoint and simply transform the returned JSON into the following dictionary hierarchy for use by the facility_status.html template partial. From 64a4ea10f3b11bd808978900dd615b92151a9175 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 21 Jul 2020 00:54:40 -0700 Subject: [PATCH 201/424] Fixed pycodestyle issues --- tom_observations/facilities/lco.py | 19 ++++++++++--------- tom_observations/widgets.py | 11 ++++++----- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index cd955414c..b3a77ebf3 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -142,8 +142,8 @@ def proposal_choices(self): class LCOBaseObservationForm(BaseRoboticObservationForm, LCOBaseForm, CadenceForm): """ - The LCOBaseObservationForm provides the base set of utilities to construct an observation at Las Cumbres - Observatory. While the forms that inherit from it provide a subset of instruments and filters, the + The LCOBaseObservationForm provides the base set of utilities to construct an observation at Las Cumbres + Observatory. While the forms that inherit from it provide a subset of instruments and filters, the LCOBaseObservationForm presents the user with all of the instrument and filter options that the facility has to offer. """ @@ -438,8 +438,8 @@ def _build_instrument_config(self): class LCOPhotometricSequenceForm(LCOBaseObservationForm, DelayedCadenceForm): """ - The LCOPhotometricSequenceForm provides a form offering a subset of the parameters in the LCOImagingObservationForm. - The form is modeled after the Supernova Exchange application's Photometric Sequence Request Form, and allows the + The LCOPhotometricSequenceForm provides a form offering a subset of the parameters in the LCOImagingObservationForm. + The form is modeled after the Supernova Exchange application's Photometric Sequence Request Form, and allows the configuration of multiple filters, as well as a more intuitive proactive cadence form. """ U_filter = FilterField(label='U', required=False) @@ -471,7 +471,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.helper.layout = Layout( Div( - Column('name'), + Column('name'), Column('cadence_type'), Column('cadence_frequency'), css_class='form-row' @@ -511,7 +511,8 @@ def clean(self): return cleaned_data def instrument_choices(self): - return [i for i in super().instrument_choices() if i[0] in ['1M0-SCICAM-SINISTRO', '0M4-SCICAM-SBIG', '2M0-SPECTRAL-AG']] + return [i for i in super().instrument_choices() + if i[0] in ['1M0-SCICAM-SINISTRO', '0M4-SCICAM-SBIG', '2M0-SPECTRAL-AG']] def layout(self): return Div( @@ -521,7 +522,7 @@ def layout(self): Column(HTML('No. of Exposures')), Column(HTML('Block No.')), ), - Row('U_filter'), + Row('U_filter'), Row('B_filter'), Row('V_filter'), Row('R_filter'), @@ -531,7 +532,7 @@ def layout(self): Row('r_filter'), Row('i_filter'), Row('z_filter'), - Row('w_filter'), + Row('w_filter'), css_class='col-md-6' ), Div( @@ -542,7 +543,7 @@ def layout(self): Row('instrument_type'), Row('proposal'), Row('observation_mode'), - Row('ipp_value'), + Row('ipp_value'), css_class='col-md-6' ), css_class='form-row' diff --git a/tom_observations/widgets.py b/tom_observations/widgets.py index 11b6e1ffd..68ed55924 100644 --- a/tom_observations/widgets.py +++ b/tom_observations/widgets.py @@ -5,11 +5,12 @@ class FilterConfigurationWidget(forms.widgets.MultiWidget): def __init__(self, attrs={}): + _default_attrs = {'class': 'form-control col-md-3', 'style': 'margin-right: 10px; display: inline-block'} _widgets = ( - forms.widgets.NumberInput(attrs={'class': 'form-control col-md-3', 'style': 'margin-right: 10px; display: inline-block'}), - forms.widgets.NumberInput(attrs={'class': 'form-control col-md-3', 'style': 'margin-right: 10px; display: inline-block'}), - forms.widgets.NumberInput(attrs={'class': 'form-control col-md-3', 'style': 'margin-right: 10px; display: inline-block'}) - ) + forms.widgets.NumberInput(attrs=_default_attrs), + forms.widgets.NumberInput(attrs=_default_attrs), + forms.widgets.NumberInput(attrs=_default_attrs) + ) super().__init__(_widgets, attrs) @@ -25,4 +26,4 @@ def __init__(self, *args, **kwargs): super().__init__(fields, *args, **kwargs) def compress(self, data_list): - return data_list \ No newline at end of file + return data_list From e69f5ecaa607203cf5afd80c610631fc1e997801 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 21 Jul 2020 16:46:53 +0000 Subject: [PATCH 202/424] consolidate app api_urls in tom_common/urls.py this removed the api_url.py files from tom_targets and tom_dataproducts to tom_common/urls.py where we set up the DRF router and it's urls. --- tom_common/urls.py | 17 +++++++++++++---- tom_dataproducts/api_urls.py | 18 ------------------ tom_targets/api_urls.py | 14 -------------- 3 files changed, 13 insertions(+), 36 deletions(-) delete mode 100644 tom_dataproducts/api_urls.py delete mode 100644 tom_targets/api_urls.py diff --git a/tom_common/urls.py b/tom_common/urls.py index 688e38c1e..04af51d5b 100644 --- a/tom_common/urls.py +++ b/tom_common/urls.py @@ -24,9 +24,18 @@ from tom_common.views import UserListView, UserPasswordChangeView, UserCreateView, UserDeleteView, UserUpdateView from tom_common.views import CommentDeleteView, GroupCreateView, GroupUpdateView, GroupDeleteView -api_urlpatterns = [ - path('', include('tom_targets.api_urls')) -] +from rest_framework import routers +from tom_targets.api_views import TargetViewSet +from tom_dataproducts.api_views import DataProductGroupViewSet, DataProductViewSet, ReducedDatumViewSet + +# for all applications +# set up the DRF router, its router.urls included in urlpatterns below +router = routers.DefaultRouter() +router.register(r'targets', TargetViewSet, 'targets') +router.register(r'dataproductgroups', DataProductGroupViewSet, 'dataproductgroups') +router.register(r'dataproducts', DataProductViewSet, 'dataproducts') +router.register(r'reduceddatums', ReducedDatumViewSet, 'reduceddatums') + urlpatterns = [ path('', TemplateView.as_view(template_name='tom_common/index.html'), name='home'), @@ -50,7 +59,7 @@ path('comment//delete', CommentDeleteView.as_view(), name='comment-delete'), path('admin/', admin.site.urls), path('api-auth/', include('rest_framework.urls')), - path('api/', include(api_urlpatterns)) + path('api/', include(router.urls)), # The static helper below only works in development see # https://docs.djangoproject.com/en/2.1/howto/static-files/#serving-files-uploaded-by-a-user-during-development ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/tom_dataproducts/api_urls.py b/tom_dataproducts/api_urls.py deleted file mode 100644 index d0a6fd8b3..000000000 --- a/tom_dataproducts/api_urls.py +++ /dev/null @@ -1,18 +0,0 @@ -from rest_framework.routers import DefaultRouter - -from .api_views import DataProductGroupViewSet, DataProductViewSet, ReducedDatumViewSet - -"""A url module specifically for api paths, separate from -the rest so it can be included in a modular fashion. -""" - -app_name = 'api' - -router = DefaultRouter() - -# prefix, ViewSet, basename -router.register(r'dataproductgroups', DataProductGroupViewSet) -router.register(r'dataproducts', DataProductViewSet) -router.register(r'reduceddatums', ReducedDatumViewSet) - -urlpatterns = router.urls diff --git a/tom_targets/api_urls.py b/tom_targets/api_urls.py deleted file mode 100644 index 0225feb0b..000000000 --- a/tom_targets/api_urls.py +++ /dev/null @@ -1,14 +0,0 @@ -from rest_framework.routers import DefaultRouter - -from .api_views import TargetViewSet - -"""A url module specifically for api paths, seperate from -the rest so it can be included modularly. -""" - -app_name = 'api' - -router = DefaultRouter() -router.register(r'targets', TargetViewSet, 'targets') - -urlpatterns = router.urls From 65f9eb479583ec45ea32b1f78dc8fe2f0f2a0ae1 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 21 Jul 2020 16:49:01 +0000 Subject: [PATCH 203/424] just making a documentation TODO --- tom_targets/api_views.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tom_targets/api_views.py b/tom_targets/api_views.py index a83140df2..94a83dc29 100644 --- a/tom_targets/api_views.py +++ b/tom_targets/api_views.py @@ -6,6 +6,9 @@ from tom_targets.models import Target from tom_targets.serializers import TargetSerializer +# TODO: The GenericViewSet (and ModelViewSet?) subclass docstrings appear on the /api// +# endpoint page. Rewrite these docstring to be useful to API consumers. + # Until we have the bandwidth to add the appropriate validation and ensure that DRF will # properly respect permissions, this class will inherit from GenericViewSet and the necessary From 61d58a28684411ae4dececd106fcafd6ad4b97fb Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 21 Jul 2020 16:49:41 +0000 Subject: [PATCH 204/424] update the queryset to be the actuall function call when just the Model sub-class name, there was an Attribute Error on 'model'. this fixes that. --- tom_dataproducts/api_views.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tom_dataproducts/api_views.py b/tom_dataproducts/api_views.py index 0edb55024..c81962eb6 100644 --- a/tom_dataproducts/api_views.py +++ b/tom_dataproducts/api_views.py @@ -7,12 +7,15 @@ # TODO: see Davids comment in tom_targets/api_views.py +# TODO: The GenericViewSet (and ModelViewset?) subclass docstrings appear on the /api// +# endpoint page. Rewrite these docstring to be useful to API consumers. + class DataProductGroupViewSet(CreateModelMixin, PermissionListMixin, GenericViewSet): """Viewset for Target objects. By default supports CRUD operations. See the docs on viewsets: https://www.django-rest-framework.org/api-guide/viewsets/ """ - queryset = DataProductGroup + queryset = DataProductGroup.objects.all() serializer_class = DataProductGroupSerializer # TODO: define filterset_class # TODO: define permission_required @@ -22,7 +25,7 @@ class DataProductViewSet(CreateModelMixin, PermissionListMixin, GenericViewSet): """Viewset for Target objects. By default supports CRUD operations. See the docs on viewsets: https://www.django-rest-framework.org/api-guide/viewsets/ """ - queryset = DataProduct + queryset = DataProduct.objects.all() serializer_class = DataProductSerializer # TODO: define filterset_class # TODO: define permission_required @@ -32,7 +35,7 @@ class ReducedDatumViewSet(CreateModelMixin, PermissionListMixin, GenericViewSet) """Viewset for Target objects. By default supports CRUD operations. See the docs on viewsets: https://www.django-rest-framework.org/api-guide/viewsets/ """ - queryset = ReducedDatum + queryset = ReducedDatum.objects.all() serializer_class = ReducedDatumSerializer # TODO: define filterset_class # TODO: define permission_required From bef8adf8b491c7047ade03cbe201828ad35bc3a9 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 21 Jul 2020 17:56:59 +0000 Subject: [PATCH 205/424] use ModelSerializer not HyperlinkedModelSerializer until we sort out how to specify the View to hyperlink, we just display the primary key instead of the link to the view --- tom_dataproducts/serializers.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tom_dataproducts/serializers.py b/tom_dataproducts/serializers.py index 439caa71c..92c6b68d8 100644 --- a/tom_dataproducts/serializers.py +++ b/tom_dataproducts/serializers.py @@ -2,13 +2,13 @@ from .models import DataProductGroup, DataProduct, ReducedDatum -class DataProductGroupSerializer(serializers.HyperlinkedModelSerializer): +class DataProductGroupSerializer(serializers.ModelSerializer): class Meta: model = DataProductGroup fields = ('name', 'created', 'modified') -class DataProductSerializer(serializers.HyperlinkedModelSerializer): +class DataProductSerializer(serializers.ModelSerializer): class Meta: model = DataProduct fields = ( @@ -25,8 +25,17 @@ class Meta: 'thumbnail' ) + # TODO: use HyperlinkedModelSerializer + # for the HyperlinkedModelSerializer use something like this + # extra_kwargs = { + # "url": { + # "view_name": ":targets:detail", + # "lookup_field": "pk", + # } + # } -class ReducedDatumSerializer(serializers.HyperlinkedModelSerializer): + +class ReducedDatumSerializer(serializers.ModelSerializer): class Meta: model = ReducedDatum fields = ( From e75aeb23b84bbc1dbdc8f6c4f1a5128fb32e6ddb Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 21 Jul 2020 19:29:19 +0000 Subject: [PATCH 206/424] remove trailing whitespace for pycodestyle travis check --- tom_targets/api_views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tom_targets/api_views.py b/tom_targets/api_views.py index 94a83dc29..70d661509 100644 --- a/tom_targets/api_views.py +++ b/tom_targets/api_views.py @@ -10,9 +10,9 @@ # endpoint page. Rewrite these docstring to be useful to API consumers. -# Until we have the bandwidth to add the appropriate validation and ensure that DRF will -# properly respect permissions, this class will inherit from GenericViewSet and the necessary -# mixins for the supported actions. Once we add the appropriate logic for all actions, we +# Until we have the bandwidth to add the appropriate validation and ensure that DRF will +# properly respect permissions, this class will inherit from GenericViewSet and the necessary +# mixins for the supported actions. Once we add the appropriate logic for all actions, we # can update it to just inherit from ModelViewSet. class TargetViewSet(CreateModelMixin, PermissionListMixin, GenericViewSet): """Viewset for Target objects. By default supports CRUD operations. From 1a4dcb19730a0c21b061bfbc12d593d163ca8957 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 21 Jul 2020 13:07:29 -0700 Subject: [PATCH 207/424] Updated form as per Curtis' request --- tom_observations/facilities/lco.py | 14 ++++++++++++-- .../tom_observations/observation_form.html | 18 +++++++++++++----- tom_observations/views.py | 2 +- tom_targets/forms.py | 6 +----- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index b3a77ebf3..7d1ca4c7c 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -476,13 +476,14 @@ def __init__(self, *args, **kwargs): Column('cadence_frequency'), css_class='form-row' ), - self.common_layout, + Layout('facility', 'target_id', 'observation_type'), self.layout(), self.button_layout() ) self.fields['cadence_type'].required = False self.fields['cadence_strategy'].required = False self.fields['cadence_frequency'].required = False + self.fields['groups'].label = 'Data granted to' def _build_instrument_config(self): instrument_config = [] @@ -515,6 +516,10 @@ def instrument_choices(self): if i[0] in ['1M0-SCICAM-SINISTRO', '0M4-SCICAM-SBIG', '2M0-SPECTRAL-AG']] def layout(self): + if settings.TARGET_PERMISSIONS_ONLY: + groups = Row('') + else: + groups = Row('groups') return Div( Div( Row( @@ -544,6 +549,7 @@ def layout(self): Row('proposal'), Row('observation_mode'), Row('ipp_value'), + groups, css_class='col-md-6' ), css_class='form-row' @@ -581,6 +587,7 @@ class LCOFacility(BaseRoboticObservationFacility): """ name = 'LCO' + default_form_class = LCOBaseObservationForm observation_types = [('IMAGING', 'Imaging'), ('SPECTRA', 'Spectroscopy'), ('SEQUENCE', 'Photometric Sequence')] # The SITES dictionary is used to calculate visibility intervals in the # planning tool. All entries should contain latitude, longitude, elevation @@ -631,7 +638,7 @@ def get_form(self, observation_type): # form_class = settings['LCO']['observation_types'][observation_type]['form_class'] # return form_class # except: - # return LCOBaseObservationForm + # return default_form_class if observation_type == 'IMAGING': return LCOImagingObservationForm elif observation_type == 'SPECTRA': @@ -641,6 +648,9 @@ def get_form(self, observation_type): else: return LCOBaseObservationForm + # def get_observation_types(self): + + def get_strategy_form(self, observation_type): return LCOObservingStrategyForm diff --git a/tom_observations/templates/tom_observations/observation_form.html b/tom_observations/templates/tom_observations/observation_form.html index bba0c887d..dfa647523 100644 --- a/tom_observations/templates/tom_observations/observation_form.html +++ b/tom_observations/templates/tom_observations/observation_form.html @@ -11,12 +11,20 @@

    Submit an observation to {{ form.facility.value }}

    {% endif %}
    -
    - {% target_data target %} +
    +
    + {% target_data target %} +
    +
    + Lunar Distance + {% moon_distance target %} +
    -
    - {% observation_type_tabs %} - {% crispy form %} +
    +
    + {% observation_type_tabs %} + {% crispy form %} +
    {% endblock %} \ No newline at end of file diff --git a/tom_observations/views.py b/tom_observations/views.py index 2e60f1d93..1d13ce646 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -176,7 +176,7 @@ def get_context_data(self, **kwargs): :rtype: dict """ context = super(ObservationCreateView, self).get_context_data(**kwargs) - context['type_choices'] = self.get_facility_class().observation_types + context['type_choices'] = self.get_facility_class().observation_types # TODO: get from settings target = Target.objects.get(pk=self.get_target_id()) context['target'] = target return context diff --git a/tom_targets/forms.py b/tom_targets/forms.py index 1590c950e..149ec5cdb 100644 --- a/tom_targets/forms.py +++ b/tom_targets/forms.py @@ -10,8 +10,7 @@ Target, TargetExtra, TargetName, SIDEREAL_FIELDS, NON_SIDEREAL_FIELDS, REQUIRED_SIDEREAL_FIELDS, REQUIRED_NON_SIDEREAL_FIELDS, REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME ) -from tom_targets.serializers import TargetSerializer -from tom_targets.validators import RequiredFieldsTogetherValidator + def extra_field_to_form_field(field_type): if field_type == 'number': @@ -128,9 +127,6 @@ def clean(self): specified field have been given """ cleaned_data = super().clean() - # serializer = TargetSerializer(data=self.cleaned_data) - # print(serializer) - # serializer.is_valid(raise_exception=True) scheme = cleaned_data['scheme'] # scheme is a required field, so this should be safe required_fields = REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME[scheme] From bd8ef5d35e3db009effa32bb744b850154762f7f Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 22 Jul 2020 20:31:30 -0700 Subject: [PATCH 208/424] Made observation forms into tabs and started refactoring views.py --- tom_observations/facilities/lco.py | 47 ++++++++++--------- .../static/tom_observations/css/main.css | 8 ++++ .../tom_observations/observation_form.html | 18 +++++-- tom_observations/views.py | 42 +++++++++-------- tom_observations/widgets.py | 1 - 5 files changed, 70 insertions(+), 46 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 7d1ca4c7c..24ea5ddd7 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -333,6 +333,7 @@ def _expand_cadence_request(self, payload): return response.json() def observation_payload(self): + print(self.cleaned_data) payload = { "name": self.cleaned_data['name'], "proposal": self.cleaned_data['proposal'], @@ -357,6 +358,7 @@ def observation_payload(self): if self.cleaned_data.get('period') and self.cleaned_data.get('jitter'): payload = self._expand_cadence_request(payload) + print(payload) return payload @@ -382,6 +384,10 @@ class LCOSpectroscopyObservationForm(LCOBaseObservationForm): """ rotator_angle = forms.FloatField(min_value=0.0, initial=0.0) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields['filter'].label = 'Slit' + def layout(self): return Div( Div( @@ -421,19 +427,20 @@ def filter_choices(self): ] + [('None', 'None')]) def _build_instrument_config(self): - instrument_config = super()._build_instrument_config() + instrument_configs = super()._build_instrument_config() if self.cleaned_data['filter'] != 'None': - instrument_config[0]['optical_elements'] = { + instrument_configs[0]['optical_elements'] = { 'slit': self.cleaned_data['filter'] } else: - instrument_config[0].pop('optical_elements') - instrument_config[0]['rotator_mode'] = 'VFLOAT' # TODO: Should be a distinct field, SKY & VFLOAT are both valid - instrument_config[0]['extra_params'] = { + instrument_configs[0].pop('optical_elements') + instrument_configs[0]['rotator_mode'] = 'VFLOAT' # TODO: Should be a distinct field, SKY & VFLOAT are both valid + instrument_configs[0]['extra_params'] = { 'rotator_angle': self.cleaned_data['rotator_angle'] } - return [instrument_config] + return [] + return instrument_configs class LCOPhotometricSequenceForm(LCOBaseObservationForm, DelayedCadenceForm): @@ -488,7 +495,6 @@ def __init__(self, *args, **kwargs): def _build_instrument_config(self): instrument_config = [] for label, filter in self.filter_mapping.items(): - filter_parameters = list(self.cleaned_data[label]) if len(self.cleaned_data[label]) > 0: instrument_config.append({ 'exposure_count': self.cleaned_data[label][1], @@ -588,7 +594,12 @@ class LCOFacility(BaseRoboticObservationFacility): name = 'LCO' default_form_class = LCOBaseObservationForm - observation_types = [('IMAGING', 'Imaging'), ('SPECTRA', 'Spectroscopy'), ('SEQUENCE', 'Photometric Sequence')] + # observation_types = [('IMAGING', 'Imaging'), ('SPECTRA', 'Spectroscopy'), ('SEQUENCE', 'Photometric Sequence')] + observation_forms = { + 'IMAGING': LCOImagingObservationForm, + 'SPECTRA': LCOSpectroscopyObservationForm, + 'PHOTSEQ': LCOPhotometricSequenceForm + } # The SITES dictionary is used to calculate visibility intervals in the # planning tool. All entries should contain latitude, longitude, elevation # and a code. @@ -634,23 +645,13 @@ class LCOFacility(BaseRoboticObservationFacility): } def get_form(self, observation_type): - # try: - # form_class = settings['LCO']['observation_types'][observation_type]['form_class'] - # return form_class - # except: - # return default_form_class - if observation_type == 'IMAGING': - return LCOImagingObservationForm - elif observation_type == 'SPECTRA': - return LCOSpectroscopyObservationForm - elif observation_type == 'SEQUENCE': - return LCOPhotometricSequenceForm - else: + print(observation_type) + try: + print(self.observation_forms[observation_type]) + return self.observation_forms[observation_type] + except KeyError: return LCOBaseObservationForm - # def get_observation_types(self): - - def get_strategy_form(self, observation_type): return LCOObservingStrategyForm diff --git a/tom_observations/static/tom_observations/css/main.css b/tom_observations/static/tom_observations/css/main.css index 492c50835..bbfd4799d 100644 --- a/tom_observations/static/tom_observations/css/main.css +++ b/tom_observations/static/tom_observations/css/main.css @@ -10,3 +10,11 @@ width: inherit; height: auto; } + +.nav-item { + cursor: pointer; +} + +span.featured { + pointer-events: none; +} \ No newline at end of file diff --git a/tom_observations/templates/tom_observations/observation_form.html b/tom_observations/templates/tom_observations/observation_form.html index dfa647523..00ff701ab 100644 --- a/tom_observations/templates/tom_observations/observation_form.html +++ b/tom_observations/templates/tom_observations/observation_form.html @@ -21,9 +21,21 @@

    Submit an observation to {{ form.facility.value }}

    -
    - {% observation_type_tabs %} - {% crispy form %} + +
    + {% for observation_type, observation_form in observation_type_choices %} +
    + {% crispy observation_form %} +
    + {% endfor %}
    diff --git a/tom_observations/views.py b/tom_observations/views.py index 1d13ce646..748e40ede 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -155,19 +155,6 @@ def get_facility_class(self): """ return get_service_class(self.get_facility()) - def get_observation_type(self): - """ - Gets the observation type from the query parameters of the request. - - :returns: observation type - :rtype: str - """ - if self.request.method == 'GET': - # TODO: This appears to not work as intended. - return self.request.GET.get('observation_type', self.get_facility_class().observation_types[0]) - elif self.request.method == 'POST': - return self.request.POST.get('observation_type') - def get_context_data(self, **kwargs): """ Adds the available observation types for the observing facility to the context object. @@ -176,8 +163,16 @@ def get_context_data(self, **kwargs): :rtype: dict """ context = super(ObservationCreateView, self).get_context_data(**kwargs) - context['type_choices'] = self.get_facility_class().observation_types # TODO: get from settings + print(context['form']) + # TODO: only update initial values for active form + # observation_type_choices = [] + # initial = self.get_initial() + # for k, v in self.get_facility_class().observation_forms.items(): + # observation_type_choices.append() + context['observation_type_choices'] = [(k, v(initial={**self.get_initial(), **{'observation_type': k}})) for k, v in self.get_facility_class().observation_forms.items()] target = Target.objects.get(pk=self.get_target_id()) + # TODO: add active to context and plumb through to template + # context['active'] = context['target'] = target return context @@ -189,9 +184,9 @@ def get_form_class(self): :rtype: subclass of GenericObservationForm """ observation_type = None - if self.request.GET: + if self.request.method == 'GET': observation_type = self.request.GET.get('observation_type') - elif self.request.POST: + elif self.request.method == 'POST': observation_type = self.request.POST.get('observation_type') return self.get_facility_class()().get_form(observation_type) @@ -202,6 +197,7 @@ def get_form(self): :returns: observation form :rtype: subclass of GenericObservationForm """ + print(self.request) form = super().get_form() if not settings.TARGET_PERMISSIONS_ONLY: form.fields['groups'].queryset = self.request.user.groups.all() @@ -218,13 +214,18 @@ def get_initial(self): :returns: initial form data :rtype: dict """ + print('get initial') + if self.request.method == 'POST': + print(self.request.POST) initial = super().get_initial() if not self.get_target_id(): raise Exception('Must provide target_id') initial['target_id'] = self.get_target_id() initial['facility'] = self.get_facility() - initial['observation_type'] = self.get_observation_type() - initial.update(self.request.GET.dict()) + if self.request.method == 'GET': + initial.update(self.request.GET.dict()) + elif self.request.method == 'POST': + initial.update(self.request.POST.dict()) return initial def form_valid(self, form): @@ -238,10 +239,13 @@ def form_valid(self, form): :param form: form containing observating request parameters :type form: subclass of GenericObservationForm """ + print('form_valid') + print(form.cleaned_data) + # TODO: Render errors properly, probably in form_invalid # Submit the observation facility = self.get_facility_class() target = self.get_target() - observation_ids = facility().submit_observation(form.observation_payload()) + # observation_ids = facility().submit_observation(form.observation_payload()) records = [] for observation_id in observation_ids: diff --git a/tom_observations/widgets.py b/tom_observations/widgets.py index 68ed55924..fbf81f55f 100644 --- a/tom_observations/widgets.py +++ b/tom_observations/widgets.py @@ -1,5 +1,4 @@ from django import forms -from django.forms.widgets import MultiWidget, NumberInput class FilterConfigurationWidget(forms.widgets.MultiWidget): From f94c55db00116a3636b9bd53c0a94284f77411b4 Mon Sep 17 00:00:00 2001 From: rachel3834 Date: Fri, 24 Jul 2020 14:35:12 -0700 Subject: [PATCH 209/424] Resolved missing GenericAlert import in antares broker which seemed to be breaking updatereduceddata. Converted gaia broker to use BS+regex for html parsing and expanded gaia broker unittests --- tom_alerts/brokers/antares.py | 4 +- tom_alerts/brokers/gaia.py | 28 +++++++---- tom_alerts/tests/tests_gaia.py | 92 ++++++++++++++++++++++++++++++---- 3 files changed, 102 insertions(+), 22 deletions(-) diff --git a/tom_alerts/brokers/antares.py b/tom_alerts/brokers/antares.py index c03b7e16b..43199c817 100644 --- a/tom_alerts/brokers/antares.py +++ b/tom_alerts/brokers/antares.py @@ -1,6 +1,6 @@ from crispy_forms.layout import Layout, HTML -from tom_alerts.alerts import GenericBroker, GenericQueryForm +from tom_alerts.alerts import GenericBroker, GenericQueryForm, GenericAlert class ANTARESQueryForm(GenericQueryForm): @@ -24,7 +24,7 @@ class ANTARESBroker(GenericBroker): def fetch_alerts(self, parameters): return iter([]) - + def process_reduced_data(self, target, alert=None): return diff --git a/tom_alerts/brokers/gaia.py b/tom_alerts/brokers/gaia.py index bc7038f14..ee88a0356 100644 --- a/tom_alerts/brokers/gaia.py +++ b/tom_alerts/brokers/gaia.py @@ -61,7 +61,7 @@ class GaiaBroker(GenericBroker): # alerts = alerts_pattern.match(str(script.string).strip()) # if alerts: # break - + # print(alerts[0]) # alert_list = json.loads(alerts[0].replace('var_alerts = ', '').replace(';', '')) @@ -96,13 +96,18 @@ def fetch_alerts(self, parameters): response = requests.get(f'{BASE_BROKER_URL}/alerts/alertsindex') response.raise_for_status() - html_data = response.text.split('\n') - for line in html_data: - if 'var alerts' in line: - alerts_data = line.replace('var alerts = ', '') - alerts_data = alerts_data.replace('\n', '').replace(';', '') + soup = BeautifulSoup(response.content, 'html.parser') + script_tags = soup.find_all('script') + alerts = None - alert_list = json.loads(alerts_data) + alerts_pattern = re.compile(r'var alerts = \[(.*?)];') + for script in script_tags: + m = alerts_pattern.match(str(script.string).strip()) + if m is not None: + alerts = '['+m.group(1)+']' + break + + alert_list = json.loads(alerts) if parameters['cone'] is not None and len(parameters['cone']) > 0: cone_params = parameters['cone'].split(',') @@ -169,13 +174,14 @@ def process_reduced_data(self, target, alert=None): except HTTPError: raise Exception('Unable to retrieve alert information from broker') - alert_url = BROKER_URL.replace('/alerts/alertsindex', - alert['per_alert']['link']) - - if alert: + if alert is not None: lc_url = path.join(base_url, alert['name'], 'lightcurve.csv') + alert_url = BROKER_URL.replace('/alerts/alertsindex', + alert['per_alert']['link']) elif target: lc_url = path.join(base_url, target.name, 'lightcurve.csv') + alert_url = BROKER_URL.replace('/alerts/alertsindex', + 'alerts/alert/'+target.name+'/') else: return diff --git a/tom_alerts/tests/tests_gaia.py b/tom_alerts/tests/tests_gaia.py index 7c1a30f91..3708e09b0 100644 --- a/tom_alerts/tests/tests_gaia.py +++ b/tom_alerts/tests/tests_gaia.py @@ -1,10 +1,14 @@ from requests import Response +from django.utils import timezone from django.test import TestCase, override_settings from django.forms import ValidationError from unittest import mock from tom_alerts.brokers.gaia import GaiaQueryForm +from tom_alerts.brokers.gaia import GaiaBroker +from tom_targets.models import Target +from tom_dataproducts.models import ReducedDatum @override_settings(TOM_ALERT_CLASSES=['tom_alerts.brokers.gaia.GaiaBroker']) class TestGaiaQueryForm(TestCase): @@ -59,24 +63,94 @@ def setUp(self): self.test_html = """ """ + self.test_html = self.test_html.replace('\n','') + self.alert_list = [ {"name": "Gaia20cpu", "tnsid": "AT2020lto", "obstime": "2020-06-04 17:25:08", "ra": "291.61247", + "dec": "13.36801", "alertMag": "20.54", "historicMag": "17.79", "historicStdDev": "0.24", + "classification": "CV", "published": "2020-06-06 12:26:16", "comment": "test comment", + "per_alert": {"link": "/alerts/alert/Gaia20cpu/", "name": "Gaia20cpu"}, "rvs": 'false'}, + {"name": "Gaia16aau", "tnsid": "AT2016dbu", "obstime": "2016-01-25 18:25:07", "ra": "12.54460", + "dec": "-69.73271", "alertMag": "15.13", "historicMag": "", "historicStdDev": "", "classification": "RCrB", + "published": "2016-01-30 13:46:16", "comment": "5mag change in 400days in Carbon Star [MH95]580, but spectrum rather blue. Candidate RCrB ?", + "per_alert": {"link": "/alerts/alert/Gaia16aau/","name": "Gaia16aau"}, "rvs": 'false'}, + {"name": "Gaia16aat", "tnsid": "AT2016dbx", "obstime": "2016-01-22 03:39:40", + "ra": "246.20861", "dec": "65.68363", "alertMag": "19.36", "historicMag": "", "historicStdDev": "", + "classification": "unknown", "published": "2016-01-30 13:22:05", "comment": + "long-term rise on a blue star seen in DSS2 and Galex", "per_alert": {"link": "/alerts/alert/Gaia16aat/", + "name": "Gaia16aat"}, "rvs": 'false'}, + {"name": "Gaia20bph", "tnsid": "AT2020ftt", "obstime": "2020-04-01 12:52:23", + "ra": "34.02266", "dec": "68.65102", "alertMag": "16.21", "historicMag": "18.39", "historicStdDev": "1.22", + "classification": "unknown", "published": "2020-04-03 09:47:04", + "comment": "candidate CV; several previous outbursts in lightcurve", + "per_alert": {"link": "/alerts/alert/Gaia20bph/", "name": "Gaia20bph"}, "rvs": 'false'} + ] + self.test_target = Target.objects.create(name=self.alert_list[0]['name']) + ReducedDatum.objects.create( + source_name='Gaia', + source_location=111111, + target=self.test_target, + data_type='photometry', + timestamp=timezone.now(), + value=12345.6789 + ) - @mock.patch('tom_alerts.brokers.mars.requests.get') + @mock.patch('tom_alerts.brokers.gaia.requests.get') def test_fetch_alerts(self, mock_requests_get): mock_response = Response() mock_response._content = self.test_html mock_response.status_code = 200 mock_requests_get.return_value = mock_response - # (\[{)(.*?)(}\]) \ No newline at end of file + search_params = {'target_name': 'Gaia20bph', 'cone': None, } + alerts = GaiaBroker().fetch_alerts(search_params) + self.assertEqual(1, sum(1 for _ in alerts)) + + search_params = {'target_name': None, 'cone': '291.61247, 13.36801, 0.002' } + alerts = GaiaBroker().fetch_alerts(search_params) + self.assertEqual(1, sum(1 for _ in alerts)) + + def test_to_generic_alert(self): + alert = GaiaBroker().to_generic_alert(self.alert_list[0]) + self.assertEqual(alert.name, self.alert_list[0]['name']) + + @mock.patch('tom_alerts.brokers.gaia.GaiaBroker.fetch_alert') + def test_process_reduced_data_with_alert(self, mock_fetch_alert): + mock_response = Response() + mock_response._content = self.test_html + mock_response.status_code = 200 + mock_fetch_alert.return_value = mock_response + + GaiaBroker().process_reduced_data(self.test_target, alert=self.alert_list[0]) + + reduced_data = ReducedDatum.objects.filter(target=self.test_target, source_name='Gaia') + self.assertGreater(reduced_data.count(), 1) + + def test_process_reduced_data_without_alert(self): + GaiaBroker().process_reduced_data(self.test_target) + + reduced_data = ReducedDatum.objects.filter(target=self.test_target, source_name='Gaia') + self.assertGreater(reduced_data.count(), 1) From 974db49fdad0ffd73c885acbf47ff1690b120c50 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 24 Jul 2020 15:11:26 -0700 Subject: [PATCH 210/424] Removing commented block --- tom_alerts/brokers/gaia.py | 43 -------------------------------------- 1 file changed, 43 deletions(-) diff --git a/tom_alerts/brokers/gaia.py b/tom_alerts/brokers/gaia.py index ee88a0356..c620c81d2 100644 --- a/tom_alerts/brokers/gaia.py +++ b/tom_alerts/brokers/gaia.py @@ -48,49 +48,6 @@ class GaiaBroker(GenericBroker): name = 'Gaia' form = GaiaQueryForm - # def fetch_alerts(self, parameters): - # response = requests.get(f'{BASE_BROKER_URL}/alerts/alertsindex') - # response.raise_for_status() - - # soup = BeautifulSoup(response.content) - # script_tags = soup.find_all('script') - # alerts = [] - - # alerts_pattern = re.compile(r'var alerts = (.*?);') - # for script in script_tags: - # alerts = alerts_pattern.match(str(script.string).strip()) - # if alerts: - # break - - # print(alerts[0]) - # alert_list = json.loads(alerts[0].replace('var_alerts = ', '').replace(';', '')) - - # cone_params = parameters.get('cone').split(',') - # parameters['cone_ra'] = float(cone_params[0]) - # parameters['cone_dec'] = float(cone_params[1]) - # parameters['cone_radius'] = float(cone_params[2])*u.deg - # parameters['cone_centre'] = SkyCoord(float(cone_params[0]), - # float(cone_params[1]), - # frame="icrs", unit="deg") - - # filtered_alerts = [] - # if parameters.get('target_name'): - # for alert in alert_list: - # if parameters['target_name'] in alert['name']: - # filtered_alerts.append(alert) - - # elif 'cone_radius' in parameters.keys(): - # for alert in alert_list: - # c = SkyCoord(float(alert['ra']), float(alert['dec']), - # frame="icrs", unit="deg") - # if parameters['cone_centre'].separation(c) <= parameters['cone_radius']: - # filtered_alerts.append(alert) - - # else: - # filtered_alerts = alert_list - - # return iter(filtered_alerts) - def fetch_alerts(self, parameters): """Must return an iterator""" response = requests.get(f'{BASE_BROKER_URL}/alerts/alertsindex') From b2636c4250f9d91802c145e29397301f0166d7c9 Mon Sep 17 00:00:00 2001 From: rachel3834 Date: Fri, 24 Jul 2020 15:22:01 -0700 Subject: [PATCH 211/424] Removed superfluous variable --- tom_alerts/brokers/gaia.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tom_alerts/brokers/gaia.py b/tom_alerts/brokers/gaia.py index 9d8c7bf15..8bf151257 100644 --- a/tom_alerts/brokers/gaia.py +++ b/tom_alerts/brokers/gaia.py @@ -120,7 +120,6 @@ def to_generic_alert(self, alert): def process_reduced_data(self, target, alert=None): base_url = BROKER_URL.replace('/alertsindex', '/alert') - query_url = f'{BASE_BROKER_URL}/alert' if not alert: try: From 833605e6ff0be595473979bccb98a88ffc807f2c Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 24 Jul 2020 16:56:33 -0700 Subject: [PATCH 212/424] Fixed some style issues and started fixing tests --- tom_alerts/brokers/gaia.py | 20 +++------ tom_alerts/tests/tests_gaia.py | 80 +++++++++++++++++----------------- 2 files changed, 47 insertions(+), 53 deletions(-) diff --git a/tom_alerts/brokers/gaia.py b/tom_alerts/brokers/gaia.py index 9d8c7bf15..96968bd46 100644 --- a/tom_alerts/brokers/gaia.py +++ b/tom_alerts/brokers/gaia.py @@ -1,6 +1,5 @@ from dateutil.parser import parse import json -from os import path import re import requests from requests.exceptions import HTTPError @@ -11,10 +10,9 @@ from bs4 import BeautifulSoup from django import forms -from tom_alerts.alerts import GenericQueryForm, GenericAlert, GenericBroker +from tom_alerts.alerts import GenericAlert, GenericBroker, GenericQueryForm from tom_dataproducts.models import ReducedDatum -BROKER_URL = 'http://gsaweb.ast.cam.ac.uk/alerts/alertsindex' BASE_BROKER_URL = 'http://gsaweb.ast.cam.ac.uk' @@ -104,7 +102,6 @@ def to_generic_alert(self, alert): timestamp = parse(alert['obstime']) alert_link = alert.get('per_alert', {})['link'] url = f'{BASE_BROKER_URL}/{alert_link}' - url = BROKER_URL.replace('/alerts/alertsindex', alert['per_alert']['link']) return GenericAlert( timestamp=timestamp, @@ -119,9 +116,6 @@ def to_generic_alert(self, alert): def process_reduced_data(self, target, alert=None): - base_url = BROKER_URL.replace('/alertsindex', '/alert') - query_url = f'{BASE_BROKER_URL}/alert' - if not alert: try: alert = self.fetch_alert(target.name) @@ -130,13 +124,13 @@ def process_reduced_data(self, target, alert=None): raise Exception('Unable to retrieve alert information from broker') if alert is not None: - lc_url = path.join(base_url, alert['name'], 'lightcurve.csv') - alert_url = BROKER_URL.replace('/alerts/alertsindex', - alert['per_alert']['link']) + alert_name = alert['name'] + alert_link = alert.get('per_alert', {})['link'] + lc_url = f'{BASE_BROKER_URL}/alerts/alert/{alert_name}/lightcurve.csv' + alert_url = f'{BASE_BROKER_URL}/{alert_link}' elif target: - lc_url = path.join(base_url, target.name, 'lightcurve.csv') - alert_url = BROKER_URL.replace('/alerts/alertsindex', - 'alerts/alert/'+target.name+'/') + lc_url = f'{BASE_BROKER_URL}/{target.name}/lightcurve.csv' + alert_url = f'{BASE_BROKER_URL}/alerts/alert/{target.name}/' else: return diff --git a/tom_alerts/tests/tests_gaia.py b/tom_alerts/tests/tests_gaia.py index 3708e09b0..caaa5a463 100644 --- a/tom_alerts/tests/tests_gaia.py +++ b/tom_alerts/tests/tests_gaia.py @@ -10,6 +10,7 @@ from tom_targets.models import Target from tom_dataproducts.models import ReducedDatum + @override_settings(TOM_ALERT_CLASSES=['tom_alerts.brokers.gaia.GaiaBroker']) class TestGaiaQueryForm(TestCase): def setUp(self): @@ -63,50 +64,41 @@ def setUp(self): self.test_html = """ """ - self.test_html = self.test_html.replace('\n','') - self.alert_list = [ {"name": "Gaia20cpu", "tnsid": "AT2020lto", "obstime": "2020-06-04 17:25:08", "ra": "291.61247", - "dec": "13.36801", "alertMag": "20.54", "historicMag": "17.79", "historicStdDev": "0.24", - "classification": "CV", "published": "2020-06-06 12:26:16", "comment": "test comment", - "per_alert": {"link": "/alerts/alert/Gaia20cpu/", "name": "Gaia20cpu"}, "rvs": 'false'}, - {"name": "Gaia16aau", "tnsid": "AT2016dbu", "obstime": "2016-01-25 18:25:07", "ra": "12.54460", - "dec": "-69.73271", "alertMag": "15.13", "historicMag": "", "historicStdDev": "", "classification": "RCrB", - "published": "2016-01-30 13:46:16", "comment": "5mag change in 400days in Carbon Star [MH95]580, but spectrum rather blue. Candidate RCrB ?", - "per_alert": {"link": "/alerts/alert/Gaia16aau/","name": "Gaia16aau"}, "rvs": 'false'}, - {"name": "Gaia16aat", "tnsid": "AT2016dbx", "obstime": "2016-01-22 03:39:40", - "ra": "246.20861", "dec": "65.68363", "alertMag": "19.36", "historicMag": "", "historicStdDev": "", - "classification": "unknown", "published": "2016-01-30 13:22:05", "comment": - "long-term rise on a blue star seen in DSS2 and Galex", "per_alert": {"link": "/alerts/alert/Gaia16aat/", - "name": "Gaia16aat"}, "rvs": 'false'}, - {"name": "Gaia20bph", "tnsid": "AT2020ftt", "obstime": "2020-04-01 12:52:23", - "ra": "34.02266", "dec": "68.65102", "alertMag": "16.21", "historicMag": "18.39", "historicStdDev": "1.22", - "classification": "unknown", "published": "2020-04-03 09:47:04", - "comment": "candidate CV; several previous outbursts in lightcurve", - "per_alert": {"link": "/alerts/alert/Gaia20bph/", "name": "Gaia20bph"}, "rvs": 'false'} + self.test_html = self.test_html.replace('\n', '') + self.alert_list = [ + { + "name": "Gaia20cpu", "tnsid": "AT2020lto", "obstime": "2020-06-04 17:25:08", + "ra": "291.61247", "dec": "13.36801", "alertMag": "20.54", "historicMag": "17.79", + "historicStdDev": "0.24", "classification": "CV", "published": "2020-06-06 12:26:16", + "comment": "test comment", "per_alert": { + "link": "/alerts/alert/Gaia20cpu/", "name": "Gaia20cpu" + }, "rvs": 'false'}, + { + "name": "Gaia20bph", "tnsid": "AT2020ftt", "obstime": "2020-04-01 12:52:23", + "ra": "34.02266", "dec": "68.65102", "alertMag": "16.21", "historicMag": "18.39", + "historicStdDev": "1.22", "classification": "unknown", + "published": "2020-04-03 09:47:04", + "comment": "candidate CV; several previous outbursts in lightcurve", "per_alert": { + "link": "/alerts/alert/Gaia20bph/", "name": "Gaia20bph"}, + "rvs": 'false'} ] self.test_target = Target.objects.create(name=self.alert_list[0]['name']) ReducedDatum.objects.create( @@ -129,7 +121,7 @@ def test_fetch_alerts(self, mock_requests_get): alerts = GaiaBroker().fetch_alerts(search_params) self.assertEqual(1, sum(1 for _ in alerts)) - search_params = {'target_name': None, 'cone': '291.61247, 13.36801, 0.002' } + search_params = {'target_name': None, 'cone': '291.61247, 13.36801, 0.002'} alerts = GaiaBroker().fetch_alerts(search_params) self.assertEqual(1, sum(1 for _ in alerts)) @@ -137,13 +129,21 @@ def test_to_generic_alert(self): alert = GaiaBroker().to_generic_alert(self.alert_list[0]) self.assertEqual(alert.name, self.alert_list[0]['name']) + # @mock.patch('tom_alerts.brokers.gaia.requests.get') @mock.patch('tom_alerts.brokers.gaia.GaiaBroker.fetch_alert') + # def test_process_reduced_data_with_alert(self, mock_fetch_alert, mock_requests_get): def test_process_reduced_data_with_alert(self, mock_fetch_alert): mock_response = Response() mock_response._content = self.test_html mock_response.status_code = 200 mock_fetch_alert.return_value = mock_response + # mock_photometry_response = Response() + # mock_response._content = str.encode('''Gaia19dzu\n#Date,JD,averagemag.\n + # 2014-08-01 00:05:24,2456870.504,19.48\n2014-08-01 06:05:38,2456870.754,19.48\n\n''') + # mock_response.status_code = 200 + # mock_requests_get.return_value = mock_photometry_response + GaiaBroker().process_reduced_data(self.test_target, alert=self.alert_list[0]) reduced_data = ReducedDatum.objects.filter(target=self.test_target, source_name='Gaia') From d3eff9a7a4a04edabd843d2152ae9e87266e4fbb Mon Sep 17 00:00:00 2001 From: David Collom Date: Sat, 25 Jul 2020 14:34:21 -0700 Subject: [PATCH 213/424] Modified view to properly fill forms after failed validation --- tom_observations/facilities/lco.py | 9 ++-- .../tom_observations/observation_form.html | 4 +- tom_observations/views.py | 41 +++++++++++-------- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 24ea5ddd7..dc008fe62 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -333,7 +333,6 @@ def _expand_cadence_request(self, payload): return response.json() def observation_payload(self): - print(self.cleaned_data) payload = { "name": self.cleaned_data['name'], "proposal": self.cleaned_data['proposal'], @@ -358,7 +357,6 @@ def observation_payload(self): if self.cleaned_data.get('period') and self.cleaned_data.get('jitter'): payload = self._expand_cadence_request(payload) - print(payload) return payload @@ -490,7 +488,8 @@ def __init__(self, *args, **kwargs): self.fields['cadence_type'].required = False self.fields['cadence_strategy'].required = False self.fields['cadence_frequency'].required = False - self.fields['groups'].label = 'Data granted to' + if self.fields.get('groups'): + self.fields['groups'].label = 'Data granted to' def _build_instrument_config(self): instrument_config = [] @@ -523,7 +522,7 @@ def instrument_choices(self): def layout(self): if settings.TARGET_PERMISSIONS_ONLY: - groups = Row('') + groups = Div() else: groups = Row('groups') return Div( @@ -645,9 +644,7 @@ class LCOFacility(BaseRoboticObservationFacility): } def get_form(self, observation_type): - print(observation_type) try: - print(self.observation_forms[observation_type]) return self.observation_forms[observation_type] except KeyError: return LCOBaseObservationForm diff --git a/tom_observations/templates/tom_observations/observation_form.html b/tom_observations/templates/tom_observations/observation_form.html index 00ff701ab..1731e6b9b 100644 --- a/tom_observations/templates/tom_observations/observation_form.html +++ b/tom_observations/templates/tom_observations/observation_form.html @@ -24,7 +24,7 @@

    Submit an observation to {{ form.facility.value }}

    {% for observation_type, observation_form in observation_type_choices %} -
    +
    {% crispy observation_form %}
    {% endfor %} diff --git a/tom_observations/views.py b/tom_observations/views.py index 748e40ede..afbdbe9eb 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -163,16 +163,22 @@ def get_context_data(self, **kwargs): :rtype: dict """ context = super(ObservationCreateView, self).get_context_data(**kwargs) - print(context['form']) - # TODO: only update initial values for active form - # observation_type_choices = [] - # initial = self.get_initial() - # for k, v in self.get_facility_class().observation_forms.items(): - # observation_type_choices.append() - context['observation_type_choices'] = [(k, v(initial={**self.get_initial(), **{'observation_type': k}})) for k, v in self.get_facility_class().observation_forms.items()] + + # Populate initial values for each form and add them to the context. If the page + # reloaded due to form errors, only repopulate the form that was submitted. + observation_type_choices = [] + initial = self.get_initial() + for k, v in self.get_facility_class().observation_forms.items(): + form_data = {**initial, **{'observation_type': k}} + if k == self.request.POST.get('observation_type'): + form_data.update(**self.request.POST.dict()) + observation_type_choices.append((k, v(initial=form_data))) + context['observation_type_choices'] = observation_type_choices + + # Ensure correct tab is active if submission is unsuccessful + context['active'] = self.request.POST.get('observation_type') + target = Target.objects.get(pk=self.get_target_id()) - # TODO: add active to context and plumb through to template - # context['active'] = context['target'] = target return context @@ -188,6 +194,7 @@ def get_form_class(self): observation_type = self.request.GET.get('observation_type') elif self.request.method == 'POST': observation_type = self.request.POST.get('observation_type') + print(self.get_facility_class()().get_form(observation_type)) return self.get_facility_class()().get_form(observation_type) def get_form(self): @@ -197,7 +204,6 @@ def get_form(self): :returns: observation form :rtype: subclass of GenericObservationForm """ - print(self.request) form = super().get_form() if not settings.TARGET_PERMISSIONS_ONLY: form.fields['groups'].queryset = self.request.user.groups.all() @@ -214,20 +220,21 @@ def get_initial(self): :returns: initial form data :rtype: dict """ - print('get initial') - if self.request.method == 'POST': - print(self.request.POST) initial = super().get_initial() if not self.get_target_id(): raise Exception('Must provide target_id') initial['target_id'] = self.get_target_id() initial['facility'] = self.get_facility() - if self.request.method == 'GET': - initial.update(self.request.GET.dict()) - elif self.request.method == 'POST': - initial.update(self.request.POST.dict()) return initial + def form_invalid(self, form): + print('form_invalid') + print(form.cleaned_data['observation_type']) + print(type(form)) + # print(form) + print(form.errors) + return super().form_invalid(form) + def form_valid(self, form): """ Runs after form validation. Submits the observation to the desired facility and creates an associated From 6432371a4e93672460c771ea455d37729c212a89 Mon Sep 17 00:00:00 2001 From: David Collom Date: Sat, 25 Jul 2020 15:34:42 -0700 Subject: [PATCH 214/424] Fixed tests and removed network dependencies --- tom_alerts/tests/tests_gaia.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/tom_alerts/tests/tests_gaia.py b/tom_alerts/tests/tests_gaia.py index caaa5a463..c70e4f0ea 100644 --- a/tom_alerts/tests/tests_gaia.py +++ b/tom_alerts/tests/tests_gaia.py @@ -129,27 +129,31 @@ def test_to_generic_alert(self): alert = GaiaBroker().to_generic_alert(self.alert_list[0]) self.assertEqual(alert.name, self.alert_list[0]['name']) - # @mock.patch('tom_alerts.brokers.gaia.requests.get') - @mock.patch('tom_alerts.brokers.gaia.GaiaBroker.fetch_alert') - # def test_process_reduced_data_with_alert(self, mock_fetch_alert, mock_requests_get): - def test_process_reduced_data_with_alert(self, mock_fetch_alert): - mock_response = Response() - mock_response._content = self.test_html - mock_response.status_code = 200 - mock_fetch_alert.return_value = mock_response + @mock.patch('tom_alerts.brokers.gaia.requests.get') + def test_process_reduced_data_with_alert(self, mock_requests_get): - # mock_photometry_response = Response() - # mock_response._content = str.encode('''Gaia19dzu\n#Date,JD,averagemag.\n - # 2014-08-01 00:05:24,2456870.504,19.48\n2014-08-01 06:05:38,2456870.754,19.48\n\n''') - # mock_response.status_code = 200 - # mock_requests_get.return_value = mock_photometry_response + mock_photometry_response = Response() + mock_photometry_response._content = str.encode('''Gaia20bph\n#Date,JD,averagemag.\n + 2014-08-01 00:05:24,2456870.504,19.48\n2014-08-01 06:05:38,2456870.754,19.48\n\n''') + mock_photometry_response.status_code = 200 + mock_requests_get.return_value = mock_photometry_response GaiaBroker().process_reduced_data(self.test_target, alert=self.alert_list[0]) reduced_data = ReducedDatum.objects.filter(target=self.test_target, source_name='Gaia') self.assertGreater(reduced_data.count(), 1) - def test_process_reduced_data_without_alert(self): + @mock.patch('tom_alerts.brokers.gaia.requests.get') + @mock.patch('tom_alerts.brokers.gaia.GaiaBroker.fetch_alerts') + def test_process_reduced_data_without_alert(self, mock_fetch_alerts, mock_requests_get): + mock_fetch_alerts.return_value = iter([self.alert_list[1]]) + + mock_photometry_response = Response() + mock_photometry_response._content = str.encode('''Gaia20bph\n#Date,JD,averagemag.\n + 2014-08-01 00:05:24,2456870.504,19.48\n2014-08-01 06:05:38,2456870.754,19.48\n\n''') + mock_photometry_response.status_code = 200 + mock_requests_get.return_value = mock_photometry_response + GaiaBroker().process_reduced_data(self.test_target) reduced_data = ReducedDatum.objects.filter(target=self.test_target, source_name='Gaia') From 34d8fe4ed7a91c35f3ec9892fd4e4350cf3552c7 Mon Sep 17 00:00:00 2001 From: David Collom Date: Sat, 25 Jul 2020 15:41:06 -0700 Subject: [PATCH 215/424] pycodestyle --- tom_alerts/tests/tests_gaia.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_alerts/tests/tests_gaia.py b/tom_alerts/tests/tests_gaia.py index c70e4f0ea..8d8e36c43 100644 --- a/tom_alerts/tests/tests_gaia.py +++ b/tom_alerts/tests/tests_gaia.py @@ -147,7 +147,7 @@ def test_process_reduced_data_with_alert(self, mock_requests_get): @mock.patch('tom_alerts.brokers.gaia.GaiaBroker.fetch_alerts') def test_process_reduced_data_without_alert(self, mock_fetch_alerts, mock_requests_get): mock_fetch_alerts.return_value = iter([self.alert_list[1]]) - + mock_photometry_response = Response() mock_photometry_response._content = str.encode('''Gaia20bph\n#Date,JD,averagemag.\n 2014-08-01 00:05:24,2456870.504,19.48\n2014-08-01 06:05:38,2456870.754,19.48\n\n''') From 84ef92aa9760751c1ef86190c122f54768b44ed2 Mon Sep 17 00:00:00 2001 From: David Collom Date: Sat, 25 Jul 2020 16:56:01 -0700 Subject: [PATCH 216/424] Fixing some code style issues just because --- tom_catalogs/models.py | 3 --- tom_publications/forms.py | 1 - tom_setup/management/commands/tom_setup.py | 1 - tom_targets/groups.py | 2 +- 4 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 tom_catalogs/models.py diff --git a/tom_catalogs/models.py b/tom_catalogs/models.py deleted file mode 100644 index 71a836239..000000000 --- a/tom_catalogs/models.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.db import models - -# Create your models here. diff --git a/tom_publications/forms.py b/tom_publications/forms.py index 9792d7d23..12496377b 100644 --- a/tom_publications/forms.py +++ b/tom_publications/forms.py @@ -1,6 +1,5 @@ from django import forms from django.apps import apps -from django.db.models.fields import Field from tom_publications.models import LatexConfiguration diff --git a/tom_setup/management/commands/tom_setup.py b/tom_setup/management/commands/tom_setup.py index 80c7cd99d..dbc5b74c3 100644 --- a/tom_setup/management/commands/tom_setup.py +++ b/tom_setup/management/commands/tom_setup.py @@ -1,7 +1,6 @@ from django.core.management.base import BaseCommand import sys import os -import mimetypes from django.conf import settings from django.template.loader import get_template from django.core.management import call_command diff --git a/tom_targets/groups.py b/tom_targets/groups.py index 8c58932ba..ec0dc0b13 100644 --- a/tom_targets/groups.py +++ b/tom_targets/groups.py @@ -22,7 +22,7 @@ def add_all_to_grouping(filter_data, grouping_object, request): failure_targets = [] try: target_queryset = TargetFilter(request=request, data=filter_data, queryset=Target.objects.all()).qs - except Exception as e: + except Exception: messages.error(request, "Error with filter parameters. No target(s) were added to group '{}'." .format(grouping_object.name)) return From f5b22f27f62e6450e8e9641f68b0d9a12a5e8817 Mon Sep 17 00:00:00 2001 From: David Collom Date: Sat, 25 Jul 2020 17:32:01 -0700 Subject: [PATCH 217/424] More codacy fixes --- tom_alerts/alerts.py | 1 - tom_alerts/brokers/gaia.py | 2 +- tom_alerts/brokers/mars.py | 2 +- tom_catalogs/admin.py | 2 -- tom_common/views.py | 2 +- tom_dataproducts/tests/tests.py | 10 +++++----- tom_dataproducts/utils.py | 2 +- tom_observations/facilities/gemini.py | 2 +- tom_publications/admin.py | 2 -- tom_publications/templatetags/publication_extras.py | 6 ------ tom_targets/filters.py | 2 +- tom_targets/tests/tests.py | 2 +- 12 files changed, 12 insertions(+), 23 deletions(-) diff --git a/tom_alerts/alerts.py b/tom_alerts/alerts.py index 38085cac3..63138695a 100644 --- a/tom_alerts/alerts.py +++ b/tom_alerts/alerts.py @@ -159,7 +159,6 @@ def fetch_alerts(self, parameters): :param parameters: JSON string of query parameters :type parameters: str """ - pass def fetch_alert(self, id): """ diff --git a/tom_alerts/brokers/gaia.py b/tom_alerts/brokers/gaia.py index 96968bd46..a7ea9e9e5 100644 --- a/tom_alerts/brokers/gaia.py +++ b/tom_alerts/brokers/gaia.py @@ -151,7 +151,7 @@ def process_reduced_data(self, target, alert=None): 'filter': 'G' } - rd, created = ReducedDatum.objects.get_or_create( + rd, _ = ReducedDatum.objects.get_or_create( timestamp=jd.to_datetime(timezone=TimezoneInfo()), value=json.dumps(value), source_name=self.name, diff --git a/tom_alerts/brokers/mars.py b/tom_alerts/brokers/mars.py index 4cbe5ccc3..2ceda4720 100644 --- a/tom_alerts/brokers/mars.py +++ b/tom_alerts/brokers/mars.py @@ -230,7 +230,7 @@ def process_reduced_data(self, target, alert=None): 'magnitude': candidate['candidate']['magpsf'], 'filter': filters[candidate['candidate']['fid']] } - rd, created = ReducedDatum.objects.get_or_create( + rd, _ = ReducedDatum.objects.get_or_create( timestamp=jd.to_datetime(timezone=TimezoneInfo()), value=json.dumps(value), source_name=self.name, diff --git a/tom_catalogs/admin.py b/tom_catalogs/admin.py index 8c38f3f3d..846f6b406 100644 --- a/tom_catalogs/admin.py +++ b/tom_catalogs/admin.py @@ -1,3 +1 @@ -from django.contrib import admin - # Register your models here. diff --git a/tom_common/views.py b/tom_common/views.py index 11fb46006..d7b39b924 100644 --- a/tom_common/views.py +++ b/tom_common/views.py @@ -142,7 +142,7 @@ def get_success_url(self): else: return reverse_lazy('user-update', kwargs={'pk': self.request.user.id}) - def get_form(self): + def get_form(self, form_class=None): """ Gets the user update form and removes the password requirement. Removes the groups field if the user is not a superuser. diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index 1df903366..2edcef18c 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -351,7 +351,7 @@ def test_serialize_spectrum(self): spectrum = Spectrum1D(spectral_axis=wavelength, flux=flux) serialized = self.serializer.serialize(spectrum) - self.assertTrue(type(serialized) is str) + self.assertTrue(isinstance(serialized, str)) serialized = json.loads(serialized) self.assertTrue(serialized['photon_flux']) self.assertTrue(serialized['photon_flux_units']) @@ -415,15 +415,15 @@ def test_process_spectroscopy_with_invalid_file_type(self): def test_process_spectrum_from_fits(self): with open('tom_dataproducts/tests/test_data/test_spectrum.fits', 'rb') as spectrum_file: self.data_product.data.save('spectrum.fits', spectrum_file) - spectrum, date_obs = self.spectrum_data_processor._process_spectrum_from_fits(self.data_product) - self.assertTrue(type(spectrum) is Spectrum1D) + spectrum, _ = self.spectrum_data_processor._process_spectrum_from_fits(self.data_product) + self.assertTrue(isinstance(spectrum, Spectrum1D)) self.assertAlmostEqual(spectrum.flux.mean().value, 2.295068e-14, places=19) self.assertAlmostEqual(spectrum.wavelength.mean().value, 6600.478789, places=5) def test_process_spectrum_from_plaintext(self): with open('tom_dataproducts/tests/test_data/test_spectrum.csv', 'rb') as spectrum_file: self.data_product.data.save('spectrum.csv', spectrum_file) - spectrum, date_obs = self.spectrum_data_processor._process_spectrum_from_plaintext(self.data_product) + spectrum, _ = self.spectrum_data_processor._process_spectrum_from_plaintext(self.data_product) self.assertTrue(type(spectrum) is Spectrum1D) self.assertAlmostEqual(spectrum.flux.mean().value, 1.166619e-14, places=19) self.assertAlmostEqual(spectrum.wavelength.mean().value, 3250.744489, places=5) @@ -443,5 +443,5 @@ def test_process_photometry_from_plaintext(self): with open('tom_dataproducts/tests/test_data/test_lightcurve.csv', 'rb') as lightcurve_file: self.data_product.data.save('lightcurve.csv', lightcurve_file) lightcurve = self.photometry_data_processor._process_photometry_from_plaintext(self.data_product) - self.assertTrue(type(lightcurve) is list) + self.assertTrue(isinstance(lightcurve, list)) self.assertEqual(len(lightcurve), 3) diff --git a/tom_dataproducts/utils.py b/tom_dataproducts/utils.py index b19cb6b17..62c7895fa 100644 --- a/tom_dataproducts/utils.py +++ b/tom_dataproducts/utils.py @@ -17,7 +17,7 @@ def create_image_dataproduct(data_product): """ tmpfile = data_product.create_thumbnail() if tmpfile: - dp, created = DataProduct.objects.get_or_create( + dp, _ = DataProduct.objects.get_or_create( product_id="{}_{}".format(data_product.product_id, "jpeg"), target=data_product.target, observation_record=data_product.observation_record, diff --git a/tom_observations/facilities/gemini.py b/tom_observations/facilities/gemini.py index d90dd8c4c..b31de4884 100644 --- a/tom_observations/facilities/gemini.py +++ b/tom_observations/facilities/gemini.py @@ -2,7 +2,7 @@ from django.conf import settings from django import forms from dateutil.parser import parse -from crispy_forms.layout import Layout, Div, HTML +from crispy_forms.layout import Div, HTML from astropy import units as u from tom_observations.facility import BaseRoboticObservationFacility, BaseRoboticObservationForm diff --git a/tom_publications/admin.py b/tom_publications/admin.py index 8c38f3f3d..846f6b406 100644 --- a/tom_publications/admin.py +++ b/tom_publications/admin.py @@ -1,3 +1 @@ -from django.contrib import admin - # Register your models here. diff --git a/tom_publications/templatetags/publication_extras.py b/tom_publications/templatetags/publication_extras.py index 522b77310..9242a6676 100644 --- a/tom_publications/templatetags/publication_extras.py +++ b/tom_publications/templatetags/publication_extras.py @@ -1,11 +1,6 @@ -import json - -from astropy.io import ascii from django import template -from django.apps import apps from tom_publications.forms import LatexTableForm -from tom_publications.latex import get_latex_processor register = template.Library() @@ -16,5 +11,4 @@ def latex_button(object): Renders a button that redirects to the LaTeX table generation page for the specified model instance. Requires an object, which is generally the object in the context for the page on which the templatetag will be used. """ - model_name = object._meta.label return {'model_name': object._meta.label, 'model_pk': object.id} diff --git a/tom_targets/filters.py b/tom_targets/filters.py index 9db0ddf65..76cb2e304 100644 --- a/tom_targets/filters.py +++ b/tom_targets/filters.py @@ -1,7 +1,7 @@ from math import radians from django.conf import settings -from django.db.models import ExpressionWrapper, F, FloatField, Q +from django.db.models import ExpressionWrapper, FloatField, Q from django.db.models.functions.math import ACos, Cos, Radians, Pi, Sin import django_filters diff --git a/tom_targets/tests/tests.py b/tom_targets/tests/tests.py index f3df7dde5..5ed97c12f 100644 --- a/tom_targets/tests/tests.py +++ b/tom_targets/tests/tests.py @@ -473,7 +473,7 @@ def test_export_filtered_targets_no_aliases(self): self.assertNotIn('M52', content) def test_export_all_targets_with_aliases(self): - st_name = TargetNameFactory.create(name='Messier 42', target=self.st) + TargetNameFactory.create(name='Messier 42', target=self.st) response = self.client.get(reverse('targets:export')) content = ''.join(line.decode('utf-8') for line in list(response.streaming_content)) self.assertIn('M42', content) From 88868198847df9185a9917f7d8d69c474929d8f3 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 3 Aug 2020 19:42:07 -0700 Subject: [PATCH 218/424] Fixed data product upload endpoint, added primarykeyrelatedfields for Targets and Observations, added data_product_type validation --- tom_common/urls.py | 3 +- tom_dataproducts/api_views.py | 36 ++++++++--------- tom_dataproducts/serializers.py | 72 ++++++++++++++++++++++++--------- tom_observations/serializers.py | 20 +++++++++ tom_targets/serializers.py | 16 +++++++- 5 files changed, 107 insertions(+), 40 deletions(-) create mode 100644 tom_observations/serializers.py diff --git a/tom_common/urls.py b/tom_common/urls.py index 04af51d5b..6c52f0e68 100644 --- a/tom_common/urls.py +++ b/tom_common/urls.py @@ -26,7 +26,7 @@ from rest_framework import routers from tom_targets.api_views import TargetViewSet -from tom_dataproducts.api_views import DataProductGroupViewSet, DataProductViewSet, ReducedDatumViewSet +from tom_dataproducts.api_views import DataProductGroupViewSet, DataProductViewSet # for all applications # set up the DRF router, its router.urls included in urlpatterns below @@ -34,7 +34,6 @@ router.register(r'targets', TargetViewSet, 'targets') router.register(r'dataproductgroups', DataProductGroupViewSet, 'dataproductgroups') router.register(r'dataproducts', DataProductViewSet, 'dataproducts') -router.register(r'reduceddatums', ReducedDatumViewSet, 'reduceddatums') urlpatterns = [ diff --git a/tom_dataproducts/api_views.py b/tom_dataproducts/api_views.py index c81962eb6..78d022dd7 100644 --- a/tom_dataproducts/api_views.py +++ b/tom_dataproducts/api_views.py @@ -1,9 +1,12 @@ from guardian.mixins import PermissionListMixin from rest_framework.mixins import CreateModelMixin +from rest_framework.parsers import FileUploadParser, MultiPartParser +from rest_framework.response import Response +from rest_framework.views import APIView from rest_framework.viewsets import GenericViewSet -from tom_dataproducts.models import DataProductGroup, DataProduct, ReducedDatum -from tom_dataproducts.serializers import DataProductGroupSerializer, DataProductSerializer, ReducedDatumSerializer +from tom_dataproducts.models import DataProductGroup, DataProduct +from tom_dataproducts.serializers import DataProductGroupSerializer, DataProductSerializer # TODO: see Davids comment in tom_targets/api_views.py @@ -11,14 +14,14 @@ # endpoint page. Rewrite these docstring to be useful to API consumers. -class DataProductGroupViewSet(CreateModelMixin, PermissionListMixin, GenericViewSet): - """Viewset for Target objects. By default supports CRUD operations. - See the docs on viewsets: https://www.django-rest-framework.org/api-guide/viewsets/ - """ - queryset = DataProductGroup.objects.all() - serializer_class = DataProductGroupSerializer - # TODO: define filterset_class - # TODO: define permission_required +# class DataProductGroupViewSet(CreateModelMixin, PermissionListMixin, GenericViewSet): +# """Viewset for Target objects. By default supports CRUD operations. +# See the docs on viewsets: https://www.django-rest-framework.org/api-guide/viewsets/ +# """ +# queryset = DataProductGroup.objects.all() +# serializer_class = DataProductGroupSerializer +# # TODO: define filterset_class +# # TODO: define permission_required class DataProductViewSet(CreateModelMixin, PermissionListMixin, GenericViewSet): @@ -29,13 +32,8 @@ class DataProductViewSet(CreateModelMixin, PermissionListMixin, GenericViewSet): serializer_class = DataProductSerializer # TODO: define filterset_class # TODO: define permission_required + parser_classes = [MultiPartParser] - -class ReducedDatumViewSet(CreateModelMixin, PermissionListMixin, GenericViewSet): - """Viewset for Target objects. By default supports CRUD operations. - See the docs on viewsets: https://www.django-rest-framework.org/api-guide/viewsets/ - """ - queryset = ReducedDatum.objects.all() - serializer_class = ReducedDatumSerializer - # TODO: define filterset_class - # TODO: define permission_required + def create(self, request, *args, **kwargs): + request.data['data'] = request.FILES['file'] + return super().create(request, *args, **kwargs) diff --git a/tom_dataproducts/serializers.py b/tom_dataproducts/serializers.py index 92c6b68d8..869f4ca88 100644 --- a/tom_dataproducts/serializers.py +++ b/tom_dataproducts/serializers.py @@ -1,5 +1,12 @@ +from django.conf import settings +from guardian.shortcuts import get_objects_for_user from rest_framework import serializers -from .models import DataProductGroup, DataProduct, ReducedDatum + +from tom_dataproducts.models import DataProductGroup, DataProduct +from tom_observations.models import ObservationRecord +from tom_observations.serializers import ObservationRecordFilteredPrimaryKeyRelatedField +from tom_targets.models import Target +from tom_targets.serializers import TargetFilteredPrimaryKeyRelatedField class DataProductGroupSerializer(serializers.ModelSerializer): @@ -9,6 +16,12 @@ class Meta: class DataProductSerializer(serializers.ModelSerializer): + target = TargetFilteredPrimaryKeyRelatedField(queryset=Target.objects.all()) + observation_record = ObservationRecordFilteredPrimaryKeyRelatedField(queryset=ObservationRecord.objects.all(), + required=False) + group = DataProductGroupSerializer(many=True, required=False) + data_product_type = serializers.CharField(allow_blank=False) + class Meta: model = DataProduct fields = ( @@ -17,12 +30,8 @@ class Meta: 'observation_record', 'data', 'extra_data', - 'group', - 'created', - 'modified', 'data_product_type', - 'featured', - 'thumbnail' + 'group', ) # TODO: use HyperlinkedModelSerializer @@ -34,16 +43,43 @@ class Meta: # } # } + # def create(self, validated_data): + # target = validated_data.get('target') + # obs_record = validated_data.get('observation_record') + # return super().create(validated_data) -class ReducedDatumSerializer(serializers.ModelSerializer): - class Meta: - model = ReducedDatum - fields = ( - 'target', - 'data_product', - 'data_type', - 'source_name', - 'source_location', - 'timestamp', - 'value' - ) + def validate_data_product_type(self, value): + for dp_type, dp_values in settings.DATA_PRODUCT_TYPES.keys(): + if not value or value == dp_values[0]: + break + else: + raise serializers.ValidationError('Not a valid data_product_type. Valid data_product_types are {0}.' + .format(', '.join(k for k in settings.DATA_PRODUCT_TYPES.keys()))) + return False + + +# class DataProductGroupFilteredPrimaryKeyRelatedField(serializers.PrimaryKeyRelatedField): +# # This PrimaryKeyRelatedField subclass is used to implement get_queryset based on the permissions of the user +# # submitting the request. The pattern was taken from this StackOverflow answer: https://stackoverflow.com/a/32683066 + +# def get_queryset(self): +# request = self.context.get('request', None) +# queryset = super().get_queryset() +# if not (request and queryset): +# return None +# return get_objects_for_user(request.user, 'tom_targets.change_target') + + +# The ReducedDatumSerializer is not necessary until we implement the DataProductDetailAPIView and DataProductListAPIView +# class ReducedDatumSerializer(serializers.ModelSerializer): +# class Meta: +# model = ReducedDatum +# fields = ( +# 'target', +# 'data_product', +# 'data_type', +# 'source_name', +# 'source_location', +# 'timestamp', +# 'value' +# ) diff --git a/tom_observations/serializers.py b/tom_observations/serializers.py new file mode 100644 index 000000000..8d1c43cf6 --- /dev/null +++ b/tom_observations/serializers.py @@ -0,0 +1,20 @@ +from django.conf import settings +from guardian.shortcuts import get_objects_for_user +from rest_framework import serializers + +from tom_observations.models import ObservationRecord + + +class ObservationRecordFilteredPrimaryKeyRelatedField(serializers.PrimaryKeyRelatedField): + # This PrimaryKeyRelatedField subclass is used to implement get_queryset based on the permissions of the user + # submitting the request. The pattern was taken from this StackOverflow answer: https://stackoverflow.com/a/32683066 + + def get_queryset(self): + request = self.context.get('request', None) + queryset = super().get_queryset() + if not (request and queryset): + return None + if settings.TARGET_PERMISSIONS_ONLY: + return ObservationRecord.objects.all() + else: + return get_objects_for_user(request.user, 'tom_observations.change_observation') diff --git a/tom_targets/serializers.py b/tom_targets/serializers.py index d6071eb68..e34d83897 100644 --- a/tom_targets/serializers.py +++ b/tom_targets/serializers.py @@ -1,5 +1,7 @@ +from guardian.shortcuts import get_objects_for_user from rest_framework import serializers -from .models import Target, TargetExtra, TargetName + +from tom_targets.models import Target, TargetExtra, TargetName class TargetNameSerializer(serializers.ModelSerializer): @@ -45,3 +47,15 @@ def create(self, validated_data): tes.save(target=target) return target + + +class TargetFilteredPrimaryKeyRelatedField(serializers.PrimaryKeyRelatedField): + # This PrimaryKeyRelatedField subclass is used to implement get_queryset based on the permissions of the user + # submitting the request. The pattern was taken from this StackOverflow answer: https://stackoverflow.com/a/32683066 + + def get_queryset(self): + request = self.context.get('request', None) + queryset = super().get_queryset() + if not (request and queryset): + return None + return get_objects_for_user(request.user, 'tom_targets.change_target') From 958273751df170844f73feffe5066f3f41774c0f Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 3 Aug 2020 19:53:40 -0700 Subject: [PATCH 219/424] Removed DataProductGroupViewSet from tom_common/urls.py, as it isn't being implemented yet --- tom_common/urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tom_common/urls.py b/tom_common/urls.py index 6c52f0e68..7dda0a66a 100644 --- a/tom_common/urls.py +++ b/tom_common/urls.py @@ -26,13 +26,13 @@ from rest_framework import routers from tom_targets.api_views import TargetViewSet -from tom_dataproducts.api_views import DataProductGroupViewSet, DataProductViewSet +from tom_dataproducts.api_views import DataProductViewSet # for all applications # set up the DRF router, its router.urls included in urlpatterns below router = routers.DefaultRouter() router.register(r'targets', TargetViewSet, 'targets') -router.register(r'dataproductgroups', DataProductGroupViewSet, 'dataproductgroups') +# router.register(r'dataproductgroups', DataProductGroupViewSet, 'dataproductgroups') router.register(r'dataproducts', DataProductViewSet, 'dataproducts') From 584017a50dc28313ea19e7ab597bfb77071fc169 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 3 Aug 2020 19:55:30 -0700 Subject: [PATCH 220/424] Fixed problem with data_product_type validation --- tom_dataproducts/serializers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tom_dataproducts/serializers.py b/tom_dataproducts/serializers.py index 869f4ca88..c18a48870 100644 --- a/tom_dataproducts/serializers.py +++ b/tom_dataproducts/serializers.py @@ -21,7 +21,7 @@ class DataProductSerializer(serializers.ModelSerializer): required=False) group = DataProductGroupSerializer(many=True, required=False) data_product_type = serializers.CharField(allow_blank=False) - + class Meta: model = DataProduct fields = ( @@ -49,13 +49,13 @@ class Meta: # return super().create(validated_data) def validate_data_product_type(self, value): - for dp_type, dp_values in settings.DATA_PRODUCT_TYPES.keys(): - if not value or value == dp_values[0]: + for dp_type in settings.DATA_PRODUCT_TYPES.keys(): + if not value or value == dp_type: break else: raise serializers.ValidationError('Not a valid data_product_type. Valid data_product_types are {0}.' .format(', '.join(k for k in settings.DATA_PRODUCT_TYPES.keys()))) - return False + return value # class DataProductGroupFilteredPrimaryKeyRelatedField(serializers.PrimaryKeyRelatedField): From 300f91372876bc372ef4f15aad003bd2a20ac068 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 3 Aug 2020 22:40:34 -0700 Subject: [PATCH 221/424] Added dataproduct tests, fixed some filters, fixed style problems --- tom_common/urls.py | 2 +- tom_dataproducts/api_views.py | 58 ++++++++++++++++++++---- tom_dataproducts/filters.py | 6 +-- tom_dataproducts/serializers.py | 15 +----- tom_dataproducts/tests/test_api.py | 73 ++++++++++++++++++++++++++++++ tom_observations/serializers.py | 2 +- tom_targets/api_views.py | 7 +-- tom_targets/serializers.py | 2 +- tom_targets/tests/test_api.py | 10 ++-- 9 files changed, 140 insertions(+), 35 deletions(-) create mode 100644 tom_dataproducts/tests/test_api.py diff --git a/tom_common/urls.py b/tom_common/urls.py index 7dda0a66a..2a1be0497 100644 --- a/tom_common/urls.py +++ b/tom_common/urls.py @@ -58,7 +58,7 @@ path('comment//delete', CommentDeleteView.as_view(), name='comment-delete'), path('admin/', admin.site.urls), path('api-auth/', include('rest_framework.urls')), - path('api/', include(router.urls)), + path('api/', include((router.urls, 'api'), namespace='api')), # The static helper below only works in development see # https://docs.djangoproject.com/en/2.1/howto/static-files/#serving-files-uploaded-by-a-user-during-development ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/tom_dataproducts/api_views.py b/tom_dataproducts/api_views.py index 78d022dd7..d8bdc5f3e 100644 --- a/tom_dataproducts/api_views.py +++ b/tom_dataproducts/api_views.py @@ -1,11 +1,18 @@ +from django.conf import settings +from django_filters import rest_framework as drf_filters from guardian.mixins import PermissionListMixin -from rest_framework.mixins import CreateModelMixin -from rest_framework.parsers import FileUploadParser, MultiPartParser +from guardian.shortcuts import assign_perm, get_objects_for_user +from rest_framework import status +from rest_framework.mixins import CreateModelMixin, DestroyModelMixin, ListModelMixin +from rest_framework.parsers import MultiPartParser +from rest_framework.permissions import DjangoObjectPermissions, IsAuthenticated from rest_framework.response import Response -from rest_framework.views import APIView from rest_framework.viewsets import GenericViewSet -from tom_dataproducts.models import DataProductGroup, DataProduct +from tom_common.hooks import run_hook +from tom_dataproducts.data_processor import run_data_processor +from tom_dataproducts.filters import DataProductFilter +from tom_dataproducts.models import DataProductGroup, DataProduct, ReducedDatum from tom_dataproducts.serializers import DataProductGroupSerializer, DataProductSerializer # TODO: see Davids comment in tom_targets/api_views.py @@ -24,16 +31,51 @@ # # TODO: define permission_required -class DataProductViewSet(CreateModelMixin, PermissionListMixin, GenericViewSet): +class DataProductViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, GenericViewSet): """Viewset for Target objects. By default supports CRUD operations. See the docs on viewsets: https://www.django-rest-framework.org/api-guide/viewsets/ """ queryset = DataProduct.objects.all() serializer_class = DataProductSerializer - # TODO: define filterset_class - # TODO: define permission_required + filter_backends = (drf_filters.DjangoFilterBackend,) + filterset_class = DataProductFilter + permission_classes = [IsAuthenticated & DjangoObjectPermissions] + # TODO: define permission required or ensure get_queryset is doing the right thing + # TODO: attempting to delete with no auth results in infinite redirect parser_classes = [MultiPartParser] def create(self, request, *args, **kwargs): request.data['data'] = request.FILES['file'] - return super().create(request, *args, **kwargs) + response = super().create(request, *args, **kwargs) + + if response.status_code == status.HTTP_201_CREATED: + dp = DataProduct.objects.get(pk=response.data['id']) + try: + run_hook('data_product_post_upload', dp) + reduced_data = run_data_processor(dp) + if not settings.TARGET_PERMISSIONS_ONLY: + for group in response.data['group']: + assign_perm('tom_dataproducts.view_dataproduct', group, dp) + assign_perm('tom_dataproducts.delete_dataproduct', group, dp) + assign_perm('tom_dataproducts.view_reduceddatum', group, reduced_data) + except Exception: + ReducedDatum.objects.filter(data_product=dp).delete() + dp.delete() + return Response({'Data processing error': '''There was an error in processing your DataProduct into \ + individual ReducedDatum objects.'''}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) + return response + + def get_queryset(self): + """ + Gets the set of ``DataProduct`` objects that the user has permission to view. + + :returns: Set of ``DataProduct`` objects + :rtype: QuerySet + """ + if settings.TARGET_PERMISSIONS_ONLY: + return super().get_queryset().filter( + target__in=get_objects_for_user(self.request.user, 'tom_targets.view_target') + ) + else: + return get_objects_for_user(self.request.user, 'tom_dataproducts.view_dataproduct') diff --git a/tom_dataproducts/filters.py b/tom_dataproducts/filters.py index dbad34fe7..d1cf167d4 100644 --- a/tom_dataproducts/filters.py +++ b/tom_dataproducts/filters.py @@ -5,12 +5,12 @@ class DataProductFilter(django_filters.FilterSet): - name = django_filters.CharFilter(label='Name', method='filter_name') + target_name = django_filters.CharFilter(label='Target Name', method='filter_name') facility = django_filters.CharFilter(field_name='observation_record__facility', label='Observation Record Facility') class Meta: model = DataProduct - fields = ['name', 'facility'] + fields = ['target_name', 'facility'] def filter_name(self, queryset, name, value): - return queryset.filter(Q(name__icontains=value) | Q(aliases__name__icontains=value)) + return queryset.filter(Q(target__name__icontains=value) | Q(target__aliases__name__icontains=value)) diff --git a/tom_dataproducts/serializers.py b/tom_dataproducts/serializers.py index c18a48870..383a230e7 100644 --- a/tom_dataproducts/serializers.py +++ b/tom_dataproducts/serializers.py @@ -17,7 +17,7 @@ class Meta: class DataProductSerializer(serializers.ModelSerializer): target = TargetFilteredPrimaryKeyRelatedField(queryset=Target.objects.all()) - observation_record = ObservationRecordFilteredPrimaryKeyRelatedField(queryset=ObservationRecord.objects.all(), + observation_record = ObservationRecordFilteredPrimaryKeyRelatedField(queryset=ObservationRecord.objects.all(), required=False) group = DataProductGroupSerializer(many=True, required=False) data_product_type = serializers.CharField(allow_blank=False) @@ -25,6 +25,7 @@ class DataProductSerializer(serializers.ModelSerializer): class Meta: model = DataProduct fields = ( + 'id', 'product_id', 'target', 'observation_record', @@ -58,18 +59,6 @@ def validate_data_product_type(self, value): return value -# class DataProductGroupFilteredPrimaryKeyRelatedField(serializers.PrimaryKeyRelatedField): -# # This PrimaryKeyRelatedField subclass is used to implement get_queryset based on the permissions of the user -# # submitting the request. The pattern was taken from this StackOverflow answer: https://stackoverflow.com/a/32683066 - -# def get_queryset(self): -# request = self.context.get('request', None) -# queryset = super().get_queryset() -# if not (request and queryset): -# return None -# return get_objects_for_user(request.user, 'tom_targets.change_target') - - # The ReducedDatumSerializer is not necessary until we implement the DataProductDetailAPIView and DataProductListAPIView # class ReducedDatumSerializer(serializers.ModelSerializer): # class Meta: diff --git a/tom_dataproducts/tests/test_api.py b/tom_dataproducts/tests/test_api.py new file mode 100644 index 000000000..76ce3523f --- /dev/null +++ b/tom_dataproducts/tests/test_api.py @@ -0,0 +1,73 @@ +from django.contrib.auth.models import User +from django.urls import reverse +from guardian.shortcuts import assign_perm +from rest_framework import status +from rest_framework.test import APITestCase + +from tom_dataproducts.models import DataProduct, ReducedDatum +from tom_observations.tests.factories import ObservingRecordFactory +from tom_targets.tests.factories import SiderealTargetFactory + + +class TestDataProductViewset(APITestCase): + def setUp(self): + user = User.objects.create(username='testuser') + self.client.force_login(user) + self.st = SiderealTargetFactory.create() + self.obsr = ObservingRecordFactory.create(target_id=self.st.id) + self.dp_data = { + 'product_id': 'test_product_id', + 'target': self.st.id, + 'data_product_type': 'photometry' + } + + assign_perm('tom_dataproducts.add_dataproduct', user) + assign_perm('tom_targets.add_target', user, self.st) + assign_perm('tom_targets.view_target', user, self.st) + assign_perm('tom_targets.change_target', user, self.st) + + def test_data_product_upload_for_target(self): + with open('tom_dataproducts/tests/test_data/test_lightcurve.csv', 'rb') as lightcurve_file: + self.dp_data['file'] = lightcurve_file + response = self.client.post(reverse('api:dataproducts-list'), self.dp_data, format='multipart') + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(DataProduct.objects.count(), 1) + self.assertEqual(ReducedDatum.objects.count(), 3) + dp = DataProduct.objects.get(pk=response.data['id']) + self.assertEqual(dp.target_id, self.st.id) + + def test_data_product_upload_for_observation(self): + self.dp_data['observation_record'] = self.obsr.id + + with open('tom_dataproducts/tests/test_data/test_lightcurve.csv', 'rb') as lightcurve_file: + self.dp_data['file'] = lightcurve_file + response = self.client.post(reverse('api:dataproducts-list'), self.dp_data, format='multipart') + + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(DataProduct.objects.count(), 1) + self.assertEqual(ReducedDatum.objects.count(), 3) + dp = DataProduct.objects.get(pk=response.data['id']) + self.assertEqual(dp.target_id, self.st.id) + self.assertEqual(dp.observation_record_id, self.obsr.id) + + def test_data_product_upload_invalid_type(self): + self.dp_data['data_product_type'] = 'invalid' + + with open('tom_dataproducts/tests/test_data/test_lightcurve.csv', 'rb') as lightcurve_file: + self.dp_data['file'] = lightcurve_file + response = self.client.post(reverse('api:dataproducts-list'), self.dp_data, format='multipart') + + self.assertContains(response, 'Not a valid data_product_type.', status_code=status.HTTP_400_BAD_REQUEST) + + def test_data_product_upload_failed_processing(self): + self.dp_data['data_product_type'] = 'spectroscopy' + + with open('tom_dataproducts/tests/test_data/test_lightcurve.csv', 'rb') as lightcurve_file: + self.dp_data['file'] = lightcurve_file + response = self.client.post(reverse('api:dataproducts-list'), self.dp_data, format='multipart') + + self.assertContains( + response.data, + 'There was an error in processing your DataProduct into individual ReducedDatum objects.', + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR + ) diff --git a/tom_observations/serializers.py b/tom_observations/serializers.py index 8d1c43cf6..726b3e49a 100644 --- a/tom_observations/serializers.py +++ b/tom_observations/serializers.py @@ -6,7 +6,7 @@ class ObservationRecordFilteredPrimaryKeyRelatedField(serializers.PrimaryKeyRelatedField): - # This PrimaryKeyRelatedField subclass is used to implement get_queryset based on the permissions of the user + # This PrimaryKeyRelatedField subclass is used to implement get_queryset based on the permissions of the user # submitting the request. The pattern was taken from this StackOverflow answer: https://stackoverflow.com/a/32683066 def get_queryset(self): diff --git a/tom_targets/api_views.py b/tom_targets/api_views.py index 70d661509..2ceb74964 100644 --- a/tom_targets/api_views.py +++ b/tom_targets/api_views.py @@ -1,6 +1,6 @@ from guardian.mixins import PermissionListMixin -from rest_framework.mixins import CreateModelMixin -from rest_framework.viewsets import GenericViewSet +from django_filters import rest_framework as drf_filters +from rest_framework.viewsets import ModelViewSet from tom_targets.filters import TargetFilter from tom_targets.models import Target @@ -14,11 +14,12 @@ # properly respect permissions, this class will inherit from GenericViewSet and the necessary # mixins for the supported actions. Once we add the appropriate logic for all actions, we # can update it to just inherit from ModelViewSet. -class TargetViewSet(CreateModelMixin, PermissionListMixin, GenericViewSet): +class TargetViewSet(PermissionListMixin, ModelViewSet): """Viewset for Target objects. By default supports CRUD operations. See the docs on viewsets: https://www.django-rest-framework.org/api-guide/viewsets/ """ queryset = Target.objects.all() serializer_class = TargetSerializer + filter_backends = (drf_filters.DjangoFilterBackend,) filterset_class = TargetFilter permission_required = 'tom_targets.view_target' diff --git a/tom_targets/serializers.py b/tom_targets/serializers.py index e34d83897..00263b900 100644 --- a/tom_targets/serializers.py +++ b/tom_targets/serializers.py @@ -50,7 +50,7 @@ def create(self, validated_data): class TargetFilteredPrimaryKeyRelatedField(serializers.PrimaryKeyRelatedField): - # This PrimaryKeyRelatedField subclass is used to implement get_queryset based on the permissions of the user + # This PrimaryKeyRelatedField subclass is used to implement get_queryset based on the permissions of the user # submitting the request. The pattern was taken from this StackOverflow answer: https://stackoverflow.com/a/32683066 def get_queryset(self): diff --git a/tom_targets/tests/test_api.py b/tom_targets/tests/test_api.py index 0304f08ae..b55cbb742 100644 --- a/tom_targets/tests/test_api.py +++ b/tom_targets/tests/test_api.py @@ -1,11 +1,11 @@ -from rest_framework.test import APITestCase -from rest_framework import status -from guardian.shortcuts import assign_perm -from django.urls import reverse from django.contrib.auth.models import User +from django.urls import reverse +from guardian.shortcuts import assign_perm +from rest_framework import status +from rest_framework.test import APITestCase +from tom_targets.tests.factories import SiderealTargetFactory, NonSiderealTargetFactory from tom_targets.models import Target -from .factories import SiderealTargetFactory, NonSiderealTargetFactory class TestTargetViewset(APITestCase): From cb4a97802cc7e201e7505a234be8c4ba062fb15e Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 3 Aug 2020 23:09:35 -0700 Subject: [PATCH 222/424] Fixed test --- tom_dataproducts/tests/test_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tom_dataproducts/tests/test_api.py b/tom_dataproducts/tests/test_api.py index 76ce3523f..f4ae92aeb 100644 --- a/tom_dataproducts/tests/test_api.py +++ b/tom_dataproducts/tests/test_api.py @@ -67,7 +67,7 @@ def test_data_product_upload_failed_processing(self): response = self.client.post(reverse('api:dataproducts-list'), self.dp_data, format='multipart') self.assertContains( - response.data, - 'There was an error in processing your DataProduct into individual ReducedDatum objects.', + response, + 'There was an error in processing your DataProduct', status_code=status.HTTP_500_INTERNAL_SERVER_ERROR ) From 3df234697530b5d7acde7ae33a0cd22cd5593ad0 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 3 Aug 2020 23:20:00 -0700 Subject: [PATCH 223/424] Removed some unnecessary code, updated doc header, and added skeleton for ReducedDatumSerializer --- tom_dataproducts/api_views.py | 25 ++++----------------- tom_dataproducts/serializers.py | 40 ++++++++++++++------------------- 2 files changed, 21 insertions(+), 44 deletions(-) diff --git a/tom_dataproducts/api_views.py b/tom_dataproducts/api_views.py index d8bdc5f3e..4d743142b 100644 --- a/tom_dataproducts/api_views.py +++ b/tom_dataproducts/api_views.py @@ -1,6 +1,5 @@ from django.conf import settings from django_filters import rest_framework as drf_filters -from guardian.mixins import PermissionListMixin from guardian.shortcuts import assign_perm, get_objects_for_user from rest_framework import status from rest_framework.mixins import CreateModelMixin, DestroyModelMixin, ListModelMixin @@ -12,35 +11,19 @@ from tom_common.hooks import run_hook from tom_dataproducts.data_processor import run_data_processor from tom_dataproducts.filters import DataProductFilter -from tom_dataproducts.models import DataProductGroup, DataProduct, ReducedDatum -from tom_dataproducts.serializers import DataProductGroupSerializer, DataProductSerializer - -# TODO: see Davids comment in tom_targets/api_views.py - -# TODO: The GenericViewSet (and ModelViewset?) subclass docstrings appear on the /api// -# endpoint page. Rewrite these docstring to be useful to API consumers. - - -# class DataProductGroupViewSet(CreateModelMixin, PermissionListMixin, GenericViewSet): -# """Viewset for Target objects. By default supports CRUD operations. -# See the docs on viewsets: https://www.django-rest-framework.org/api-guide/viewsets/ -# """ -# queryset = DataProductGroup.objects.all() -# serializer_class = DataProductGroupSerializer -# # TODO: define filterset_class -# # TODO: define permission_required +from tom_dataproducts.models import DataProduct, ReducedDatum +from tom_dataproducts.serializers import DataProductSerializer class DataProductViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, GenericViewSet): - """Viewset for Target objects. By default supports CRUD operations. - See the docs on viewsets: https://www.django-rest-framework.org/api-guide/viewsets/ + """ + Viewset for DataProduct objects. Supports list, create, and delete. """ queryset = DataProduct.objects.all() serializer_class = DataProductSerializer filter_backends = (drf_filters.DjangoFilterBackend,) filterset_class = DataProductFilter permission_classes = [IsAuthenticated & DjangoObjectPermissions] - # TODO: define permission required or ensure get_queryset is doing the right thing # TODO: attempting to delete with no auth results in infinite redirect parser_classes = [MultiPartParser] diff --git a/tom_dataproducts/serializers.py b/tom_dataproducts/serializers.py index 383a230e7..98d0a75c3 100644 --- a/tom_dataproducts/serializers.py +++ b/tom_dataproducts/serializers.py @@ -1,8 +1,7 @@ from django.conf import settings -from guardian.shortcuts import get_objects_for_user from rest_framework import serializers -from tom_dataproducts.models import DataProductGroup, DataProduct +from tom_dataproducts.models import DataProductGroup, DataProduct, ReducedDatum from tom_observations.models import ObservationRecord from tom_observations.serializers import ObservationRecordFilteredPrimaryKeyRelatedField from tom_targets.models import Target @@ -15,11 +14,26 @@ class Meta: fields = ('name', 'created', 'modified') +# The ReducedDatumSerializer is not necessary until we implement the DataProductDetailAPIView and DataProductListAPIView +# class ReducedDatumSerializer(serializers.ModelSerializer): +# class Meta: +# model = ReducedDatum +# fields = ( +# 'data_product', +# 'data_type', +# 'source_name', +# 'source_location', +# 'timestamp', +# 'value' +# ) + + class DataProductSerializer(serializers.ModelSerializer): target = TargetFilteredPrimaryKeyRelatedField(queryset=Target.objects.all()) observation_record = ObservationRecordFilteredPrimaryKeyRelatedField(queryset=ObservationRecord.objects.all(), required=False) group = DataProductGroupSerializer(many=True, required=False) + # reduced_datum_set = ReducedDatumSerializer(many=True, required=False) data_product_type = serializers.CharField(allow_blank=False) class Meta: @@ -32,7 +46,7 @@ class Meta: 'data', 'extra_data', 'data_product_type', - 'group', + 'group' ) # TODO: use HyperlinkedModelSerializer @@ -44,11 +58,6 @@ class Meta: # } # } - # def create(self, validated_data): - # target = validated_data.get('target') - # obs_record = validated_data.get('observation_record') - # return super().create(validated_data) - def validate_data_product_type(self, value): for dp_type in settings.DATA_PRODUCT_TYPES.keys(): if not value or value == dp_type: @@ -57,18 +66,3 @@ def validate_data_product_type(self, value): raise serializers.ValidationError('Not a valid data_product_type. Valid data_product_types are {0}.' .format(', '.join(k for k in settings.DATA_PRODUCT_TYPES.keys()))) return value - - -# The ReducedDatumSerializer is not necessary until we implement the DataProductDetailAPIView and DataProductListAPIView -# class ReducedDatumSerializer(serializers.ModelSerializer): -# class Meta: -# model = ReducedDatum -# fields = ( -# 'target', -# 'data_product', -# 'data_type', -# 'source_name', -# 'source_location', -# 'timestamp', -# 'value' -# ) From bb75e77c351d977b4862907097838bbb234ed9d6 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 4 Aug 2020 23:16:09 +0000 Subject: [PATCH 224/424] pin requirements to exact, not minimum, versions This ensures we don't pick up a new version of a dependency before downstream dependencies are ready. I'm looking at you, django-filter. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 1de66c27f..d3bcb1179 100644 --- a/setup.py +++ b/setup.py @@ -29,11 +29,11 @@ setup_requires=['setuptools_scm', 'wheel'], install_requires=[ 'beautifulsoup4==4.9.1', - 'django>=3.0.7', # TOM Toolkit requires db math functions + 'django==3.0.7', # TOM Toolkit requires db math functions 'django-bootstrap4==1.1.1', 'django-extensions==2.2.9', 'django-filter==2.2.0', - 'django-contrib-comments>=1.9.2', # Earlier version are incompatible with Django >= 3.0 + 'django-contrib-comments==1.9.2', # Earlier version are incompatible with Django >= 3.0 'django-gravatar2==1.4.3', 'django-crispy-forms==1.9.0', 'django-guardian==2.2.0', From 32ffb22022d75b2a061f8f9ce0644d9ee4437e4d Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 4 Aug 2020 23:21:07 +0000 Subject: [PATCH 225/424] point folks to list of requirements in setup.py --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index a4af5aa10..3595830b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -e .[test] +# see setup.py install_requires for list From cb013834d55d9d519e02d86ef455c31d2865b194 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 4 Aug 2020 20:55:49 -0700 Subject: [PATCH 226/424] Added reduceddatums to DataProductSerializer --- tom_dataproducts/serializers.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tom_dataproducts/serializers.py b/tom_dataproducts/serializers.py index 98d0a75c3..db59885f7 100644 --- a/tom_dataproducts/serializers.py +++ b/tom_dataproducts/serializers.py @@ -15,17 +15,17 @@ class Meta: # The ReducedDatumSerializer is not necessary until we implement the DataProductDetailAPIView and DataProductListAPIView -# class ReducedDatumSerializer(serializers.ModelSerializer): -# class Meta: -# model = ReducedDatum -# fields = ( -# 'data_product', -# 'data_type', -# 'source_name', -# 'source_location', -# 'timestamp', -# 'value' -# ) +class ReducedDatumSerializer(serializers.ModelSerializer): + class Meta: + model = ReducedDatum + fields = ( + 'data_product', + 'data_type', + 'source_name', + 'source_location', + 'timestamp', + 'value' + ) class DataProductSerializer(serializers.ModelSerializer): @@ -33,7 +33,7 @@ class DataProductSerializer(serializers.ModelSerializer): observation_record = ObservationRecordFilteredPrimaryKeyRelatedField(queryset=ObservationRecord.objects.all(), required=False) group = DataProductGroupSerializer(many=True, required=False) - # reduced_datum_set = ReducedDatumSerializer(many=True, required=False) + reduceddatum_set = ReducedDatumSerializer(many=True, required=False) data_product_type = serializers.CharField(allow_blank=False) class Meta: @@ -46,7 +46,8 @@ class Meta: 'data', 'extra_data', 'data_product_type', - 'group' + 'group', + 'reduceddatum_set' ) # TODO: use HyperlinkedModelSerializer From 0bebed0537c9e8952e726d86ba01074c3d10290e Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 5 Aug 2020 15:27:25 -0700 Subject: [PATCH 227/424] Added tests for list and delete dataproduct --- tom_dataproducts/tests/test_api.py | 35 +++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/tom_dataproducts/tests/test_api.py b/tom_dataproducts/tests/test_api.py index f4ae92aeb..9c0d49a6f 100644 --- a/tom_dataproducts/tests/test_api.py +++ b/tom_dataproducts/tests/test_api.py @@ -1,4 +1,5 @@ from django.contrib.auth.models import User +from django.core.files.uploadedfile import SimpleUploadedFile from django.urls import reverse from guardian.shortcuts import assign_perm from rest_framework import status @@ -11,8 +12,8 @@ class TestDataProductViewset(APITestCase): def setUp(self): - user = User.objects.create(username='testuser') - self.client.force_login(user) + self.user = User.objects.create(username='testuser') + self.client.force_login(self.user) self.st = SiderealTargetFactory.create() self.obsr = ObservingRecordFactory.create(target_id=self.st.id) self.dp_data = { @@ -21,10 +22,10 @@ def setUp(self): 'data_product_type': 'photometry' } - assign_perm('tom_dataproducts.add_dataproduct', user) - assign_perm('tom_targets.add_target', user, self.st) - assign_perm('tom_targets.view_target', user, self.st) - assign_perm('tom_targets.change_target', user, self.st) + assign_perm('tom_dataproducts.add_dataproduct', self.user) + assign_perm('tom_targets.add_target', self.user, self.st) + assign_perm('tom_targets.view_target', self.user, self.st) + assign_perm('tom_targets.change_target', self.user, self.st) def test_data_product_upload_for_target(self): with open('tom_dataproducts/tests/test_data/test_lightcurve.csv', 'rb') as lightcurve_file: @@ -71,3 +72,25 @@ def test_data_product_upload_failed_processing(self): 'There was an error in processing your DataProduct', status_code=status.HTTP_500_INTERNAL_SERVER_ERROR ) + + # TODO: Test currently returns a 302, indicating redirection to login due to lack of AuthZ + def test_data_product_delete(self): + dp = DataProduct.objects.create( + product_id='testproductid', + target=self.st, + data=SimpleUploadedFile('afile.fits', b'somedata') + ) + assign_perm('tom_dataproducts.delete_dataproduct', self.user, dp) + + response = self.client.delete(reverse('api:dataproducts-detail', args=(dp.id,))) + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + + def test_data_product_list(self): + dp = DataProduct.objects.create( + product_id='testproductid', + target=self.st, + data=SimpleUploadedFile('afile.fits', b'somedata') + ) + + response = self.client.get(reverse('api:dataproducts-list')) + self.assertContains(response, dp.product_id, status_code=status.HTTP_200_OK) From 4e3188052459b967551158e82c4946d793b94d2f Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 7 Aug 2020 11:37:24 -0700 Subject: [PATCH 228/424] Fixed permissions and tests for targets, targetnames, targetextras, and dataproducts --- tom_common/urls.py | 4 +- tom_dataproducts/api_views.py | 6 +- tom_targets/api_views.py | 67 ++++++++++++++---- tom_targets/models.py | 2 +- tom_targets/serializers.py | 7 +- tom_targets/tests/test_api.py | 127 ++++++++++++++++++++++++---------- tom_targets/validators.py | 3 +- 7 files changed, 158 insertions(+), 58 deletions(-) diff --git a/tom_common/urls.py b/tom_common/urls.py index 2a1be0497..1193817d0 100644 --- a/tom_common/urls.py +++ b/tom_common/urls.py @@ -25,13 +25,15 @@ from tom_common.views import CommentDeleteView, GroupCreateView, GroupUpdateView, GroupDeleteView from rest_framework import routers -from tom_targets.api_views import TargetViewSet +from tom_targets.api_views import TargetViewSet, TargetNameViewSet, TargetExtraViewSet from tom_dataproducts.api_views import DataProductViewSet # for all applications # set up the DRF router, its router.urls included in urlpatterns below router = routers.DefaultRouter() router.register(r'targets', TargetViewSet, 'targets') +router.register(r'targetextra', TargetExtraViewSet, 'targetextra') +router.register(r'targetname', TargetNameViewSet, 'targetname') # router.register(r'dataproductgroups', DataProductGroupViewSet, 'dataproductgroups') router.register(r'dataproducts', DataProductViewSet, 'dataproducts') diff --git a/tom_dataproducts/api_views.py b/tom_dataproducts/api_views.py index 4d743142b..e031b3068 100644 --- a/tom_dataproducts/api_views.py +++ b/tom_dataproducts/api_views.py @@ -1,5 +1,6 @@ from django.conf import settings from django_filters import rest_framework as drf_filters +from guardian.mixins import PermissionListMixin from guardian.shortcuts import assign_perm, get_objects_for_user from rest_framework import status from rest_framework.mixins import CreateModelMixin, DestroyModelMixin, ListModelMixin @@ -15,7 +16,7 @@ from tom_dataproducts.serializers import DataProductSerializer -class DataProductViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, GenericViewSet): +class DataProductViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, GenericViewSet, PermissionListMixin): """ Viewset for DataProduct objects. Supports list, create, and delete. """ @@ -23,8 +24,7 @@ class DataProductViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, Ge serializer_class = DataProductSerializer filter_backends = (drf_filters.DjangoFilterBackend,) filterset_class = DataProductFilter - permission_classes = [IsAuthenticated & DjangoObjectPermissions] - # TODO: attempting to delete with no auth results in infinite redirect + permission_required = 'tom_dataproducts.view_dataproduct' parser_classes = [MultiPartParser] def create(self, request, *args, **kwargs): diff --git a/tom_targets/api_views.py b/tom_targets/api_views.py index 2c2ccd8c3..970e67f79 100644 --- a/tom_targets/api_views.py +++ b/tom_targets/api_views.py @@ -2,32 +2,73 @@ from guardian.mixins import PermissionListMixin from guardian.shortcuts import get_objects_for_user from rest_framework.mixins import DestroyModelMixin, RetrieveModelMixin -from rest_framework.permissions import DjangoObjectPermissions, IsAuthenticated from rest_framework.viewsets import GenericViewSet, ModelViewSet from tom_targets.filters import TargetFilter -from tom_targets.models import TargetName -from tom_targets.serializers import TargetSerializer, TargetNameSerializer +from tom_targets.models import TargetExtra, TargetName +from tom_targets.serializers import TargetSerializer, TargetExtraSerializer, TargetNameSerializer -# TODO: The GenericViewSet (and ModelViewSet?) subclass docstrings appear on the /api// -# endpoint page. Rewrite these docstring to be useful to API consumers. +permissions_map = { # TODO: Use the built-in DRF mapping or just switch to DRF entirely. + 'GET': 'view_target', + 'OPTIONS': [], + 'HEAD': [], + 'POST': 'add_target', + 'PATCH': 'change_target', + 'PUT': 'change_target', + 'DELETE': 'delete_target' + } -class TargetViewSet(ModelViewSet): - """Viewset for Target objects. By default supports CRUD operations. + +# Though DRF supports using django-guardian as a permission backend without explicitly using PermissionListMixin, we +# chose to use it because it removes the requirement that a user be granted both object- and model-level permissions, +# and a user that has object-level permissions is understood to also have model-level permissions. +# For whatever reason, get_queryset has to be explicitly defined, and can't be set as a property, else the API won't +# respect permissions. +# +# At present, create is restricted at all. This appears to be a limitation of django-guardian and should be revisited. +class TargetViewSet(ModelViewSet, PermissionListMixin): + """ + Viewset for Target objects. By default supports CRUD operations. See the docs on viewsets: https://www.django-rest-framework.org/api-guide/viewsets/ + + In order to create new ``TargetName`` or ``TargetExtra`` objects, a dictionary with the new values must be appended + to the ``aliases`` or ``targetextra_set`` lists. If ``id`` is included, the API will attempt to update an existing + ``TargetName`` or ``TargetExtra``. If no ``id`` is provided, the API will attempt to create new entries. + + ``TargetName`` and ``TargetExtra`` objects can only be deleted or specifically retrieved via the + ``/api/targetname/`` or ``/api/targetextra/`` endpoints. """ serializer_class = TargetSerializer filter_backends = (drf_filters.DjangoFilterBackend,) filterset_class = TargetFilter - permission_classes = [IsAuthenticated & DjangoObjectPermissions] def get_queryset(self): - return get_objects_for_user(self.request.user, 'tom_targets.view_target') + permission_required = permissions_map.get(self.request.method) + return get_objects_for_user(self.request.user, f'tom_targets.{permission_required}') -class TargetNamesViewSet(DestroyModelMixin, PermissionListMixin, RetrieveModelMixin, GenericViewSet): - queryset = TargetName.objects.all() +class TargetNameViewSet(DestroyModelMixin, PermissionListMixin, RetrieveModelMixin, GenericViewSet): + """ + Viewset for TargetName objects. Only ``GET`` and ``DELETE`` operations are permitted. + """ serializer_class = TargetNameSerializer - permission_classes = [DjangoObjectPermissions] - permission_required = 'tom_targets.change_target' + + def get_queryset(self): + permission_required = permissions_map.get(self.request.method) + return TargetName.objects.filter( + target__in=get_objects_for_user(self.request.user, f'tom_targets.{permission_required}') + ) + + +class TargetExtraViewSet(DestroyModelMixin, PermissionListMixin, RetrieveModelMixin, GenericViewSet): + """ + Viewset for TargetExtra objects. Only ``GET`` and ``DELETE`` operations are permitted. + """ + serializer_class = TargetExtraSerializer + + def get_queryset(self): + permission_required = permissions_map.get(self.request.method) + return TargetExtra.objects.filter( + target__in=get_objects_for_user(self.request.user, f'tom_targets.{permission_required}') + ) diff --git a/tom_targets/models.py b/tom_targets/models.py index f84d45e71..dd98a98a8 100644 --- a/tom_targets/models.py +++ b/tom_targets/models.py @@ -387,7 +387,7 @@ def validate_unique(self, *args, **kwargs): """ super().validate_unique(*args, **kwargs) if self.name == self.target.name: - raise ValidationError(f'''Alias {self.name} has a conflict with the primary name of the target + raise ValidationError(f'''Alias {self.name} has a conflict with the primary name of the target {self.target.name} (id={self.target.id})''') diff --git a/tom_targets/serializers.py b/tom_targets/serializers.py index 3e1446026..8528c946d 100644 --- a/tom_targets/serializers.py +++ b/tom_targets/serializers.py @@ -35,13 +35,12 @@ class Meta: # TODO: We should investigate if this validator logic can be reused in the forms to reduce code duplication. # TODO: Try to put validators in settings to allow user changes validators = [RequiredFieldsTogetherValidator('type', 'SIDEREAL', 'ra', 'dec'), - RequiredFieldsTogetherValidator('type', 'NON_SIDEREAL', 'epoch_of_elements', 'inclination', + RequiredFieldsTogetherValidator('type', 'NON_SIDEREAL', 'epoch_of_elements', 'inclination', 'lng_asc_node', 'arg_of_perihelion', 'eccentricity'), RequiredFieldsTogetherValidator('scheme', 'MPC_COMET', 'perihdist', 'epoch_of_perihelion'), RequiredFieldsTogetherValidator('scheme', 'MPC_MINOR_PLANET', 'mean_anomaly', 'semimajor_axis'), RequiredFieldsTogetherValidator('scheme', 'JPL_MAJOR_PLANET', 'mean_daily_motion', 'mean_anomaly', - 'semimajor_axis') - ] + 'semimajor_axis')] def create(self, validated_data): """DRF requires explicitly handling writeable nested serializers, @@ -65,7 +64,7 @@ def create(self, validated_data): def update(self, instance, validated_data): """ - For TargetExtra and TargetName objects, if the ID is present, it will update the corresponding row. If the ID is + For TargetExtra and TargetName objects, if the ID is present, it will update the corresponding row. If the ID is not present, it will attempt to create a new TargetExtra or TargetName associated with this Target. """ aliases = validated_data.pop('aliases', []) diff --git a/tom_targets/tests/test_api.py b/tom_targets/tests/test_api.py index 23379abad..515a277b5 100644 --- a/tom_targets/tests/test_api.py +++ b/tom_targets/tests/test_api.py @@ -1,40 +1,45 @@ from django.contrib.auth.models import User from django.urls import reverse -from guardian.shortcuts import assign_perm +from guardian.shortcuts import assign_perm, get_objects_for_user from rest_framework import status from rest_framework.test import APITestCase from tom_targets.tests.factories import SiderealTargetFactory, NonSiderealTargetFactory -from tom_targets.models import Target +from tom_targets.tests.factories import TargetExtraFactory, TargetNameFactory +from tom_targets.models import Target, TargetExtra, TargetName class TestTargetViewset(APITestCase): def setUp(self): - self.user = User.objects.create(username='testuser') - self.user2 = User.objects.create(username='testuser2') - # self.client.force_login(user) + # Create test user with permissions + user = User.objects.create(username='testuser') self.st = SiderealTargetFactory.create() + self.st2 = SiderealTargetFactory.create() self.nst = NonSiderealTargetFactory.create() - assign_perm('tom_targets.view_target', self.user, self.st) - assign_perm('tom_targets.add_target', self.user, self.st) - assign_perm('tom_targets.change_target', self.user, self.st) - assign_perm('tom_targets.delete_target', self.user, self.st) - assign_perm('tom_targets.view_target', self.user, self.nst) + assign_perm('tom_targets.view_target', user, self.st) + assign_perm('tom_targets.view_target', user, self.nst) + assign_perm('tom_targets.add_target', user) + assign_perm('tom_targets.change_target', user, self.st) + assign_perm('tom_targets.delete_target', user, self.st) + + # Create test user with subset of permissions + self.user2 = User.objects.create(username='testuser2') assign_perm('tom_targets.view_target', self.user2, self.st) + # Login with privileged user + self.client.force_login(user) + def test_target_list(self): - self.client.force_login(self.user) response = self.client.get(reverse('api:targets-list')) self.assertEqual(response.json()['count'], 2) - # Ensure that a user without view_target permission on all targets can only retrieve the subset of targets for + # Ensure that a user without view_target permission on all targets can only retrieve the subset of targets for # which they have permission self.client.force_login(self.user2) response = self.client.get(reverse('api:targets-list')) self.assertEqual(response.json()['count'], 1) def test_target_detail(self): - self.client.force_login(self.user) response = self.client.get(reverse('api:targets-detail', args=(self.st.id,))) self.assertEqual(response.json()['name'], self.st.name) @@ -57,16 +62,20 @@ def test_target_create(self): {'name': 'alternative name'} ] } - self.client.force_login(self.user) response = self.client.post(reverse('api:targets-list'), data=target_data) self.assertEqual(response.status_code, status.HTTP_201_CREATED) self.assertEqual(response.json()['name'], target_data['name']) self.assertEqual(response.json()['aliases'][0]['name'], target_data['aliases'][0]['name']) + # TODO: For whatever reason, in django-guardian, authenticated users have permission to create objects, + # regardless of their row-level permissions. This should be addressed eventually--however, we don't provide a + # way for PIs to restrict create/update ability, simply target access, so this can be ignored at present. + # # self.client.force_login(self.user2) # target_data['name'] = 'test_target_create_bad_permissions' + # target_data['aliases'] = [] # response = self.client.post(reverse('api:targets-list'), data=target_data) - # self.assertEqual(response.status_code, status.HTTP_302_REDIRECT) + # self.assertEqual(response.status_code, status.HTTP_302_FOUND) def test_target_create_sidereal_missing_parameters(self): target_data = { @@ -81,16 +90,18 @@ def test_target_create_sidereal_missing_parameters(self): ] } response = self.client.post(reverse('api:targets-list'), data=target_data) - self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertEqual(response.json()['name'], target_data['name']) - self.assertEqual(response.json()['aliases'][0]['name'], target_data['aliases'][0]['name']) + self.assertContains(response, + 'The following fields are required for SIDEREAL targets: [\'dec\']', + status_code=status.HTTP_400_BAD_REQUEST) def test_target_create_non_sidereal_missing_parameters(self): target_data = { 'name': 'test_target_name_wtf', - 'type': Target.SIDEREAL, - 'ra': 123.456, - 'dec': -32.1, + 'type': Target.NON_SIDEREAL, + 'epoch_of_elements': 2000, + 'inclination': '0.0005', + 'lng_asc_node': '0.12345', + 'arg_of_perihelion': '57', 'targetextra_set': [ {'key': 'foo', 'value': 5} ], @@ -99,36 +110,82 @@ def test_target_create_non_sidereal_missing_parameters(self): ] } response = self.client.post(reverse('api:targets-list'), data=target_data) - self.assertEqual(response.status_code, status.HTTP_201_CREATED) - self.assertEqual(response.json()['name'], target_data['name']) - self.assertEqual(response.json()['aliases'][0]['name'], target_data['aliases'][0]['name']) + self.assertContains(response, + 'The following fields are required for NON_SIDEREAL targets: [\'eccentricity\']', + status_code=status.HTTP_400_BAD_REQUEST) def test_target_update(self): - self.client.force_login(self.user) updates = {'ra': 123.456} - response = self.client.patch(reverse('api:targets-detail', args=(self.st.id,)), data=updates, follow=True) - print(response.content) + response = self.client.patch(reverse('api:targets-detail', args=(self.st.id,)), data=updates) self.assertEqual(response.status_code, status.HTTP_200_OK) self.st.refresh_from_db() self.assertEqual(self.st.ra, updates['ra']) - # self.client.force_login(self.user2) - # updates = {'ra': 654.321} - # response = self.client.patch(reverse('api:targets-detail', args=(self.st.id,)), data=updates) - # self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + self.client.force_login(self.user2) + updates = {'ra': 654.321} + response = self.client.patch(reverse('api:targets-detail', args=(self.st.id,)), data=updates) + self.st.refresh_from_db() + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) def test_target_delete(self): response = self.client.delete(reverse('api:targets-detail', args=(self.st.id,))) - print(response.content) self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) self.assertFalse(Target.objects.filter(pk=self.st.id).exists()) + self.client.force_login(self.user2) + response = self.client.delete(reverse('api:targets-detail', args=(self.nst.id,))) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + -class TestTargetDetailAPIView(APITestCase): +class TestTargetNameViewset(APITestCase): def setUp(self): user = User.objects.create(username='testuser') + self.st = SiderealTargetFactory.create() + self.alias = TargetNameFactory.create(target=self.st) + assign_perm('tom_targets.view_target', user, self.st) + assign_perm('tom_targets.delete_target', user, self.st) + + self.user2 = User.objects.create(username='testuser2') + self.client.force_login(user) + + def test_targetname_detail(self): + response = self.client.get(reverse('api:targetname-detail', args=(self.alias.id,))) + self.assertEqual(response.json()['name'], self.alias.name) + + # Ensure that a user without view_target permission cannot access the target + self.client.force_login(self.user2) + response = self.client.get(reverse('api:targetname-detail', args=(self.alias.id,))) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + def test_targetname_delete(self): + response = self.client.delete(reverse('api:targetname-detail', args=(self.alias.id,))) + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + self.assertFalse(TargetName.objects.filter(pk=self.alias.id).exists()) + + +class TestTargetExtraViewset(APITestCase): + def setUp(self): + user = User.objects.create(username='testuser') self.st = SiderealTargetFactory.create() - self.nst = NonSiderealTargetFactory.create() + self.extra = TargetExtraFactory.create(target=self.st) assign_perm('tom_targets.view_target', user, self.st) - assign_perm('tom_targets.view_target', user, self.nst) \ No newline at end of file + assign_perm('tom_targets.delete_target', user, self.st) + + self.user2 = User.objects.create(username='testuser2') + + self.client.force_login(user) + + def test_targetextra_detail(self): + response = self.client.get(reverse('api:targetextra-detail', args=(self.extra.id,))) + self.assertEqual(response.json()['id'], self.extra.id) + + # Ensure that a user without view_target permission cannot access the target + self.client.force_login(self.user2) + response = self.client.get(reverse('api:targetextra-detail', args=(self.extra.id,))) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + def test_targetextra_delete(self): + response = self.client.delete(reverse('api:targetextra-detail', args=(self.extra.id,))) + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + self.assertFalse(TargetExtra.objects.filter(pk=self.extra.id).exists()) diff --git a/tom_targets/validators.py b/tom_targets/validators.py index b785bed76..e0bf466e7 100644 --- a/tom_targets/validators.py +++ b/tom_targets/validators.py @@ -1,5 +1,6 @@ from rest_framework.serializers import ValidationError + class RequiredFieldsTogetherValidator(object): def __init__(self, type_name, type_value, *args): @@ -17,6 +18,6 @@ def __call__(self, attrs): for field in self.required_fields: if not values.get(field): missing_fields.append(field) - + if missing_fields: raise ValidationError(f'The following fields are required for {self.type_value} targets: {missing_fields}') From b86c87aadb913ae78062fc434954e742987440f9 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 7 Aug 2020 12:03:35 -0700 Subject: [PATCH 229/424] Removed superfluous code blocks, fixed codacy issues, added autodoc --- docs/api/tom_dataproducts/api_views.rst | 5 +++++ docs/api/tom_dataproducts/index.rst | 3 ++- docs/api/tom_targets/api_views.rst | 5 +++++ docs/api/tom_targets/index.rst | 1 + tom_common/urls.py | 4 +--- tom_dataproducts/api_views.py | 1 - tom_dataproducts/serializers.py | 10 ---------- tom_dataproducts/tests/test_api.py | 1 - tom_targets/api_views.py | 2 +- tom_targets/tests/test_api.py | 2 +- tom_targets/views.py | 1 - 11 files changed, 16 insertions(+), 19 deletions(-) create mode 100644 docs/api/tom_dataproducts/api_views.rst create mode 100644 docs/api/tom_targets/api_views.rst diff --git a/docs/api/tom_dataproducts/api_views.rst b/docs/api/tom_dataproducts/api_views.rst new file mode 100644 index 000000000..8241a90f1 --- /dev/null +++ b/docs/api/tom_dataproducts/api_views.rst @@ -0,0 +1,5 @@ +API Views +========= + +.. automodule:: tom_dataproducts.api_views + :members: \ No newline at end of file diff --git a/docs/api/tom_dataproducts/index.rst b/docs/api/tom_dataproducts/index.rst index fd1774445..ef1f3490a 100644 --- a/docs/api/tom_dataproducts/index.rst +++ b/docs/api/tom_dataproducts/index.rst @@ -8,4 +8,5 @@ Data Products models templatetags utils - views \ No newline at end of file + views + api_views \ No newline at end of file diff --git a/docs/api/tom_targets/api_views.rst b/docs/api/tom_targets/api_views.rst new file mode 100644 index 000000000..26b9632cb --- /dev/null +++ b/docs/api/tom_targets/api_views.rst @@ -0,0 +1,5 @@ +API Views +========= + +.. automodule:: tom_targets.api_views + :members: \ No newline at end of file diff --git a/docs/api/tom_targets/index.rst b/docs/api/tom_targets/index.rst index daa72898c..869d2c6be 100644 --- a/docs/api/tom_targets/index.rst +++ b/docs/api/tom_targets/index.rst @@ -9,3 +9,4 @@ Targets templatetags utils views + api_views \ No newline at end of file diff --git a/tom_common/urls.py b/tom_common/urls.py index 1193817d0..f9c183fc1 100644 --- a/tom_common/urls.py +++ b/tom_common/urls.py @@ -28,13 +28,11 @@ from tom_targets.api_views import TargetViewSet, TargetNameViewSet, TargetExtraViewSet from tom_dataproducts.api_views import DataProductViewSet -# for all applications -# set up the DRF router, its router.urls included in urlpatterns below +# For all applications, set up the DRF router, its router.urls is included in urlpatterns below router = routers.DefaultRouter() router.register(r'targets', TargetViewSet, 'targets') router.register(r'targetextra', TargetExtraViewSet, 'targetextra') router.register(r'targetname', TargetNameViewSet, 'targetname') -# router.register(r'dataproductgroups', DataProductGroupViewSet, 'dataproductgroups') router.register(r'dataproducts', DataProductViewSet, 'dataproducts') diff --git a/tom_dataproducts/api_views.py b/tom_dataproducts/api_views.py index e031b3068..15385193b 100644 --- a/tom_dataproducts/api_views.py +++ b/tom_dataproducts/api_views.py @@ -5,7 +5,6 @@ from rest_framework import status from rest_framework.mixins import CreateModelMixin, DestroyModelMixin, ListModelMixin from rest_framework.parsers import MultiPartParser -from rest_framework.permissions import DjangoObjectPermissions, IsAuthenticated from rest_framework.response import Response from rest_framework.viewsets import GenericViewSet diff --git a/tom_dataproducts/serializers.py b/tom_dataproducts/serializers.py index db59885f7..896443765 100644 --- a/tom_dataproducts/serializers.py +++ b/tom_dataproducts/serializers.py @@ -14,7 +14,6 @@ class Meta: fields = ('name', 'created', 'modified') -# The ReducedDatumSerializer is not necessary until we implement the DataProductDetailAPIView and DataProductListAPIView class ReducedDatumSerializer(serializers.ModelSerializer): class Meta: model = ReducedDatum @@ -50,15 +49,6 @@ class Meta: 'reduceddatum_set' ) - # TODO: use HyperlinkedModelSerializer - # for the HyperlinkedModelSerializer use something like this - # extra_kwargs = { - # "url": { - # "view_name": ":targets:detail", - # "lookup_field": "pk", - # } - # } - def validate_data_product_type(self, value): for dp_type in settings.DATA_PRODUCT_TYPES.keys(): if not value or value == dp_type: diff --git a/tom_dataproducts/tests/test_api.py b/tom_dataproducts/tests/test_api.py index 9c0d49a6f..9cc09690c 100644 --- a/tom_dataproducts/tests/test_api.py +++ b/tom_dataproducts/tests/test_api.py @@ -73,7 +73,6 @@ def test_data_product_upload_failed_processing(self): status_code=status.HTTP_500_INTERNAL_SERVER_ERROR ) - # TODO: Test currently returns a 302, indicating redirection to login due to lack of AuthZ def test_data_product_delete(self): dp = DataProduct.objects.create( product_id='testproductid', diff --git a/tom_targets/api_views.py b/tom_targets/api_views.py index 970e67f79..14c90ad16 100644 --- a/tom_targets/api_views.py +++ b/tom_targets/api_views.py @@ -26,7 +26,7 @@ # For whatever reason, get_queryset has to be explicitly defined, and can't be set as a property, else the API won't # respect permissions. # -# At present, create is restricted at all. This appears to be a limitation of django-guardian and should be revisited. +# At present, create is not restricted at all. This seems to be a limitation of django-guardian and should be revisited. class TargetViewSet(ModelViewSet, PermissionListMixin): """ Viewset for Target objects. By default supports CRUD operations. diff --git a/tom_targets/tests/test_api.py b/tom_targets/tests/test_api.py index 515a277b5..3ac6924f1 100644 --- a/tom_targets/tests/test_api.py +++ b/tom_targets/tests/test_api.py @@ -1,6 +1,6 @@ from django.contrib.auth.models import User from django.urls import reverse -from guardian.shortcuts import assign_perm, get_objects_for_user +from guardian.shortcuts import assign_perm from rest_framework import status from rest_framework.test import APITestCase diff --git a/tom_targets/views.py b/tom_targets/views.py index 4c0c97c14..841e9c603 100644 --- a/tom_targets/views.py +++ b/tom_targets/views.py @@ -21,7 +21,6 @@ from django.views.generic.list import ListView from django.views.generic import TemplateView, View from django_filters.views import FilterView -from rest_framework import viewsets from guardian.mixins import PermissionListMixin from guardian.shortcuts import get_objects_for_user, get_groups_with_perms, assign_perm From e614ec914d651097ba27cdb110d433fde815eb55 Mon Sep 17 00:00:00 2001 From: David Collom Date: Sun, 9 Aug 2020 16:55:10 -0700 Subject: [PATCH 230/424] Added groups to create/update --- tom_common/serializers.py | 11 +++++ tom_observations/facility.py | 2 - tom_targets/serializers.py | 85 ++++++++++++++++++++------------ tom_targets/tests/test_api.py | 93 ++++++++++++++++++++++++++++++----- 4 files changed, 145 insertions(+), 46 deletions(-) create mode 100644 tom_common/serializers.py diff --git a/tom_common/serializers.py b/tom_common/serializers.py new file mode 100644 index 000000000..fca7ad909 --- /dev/null +++ b/tom_common/serializers.py @@ -0,0 +1,11 @@ +from django.contrib.auth.models import Group +from rest_framework import serializers + + +class GroupSerializer(serializers.ModelSerializer): + id = serializers.IntegerField(required=False) + name = serializers.CharField(required=False) + + class Meta: + model = Group + fields = ('id', 'name',) diff --git a/tom_observations/facility.py b/tom_observations/facility.py index b0f39eed2..d32ece7a3 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -422,7 +422,5 @@ class BaseManualObservationFacility(BaseObservationFacility): ``TOM_FACILITY_CLASSES`` in your ``settings.py``. This specific class is intended for use with classical-style manual facilities. - - TODO: Add an implementation example. """ name = 'BaseManual' # rename in concrete subclasses diff --git a/tom_targets/serializers.py b/tom_targets/serializers.py index 8528c946d..8f5df63bd 100644 --- a/tom_targets/serializers.py +++ b/tom_targets/serializers.py @@ -1,6 +1,8 @@ -from guardian.shortcuts import get_objects_for_user +from django.contrib.auth.models import Group +from guardian.shortcuts import assign_perm, get_groups_with_perms, get_objects_for_user, remove_perm from rest_framework import serializers +from tom_common.serializers import GroupSerializer from tom_targets.models import Target, TargetExtra, TargetName from tom_targets.validators import RequiredFieldsTogetherValidator @@ -28,6 +30,8 @@ class TargetSerializer(serializers.ModelSerializer): """ targetextra_set = TargetExtraSerializer(many=True) aliases = TargetNameSerializer(many=True) + groups = GroupSerializer(many=True, required=False) # TODO: return groups in detail and list + # groups = serializers.SerializerMethodField() class Meta: model = Target @@ -42,18 +46,40 @@ class Meta: RequiredFieldsTogetherValidator('scheme', 'JPL_MAJOR_PLANET', 'mean_daily_motion', 'mean_anomaly', 'semimajor_axis')] + def get_groups(self, obj): + print('get groups') + print(obj.id) + return get_groups_with_perms(obj) + def create(self, validated_data): """DRF requires explicitly handling writeable nested serializers, here we pop the alias/tag data and save it using their respective serializers """ + aliases = validated_data.pop('aliases', []) targetextras = validated_data.pop('targetextra_set', []) + groups = validated_data.pop('groups', []) target = Target.objects.create(**validated_data) + # Save groups for this target + group_serializer = GroupSerializer(data=groups, many=True) + if group_serializer.is_valid(): + for group in groups: + group_instance = Group.objects.get(pk=group['id']) + assign_perm('tom_targets.view_target', group_instance, target) + assign_perm('tom_targets.change_target', group_instance, target) + assign_perm('tom_targets.delete_target', group_instance, target) + tns = TargetNameSerializer(data=aliases, many=True) if tns.is_valid(): + for alias in aliases: + if alias['name'] == target.name: + target.delete() + alias_value = alias['name'] + raise serializers.ValidationError( + f'Alias \'{alias_value}\' conflicts with Target name \'{target.name}\'.') tns.save(target=target) tes = TargetExtraSerializer(data=targetextras, many=True) @@ -69,42 +95,30 @@ def update(self, instance, validated_data): """ aliases = validated_data.pop('aliases', []) targetextras = validated_data.pop('targetextra_set', []) + groups = validated_data.pop('groups', []) - instance.name = validated_data.get('name', instance.name) - instance.type = validated_data.get('type', instance.type) - instance.ra = validated_data.get('ra', instance.ra) - instance.dec = validated_data.get('dec', instance.dec) - instance.epoch = validated_data.get('epoch', instance.epoch) - instance.parallax = validated_data.get('parallax', instance.parallax) - instance.pm_ra = validated_data.get('pm_ra', instance.pm_ra) - instance.pm_dec = validated_data.get('pm_dec', instance.pm_dec) - instance.galactic_lng = validated_data.get('galactic_lng', instance.galactic_lng) - instance.galactic_lat = validated_data.get('galactic_lat', instance.galactic_lat) - instance.distance = validated_data.get('distance', instance.distance) - instance.distance_err = validated_data.get('distance_err', instance.distance_err) - instance.scheme = validated_data.get('scheme', instance.scheme) - instance.epoch_of_elements = validated_data.get('epoch_of_elements', instance.epoch_of_elements) - instance.mean_anomaly = validated_data.get('mean_anomaly', instance.mean_anomaly) - instance.arg_of_perihelion = validated_data.get('arg_of_perihelion', instance.arg_of_perihelion) - instance.eccentricity = validated_data.get('eccentricity', instance.eccentricity) - instance.lng_asc_node = validated_data.get('lng_asc_node', instance.lng_asc_node) - instance.inclination = validated_data.get('inclination', instance.inclination) - instance.mean_daily_motion = validated_data.get('mean_daily_motion', instance.mean_daily_motion) - instance.semimajor_axis = validated_data.get('semimajor_axis', instance.semimajor_axis) - instance.epoch_of_perihelion = validated_data.get('epoch_of_perihelion', instance.epoch_of_perihelion) - instance.ephemeris_period = validated_data.get('ephemeris_period', instance.ephemeris_period) - instance.ephemeris_period_err = validated_data.get('ephemeris_period_err', instance.ephemeris_period_err) - instance.ephemeris_epoch = validated_data.get('ephemeris_epoch', instance.ephemeris_epoch) - instance.ephemeris_epoch_err = validated_data.get('ephemeris_epoch_err', instance.ephemeris_epoch_err) - instance.perihdist = validated_data.get('perihdist', instance.perihdist) - instance.save() + # Save groups for this target + group_serializer = GroupSerializer(data=groups, many=True) + if group_serializer.is_valid(): + for group in groups: + group_instance = Group.objects.get(pk=group['id']) + assign_perm('tom_targets.view_target', group_instance, instance) + assign_perm('tom_targets.change_target', group_instance, instance) + assign_perm('tom_targets.delete_target', group_instance, instance) # TODO: add tests - # TODO: updating an existing TargetName with the same value results in an integrity error - # TODO: validate_unique is not called on TargetName for alias_data in aliases: alias = dict(alias_data) + if alias['name'] == instance.name: # Alias shouldn't conflict with target name + alias_name = alias['name'] + raise serializers.ValidationError( + f'Alias \'{alias_name}\' conflicts with Target name \'{instance.name}\'.') if alias.get('id'): tn_instance = TargetName.objects.get(pk=alias['id']) + if tn_instance.target != instance: # Alias should correspond with target to be updated + raise serializers.ValidationError(f'''TargetName identified by id \'{tn_instance.id}\' is not an + alias of Target \'{instance.name}\'''') + elif alias['name'] == tn_instance.name: + break # Don't update if value doesn't change, because it will throw an error tns = TargetNameSerializer(tn_instance, data=alias_data) else: tns = TargetNameSerializer(data=alias_data) @@ -121,6 +135,15 @@ def update(self, instance, validated_data): if tes.is_valid(): tes.save(target=instance) + fields_to_validate = ['name', 'type', 'ra', 'dec', 'epoch', 'parallax', 'pm_ra', 'pm_dec', 'galactic_lng', + 'galactic_lat', 'distance', 'distance_err', 'scheme', 'epoch_of_elements', + 'mean_anomaly', 'arg_of_perihelion', 'eccentricity', 'lng_asc_node', 'inclination', + 'mean_daily_motion', 'semimajor_axis', 'epoch_of_perihelion', 'ephemeris_period', + 'ephemeris_period_err', 'ephemeris_epoch', 'ephemeris_epoch_err', 'perihdist'] + for field in fields_to_validate: + setattr(instance, field, validated_data.get(field, getattr(instance, field))) + instance.save() + return instance diff --git a/tom_targets/tests/test_api.py b/tom_targets/tests/test_api.py index 3ac6924f1..7c2261319 100644 --- a/tom_targets/tests/test_api.py +++ b/tom_targets/tests/test_api.py @@ -1,6 +1,8 @@ -from django.contrib.auth.models import User +import copy + +from django.contrib.auth.models import User, Group from django.urls import reverse -from guardian.shortcuts import assign_perm +from guardian.shortcuts import assign_perm, get_objects_for_user from rest_framework import status from rest_framework.test import APITestCase @@ -12,22 +14,23 @@ class TestTargetViewset(APITestCase): def setUp(self): # Create test user with permissions - user = User.objects.create(username='testuser') - self.st = SiderealTargetFactory.create() + self.user = User.objects.create(username='testuser') + self.st = SiderealTargetFactory.create(name='test target', targetextra_set=None, aliases=None) self.st2 = SiderealTargetFactory.create() self.nst = NonSiderealTargetFactory.create() - assign_perm('tom_targets.view_target', user, self.st) - assign_perm('tom_targets.view_target', user, self.nst) - assign_perm('tom_targets.add_target', user) - assign_perm('tom_targets.change_target', user, self.st) - assign_perm('tom_targets.delete_target', user, self.st) + assign_perm('tom_targets.view_target', self.user, self.st) + assign_perm('tom_targets.view_target', self.user, self.nst) + assign_perm('tom_targets.add_target', self.user) + assign_perm('tom_targets.change_target', self.user, self.st) + assign_perm('tom_targets.change_target', self.user, self.nst) + assign_perm('tom_targets.delete_target', self.user, self.st) # Create test user with subset of permissions self.user2 = User.objects.create(username='testuser2') assign_perm('tom_targets.view_target', self.user2, self.st) # Login with privileged user - self.client.force_login(user) + self.client.force_login(self.user) def test_target_list(self): response = self.client.get(reverse('api:targets-list')) @@ -50,11 +53,19 @@ def test_target_detail(self): self.assertEqual(response.json()['detail'], 'Not found.') def test_target_create(self): + collaborator = User.objects.create(username='test collaborator') + group = Group.objects.create(name='bourgeoisie') + group.user_set.add(self.user) + group.user_set.add(collaborator) + target_data = { 'name': 'test_target_name_wtf', 'type': Target.SIDEREAL, 'ra': 123.456, 'dec': -32.1, + 'groups': [ + {'id': group.id} + ], 'targetextra_set': [ {'key': 'foo', 'value': 5} ], @@ -66,6 +77,8 @@ def test_target_create(self): self.assertEqual(response.status_code, status.HTTP_201_CREATED) self.assertEqual(response.json()['name'], target_data['name']) self.assertEqual(response.json()['aliases'][0]['name'], target_data['aliases'][0]['name']) + self.assertEqual(get_objects_for_user(collaborator, 'tom_targets.view_target').first().name, + target_data['name']) # Test that group permissions are respected # TODO: For whatever reason, in django-guardian, authenticated users have permission to create objects, # regardless of their row-level permissions. This should be addressed eventually--however, we don't provide a @@ -77,6 +90,15 @@ def test_target_create(self): # response = self.client.post(reverse('api:targets-list'), data=target_data) # self.assertEqual(response.status_code, status.HTTP_302_FOUND) + # Ensure targets can't be created with duplicate aliases + invalid_target_data = copy.deepcopy(target_data) + invalid_target_data['name'] = 'invalid_name' + invalid_target_data['aliases'][0]['name'] = 'invalid_name' + response = self.client.post(reverse('api:targets-list'), data=invalid_target_data) + self.assertContains(response, + 'Alias \'invalid_name\' conflicts with Target name \'invalid_name\'.', + status_code=status.HTTP_400_BAD_REQUEST) + def test_target_create_sidereal_missing_parameters(self): target_data = { 'name': 'test_target_name_wtf', @@ -115,11 +137,18 @@ def test_target_create_non_sidereal_missing_parameters(self): status_code=status.HTTP_400_BAD_REQUEST) def test_target_update(self): - updates = {'ra': 123.456} + collaborator = User.objects.create(username='test collaborator') + group = Group.objects.create(name='bourgeoisie') + group.user_set.add(self.user) + group.user_set.add(collaborator) + + updates = {'ra': 123.456, 'groups': [{'id': group.id}]} response = self.client.patch(reverse('api:targets-detail', args=(self.st.id,)), data=updates) self.assertEqual(response.status_code, status.HTTP_200_OK) self.st.refresh_from_db() self.assertEqual(self.st.ra, updates['ra']) + self.assertEqual(get_objects_for_user(collaborator, 'tom_targets.view_target').first().name, + self.st.name) # Test that group permissions are respected self.client.force_login(self.user2) updates = {'ra': 654.321} @@ -127,6 +156,44 @@ def test_target_update(self): self.st.refresh_from_db() self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + def test_targetname_update(self): + # Test both create new alias and update alias + alias = TargetNameFactory.create(name='alias', target=self.st) + updates = { + 'aliases': [ + {'id': alias.id, 'name': 'update alias'}, + {'name': 'create alias'} + ] + } + response = self.client.patch(reverse('api:targets-detail', args=(self.st.id,)), data=updates) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.st.refresh_from_db() + alias.refresh_from_db() + self.assertEqual(self.st.aliases.count(), 2) + self.assertEqual(alias.name, 'update alias') + + # Ensure a user can't inadvertently update a TargetName for a different Target + invalid_data = { + 'aliases': [ + {'id': alias.id, 'name': 'alias for wrong target'} + ] + } + response = self.client.patch(reverse('api:targets-detail', args=(self.nst.id,)), data=invalid_data) + self.assertContains(response, + f'TargetName identified by id \'{alias.id}\' is not an', + status_code=status.HTTP_400_BAD_REQUEST) + + # Ensure proper exception handling when updating or creating with the same name + updates = { + 'aliases': [ + {'name': self.st.name} + ] + } + response = self.client.patch(reverse('api:targets-detail', args=(self.st.id,)), data=updates) + self.assertContains(response, + f'Alias \'{self.st.name}\' conflicts with Target name \'{self.st.name}\'', + status_code=status.HTTP_400_BAD_REQUEST) + def test_target_delete(self): response = self.client.delete(reverse('api:targets-detail', args=(self.st.id,))) self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) @@ -153,7 +220,7 @@ def test_targetname_detail(self): response = self.client.get(reverse('api:targetname-detail', args=(self.alias.id,))) self.assertEqual(response.json()['name'], self.alias.name) - # Ensure that a user without view_target permission cannot access the target + # Ensure that a user without view_target permission cannot access the target name self.client.force_login(self.user2) response = self.client.get(reverse('api:targetname-detail', args=(self.alias.id,))) self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) @@ -180,7 +247,7 @@ def test_targetextra_detail(self): response = self.client.get(reverse('api:targetextra-detail', args=(self.extra.id,))) self.assertEqual(response.json()['id'], self.extra.id) - # Ensure that a user without view_target permission cannot access the target + # Ensure that a user without view_target permission cannot access the target extra self.client.force_login(self.user2) response = self.client.get(reverse('api:targetextra-detail', args=(self.extra.id,))) self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) From 53fe639483caa7d45c8392e9c7625c7ca1e3f7b8 Mon Sep 17 00:00:00 2001 From: David Collom Date: Sun, 9 Aug 2020 17:04:31 -0700 Subject: [PATCH 231/424] Added group assignment to data product endpoints --- tom_dataproducts/serializers.py | 45 +++++++++++++++++++++++++++++++-- tom_targets/serializers.py | 4 +-- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/tom_dataproducts/serializers.py b/tom_dataproducts/serializers.py index 896443765..44e4b1702 100644 --- a/tom_dataproducts/serializers.py +++ b/tom_dataproducts/serializers.py @@ -1,6 +1,9 @@ from django.conf import settings +from django.contrib.auth.models import Group +from guardian.shortcuts import assign_perm from rest_framework import serializers +from tom_common.serializers import GroupSerializer from tom_dataproducts.models import DataProductGroup, DataProduct, ReducedDatum from tom_observations.models import ObservationRecord from tom_observations.serializers import ObservationRecordFilteredPrimaryKeyRelatedField @@ -31,7 +34,8 @@ class DataProductSerializer(serializers.ModelSerializer): target = TargetFilteredPrimaryKeyRelatedField(queryset=Target.objects.all()) observation_record = ObservationRecordFilteredPrimaryKeyRelatedField(queryset=ObservationRecord.objects.all(), required=False) - group = DataProductGroupSerializer(many=True, required=False) + groups = GroupSerializer(many=True, required=False) + data_product_group = DataProductGroupSerializer(many=True, required=False) reduceddatum_set = ReducedDatumSerializer(many=True, required=False) data_product_type = serializers.CharField(allow_blank=False) @@ -45,10 +49,47 @@ class Meta: 'data', 'extra_data', 'data_product_type', - 'group', + 'groups', + 'data_product_group', 'reduceddatum_set' ) + def create(self, validated_data): + """DRF requires explicitly handling writeable nested serializers, + here we pop the groups data and save it using its serializer. + """ + + groups = validated_data.pop('groups', []) + + dp = DataProduct.objects.create(**validated_data) + + # Save groups for this target + group_serializer = GroupSerializer(data=groups, many=True) + if group_serializer.is_valid() and not settings.TARGET_PERMISSIONS_ONLY: + for group in groups: + group_instance = Group.objects.get(pk=group['id']) + assign_perm('tom_dataproducts.view_dataproduct', group_instance, dp) + assign_perm('tom_dataproducts.change_dataproduct', group_instance, dp) + assign_perm('tom_dataproducts.delete_dataproduct', group_instance, dp) + + return dp + + def update(self, instance, validated_data): + groups = validated_data.pop('groups', []) + + super().save(instance, validated_data) + + # Save groups for this dataproduct + group_serializer = GroupSerializer(data=groups, many=True) + if group_serializer.is_valid() and not settings.TARGET_PERMISSIONS_ONLY: + for group in groups: + group_instance = Group.objects.get(pk=group['id']) + assign_perm('tom_dataproducts.view_dataproduct', group_instance, instance) + assign_perm('tom_dataproducts.change_dataproduct', group_instance, instance) + assign_perm('tom_dataproducts.delete_dataproduct', group_instance, instance) + + return instance + def validate_data_product_type(self, value): for dp_type in settings.DATA_PRODUCT_TYPES.keys(): if not value or value == dp_type: diff --git a/tom_targets/serializers.py b/tom_targets/serializers.py index 8f5df63bd..350897a72 100644 --- a/tom_targets/serializers.py +++ b/tom_targets/serializers.py @@ -1,5 +1,5 @@ from django.contrib.auth.models import Group -from guardian.shortcuts import assign_perm, get_groups_with_perms, get_objects_for_user, remove_perm +from guardian.shortcuts import assign_perm, get_groups_with_perms, get_objects_for_user from rest_framework import serializers from tom_common.serializers import GroupSerializer @@ -53,7 +53,7 @@ def get_groups(self, obj): def create(self, validated_data): """DRF requires explicitly handling writeable nested serializers, - here we pop the alias/tag data and save it using their respective + here we pop the alias/tag/group data and save it using their respective serializers """ From 7c97a1e55ca65debb04498bebe7c71603576c511 Mon Sep 17 00:00:00 2001 From: David Collom Date: Sun, 9 Aug 2020 17:48:32 -0700 Subject: [PATCH 232/424] Added permission logic to dataproducts and corresponding tests --- tom_dataproducts/serializers.py | 10 +++++++++- tom_dataproducts/tests/test_api.py | 13 +++++++++++-- tom_targets/serializers.py | 14 ++++++++------ 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/tom_dataproducts/serializers.py b/tom_dataproducts/serializers.py index 44e4b1702..11d1f1a0c 100644 --- a/tom_dataproducts/serializers.py +++ b/tom_dataproducts/serializers.py @@ -1,6 +1,6 @@ from django.conf import settings from django.contrib.auth.models import Group -from guardian.shortcuts import assign_perm +from guardian.shortcuts import assign_perm, get_groups_with_perms from rest_framework import serializers from tom_common.serializers import GroupSerializer @@ -74,6 +74,14 @@ def create(self, validated_data): return dp + def to_representation(self, instance): + representation = super().to_representation(instance) + groups = [] + for group in get_groups_with_perms(instance): + groups.append(GroupSerializer(group).data) + representation['groups'] = groups + return representation + def update(self, instance, validated_data): groups = validated_data.pop('groups', []) diff --git a/tom_dataproducts/tests/test_api.py b/tom_dataproducts/tests/test_api.py index 9cc09690c..530fa657b 100644 --- a/tom_dataproducts/tests/test_api.py +++ b/tom_dataproducts/tests/test_api.py @@ -1,7 +1,7 @@ -from django.contrib.auth.models import User +from django.contrib.auth.models import Group, User from django.core.files.uploadedfile import SimpleUploadedFile from django.urls import reverse -from guardian.shortcuts import assign_perm +from guardian.shortcuts import assign_perm, get_objects_for_user from rest_framework import status from rest_framework.test import APITestCase @@ -28,6 +28,11 @@ def setUp(self): assign_perm('tom_targets.change_target', self.user, self.st) def test_data_product_upload_for_target(self): + collaborator = User.objects.create(username='test collaborator') + group = Group.objects.create(name='bourgeoisie') + group.user_set.add(self.user) + group.user_set.add(collaborator) + with open('tom_dataproducts/tests/test_data/test_lightcurve.csv', 'rb') as lightcurve_file: self.dp_data['file'] = lightcurve_file response = self.client.post(reverse('api:dataproducts-list'), self.dp_data, format='multipart') @@ -37,6 +42,10 @@ def test_data_product_upload_for_target(self): dp = DataProduct.objects.get(pk=response.data['id']) self.assertEqual(dp.target_id, self.st.id) + # Test that group permissions are respected + response = self.client.get(reverse('api:dataproducts-list')) + self.assertContains(response, self.dp_data['product_id'], status_code=status.HTTP_200_OK) + def test_data_product_upload_for_observation(self): self.dp_data['observation_record'] = self.obsr.id diff --git a/tom_targets/serializers.py b/tom_targets/serializers.py index 350897a72..6a552f092 100644 --- a/tom_targets/serializers.py +++ b/tom_targets/serializers.py @@ -31,7 +31,6 @@ class TargetSerializer(serializers.ModelSerializer): targetextra_set = TargetExtraSerializer(many=True) aliases = TargetNameSerializer(many=True) groups = GroupSerializer(many=True, required=False) # TODO: return groups in detail and list - # groups = serializers.SerializerMethodField() class Meta: model = Target @@ -46,11 +45,6 @@ class Meta: RequiredFieldsTogetherValidator('scheme', 'JPL_MAJOR_PLANET', 'mean_daily_motion', 'mean_anomaly', 'semimajor_axis')] - def get_groups(self, obj): - print('get groups') - print(obj.id) - return get_groups_with_perms(obj) - def create(self, validated_data): """DRF requires explicitly handling writeable nested serializers, here we pop the alias/tag/group data and save it using their respective @@ -88,6 +82,14 @@ def create(self, validated_data): return target + def to_representation(self, instance): + representation = super().to_representation(instance) + groups = [] + for group in get_groups_with_perms(instance): + groups.append(GroupSerializer(group).data) + representation['groups'] = groups + return representation + def update(self, instance, validated_data): """ For TargetExtra and TargetName objects, if the ID is present, it will update the corresponding row. If the ID is From a604d034ea3ac6c9d42b6702a181d113b967ed72 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 10 Aug 2020 10:13:25 -0700 Subject: [PATCH 233/424] Added drf to docs requirements.txt --- docs/requirements.txt | 1 + tom_dataproducts/tests/test_api.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index db6f1a037..664569358 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -9,6 +9,7 @@ django-extensions django-filter django-gravatar2 django-guardian +djangorestframework fits2image numpy plotly diff --git a/tom_dataproducts/tests/test_api.py b/tom_dataproducts/tests/test_api.py index 530fa657b..fb86fdecd 100644 --- a/tom_dataproducts/tests/test_api.py +++ b/tom_dataproducts/tests/test_api.py @@ -1,7 +1,7 @@ from django.contrib.auth.models import Group, User from django.core.files.uploadedfile import SimpleUploadedFile from django.urls import reverse -from guardian.shortcuts import assign_perm, get_objects_for_user +from guardian.shortcuts import assign_perm from rest_framework import status from rest_framework.test import APITestCase From 70b97abb2f6588a522d6c500cf4f9d52f5f8932d Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 10 Aug 2020 11:31:12 -0700 Subject: [PATCH 234/424] Fixed missing target name in comment list --- tom_common/templates/comments/list.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tom_common/templates/comments/list.html b/tom_common/templates/comments/list.html index aebf4e071..3af34235b 100644 --- a/tom_common/templates/comments/list.html +++ b/tom_common/templates/comments/list.html @@ -7,9 +7,9 @@ {{ comment.user.first_name }} {{ comment.user.last_name }} {% if not object %} - on {{ comment.content_object.identifier }} + about {{ comment.content_object.name }} {% endif %} - {{ comment.submit_date|date }} + on {{ comment.submit_date|date }} {% if comment.user == user or user.is_superuser %} ✖ {% endif %} From f0ffd87c92731360cd5fb1c4147ae1f137b11458 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 12 Aug 2020 18:37:09 -0700 Subject: [PATCH 235/424] Added warnings on groups for API calls --- docs/api/tom_dataproducts/api_views.rst | 12 ++++++++++++ docs/api/tom_targets/api_views.rst | 11 +++++++++++ setup.py | 3 ++- tom_dataproducts/api_views.py | 7 +++++++ tom_targets/api_views.py | 10 ++++++++++ 5 files changed, 42 insertions(+), 1 deletion(-) diff --git a/docs/api/tom_dataproducts/api_views.rst b/docs/api/tom_dataproducts/api_views.rst index 8241a90f1..27cd50c48 100644 --- a/docs/api/tom_dataproducts/api_views.rst +++ b/docs/api/tom_dataproducts/api_views.rst @@ -1,5 +1,17 @@ API Views ========= +.. warning:: Check your groups! + + When creating a ``DataProduct`` via the API and you have set ``TARGET_PERMISSIONS_ONLY`` to ``False``, one of the + accepted parameters is a list of groups that will have permission to view the ``DataProduct``. If you neglect to + specify any groups, your ``DataProduct`` will only be visible to the user that created the ``DataProduct``. Please + be sure to specify groups!! + +.. tip:: Better API documentation + + The available parameters for RESTful API calls are not available here. However, if you navigate to ``/api/targets/`` + and click the ``OPTIONS`` button, you can easily view all of the available parameters. + .. automodule:: tom_dataproducts.api_views :members: \ No newline at end of file diff --git a/docs/api/tom_targets/api_views.rst b/docs/api/tom_targets/api_views.rst index 26b9632cb..219e6dbf2 100644 --- a/docs/api/tom_targets/api_views.rst +++ b/docs/api/tom_targets/api_views.rst @@ -1,5 +1,16 @@ API Views ========= +.. warning:: Check your groups! + + When creating a ``Target`` via the API, one of the accepted parameters is a list of groups that will have permission + to view the ``Target``. If you neglect to specify any groups, your ``Target`` will only be visible to the user that + created the ``Target``. Please be sure to specify groups!! + +.. tip:: Better API documentation + + The available parameters for RESTful API calls are not available here. However, if you navigate to ``/api/targets/`` + and click the ``OPTIONS`` button, you can easily view all of the available parameters. + .. automodule:: tom_targets.api_views :members: \ No newline at end of file diff --git a/setup.py b/setup.py index f914ccf39..c5c1f9c62 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ 'beautifulsoup4==4.9.1', 'dataclasses; python_version < "3.7"', 'django==3.0.7', # TOM Toolkit requires db math functions - 'djangorestframework', + 'djangorestframework==3.11.0', 'django-bootstrap4==1.1.1', 'django-contrib-comments>=1.9.2', # Earlier version are incompatible with Django >= 3.0 'django-crispy-forms==1.9.0', @@ -43,6 +43,7 @@ 'django-gravatar2==1.4.3', 'django-guardian==2.2.0', 'fits2image==0.4.3', + 'Markdown==3.2.2', # django-rest-framework doc headers require this to support Markdown 'numpy==1.18.2', 'pillow==7.1.0', 'plotly==4.6.0', diff --git a/tom_dataproducts/api_views.py b/tom_dataproducts/api_views.py index 15385193b..8f59138ae 100644 --- a/tom_dataproducts/api_views.py +++ b/tom_dataproducts/api_views.py @@ -18,6 +18,13 @@ class DataProductViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, GenericViewSet, PermissionListMixin): """ Viewset for DataProduct objects. Supports list, create, and delete. + + To view supported query parameters, please use the OPTIONS endpoint, which can be accessed through the web UI. + + **Please note that ``groups`` are an accepted query parameters for the ``CREATE`` endpoint. The groups parameter + will specify which ``groups`` can view the created ``DataProduct``. If no ``groups`` are specified, the + ``DataProduct`` will only be visible to the user that created the ``DataProduct``. Make sure to check your + ``groups``!!** """ queryset = DataProduct.objects.all() serializer_class = DataProductSerializer diff --git a/tom_targets/api_views.py b/tom_targets/api_views.py index 14c90ad16..6e87576ac 100644 --- a/tom_targets/api_views.py +++ b/tom_targets/api_views.py @@ -32,6 +32,12 @@ class TargetViewSet(ModelViewSet, PermissionListMixin): Viewset for Target objects. By default supports CRUD operations. See the docs on viewsets: https://www.django-rest-framework.org/api-guide/viewsets/ + To view supported query parameters, please use the ``OPTIONS`` endpoint, which can be accessed through the web UI. + + **Please note that ``groups`` are an accepted query parameters for the ``CREATE`` endpoint. The ``groups`` parameter + will specify which ``groups`` can view the created Target. If no ``groups`` are specified, the ``Target`` will only + be visible to the user that created the ``Target``. Make sure to check your ``groups``!!** + In order to create new ``TargetName`` or ``TargetExtra`` objects, a dictionary with the new values must be appended to the ``aliases`` or ``targetextra_set`` lists. If ``id`` is included, the API will attempt to update an existing ``TargetName`` or ``TargetExtra``. If no ``id`` is provided, the API will attempt to create new entries. @@ -51,6 +57,8 @@ def get_queryset(self): class TargetNameViewSet(DestroyModelMixin, PermissionListMixin, RetrieveModelMixin, GenericViewSet): """ Viewset for TargetName objects. Only ``GET`` and ``DELETE`` operations are permitted. + + To view available query parameters, please use the OPTIONS endpoint, which can be accessed through the web UI. """ serializer_class = TargetNameSerializer @@ -64,6 +72,8 @@ def get_queryset(self): class TargetExtraViewSet(DestroyModelMixin, PermissionListMixin, RetrieveModelMixin, GenericViewSet): """ Viewset for TargetExtra objects. Only ``GET`` and ``DELETE`` operations are permitted. + + To view available query parameters, please use the OPTIONS endpoint, which can be accessed through the web UI. """ serializer_class = TargetExtraSerializer From ec9a6077f90e97d248b5ef1d5084d84c0e1f8cde Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 13 Aug 2020 08:33:42 -0700 Subject: [PATCH 236/424] Upgrading and pinning factory boy to fix the bug introduced by 3.0 --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index c5c1f9c62..7795b1ca0 100644 --- a/setup.py +++ b/setup.py @@ -33,13 +33,13 @@ 'astropy==4.0', 'beautifulsoup4==4.9.1', 'dataclasses; python_version < "3.7"', - 'django==3.0.7', # TOM Toolkit requires db math functions - 'djangorestframework==3.11.0', + 'django==3.1.0', # TOM Toolkit requires db math functions + 'djangorestframework==3.11.1', 'django-bootstrap4==1.1.1', 'django-contrib-comments>=1.9.2', # Earlier version are incompatible with Django >= 3.0 'django-crispy-forms==1.9.0', 'django-extensions==2.2.9', - 'django-filter==2.2.0', + 'django-filter==2.3.0', 'django-gravatar2==1.4.3', 'django-guardian==2.2.0', 'fits2image==0.4.3', @@ -52,7 +52,7 @@ 'specutils==1.0', ], extras_require={ - 'test': ['factory_boy'] + 'test': ['factory_boy==3.0.1'] }, include_package_data=True, ) From 6edc1a6a97726cf1b4f74d394264e35b771a90ab Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 13 Aug 2020 13:18:20 -0700 Subject: [PATCH 237/424] Pin django-contrib-comments --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5730459a5..43e1a529e 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ 'django==3.1.0', # TOM Toolkit requires db math functions 'djangorestframework==3.11.1', 'django-bootstrap4==1.1.1', - 'django-contrib-comments>=1.9.2', # Earlier version are incompatible with Django >= 3.0 + 'django-contrib-comments==1.9.2', # Earlier version are incompatible with Django >= 3.0 'django-crispy-forms==1.9.0', 'django-extensions==2.2.9', 'django-gravatar2==1.4.3', From 3f72cdd2285fc045e4d8bd7e45ba6bebcecd9139 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 13 Aug 2020 21:07:36 +0000 Subject: [PATCH 238/424] Bump django-bootstrap4 from 1.1.1 to 2.2.0 Bumps [django-bootstrap4](https://github.com/zostera/django-bootstrap4) from 1.1.1 to 2.2.0. - [Release notes](https://github.com/zostera/django-bootstrap4/releases) - [Changelog](https://github.com/zostera/django-bootstrap4/blob/main/CHANGELOG.md) - [Commits](https://github.com/zostera/django-bootstrap4/compare/v1.1.1...v2.2.0) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 43e1a529e..566d69e7c 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ 'dataclasses; python_version < "3.7"', 'django==3.1.0', # TOM Toolkit requires db math functions 'djangorestframework==3.11.1', - 'django-bootstrap4==1.1.1', + 'django-bootstrap4==2.2.0', 'django-contrib-comments==1.9.2', # Earlier version are incompatible with Django >= 3.0 'django-crispy-forms==1.9.0', 'django-extensions==2.2.9', From 76926391f5805a329c198dc73a09f9f0149432e8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 13 Aug 2020 21:07:36 +0000 Subject: [PATCH 239/424] Bump astropy from 4.0 to 4.0.1.post1 Bumps [astropy](https://github.com/astropy/astropy) from 4.0 to 4.0.1.post1. - [Release notes](https://github.com/astropy/astropy/releases) - [Changelog](https://github.com/astropy/astropy/blob/master/CHANGES.rst) - [Commits](https://github.com/astropy/astropy/compare/v4.0...v4.0.1.post1) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 43e1a529e..e1b57a28c 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ install_requires=[ 'astroquery==0.4', 'astroplan==0.6', - 'astropy==4.0', + 'astropy==4.0.1.post1', 'beautifulsoup4==4.9.1', 'dataclasses; python_version < "3.7"', 'django==3.1.0', # TOM Toolkit requires db math functions From bac8f9d3fa8b02b1c4da083a8341164c00c54212 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 13 Aug 2020 21:07:36 +0000 Subject: [PATCH 240/424] Bump requests from 2.23.0 to 2.24.0 Bumps [requests](https://github.com/psf/requests) from 2.23.0 to 2.24.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/master/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.23.0...v2.24.0) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 43e1a529e..6b613950e 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ 'pillow==7.1.0', 'plotly==4.6.0', 'python-dateutil==2.8.1', - 'requests==2.23.0', + 'requests==2.24.0', 'specutils==1.0', ], extras_require={ From 55435d2cde2c0b0dd99c8f82b4c0c94a3fd84f50 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 13 Aug 2020 21:07:37 +0000 Subject: [PATCH 241/424] Bump django-extensions from 2.2.9 to 3.0.5 Bumps [django-extensions](https://github.com/django-extensions/django-extensions) from 2.2.9 to 3.0.5. - [Release notes](https://github.com/django-extensions/django-extensions/releases) - [Changelog](https://github.com/django-extensions/django-extensions/blob/master/CHANGELOG.md) - [Commits](https://github.com/django-extensions/django-extensions/compare/2.2.9...3.0.5) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 43e1a529e..52647a751 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ 'django-bootstrap4==1.1.1', 'django-contrib-comments==1.9.2', # Earlier version are incompatible with Django >= 3.0 'django-crispy-forms==1.9.0', - 'django-extensions==2.2.9', + 'django-extensions==3.0.5', 'django-gravatar2==1.4.3', 'django-filter==2.3.0', 'django-guardian==2.2.0', From dea16187bd11da48bd2a017c9a5cb6ee1a1e9a0a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 13 Aug 2020 21:07:42 +0000 Subject: [PATCH 242/424] Bump astroquery from 0.4 to 0.4.1 Bumps [astroquery](https://github.com/astropy/astroquery) from 0.4 to 0.4.1. - [Release notes](https://github.com/astropy/astroquery/releases) - [Changelog](https://github.com/astropy/astroquery/blob/master/CHANGES.rst) - [Commits](https://github.com/astropy/astroquery/compare/v0.4...v0.4.1) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 43e1a529e..47e978584 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ use_scm_version=True, setup_requires=['setuptools_scm', 'wheel'], install_requires=[ - 'astroquery==0.4', + 'astroquery==0.4.1', 'astroplan==0.6', 'astropy==4.0', 'beautifulsoup4==4.9.1', From 92de631a06246c30312109216ffbfc022156cdb9 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 13 Aug 2020 14:27:28 -0700 Subject: [PATCH 243/424] Removed comments --- tom_base/settings.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index 651f63e9b..69d37ad82 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -123,7 +123,6 @@ }, ] -# TODO: Release notes MUST document this change!! LOGIN_URL = '/accounts/login/' LOGIN_REDIRECT_URL = '/' LOGOUT_REDIRECT_URL = '/' @@ -265,7 +264,6 @@ HINTS_ENABLED = False HINT_LEVEL = 20 -# TODO: Release notes MUST document this change!! REST_FRAMEWORK = { 'DEFAULT_PERMISSION_CLASSES': [ ], From 76e932392ae12224e1c3e4eab614004a64c34075 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 13 Aug 2020 23:40:08 +0000 Subject: [PATCH 244/424] Bump django-crispy-forms from 1.9.0 to 1.9.2 Bumps [django-crispy-forms](https://github.com/django-crispy-forms/django-crispy-forms) from 1.9.0 to 1.9.2. - [Release notes](https://github.com/django-crispy-forms/django-crispy-forms/releases) - [Changelog](https://github.com/django-crispy-forms/django-crispy-forms/blob/master/CHANGELOG.md) - [Commits](https://github.com/django-crispy-forms/django-crispy-forms/compare/1.9.0...1.9.2) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 54f207c36..7b83a1097 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ 'djangorestframework==3.11.1', 'django-bootstrap4==2.2.0', 'django-contrib-comments==1.9.2', # Earlier version are incompatible with Django >= 3.0 - 'django-crispy-forms==1.9.0', + 'django-crispy-forms==1.9.2', 'django-extensions==3.0.5', 'django-gravatar2==1.4.3', 'django-filter==2.3.0', From a6c9977d48f25d82dc915ed6bfca1d942fd2ccc3 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 13 Aug 2020 23:40:11 +0000 Subject: [PATCH 245/424] Bump django-guardian from 2.2.0 to 2.3.0 Bumps [django-guardian](https://github.com/django-guardian/django-guardian) from 2.2.0 to 2.3.0. - [Release notes](https://github.com/django-guardian/django-guardian/releases) - [Changelog](https://github.com/django-guardian/django-guardian/blob/devel/CHANGES) - [Commits](https://github.com/django-guardian/django-guardian/compare/v2.2.0...v2.3.0) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 54f207c36..aace9e83f 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ 'django-extensions==3.0.5', 'django-gravatar2==1.4.3', 'django-filter==2.3.0', - 'django-guardian==2.2.0', + 'django-guardian==2.3.0', 'fits2image==0.4.3', 'Markdown==3.2.2', # django-rest-framework doc headers require this to support Markdown 'numpy==1.18.2', From 5c357aa718b95b6358165848ee62c88f1b097cc1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2020 03:10:54 +0000 Subject: [PATCH 246/424] Bump numpy from 1.18.2 to 1.19.1 Bumps [numpy](https://github.com/numpy/numpy) from 1.18.2 to 1.19.1. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/master/doc/HOWTO_RELEASE.rst.txt) - [Commits](https://github.com/numpy/numpy/compare/v1.18.2...v1.19.1) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 20cafd8ce..af555d8d1 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ 'django-guardian==2.3.0', 'fits2image==0.4.3', 'Markdown==3.2.2', # django-rest-framework doc headers require this to support Markdown - 'numpy==1.18.2', + 'numpy==1.19.1', 'pillow==7.1.0', 'plotly==4.6.0', 'python-dateutil==2.8.1', From a9948a629882ae6c94b99806a90b46102cae05b6 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2020 13:41:19 +0000 Subject: [PATCH 247/424] Bump django-gravatar2 from 1.4.3 to 1.4.4 Bumps [django-gravatar2](https://github.com/twaddington/django-gravatar) from 1.4.3 to 1.4.4. - [Release notes](https://github.com/twaddington/django-gravatar/releases) - [Commits](https://github.com/twaddington/django-gravatar/commits) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index af555d8d1..1f1ab94bc 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ 'django-contrib-comments==1.9.2', # Earlier version are incompatible with Django >= 3.0 'django-crispy-forms==1.9.2', 'django-extensions==3.0.5', - 'django-gravatar2==1.4.3', + 'django-gravatar2==1.4.4', 'django-filter==2.3.0', 'django-guardian==2.3.0', 'fits2image==0.4.3', From 552575aebbc2b3d4fd7411af7e4e10dbf9696363 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2020 13:42:00 +0000 Subject: [PATCH 248/424] Bump plotly from 4.6.0 to 4.9.0 Bumps [plotly](https://github.com/plotly/plotly.py) from 4.6.0 to 4.9.0. - [Release notes](https://github.com/plotly/plotly.py/releases) - [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.py/compare/v4.6.0...v4.9.0) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index af555d8d1..0f8621094 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ 'Markdown==3.2.2', # django-rest-framework doc headers require this to support Markdown 'numpy==1.19.1', 'pillow==7.1.0', - 'plotly==4.6.0', + 'plotly==4.9.0', 'python-dateutil==2.8.1', 'requests==2.24.0', 'specutils==1.0', From 80764e03c7487d19d4aa9d9e7db665f4446b3fcd Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 14 Aug 2020 14:45:54 -0700 Subject: [PATCH 249/424] Got errors to display --- tom_observations/facilities/lco.py | 7 +++--- .../tom_observations/observation_form.html | 1 + tom_observations/views.py | 23 +++++++------------ 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index dc008fe62..62f6ce4a1 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -485,9 +485,10 @@ def __init__(self, *args, **kwargs): self.layout(), self.button_layout() ) - self.fields['cadence_type'].required = False - self.fields['cadence_strategy'].required = False - self.fields['cadence_frequency'].required = False + for field_name in ['cadence_type', 'cadence_strategy', 'cadence_frequency']: + self.fields[field_name].required = False + for field_name in ['exposure_time', 'exposure_count', 'start', 'end', 'filter']: + self.fields.pop(field_name) if self.fields.get('groups'): self.fields['groups'].label = 'Data granted to' diff --git a/tom_observations/templates/tom_observations/observation_form.html b/tom_observations/templates/tom_observations/observation_form.html index 1731e6b9b..fa884a77d 100644 --- a/tom_observations/templates/tom_observations/observation_form.html +++ b/tom_observations/templates/tom_observations/observation_form.html @@ -2,6 +2,7 @@ {% load bootstrap4 crispy_forms_tags observation_extras targets_extras %} {% block title %}Submit Observation{% endblock %} {% block content %} +{{ form|as_crispy_errors }}

    Submit an observation to {{ form.facility.value }}

    {% if target.type == 'SIDEREAL' %}
    diff --git a/tom_observations/views.py b/tom_observations/views.py index afbdbe9eb..c97d7c785 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -162,9 +162,13 @@ def get_context_data(self, **kwargs): :returns: context dictionary :rtype: dict """ + # TODO: ensure one form is active at least + # TODO: add spectroscopic sequence form + # TODO: style form pointers properly + # TODO: add display tab title for form context = super(ObservationCreateView, self).get_context_data(**kwargs) - - # Populate initial values for each form and add them to the context. If the page + + # Populate initial values for each form and add them to the context. If the page # reloaded due to form errors, only repopulate the form that was submitted. observation_type_choices = [] initial = self.get_initial() @@ -194,7 +198,6 @@ def get_form_class(self): observation_type = self.request.GET.get('observation_type') elif self.request.method == 'POST': observation_type = self.request.POST.get('observation_type') - print(self.get_facility_class()().get_form(observation_type)) return self.get_facility_class()().get_form(observation_type) def get_form(self): @@ -204,6 +207,7 @@ def get_form(self): :returns: observation form :rtype: subclass of GenericObservationForm """ + form = super().get_form() if not settings.TARGET_PERMISSIONS_ONLY: form.fields['groups'].queryset = self.request.user.groups.all() @@ -227,14 +231,6 @@ def get_initial(self): initial['facility'] = self.get_facility() return initial - def form_invalid(self, form): - print('form_invalid') - print(form.cleaned_data['observation_type']) - print(type(form)) - # print(form) - print(form.errors) - return super().form_invalid(form) - def form_valid(self, form): """ Runs after form validation. Submits the observation to the desired facility and creates an associated @@ -246,13 +242,10 @@ def form_valid(self, form): :param form: form containing observating request parameters :type form: subclass of GenericObservationForm """ - print('form_valid') - print(form.cleaned_data) - # TODO: Render errors properly, probably in form_invalid # Submit the observation facility = self.get_facility_class() target = self.get_target() - # observation_ids = facility().submit_observation(form.observation_payload()) + observation_ids = facility().submit_observation(form.observation_payload()) records = [] for observation_id in observation_ids: From b50660ec0cdb540c0cce1e65e0028f8fd3a88c3d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2020 21:51:53 +0000 Subject: [PATCH 250/424] Bump pillow from 7.1.0 to 7.2.0 Bumps [pillow](https://github.com/python-pillow/Pillow) from 7.1.0 to 7.2.0. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/master/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/7.1.0...7.2.0) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0f8621094..cc663d8ad 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ 'fits2image==0.4.3', 'Markdown==3.2.2', # django-rest-framework doc headers require this to support Markdown 'numpy==1.19.1', - 'pillow==7.1.0', + 'pillow==7.2.0', 'plotly==4.9.0', 'python-dateutil==2.8.1', 'requests==2.24.0', From 931a50783faf9995bc46365cc4fee14a4d31ec98 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 14 Aug 2020 15:15:14 -0700 Subject: [PATCH 251/424] Ensured active form on page load --- tom_observations/facilities/lco.py | 2 +- .../templates/tom_observations/observation_form.html | 4 ++-- tom_observations/views.py | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 62f6ce4a1..80de955f1 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -432,7 +432,7 @@ def _build_instrument_config(self): } else: instrument_configs[0].pop('optical_elements') - instrument_configs[0]['rotator_mode'] = 'VFLOAT' # TODO: Should be a distinct field, SKY & VFLOAT are both valid + instrument_configs[0]['rotator_mode'] = 'VFLOAT' # TODO: Should be distinct field, SKY & VFLOAT are both valid instrument_configs[0]['extra_params'] = { 'rotator_angle': self.cleaned_data['rotator_angle'] } diff --git a/tom_observations/templates/tom_observations/observation_form.html b/tom_observations/templates/tom_observations/observation_form.html index fa884a77d..650bf0d34 100644 --- a/tom_observations/templates/tom_observations/observation_form.html +++ b/tom_observations/templates/tom_observations/observation_form.html @@ -25,7 +25,7 @@

    Submit an observation to {{ form.facility.value }}

    {% for observation_type, observation_form in observation_type_choices %} -
    +
    {% crispy observation_form %}
    {% endfor %} diff --git a/tom_observations/views.py b/tom_observations/views.py index c97d7c785..fe929dffd 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -162,7 +162,6 @@ def get_context_data(self, **kwargs): :returns: context dictionary :rtype: dict """ - # TODO: ensure one form is active at least # TODO: add spectroscopic sequence form # TODO: style form pointers properly # TODO: add display tab title for form From 9edb8c4e79ca43f6cb05bcae4b2a40246465392b Mon Sep 17 00:00:00 2001 From: David Collom Date: Sun, 16 Aug 2020 12:59:20 -0700 Subject: [PATCH 252/424] Fixed cursor for tabs --- tom_common/static/tom_common/css/main.css | 4 ++++ .../templates/tom_observations/observation_form.html | 5 ++++- tom_targets/static/tom_targets/css/main.css | 4 ---- tom_targets/templates/tom_targets/target_detail.html | 1 + 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tom_common/static/tom_common/css/main.css b/tom_common/static/tom_common/css/main.css index daf313be9..3caae4a29 100644 --- a/tom_common/static/tom_common/css/main.css +++ b/tom_common/static/tom_common/css/main.css @@ -13,3 +13,7 @@ body { .input-group-text { font-family: monospace; } + +.nav-item { + cursor: pointer; +} \ No newline at end of file diff --git a/tom_observations/templates/tom_observations/observation_form.html b/tom_observations/templates/tom_observations/observation_form.html index 650bf0d34..45ae4ead3 100644 --- a/tom_observations/templates/tom_observations/observation_form.html +++ b/tom_observations/templates/tom_observations/observation_form.html @@ -1,6 +1,9 @@ {% extends 'tom_common/base.html' %} -{% load bootstrap4 crispy_forms_tags observation_extras targets_extras %} +{% load bootstrap4 static crispy_forms_tags observation_extras targets_extras %} {% block title %}Submit Observation{% endblock %} +{% block additional_css %} + +{% endblock %} {% block content %} {{ form|as_crispy_errors }}

    Submit an observation to {{ form.facility.value }}

    diff --git a/tom_targets/static/tom_targets/css/main.css b/tom_targets/static/tom_targets/css/main.css index b6eec2ab5..c28bac3f3 100644 --- a/tom_targets/static/tom_targets/css/main.css +++ b/tom_targets/static/tom_targets/css/main.css @@ -12,10 +12,6 @@ dl { padding: 2% 0 0 0; } -.nav-item { - cursor: pointer; -} - .light-curve { height: 600px; width: inherit; diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index 569fc95d6..654b5959e 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -2,6 +2,7 @@ {% load comments bootstrap4 tom_common_extras targets_extras observation_extras dataproduct_extras publication_extras static cache %} {% block title %}Target {{ object.name }}{% endblock %} {% block additional_css %} + {% endblock %} {% bootstrap_javascript jquery='True' %} From d49c67fe7519f7b14f6f74fefe6e61f10b3e3649 Mon Sep 17 00:00:00 2001 From: David Collom Date: Sun, 16 Aug 2020 13:12:39 -0700 Subject: [PATCH 253/424] Added templatetag to display observation type in title case --- tom_observations/facilities/lco.py | 3 +-- .../templates/tom_observations/observation_form.html | 2 +- tom_observations/templatetags/observation_extras.py | 5 +++++ tom_observations/views.py | 3 --- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 80de955f1..9926e7ef1 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -594,11 +594,10 @@ class LCOFacility(BaseRoboticObservationFacility): name = 'LCO' default_form_class = LCOBaseObservationForm - # observation_types = [('IMAGING', 'Imaging'), ('SPECTRA', 'Spectroscopy'), ('SEQUENCE', 'Photometric Sequence')] observation_forms = { 'IMAGING': LCOImagingObservationForm, 'SPECTRA': LCOSpectroscopyObservationForm, - 'PHOTSEQ': LCOPhotometricSequenceForm + 'PHOTOMETRIC_SEQUENCE': LCOPhotometricSequenceForm } # The SITES dictionary is used to calculate visibility intervals in the # planning tool. All entries should contain latitude, longitude, elevation diff --git a/tom_observations/templates/tom_observations/observation_form.html b/tom_observations/templates/tom_observations/observation_form.html index 45ae4ead3..0791bfc62 100644 --- a/tom_observations/templates/tom_observations/observation_form.html +++ b/tom_observations/templates/tom_observations/observation_form.html @@ -29,7 +29,7 @@

    Submit an observation to {{ form.facility.value }}

    {% for observation_type, observation_form in observation_type_choices %}
  • - {{ observation_type }} + {{ observation_type|display_obs_type }}
    • {% endfor %} diff --git a/tom_observations/templatetags/observation_extras.py b/tom_observations/templatetags/observation_extras.py index 0237d9fa2..7d618a10d 100644 --- a/tom_observations/templatetags/observation_extras.py +++ b/tom_observations/templatetags/observation_extras.py @@ -19,6 +19,11 @@ register = template.Library() +@register.filter +def display_obs_type(value): + return value.replace('_', ' ').title() + + @register.inclusion_tag('tom_observations/partials/observing_buttons.html') def observing_buttons(target): """ diff --git a/tom_observations/views.py b/tom_observations/views.py index fe929dffd..e56639c99 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -162,9 +162,6 @@ def get_context_data(self, **kwargs): :returns: context dictionary :rtype: dict """ - # TODO: add spectroscopic sequence form - # TODO: style form pointers properly - # TODO: add display tab title for form context = super(ObservationCreateView, self).get_context_data(**kwargs) # Populate initial values for each form and add them to the context. If the page From 40c11ac983f16e8bbd0da1a045cd83bc7ee063c2 Mon Sep 17 00:00:00 2001 From: David Collom Date: Sun, 16 Aug 2020 15:06:33 -0700 Subject: [PATCH 254/424] Fixed tests --- tom_observations/facilities/lco.py | 1 - tom_observations/tests/factories.py | 4 ++-- tom_observations/tests/tests.py | 3 ++- tom_observations/tests/utils.py | 6 ++++-- tom_targets/tests/factories.py | 4 ++-- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 9926e7ef1..f5ff4227e 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -593,7 +593,6 @@ class LCOFacility(BaseRoboticObservationFacility): """ name = 'LCO' - default_form_class = LCOBaseObservationForm observation_forms = { 'IMAGING': LCOImagingObservationForm, 'SPECTRA': LCOSpectroscopyObservationForm, diff --git a/tom_observations/tests/factories.py b/tom_observations/tests/factories.py index 3499fc2df..f5e4c6563 100644 --- a/tom_observations/tests/factories.py +++ b/tom_observations/tests/factories.py @@ -17,8 +17,8 @@ class Meta: model = Target name = factory.Faker('pystr') - ra = factory.Faker('pyfloat') - dec = factory.Faker('pyfloat') + ra = factory.Faker('pyfloat', min_value=-90, max_value=90) + dec = factory.Faker('pyfloat', min_value=-90, max_value=90) epoch = factory.Faker('pyfloat') pm_ra = factory.Faker('pyfloat') pm_dec = factory.Faker('pyfloat') diff --git a/tom_observations/tests/tests.py b/tom_observations/tests/tests.py index 183c420f4..ca264d61e 100644 --- a/tom_observations/tests/tests.py +++ b/tom_observations/tests/tests.py @@ -72,7 +72,7 @@ def test_update_observations(self): def test_get_observation_form(self): url = f"{reverse('tom_observations:create', kwargs={'facility': 'FakeRoboticFacility'})}" \ - f"?target_id={self.target.id}" + f"?target_id={self.target.id}&observation_type=OBSERVATION" response = self.client.get(url) # self.assertContains(response, 'fake form input') self.assertContains(response, 'FakeRoboticFacility') @@ -107,6 +107,7 @@ def test_submit_observation_robotic(self): 'target_id': self.target.id, 'test_input': 'gnomes', 'facility': 'FakeRoboticFacility', + 'observation_type': 'OBSERVATION' } self.client.post( '{}?target_id={}'.format( diff --git a/tom_observations/tests/utils.py b/tom_observations/tests/utils.py index 93c1d7cb5..af3203b29 100644 --- a/tom_observations/tests/utils.py +++ b/tom_observations/tests/utils.py @@ -33,10 +33,12 @@ class FakeFacilityStrategyForm(GenericStrategyForm): class FakeRoboticFacility(BaseRoboticObservationFacility): name = 'FakeRoboticFacility' - observation_types = [('FakeRoboticFacility Observation', 'OBSERVATION')] + observation_forms = { + 'OBSERVATION': FakeFacilityForm + } def get_form(self, observation_type): - return FakeFacilityForm + return self.observation_forms[observation_type] def get_strategy_form(self, observation_type): return FakeFacilityStrategyForm diff --git a/tom_targets/tests/factories.py b/tom_targets/tests/factories.py index 9ae657455..d66308efa 100644 --- a/tom_targets/tests/factories.py +++ b/tom_targets/tests/factories.py @@ -24,8 +24,8 @@ class Meta: name = factory.Faker('pystr') type = Target.SIDEREAL - ra = factory.Faker('pyfloat') - dec = factory.Faker('pyfloat') + ra = factory.Faker('pyfloat', min_value=-90, max_value=90) + dec = factory.Faker('pyfloat', min_value=-90, max_value=90) epoch = factory.Faker('pyfloat') pm_ra = factory.Faker('pyfloat') pm_dec = factory.Faker('pyfloat') From 69be8a65b0879252fa6ead0db95ee52a5d68e948 Mon Sep 17 00:00:00 2001 From: David Collom Date: Sun, 16 Aug 2020 15:22:29 -0700 Subject: [PATCH 255/424] Fixed some codacy stuff --- tom_common/static/tom_common/css/main.css | 2 +- tom_observations/facilities/gemini.py | 4 +++- tom_observations/facilities/lco.py | 4 +--- tom_publications/forms.py | 1 - tom_targets/tests/tests.py | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/tom_common/static/tom_common/css/main.css b/tom_common/static/tom_common/css/main.css index 3caae4a29..b2589c6a0 100644 --- a/tom_common/static/tom_common/css/main.css +++ b/tom_common/static/tom_common/css/main.css @@ -16,4 +16,4 @@ body { .nav-item { cursor: pointer; -} \ No newline at end of file +} diff --git a/tom_observations/facilities/gemini.py b/tom_observations/facilities/gemini.py index b31de4884..4b496e749 100644 --- a/tom_observations/facilities/gemini.py +++ b/tom_observations/facilities/gemini.py @@ -426,7 +426,9 @@ class GEMFacility(BaseRoboticObservationFacility): """ name = 'GEM' - observation_types = [('OBSERVATION', 'Gemini Observation')] + observation_forms = { + 'OBSERVATION': GEMObservationForm + } def get_form(self, observation_type): return GEMObservationForm diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index f5ff4227e..1ca441bfd 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -1,5 +1,4 @@ from datetime import datetime, timedelta -import json import requests from astropy import units as u @@ -437,7 +436,6 @@ def _build_instrument_config(self): 'rotator_angle': self.cleaned_data['rotator_angle'] } - return [] return instrument_configs @@ -494,7 +492,7 @@ def __init__(self, *args, **kwargs): def _build_instrument_config(self): instrument_config = [] - for label, filter in self.filter_mapping.items(): + for label, _ in self.filter_mapping.items(): if len(self.cleaned_data[label]) > 0: instrument_config.append({ 'exposure_count': self.cleaned_data[label][1], diff --git a/tom_publications/forms.py b/tom_publications/forms.py index 12496377b..02c836b0c 100644 --- a/tom_publications/forms.py +++ b/tom_publications/forms.py @@ -1,5 +1,4 @@ from django import forms -from django.apps import apps from tom_publications.models import LatexConfiguration diff --git a/tom_targets/tests/tests.py b/tom_targets/tests/tests.py index 41678b9ee..23ef18d20 100644 --- a/tom_targets/tests/tests.py +++ b/tom_targets/tests/tests.py @@ -480,7 +480,7 @@ def test_export_all_targets_with_aliases(self): self.assertIn('M52', content) def test_export_filtered_targets_with_aliases(self): - st_name = TargetNameFactory.create(name='Messier 42', target=self.st) + TargetNameFactory.create(name='Messier 42', target=self.st) response = self.client.get(reverse('targets:export') + '?name=M42') content = ''.join(line.decode('utf-8') for line in list(response.streaming_content)) self.assertIn('M42', content) From bb750d40146b3817fc32fdfb31332e5a8acb4e25 Mon Sep 17 00:00:00 2001 From: David Collom Date: Sun, 16 Aug 2020 15:31:05 -0700 Subject: [PATCH 256/424] More codacy fixes --- tom_common/admin.py | 3 --- tom_dataproducts/models.py | 2 +- tom_observations/static/tom_observations/css/main.css | 2 +- tom_observations/widgets.py | 11 +++++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tom_common/admin.py b/tom_common/admin.py index 8c38f3f3d..e69de29bb 100644 --- a/tom_common/admin.py +++ b/tom_common/admin.py @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index 45f7e0961..05eeec306 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -190,7 +190,7 @@ def save(self, *args, **kwargs): Saves the current `DataProduct` instance. Before saving, validates the `data_product_type` against those specified in `settings.py`. """ - for dp_type, dp_values in settings.DATA_PRODUCT_TYPES.items(): + for _, dp_values in settings.DATA_PRODUCT_TYPES.items(): if not self.data_product_type or self.data_product_type == dp_values[0]: break else: diff --git a/tom_observations/static/tom_observations/css/main.css b/tom_observations/static/tom_observations/css/main.css index bbfd4799d..72d4496b5 100644 --- a/tom_observations/static/tom_observations/css/main.css +++ b/tom_observations/static/tom_observations/css/main.css @@ -17,4 +17,4 @@ span.featured { pointer-events: none; -} \ No newline at end of file +} diff --git a/tom_observations/widgets.py b/tom_observations/widgets.py index fbf81f55f..8c3a31abb 100644 --- a/tom_observations/widgets.py +++ b/tom_observations/widgets.py @@ -3,12 +3,15 @@ class FilterConfigurationWidget(forms.widgets.MultiWidget): - def __init__(self, attrs={}): + def __init__(self, attrs=None): + if not attrs: + attrs = {} _default_attrs = {'class': 'form-control col-md-3', 'style': 'margin-right: 10px; display: inline-block'} + attrs.update(_default_attrs) _widgets = ( - forms.widgets.NumberInput(attrs=_default_attrs), - forms.widgets.NumberInput(attrs=_default_attrs), - forms.widgets.NumberInput(attrs=_default_attrs) + forms.widgets.NumberInput(attrs=attrs), + forms.widgets.NumberInput(attrs=attrs), + forms.widgets.NumberInput(attrs=attrs) ) super().__init__(_widgets, attrs) From 000af501e212413b2ab344a45cfa6badd1b6c0d2 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 17 Aug 2020 16:07:47 -0700 Subject: [PATCH 257/424] Added support for immediate obs submission, condensed filter field construction, added some docs --- tom_observations/cadence.py | 16 ---- tom_observations/facilities/lco.py | 122 ++++++++++++++++------------- tom_observations/views.py | 1 + 3 files changed, 68 insertions(+), 71 deletions(-) diff --git a/tom_observations/cadence.py b/tom_observations/cadence.py index c96bc99b2..de2d8e59a 100644 --- a/tom_observations/cadence.py +++ b/tom_observations/cadence.py @@ -219,19 +219,3 @@ def cadence_layout(self): css_class='form-row' ) ) - - -class DelayedCadenceForm(CadenceForm): - cadence_type = forms.ChoiceField(choices=[('', ''), ('repeat', 'Repeating every')]) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.fields['cadence_strategy'].widget = forms.HiddenInput() - self.fields['cadence_frequency'].widget.attrs['readonly'] = False - - def cadence_layout(self): - return Layout( - Row( - Column('cadence_type'), Column('cadence_frequency') - ) - ) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 1ca441bfd..eab3cfff6 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -10,7 +10,7 @@ from django.core.cache import cache from tom_common.exceptions import ImproperCredentialsException -from tom_observations.cadence import CadenceForm, DelayedCadenceForm +from tom_observations.cadence import CadenceForm from tom_observations.facility import BaseRoboticObservationFacility, BaseRoboticObservationForm, get_service_class from tom_observations.observing_strategy import GenericStrategyForm from tom_observations.widgets import FilterField @@ -439,39 +439,37 @@ def _build_instrument_config(self): return instrument_configs -class LCOPhotometricSequenceForm(LCOBaseObservationForm, DelayedCadenceForm): +class LCOPhotometricSequenceForm(LCOBaseObservationForm): """ The LCOPhotometricSequenceForm provides a form offering a subset of the parameters in the LCOImagingObservationForm. The form is modeled after the Supernova Exchange application's Photometric Sequence Request Form, and allows the configuration of multiple filters, as well as a more intuitive proactive cadence form. """ - U_filter = FilterField(label='U', required=False) - B_filter = FilterField(label='B', required=False) - V_filter = FilterField(label='V', required=False) - R_filter = FilterField(label='R', required=False) - I_filter = FilterField(label='I', required=False) - u_filter = FilterField(label='u', required=False) - g_filter = FilterField(label='g', required=False) - r_filter = FilterField(label='r', required=False) - i_filter = FilterField(label='i', required=False) - z_filter = FilterField(label='z', required=False) - w_filter = FilterField(label='w', required=False) - filter_mapping = { - 'U_filter': 'U', - 'B_filter': 'B', - 'V_filter': 'v', - 'R_filter': 'R', - 'I_filter': 'I', - 'u_filter': 'up', - 'g_filter': 'gp', - 'r_filter': 'rp', - 'i_filter': 'ip', - 'z_filter': 'zs', - 'w_filter': 'w' - } + filters = ['U', 'B', 'V', 'R', 'I', 'u', 'g', 'r', 'i', 'z', 'w'] + cadence_type = forms.ChoiceField( + choices=[('once', 'Once in the next'), ('repeat', 'Repeating every')], + required=True + ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + + # Add fields for each available filter as specified in the filters property + for filter_name in self.filters: + self.fields[filter_name] = FilterField(label=filter_name, required=False) + + # Massage cadence form to be SNEx-styled + self.fields['cadence_strategy'].widget = forms.HiddenInput() + self.fields['cadence_strategy'].required = False + self.fields['cadence_frequency'].required = True + self.fields['cadence_frequency'].widget.attrs['readonly'] = False + self.fields['cadence_frequency'].widget.attrs['help_text'] = 'in hours' + + for field_name in ['exposure_time', 'exposure_count', 'start', 'end', 'filter']: + self.fields.pop(field_name) + if self.fields.get('groups'): + self.fields['groups'].label = 'Data granted to' + self.helper.layout = Layout( Div( Column('name'), @@ -483,65 +481,79 @@ def __init__(self, *args, **kwargs): self.layout(), self.button_layout() ) - for field_name in ['cadence_type', 'cadence_strategy', 'cadence_frequency']: - self.fields[field_name].required = False - for field_name in ['exposure_time', 'exposure_count', 'start', 'end', 'filter']: - self.fields.pop(field_name) - if self.fields.get('groups'): - self.fields['groups'].label = 'Data granted to' def _build_instrument_config(self): + """ + Because the photometric sequence form provides form inputs for 10 different filters, they must be + constructed into a list of instrument configurations as per the LCO API. This method constructs the + instrument configurations in the appropriate manner. + """ instrument_config = [] - for label, _ in self.filter_mapping.items(): - if len(self.cleaned_data[label]) > 0: + for filter_name in self.filters: + if len(self.cleaned_data[filter_name]) > 0: instrument_config.append({ - 'exposure_count': self.cleaned_data[label][1], - 'exposure_time': self.cleaned_data[label][0], + 'exposure_count': self.cleaned_data[filter_name][1], + 'exposure_time': self.cleaned_data[filter_name][0], 'optical_elements': { - 'filter': self.filter_mapping[label] + 'filter': filter_name } }) return instrument_config def clean(self): + """ + This clean method does the following: + - Adds a start time of "right now", as the photometric sequence form does not allow for specification + of a start time. + - Adds an end time that corresponds with the cadence frequency + - Adds the cadence strategy to the form if "repeat" was the selected "cadence_type". If "once" was + selected, the observation is submitted as a single observation. + """ cleaned_data = super().clean() now = datetime.now() cleaned_data['start'] = datetime.strftime(datetime.now(), '%Y-%m-%dT%H:%M:%S') - cadence_frequency = 24 if not cleaned_data.get('cadence_frequency') else cleaned_data.get('cadence_frequency') - cleaned_data['end'] = datetime.strftime(now + timedelta(hours=cadence_frequency), '%Y-%m-%dT%H:%M:%S') + cleaned_data['end'] = datetime.strftime(now + timedelta(hours=cleaned_data['cadence_frequency']), + '%Y-%m-%dT%H:%M:%S') if cleaned_data['cadence_type'] == 'repeat': cleaned_data['cadence_strategy'] = 'Resume Cadence After Failure' return cleaned_data def instrument_choices(self): + """ + This method returns only the instrument choices available in the current SNEx photometric sequence form. + """ return [i for i in super().instrument_choices() if i[0] in ['1M0-SCICAM-SINISTRO', '0M4-SCICAM-SBIG', '2M0-SPECTRAL-AG']] + def cadence_layout(self): + return Layout( + Row( + Column('cadence_type'), Column('cadence_frequency') + ) + ) + def layout(self): if settings.TARGET_PERMISSIONS_ONLY: groups = Div() else: groups = Row('groups') + + # Add filters to layout + filter_layout = Layout( + Row( + Column(HTML('Exposure Time')), + Column(HTML('No. of Exposures')), + Column(HTML('Block No.')), + ) + ) + for filter_name in self.filters: + filter_layout.append(Row(filter_name)) + return Div( Div( - Row( - Column(HTML('Exposure Time')), - Column(HTML('No. of Exposures')), - Column(HTML('Block No.')), - ), - Row('U_filter'), - Row('B_filter'), - Row('V_filter'), - Row('R_filter'), - Row('I_filter'), - Row('u_filter'), - Row('g_filter'), - Row('r_filter'), - Row('i_filter'), - Row('z_filter'), - Row('w_filter'), + filter_layout, css_class='col-md-6' ), Div( diff --git a/tom_observations/views.py b/tom_observations/views.py index e56639c99..390ad7bf4 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -170,6 +170,7 @@ def get_context_data(self, **kwargs): initial = self.get_initial() for k, v in self.get_facility_class().observation_forms.items(): form_data = {**initial, **{'observation_type': k}} + # Repopulate the appropriate form with form data if the original submission was invalid if k == self.request.POST.get('observation_type'): form_data.update(**self.request.POST.dict()) observation_type_choices.append((k, v(initial=form_data))) From 7d2a60e28453bcf1c6dc4735df2a5400778e3067 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 17 Aug 2020 16:17:11 -0700 Subject: [PATCH 258/424] Added docstring for display_obs_type --- tom_observations/templatetags/observation_extras.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tom_observations/templatetags/observation_extras.py b/tom_observations/templatetags/observation_extras.py index 7d618a10d..318ec162e 100644 --- a/tom_observations/templatetags/observation_extras.py +++ b/tom_observations/templatetags/observation_extras.py @@ -21,6 +21,10 @@ @register.filter def display_obs_type(value): + """ + This converts SAMPLE_TITLE into Sample Title. Used for display all-caps observation type in the + tabs as titles. + """ return value.replace('_', ' ').title() From f0b68b59f1fcde4756a54dc2f03dc8072aef86d9 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 17 Aug 2020 16:34:24 -0700 Subject: [PATCH 259/424] Added skeleton of spectroscopic sequence form --- tom_observations/facilities/lco.py | 72 +++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index eab3cfff6..0f8472ed6 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -572,6 +572,75 @@ def layout(self): ) +class LCOSpectroscopicSequenceForm(LCOBaseObservationForm): + site = forms.ChoiceField(choices=(('any', 'Any'), ('ogg', 'Hawaii'), ('coj', 'Australia'))) + acquisition_radius = forms.FloatField(min_value=0) + guider_mode = forms.BooleanField(widget=forms.Select(choices=[(True, 'On'), (False, 'Optional')]), required=True) + guider_exposure_time = forms.IntegerField(min_value=0) + cadence_type = forms.ChoiceField( + choices=[('once', 'Once in the next'), ('repeat', 'Repeating every')], + required=True + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Massage cadence form to be SNEx-styled + self.fields['cadence_strategy'].widget = forms.HiddenInput() + self.fields['cadence_strategy'].required = False + self.fields['cadence_frequency'].required = True + self.fields['cadence_frequency'].widget.attrs['readonly'] = False + self.fields['cadence_frequency'].widget.attrs['help_text'] = 'in hours' + + for field_name in ['exposure_count', 'start', 'end']: + self.fields.pop(field_name) + if self.fields.get('groups'): + self.fields['groups'].label = 'Data granted to' + + self.helper.layout = Layout( + Div( + Column('name'), + Column('cadence_type'), + Column('cadence_frequency'), + css_class='form-row' + ), + Layout('facility', 'target_id', 'observation_type'), + self.layout(), + self.button_layout() + ) + + # def clean(self): + # cleaned_data = super().clean() + # now = datetime.now() + # cleaned_data['start'] = datetime.strftime(datetime.now(), '%Y-%m-%dT%H:%M:%S') + # cadence_frequency = 24 if not cleaned_data.get('cadence_frequency') else cleaned_data.get('cadence_frequency') + # cleaned_data['end'] = datetime.strftime(now + timedelta(hours=cadence_frequency), '%Y-%m-%dT%H:%M:%S') + # if cleaned_data['cadence_type'] == 'repeat': + # cleaned_data['cadence_strategy'] = 'Resume Cadence After Failure' + + # return cleaned_data + + def layout(self): + if settings.TARGET_PERMISSIONS_ONLY: + groups = Div() + else: + groups = Row('groups') + return Div( + Row('exposure_time'), + Row('max_airmass'), + Row(PrependedText('min_lunar_distance', '>')), + Row('site'), + Row('filter'), # TODO: convert this to slit, figure out what instrument to use + Row('acquisition_radius'), + Row('guider_mode'), # TODO: ensure this gets the correct boolean value + Row('guider_exposure_time'), + Row('proposal'), + Row('observation_mode'), + Row('ipp_value'), + groups, + ) + + class LCOObservingStrategyForm(GenericStrategyForm, LCOBaseForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -606,7 +675,8 @@ class LCOFacility(BaseRoboticObservationFacility): observation_forms = { 'IMAGING': LCOImagingObservationForm, 'SPECTRA': LCOSpectroscopyObservationForm, - 'PHOTOMETRIC_SEQUENCE': LCOPhotometricSequenceForm + 'PHOTOMETRIC_SEQUENCE': LCOPhotometricSequenceForm, + 'SPECTROSCOPIC_SEQUENCE': LCOSpectroscopicSequenceForm } # The SITES dictionary is used to calculate visibility intervals in the # planning tool. All entries should contain latitude, longitude, elevation From 8b7a958f01aba4a356eba59983ec1d366b73a333 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 18 Aug 2020 09:06:23 -0700 Subject: [PATCH 260/424] Updated moon distance plot to only render for sidereal targets --- tom_targets/templatetags/targets_extras.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tom_targets/templatetags/targets_extras.py b/tom_targets/templatetags/targets_extras.py index 95fcee842..c9b9f8cbc 100644 --- a/tom_targets/templatetags/targets_extras.py +++ b/tom_targets/templatetags/targets_extras.py @@ -116,7 +116,7 @@ def target_plan(context): @register.inclusion_tag('tom_targets/partials/moon_distance.html') def moon_distance(target, day_range=30): """ - Renders plot for lunar distance from target. + Renders plot for lunar distance from sidereal target. Adapted from Jamison Frost Burke's moon visibility code in Supernova Exchange 2.0, as seen here: https://github.com/jfrostburke/snex2/blob/0c1eb184c942cb10f7d54084e081d8ac11700edf/custom_code/templatetags/custom_code_tags.py#L196 @@ -127,6 +127,8 @@ def moon_distance(target, day_range=30): :param day_range: Number of days to plot lunar distance :type day_range: int """ + if target.type != 'SIDEREAL': + return {'plot': None} day_range = 30 times = Time( From 422a824995e581e8eedef13327e9ae2d26d9b22c Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 18 Aug 2020 17:13:41 -0700 Subject: [PATCH 261/424] Added user relationship to obsr, added logic to create relationship after submission, updated tests --- .../migrations/0009_observationrecord_user.py | 21 +++++++++++++++++++ tom_observations/models.py | 2 ++ tom_observations/tests/tests.py | 8 ++++--- tom_observations/views.py | 1 + 4 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 tom_observations/migrations/0009_observationrecord_user.py diff --git a/tom_observations/migrations/0009_observationrecord_user.py b/tom_observations/migrations/0009_observationrecord_user.py new file mode 100644 index 000000000..5da977a4c --- /dev/null +++ b/tom_observations/migrations/0009_observationrecord_user.py @@ -0,0 +1,21 @@ +# Generated by Django 3.1 on 2020-08-18 20:34 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('tom_observations', '0008_observationgroup_cadence_parameters'), + ] + + operations = [ + migrations.AddField( + model_name='observationrecord', + name='user', + field=models.ForeignKey(default=None, null=True, on_delete=django.db.models.deletion.DO_NOTHING, to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/tom_observations/models.py b/tom_observations/models.py index 7bd216c3e..6cd6c0d8c 100644 --- a/tom_observations/models.py +++ b/tom_observations/models.py @@ -1,3 +1,4 @@ +from django.contrib.auth.models import User from django.db import models import json @@ -41,6 +42,7 @@ class ObservationRecord(models.Model): :type modified: datetime """ target = models.ForeignKey(Target, on_delete=models.CASCADE) + user = models.ForeignKey(User, null=True, default=None, on_delete=models.DO_NOTHING) facility = models.CharField(max_length=50) parameters = models.TextField() observation_id = models.CharField(max_length=255) diff --git a/tom_observations/tests/tests.py b/tom_observations/tests/tests.py index ca264d61e..ad4af63ff 100644 --- a/tom_observations/tests/tests.py +++ b/tom_observations/tests/tests.py @@ -29,10 +29,10 @@ def setUp(self): facility=FakeRoboticFacility.name, parameters='{}' ) - user = User.objects.create_user(username='vincent_adultman', password='important') + self.user = User.objects.create_user(username='vincent_adultman', password='important') self.user2 = User.objects.create_user(username='peon', password='plebian') - assign_perm('tom_targets.view_target', user, self.target) - self.client.force_login(user) + assign_perm('tom_targets.view_target', self.user, self.target) + self.client.force_login(self.user) def test_observation_list(self): response = self.client.get(reverse('tom_observations:list')) @@ -118,6 +118,7 @@ def test_submit_observation_robotic(self): follow=True ) self.assertTrue(ObservationRecord.objects.filter(observation_id='fakeid').exists()) + self.assertEqual(ObservationRecord.objects.filter(observation_id='fakeid').first().user, self.user) def test_submit_observation_manual(self): form_data = { @@ -129,6 +130,7 @@ def test_submit_observation_manual(self): f"?target_id={self.target.id}" self.client.post(url, data=form_data, follow=True) self.assertTrue(ObservationRecord.objects.filter(observation_id='fakeid').exists()) + self.assertEqual(ObservationRecord.objects.filter(observation_id='fakeid').first().user, self.user) @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], diff --git a/tom_observations/views.py b/tom_observations/views.py index 390ad7bf4..39950e0d2 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -249,6 +249,7 @@ def form_valid(self, form): # Create Observation record record = ObservationRecord.objects.create( target=target, + user=self.request.user, facility=facility.name, parameters=form.serialize_parameters(), observation_id=observation_id From ce59a0aa30dcc25a88e0118b2ec42066feb4d936 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 19 Aug 2020 16:28:34 -0700 Subject: [PATCH 262/424] Update examples.rst --- docs/examples.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/examples.rst b/docs/examples.rst index ebc8f4a05..4f36b8800 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -43,6 +43,23 @@ LCO is rewriting an existing piece of software that automatically schedules nightly telescope calibrations using the TOM Toolkit called the `Calibration TOM `__. +PANOPTES TOM +~~~~~~~~~~~~ + +The `PANOPTES TOM `__ is being +built to enable their community to coordinate observations for the +`PANOPTES citizen science project `__, which +aims to detect transiting exoplanets. + +MOP +~~~ + +AMON TOM +~~~~~~~~ + +Black Hole TOM +~~~~~~~~~~~~~~ + Others ~~~~~~ From 400d6294115d3dea19d02b5f16b51be83b9bfdf0 Mon Sep 17 00:00:00 2001 From: Rachel Street Date: Thu, 20 Aug 2020 09:00:46 -0700 Subject: [PATCH 263/424] Added text to describe microlensing TOMs --- docs/examples.rst | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/examples.rst b/docs/examples.rst index 4f36b8800..95fbfef70 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -22,12 +22,11 @@ resulting images. Originally built from scratch, it’s being rewritten using the TOM Toolkit, which will allow the underlying TOM to be used with multiple front-ends for completely different educational purposes. -Microlensing TOM -~~~~~~~~~~~~~~~~ +Microlensing TOM (MOP) +~~~~~~~~~~~~~~~~~~~~~~ + +The `Microlensing Observing Platform `__ is the core interface of the OMEGA Key Project. It is designed to harvest and prioritize microlensing events from various surveys, then submit additional observations with the Las Cumbres Observatory telescopes automatically. -The `Microlensing TOM `__ is -being written in order to identify microlensing events from ZTF and -conduct follow-up observations. PhotTOM ~~~~~~~ @@ -51,14 +50,13 @@ built to enable their community to coordinate observations for the `PANOPTES citizen science project `__, which aims to detect transiting exoplanets. -MOP -~~~ AMON TOM ~~~~~~~~ Black Hole TOM ~~~~~~~~~~~~~~ +Black Hole TOM (BHTOM) aims at coordinating the photometric and spectroscopic follow-up observations of targets requiring long-term monitoring. This includes long lasting microlensing events reported by Gaia and other surveys, likely caused by galactic black holes. The system lists targets according to their priorities and allows for triggering robotic observations. It also allows users of any partner observatory to submit their raw photometric and spectroscopic data, which gets automatically processed and calibrated. BHTOM is developed as part of the Time Domain Astronony work package of the European OPTICON grant by the team at the University of Warsaw, Poland, with support from LCO. Website: http://visata.astrouw.edu.pl:8080 Others ~~~~~~ From 24af3379ab91db5afdb5cceebe08c06e063e8c4f Mon Sep 17 00:00:00 2001 From: fraserw Date: Thu, 20 Aug 2020 13:48:08 -0700 Subject: [PATCH 264/424] --added input of ephemeris uncertainties --- tom_targets/utils.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tom_targets/utils.py b/tom_targets/utils.py index 1c8f4d3b6..f5e3b0068 100644 --- a/tom_targets/utils.py +++ b/tom_targets/utils.py @@ -127,6 +127,8 @@ def import_ephemeris_target(stream): jpl_ra_key = 'R.A._____(ICRF)_____DEC' jpl_jd_key = 'Date_________JDUT' + jpl_dr_key = 'RA_3sigma' + jpl_dd_key = 'DEC_3sigma' eph = stream.getvalue().split('\n') @@ -147,6 +149,8 @@ def import_ephemeris_target(stream): name = 'custom' jd_inds = None ra_inds = None + dr_inds = None + dd_inds = None loop_inds = [-1, -1] for i in range(end_ind, len(eph)): if 'Center-site name' in eph[i]: @@ -162,9 +166,11 @@ def import_ephemeris_target(stream): if 'Target body name' in eph[i]: name = "-".join(eph[i].split(': ')[1].split('{source')[0].split()) - if jpl_ra_key in eph[i] and jpl_jd_key in eph[i]: + if jpl_ra_key in eph[i] and jpl_jd_key in eph[i] and jpl_dr_key in eph[i] and jpl_dd_key in eph[i]: ra_inds = [eph[i].index(jpl_ra_key), eph[i].index(jpl_ra_key)+len(jpl_ra_key)] jd_inds = [eph[i].index(jpl_jd_key), eph[i].index(jpl_jd_key)+len(jpl_jd_key)] + dr_inds = [eph[i].index(jpl_dr_key), eph[i].index(jpl_dr_key)+len(jpl_dr_key)] + dd_inds = [eph[i].index(jpl_dd_key), eph[i].index(jpl_dd_key)+len(jpl_dd_key)] if '$$SOE' in eph[i]: if ra_inds is not None and loop_inds[0] == -1: loop_inds[0] = i+1 @@ -188,18 +194,24 @@ def import_ephemeris_target(stream): mjds = [] ras = [] decs = [] + drs = [] + dds = [] R = 0.0 D = 0.0 n = 0.0 for i in range(loop_inds[0], loop_inds[1]): mjds.append(str(float(eph[i][jd_inds[0]:jd_inds[1]])-2400000.5)) + s = eph[i][ra_inds[0]:ra_inds[1]].split() r = 15.0*(float(s[0])+float(s[1])/60.0+float(s[2])/3600.0) - ras.append("{:11.7f}".format(r)) + ras.append("{:.7f}".format(r)) d = abs(float(s[3]))+float(s[4])/60.0+float(s[5])/3600.0 if '-' in s[3]: d *= -1.0 - decs.append("{:10.6f}".format(d)) + decs.append("{:.6f}".format(d)) + + drs.append('{:.7f}'.format( float(eph[i][dr_inds[0]:dr_inds[1]])/3600.0 )) + dds.append('{:.6f}'.format( float(eph[i][dd_inds[0]:dd_inds[1]])/3600.0 )) R += r D += d @@ -211,8 +223,8 @@ def import_ephemeris_target(stream): entry['t'] = mjds[i] entry['R'] = ras[i] entry['D'] = decs[i] - entry['dR'] = 0.0 - entry['dD'] = 0.0 + entry['dR'] = drs[i] + entry['dD'] = dds[i] eph_json[centre_site_name].append(entry) From 9f3c6246c53b4162726dd7d0faa9a762bb267b03 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 27 Aug 2020 15:10:17 -0700 Subject: [PATCH 265/424] Only displaying comments for targets for which logged-in user has permission to view --- tom_common/templatetags/tom_common_extras.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tom_common/templatetags/tom_common_extras.py b/tom_common/templatetags/tom_common_extras.py index 8e2c73a08..8d0179d94 100644 --- a/tom_common/templatetags/tom_common_extras.py +++ b/tom_common/templatetags/tom_common_extras.py @@ -1,6 +1,7 @@ from django import template from django.conf import settings from django_comments.models import Comment +from guardian.shortcuts import get_objects_for_user register = template.Library() @@ -24,12 +25,18 @@ def verbose_name(instance, field_name): return instance._meta.get_field(field_name).verbose_name.title() -@register.inclusion_tag('comments/list.html') -def recent_comments(limit=10): +@register.inclusion_tag('comments/list.html', takes_context=True) +def recent_comments(context, limit=10): """ Displays a list of the most recent comments in the TOM up to the given limit, or 10 if not specified. + + Comments will only be displayed for targets which the logged-in user has permission to view. """ - return {'comment_list': Comment.objects.all().order_by('-submit_date')[:limit]} + user = context['request'].user + targets_for_user = get_objects_for_user(user, 'tom_targets.view_target') + return { + 'comment_list': Comment.objects.filter(object_pk__in=targets_for_user).order_by('-submit_date')[:limit] + } @register.filter From 201575af4ad2f466bf59aaca483aaf645928d656 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 27 Aug 2020 15:12:42 -0700 Subject: [PATCH 266/424] Potentially fixed ordering issues --- tom_observations/facilities/lco.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index eab3cfff6..13149a98b 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -1,3 +1,4 @@ +from collections import OrderedDict from datetime import datetime, timedelta import requests @@ -112,7 +113,7 @@ def _get_instruments(self): PORTAL_URL + '/api/instruments/', headers={'Authorization': 'Token {0}'.format(LCO_SETTINGS['api_key'])} ) - cached_instruments = {k: v for k, v in response.json().items() if 'SOAR' not in k} + cached_instruments = OrderedDict((k, v) for k, v in response.json().items() if 'SOAR' not in k) cache.set('lco_instruments', cached_instruments) return cached_instruments From 309ce170550765a648b52225389e9a3b60a79172 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 27 Aug 2020 16:35:55 -0700 Subject: [PATCH 267/424] Added some boilerplate for configuring more specific parts of observation payloads --- tom_observations/facilities/lco.py | 67 ++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 0f8472ed6..787754ea6 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -298,6 +298,13 @@ def _build_instrument_config(self): return [instrument_config] + def _build_guiding_config(self): + guiding_config = { + + } + + return guiding_config + def _build_configuration(self): return { 'type': self.instrument_to_type(self.cleaned_data['instrument_type']), @@ -307,14 +314,15 @@ def _build_configuration(self): 'acquisition_config': { }, - 'guiding_config': { - - }, + 'guiding_config': self._build_guiding_config(), 'constraints': { 'max_airmass': self.cleaned_data['max_airmass'] } } + def _build_location(self): + return {'telescope_class': self._get_instruments()[self.cleaned_data['instrument_type']]['class']} + def _expand_cadence_request(self, payload): payload['requests'][0]['cadence'] = { 'start': self.cleaned_data['start'], @@ -347,16 +355,15 @@ def observation_payload(self): "end": self.cleaned_data['end'] } ], - "location": { - "telescope_class": self._get_instruments()[self.cleaned_data['instrument_type']]['class'] - } + "location": self._build_location() } ] } if self.cleaned_data.get('period') and self.cleaned_data.get('jitter'): payload = self._expand_cadence_request(payload) - return payload + print(payload) + # return payload class LCOImagingObservationForm(LCOBaseObservationForm): @@ -573,7 +580,7 @@ def layout(self): class LCOSpectroscopicSequenceForm(LCOBaseObservationForm): - site = forms.ChoiceField(choices=(('any', 'Any'), ('ogg', 'Hawaii'), ('coj', 'Australia'))) + site = forms.ChoiceField(choices=(('None', 'Any'), ('ogg', 'Hawaii'), ('coj', 'Australia'))) acquisition_radius = forms.FloatField(min_value=0) guider_mode = forms.BooleanField(widget=forms.Select(choices=[(True, 'On'), (False, 'Optional')]), required=True) guider_exposure_time = forms.IntegerField(min_value=0) @@ -601,7 +608,7 @@ def __init__(self, *args, **kwargs): Div( Column('name'), Column('cadence_type'), - Column('cadence_frequency'), + Column('cadence_frequency'), # TODO: Add placeholder text for "Once in the next", etc css_class='form-row' ), Layout('facility', 'target_id', 'observation_type'), @@ -609,16 +616,40 @@ def __init__(self, *args, **kwargs): self.button_layout() ) - # def clean(self): - # cleaned_data = super().clean() - # now = datetime.now() - # cleaned_data['start'] = datetime.strftime(datetime.now(), '%Y-%m-%dT%H:%M:%S') - # cadence_frequency = 24 if not cleaned_data.get('cadence_frequency') else cleaned_data.get('cadence_frequency') - # cleaned_data['end'] = datetime.strftime(now + timedelta(hours=cadence_frequency), '%Y-%m-%dT%H:%M:%S') - # if cleaned_data['cadence_type'] == 'repeat': - # cleaned_data['cadence_strategy'] = 'Resume Cadence After Failure' + def _build_guiding_config(self): + # TODO: This + print(self.cleaned_data) + return { + + } + + def _build_location(self): + location = super()._build_location() + site = self.cleaned_data['site'] + if site: + location['site'] = site + return location + + def clean(self): + """ + This clean method does the following: + - Adds a start time of "right now", as the spectroscopic sequence form does not allow for specification + of a start time. + - Adds an end time that corresponds with the cadence frequency + - Adds the cadence strategy to the form if "repeat" was the selected "cadence_type". If "once" was + selected, the observation is submitted as a single observation. + """ + cleaned_data = super().clean() + cleaned_data['instrument_type'] = '1M0-SCICAM-SINISTRO' + cleaned_data['exposure_count'] = 1 + now = datetime.now() + cleaned_data['start'] = datetime.strftime(datetime.now(), '%Y-%m-%dT%H:%M:%S') + cleaned_data['end'] = datetime.strftime(now + timedelta(hours=cleaned_data['cadence_frequency']), + '%Y-%m-%dT%H:%M:%S') + if cleaned_data['cadence_type'] == 'repeat': + cleaned_data['cadence_strategy'] = 'Resume Cadence After Failure' - # return cleaned_data + return cleaned_data def layout(self): if settings.TARGET_PERMISSIONS_ONLY: From 76b2237ae9dbba94dd353aefd9eb95782a2d6960 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 28 Aug 2020 11:40:12 -0700 Subject: [PATCH 268/424] Updated specseq form to be 100% complete --- tom_observations/facilities/lco.py | 75 +++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 21 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 787754ea6..af4525d96 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -2,7 +2,7 @@ import requests from astropy import units as u -from crispy_forms.bootstrap import PrependedText +from crispy_forms.bootstrap import AppendedText, PrependedText from crispy_forms.layout import Column, Div, HTML, Layout, Row from dateutil.parser import parse from django import forms @@ -105,6 +105,7 @@ def __init__(self, *args, **kwargs): def _get_instruments(self): cached_instruments = cache.get('lco_instruments') + cached_instruments = None if not cached_instruments: response = make_request( @@ -298,11 +299,14 @@ def _build_instrument_config(self): return [instrument_config] + def _build_acquisition_config(self): + acquisition_config = {} + + return acquisition_config + def _build_guiding_config(self): - guiding_config = { + guiding_config = {} - } - return guiding_config def _build_configuration(self): @@ -311,9 +315,7 @@ def _build_configuration(self): 'instrument_type': self.cleaned_data['instrument_type'], 'target': self._build_target_fields(), 'instrument_configs': self._build_instrument_config(), - 'acquisition_config': { - - }, + 'acquisition_config': self._build_acquisition_config(), 'guiding_config': self._build_guiding_config(), 'constraints': { 'max_airmass': self.cleaned_data['max_airmass'] @@ -580,26 +582,32 @@ def layout(self): class LCOSpectroscopicSequenceForm(LCOBaseObservationForm): - site = forms.ChoiceField(choices=(('None', 'Any'), ('ogg', 'Hawaii'), ('coj', 'Australia'))) + site = forms.ChoiceField(choices=(('any', 'Any'), ('ogg', 'Hawaii'), ('coj', 'Australia'))) acquisition_radius = forms.FloatField(min_value=0) - guider_mode = forms.BooleanField(widget=forms.Select(choices=[(True, 'On'), (False, 'Optional')]), required=True) + guider_mode = forms.ChoiceField(choices=[('on', 'On'), ('off', 'Off'), ('optional', 'Optional')], required=True) guider_exposure_time = forms.IntegerField(min_value=0) cadence_type = forms.ChoiceField( choices=[('once', 'Once in the next'), ('repeat', 'Repeating every')], - required=True + required=True, + label='' ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # Massage cadence form to be SNEx-styled + self.fields['name'].label = '' + self.fields['name'].widget.attrs['placeholder'] = 'Name' + self.fields['min_lunar_distance'].widget.attrs['placeholder'] = 'Degrees' self.fields['cadence_strategy'].widget = forms.HiddenInput() self.fields['cadence_strategy'].required = False self.fields['cadence_frequency'].required = True + self.fields['cadence_frequency'].label = '' self.fields['cadence_frequency'].widget.attrs['readonly'] = False - self.fields['cadence_frequency'].widget.attrs['help_text'] = 'in hours' + self.fields['cadence_frequency'].widget.attrs['placeholder'] = 'Hours' + self.fields['cadence_frequency'].help_text = None - for field_name in ['exposure_count', 'start', 'end']: + for field_name in ['start', 'end']: self.fields.pop(field_name) if self.fields.get('groups'): self.fields['groups'].label = 'Data granted to' @@ -608,7 +616,7 @@ def __init__(self, *args, **kwargs): Div( Column('name'), Column('cadence_type'), - Column('cadence_frequency'), # TODO: Add placeholder text for "Once in the next", etc + Column(AppendedText('cadence_frequency', 'Hours')), css_class='form-row' ), Layout('facility', 'target_id', 'observation_type'), @@ -616,17 +624,31 @@ def __init__(self, *args, **kwargs): self.button_layout() ) + def _build_acquisition_config(self): + # SNEx uses WCS mode if no acquisition radius is specified, and BRIGHTEST otherwise + acquisition_mode = 'BRIGHTEST' + if not self.cleaned_data['acquisition_radius']: + acquisition_mode = 'WCS' + + return { + 'mode': acquisition_mode, + 'extra_params': { + 'acquire_radius': self.cleaned_data['acquisition_radius'] + } + } + def _build_guiding_config(self): - # TODO: This - print(self.cleaned_data) + mode = 'ON' if self.cleaned_data['guider_mode'] in ['on', 'optional'] else 'OFF' + optional = 'true' if self.cleaned_data['guider_mode'] == 'optional' else 'false' return { - + 'mode': mode, + 'optional': optional } def _build_location(self): location = super()._build_location() site = self.cleaned_data['site'] - if site: + if site != 'any': location['site'] = site return location @@ -640,8 +662,7 @@ def clean(self): selected, the observation is submitted as a single observation. """ cleaned_data = super().clean() - cleaned_data['instrument_type'] = '1M0-SCICAM-SINISTRO' - cleaned_data['exposure_count'] = 1 + self.cleaned_data['instrument_type'] = '2M0-FLOYDS-SCICAM' # SNEx only submits spectra to FLOYDS now = datetime.now() cleaned_data['start'] = datetime.strftime(datetime.now(), '%Y-%m-%dT%H:%M:%S') cleaned_data['end'] = datetime.strftime(now + timedelta(hours=cleaned_data['cadence_frequency']), @@ -651,19 +672,31 @@ def clean(self): return cleaned_data + def instrument_choices(self): + # SNEx only uses the Spectroscopic Sequence Form with FLOYDS + return [(k, v['name']) for k, v in self._get_instruments().items() if k == '2M0-FLOYDS-SCICAM'] + + def filter_choices(self): + # SNEx only uses the Spectroscopic Sequence Form with FLOYDS + return set([ + (f['code'], f['name']) for name, ins in self._get_instruments().items() for f in + ins['optical_elements'].get('slits', []) if name == '2M0-FLOYDS-SCICAM' + ]) + def layout(self): if settings.TARGET_PERMISSIONS_ONLY: groups = Div() else: groups = Row('groups') return Div( + Row('exposure_count'), Row('exposure_time'), Row('max_airmass'), Row(PrependedText('min_lunar_distance', '>')), Row('site'), - Row('filter'), # TODO: convert this to slit, figure out what instrument to use + Row('filter'), Row('acquisition_radius'), - Row('guider_mode'), # TODO: ensure this gets the correct boolean value + Row('guider_mode'), Row('guider_exposure_time'), Row('proposal'), Row('observation_mode'), From 793e495e62d9d77a6eba57155cbc61d59d1c7a9b Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 28 Aug 2020 11:49:35 -0700 Subject: [PATCH 269/424] Fixed issue with slit --- tom_observations/facilities/lco.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index af4525d96..76e33ca64 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -364,8 +364,7 @@ def observation_payload(self): if self.cleaned_data.get('period') and self.cleaned_data.get('jitter'): payload = self._expand_cadence_request(payload) - print(payload) - # return payload + return payload class LCOImagingObservationForm(LCOBaseObservationForm): @@ -624,6 +623,14 @@ def __init__(self, *args, **kwargs): self.button_layout() ) + def _build_instrument_config(self): + instrument_configs = super()._build_instrument_config() + instrument_configs[0]['optical_elements'] = { + 'slit': self.cleaned_data['filter'] + } + + return instrument_configs + def _build_acquisition_config(self): # SNEx uses WCS mode if no acquisition radius is specified, and BRIGHTEST otherwise acquisition_mode = 'BRIGHTEST' From 445358a70b0d77ad98689dab04bf097fffcfb3ff Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 28 Aug 2020 12:19:23 -0700 Subject: [PATCH 270/424] Removing cache bypass --- tom_observations/facilities/lco.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 1deca3b11..f3efa45bc 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -106,7 +106,6 @@ def __init__(self, *args, **kwargs): def _get_instruments(self): cached_instruments = cache.get('lco_instruments') - cached_instruments = None if not cached_instruments: response = make_request( From 1ef6949d80a7182d073a5468022fe735d7d6364c Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 28 Aug 2020 13:52:03 -0700 Subject: [PATCH 271/424] Fixed sorting for all forms --- tom_observations/facilities/lco.py | 31 +++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index f3efa45bc..97326e47d 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -119,13 +119,13 @@ def _get_instruments(self): return cached_instruments def instrument_choices(self): - return [(k, v['name']) for k, v in self._get_instruments().items()] + return sorted([(k, v['name']) for k, v in self._get_instruments().items()], key=lambda inst: inst[1]) def filter_choices(self): - return set([ + return sorted(set([ (f['code'], f['name']) for ins in self._get_instruments().values() for f in ins['optical_elements'].get('filters', []) + ins['optical_elements'].get('slits', []) - ]) + ]), key=lambda filter_tuple: filter_tuple[1]) def proposal_choices(self): response = make_request( @@ -373,13 +373,14 @@ class LCOImagingObservationForm(LCOBaseObservationForm): Imagers and their details can be found here: https://lco.global/observatory/instruments/ """ def instrument_choices(self): - return [(k, v['name']) for k, v in self._get_instruments().items() if 'IMAGE' in v['type']] + return sorted([(k, v['name']) for k, v in self._get_instruments().items() if 'IMAGE' in v['type']], + key=lambda inst: inst[1]) def filter_choices(self): - return set([ + return sorted(set([ (f['code'], f['name']) for ins in self._get_instruments().values() for f in ins['optical_elements'].get('filters', []) - ]) + ]), key=lambda filter_tuple: filter_tuple[1]) class LCOSpectroscopyObservationForm(LCOBaseObservationForm): @@ -422,14 +423,16 @@ def layout(self): ) def instrument_choices(self): - return [(k, v['name']) for k, v in self._get_instruments().items() if 'SPECTRA' in v['type']] + return sorted([(k, v['name']) for k, v in self._get_instruments().items() if 'SPECTRA' in v['type']], + key=lambda inst: inst[1]) # NRES does not take a slit, and therefore needs an option of None def filter_choices(self): - return set([ + return sorted(set([ (f['code'], f['name']) for ins in self._get_instruments().values() for f in ins['optical_elements'].get('slits', []) - ] + [('None', 'None')]) + ] + [('None', 'None')]), + key=lambda filter_tuple: filter_tuple[1]) def _build_instrument_config(self): instrument_configs = super()._build_instrument_config() @@ -453,6 +456,7 @@ class LCOPhotometricSequenceForm(LCOBaseObservationForm): The form is modeled after the Supernova Exchange application's Photometric Sequence Request Form, and allows the configuration of multiple filters, as well as a more intuitive proactive cadence form. """ + valid_instruments = ['1M0-SCICAM-SINISTRO', '0M4-SCICAM-SBIG', '2M0-SPECTRAL-AG'] filters = ['U', 'B', 'V', 'R', 'I', 'u', 'g', 'r', 'i', 'z', 'w'] cadence_type = forms.ChoiceField( choices=[('once', 'Once in the next'), ('repeat', 'Repeating every')], @@ -532,8 +536,8 @@ def instrument_choices(self): """ This method returns only the instrument choices available in the current SNEx photometric sequence form. """ - return [i for i in super().instrument_choices() - if i[0] in ['1M0-SCICAM-SINISTRO', '0M4-SCICAM-SBIG', '2M0-SPECTRAL-AG']] + return sorted([(k, v['name']) for k, v in self._get_instruments().items() if k in self.valid_instruments], + key=lambda inst: inst[1]) def cadence_layout(self): return Layout( @@ -681,14 +685,15 @@ def clean(self): def instrument_choices(self): # SNEx only uses the Spectroscopic Sequence Form with FLOYDS + # This doesn't need to be sorted because it will only return one instrument return [(k, v['name']) for k, v in self._get_instruments().items() if k == '2M0-FLOYDS-SCICAM'] def filter_choices(self): # SNEx only uses the Spectroscopic Sequence Form with FLOYDS - return set([ + return sorted(set([ (f['code'], f['name']) for name, ins in self._get_instruments().items() for f in ins['optical_elements'].get('slits', []) if name == '2M0-FLOYDS-SCICAM' - ]) + ]), key=lambda filter_tuple: filter_tuple[1]) def layout(self): if settings.TARGET_PERMISSIONS_ONLY: From a5d1a1ba9ee37b311d5e2ad4ad71be5c689c6eb2 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 28 Aug 2020 14:01:50 -0700 Subject: [PATCH 272/424] Adding padding to forms so they look less bad --- tom_observations/static/tom_observations/css/main.css | 4 ++++ .../templates/tom_observations/observation_form.html | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tom_observations/static/tom_observations/css/main.css b/tom_observations/static/tom_observations/css/main.css index 72d4496b5..27e5495ee 100644 --- a/tom_observations/static/tom_observations/css/main.css +++ b/tom_observations/static/tom_observations/css/main.css @@ -18,3 +18,7 @@ span.featured { pointer-events: none; } + +div.observation-form { + padding: 20px 0px; +} \ No newline at end of file diff --git a/tom_observations/templates/tom_observations/observation_form.html b/tom_observations/templates/tom_observations/observation_form.html index 0791bfc62..f1c06dc9f 100644 --- a/tom_observations/templates/tom_observations/observation_form.html +++ b/tom_observations/templates/tom_observations/observation_form.html @@ -3,6 +3,7 @@ {% block title %}Submit Observation{% endblock %} {% block additional_css %} + {% endblock %} {% block content %} {{ form|as_crispy_errors }} @@ -34,7 +35,7 @@

    Submit an observation to {{ form.facility.value }}

    {% endfor %} -
    +
    {% for observation_type, observation_form in observation_type_choices %}
    {% crispy observation_form %} From 70bf3c85f6c4a2de5b1bfc307610ef5135d0a331 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 28 Aug 2020 14:09:38 -0700 Subject: [PATCH 273/424] Fixing two new Codacy issues --- tom_observations/static/tom_observations/css/main.css | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tom_observations/static/tom_observations/css/main.css b/tom_observations/static/tom_observations/css/main.css index 27e5495ee..0e0ad7bf8 100644 --- a/tom_observations/static/tom_observations/css/main.css +++ b/tom_observations/static/tom_observations/css/main.css @@ -20,5 +20,5 @@ span.featured { } div.observation-form { - padding: 20px 0px; -} \ No newline at end of file + padding-top: 20px; +} From aded2b01a46da20dd2e8a6e3d9a7cf0e13af233a Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 28 Aug 2020 14:59:04 -0700 Subject: [PATCH 274/424] Made PR changes from Lindy's feedback, just a little late --- tom_observations/facilities/lco.py | 135 +++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index eab3cfff6..3f76d2b51 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -572,6 +572,141 @@ def layout(self): ) +<<<<<<< Updated upstream +======= +class LCOSpectroscopicSequenceForm(LCOBaseObservationForm): + site = forms.ChoiceField(choices=(('any', 'Any'), ('ogg', 'Hawaii'), ('coj', 'Australia'))) + acquisition_radius = forms.FloatField(min_value=0) + guider_mode = forms.ChoiceField(choices=[('on', 'On'), ('off', 'Off'), ('optional', 'Optional')], required=True) + guider_exposure_time = forms.IntegerField(min_value=0) + cadence_type = forms.ChoiceField( + choices=[('once', 'Once in the next'), ('repeat', 'Repeating every')], + required=True, + label='' + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Massage cadence form to be SNEx-styled + self.fields['name'].label = '' + self.fields['name'].widget.attrs['placeholder'] = 'Name' + self.fields['min_lunar_distance'].widget.attrs['placeholder'] = 'Degrees' + self.fields['cadence_strategy'].widget = forms.HiddenInput() + self.fields['cadence_strategy'].required = False + self.fields['cadence_frequency'].required = True + self.fields['cadence_frequency'].label = '' + self.fields['cadence_frequency'].widget.attrs['readonly'] = False + self.fields['cadence_frequency'].widget.attrs['placeholder'] = 'Hours' + self.fields['cadence_frequency'].help_text = None + + # Remove start and end because those are determined by the cadence + for field_name in ['start', 'end']: + self.fields.pop(field_name) + if self.fields.get('groups'): + self.fields['groups'].label = 'Data granted to' + + self.helper.layout = Layout( + Div( + Column('name'), + Column('cadence_type'), + Column(AppendedText('cadence_frequency', 'Hours')), + css_class='form-row' + ), + Layout('facility', 'target_id', 'observation_type'), + self.layout(), + self.button_layout() + ) + + def _build_instrument_config(self): + instrument_configs = super()._build_instrument_config() + instrument_configs[0]['optical_elements'].pop('filter') + instrument_configs[0]['optical_elements']['slit'] = self.cleaned_data['filter'] + + return instrument_configs + + def _build_acquisition_config(self): + acquisition_config = super()._build_acquisition_config() + # SNEx uses WCS mode if no acquisition radius is specified, and BRIGHTEST otherwise + acquisition_mode = 'BRIGHTEST' + if not self.cleaned_data['acquisition_radius']: + acquisition_mode = 'WCS' + acquisition_config['mode'] = acquisition_mode + acquisition_config['extra_params'] = { + 'acquire_radius': self.cleaned_data['acquisition_radius'] + } + + return acquisition_config + + def _build_guiding_config(self): + guiding_config = super()._build_guiding_config() + guiding_config['mode'] = 'ON' if self.cleaned_data['guider_mode'] in ['on', 'optional'] else 'OFF' + guiding_config['optional'] = 'true' if self.cleaned_data['guider_mode'] == 'optional' else 'false' + return guiding_config + + def _build_location(self): + location = super()._build_location() + site = self.cleaned_data['site'] + if site != 'any': + location['site'] = site + return location + + def clean(self): + """ + This clean method does the following: + - Hardcodes instrument type as "2M0-FLOYDS-SCICAM" because it's the only instrument this form uses + - Adds a start time of "right now", as the spectroscopic sequence form does not allow for specification + of a start time. + - Adds an end time that corresponds with the cadence frequency + - Adds the cadence strategy to the form if "repeat" was the selected "cadence_type". If "once" was + selected, the observation is submitted as a single observation. + """ + cleaned_data = super().clean() + self.cleaned_data['instrument_type'] = '2M0-FLOYDS-SCICAM' # SNEx only submits spectra to FLOYDS + now = datetime.now() + cleaned_data['start'] = datetime.strftime(datetime.now(), '%Y-%m-%dT%H:%M:%S') + cleaned_data['end'] = datetime.strftime(now + timedelta(hours=cleaned_data['cadence_frequency']), + '%Y-%m-%dT%H:%M:%S') + if cleaned_data['cadence_type'] == 'repeat': + cleaned_data['cadence_strategy'] = 'Resume Cadence After Failure' + + return cleaned_data + + def instrument_choices(self): + # SNEx only uses the Spectroscopic Sequence Form with FLOYDS + # This doesn't need to be sorted because it will only return one instrument + return [(k, v['name']) for k, v in self._get_instruments().items() if k == '2M0-FLOYDS-SCICAM'] + + def filter_choices(self): + # SNEx only uses the Spectroscopic Sequence Form with FLOYDS + return sorted(set([ + (f['code'], f['name']) for name, ins in self._get_instruments().items() for f in + ins['optical_elements'].get('slits', []) if name == '2M0-FLOYDS-SCICAM' + ]), key=lambda filter_tuple: filter_tuple[1]) + + def layout(self): + if settings.TARGET_PERMISSIONS_ONLY: + groups = Div() + else: + groups = Row('groups') + return Div( + Row('exposure_count'), + Row('exposure_time'), + Row('max_airmass'), + Row(PrependedText('min_lunar_distance', '>')), + Row('site'), + Row('filter'), + Row('acquisition_radius'), + Row('guider_mode'), + Row('guider_exposure_time'), + Row('proposal'), + Row('observation_mode'), + Row('ipp_value'), + groups, + ) + + +>>>>>>> Stashed changes class LCOObservingStrategyForm(GenericStrategyForm, LCOBaseForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) From 8da37c2a6c2d2d83ad1bd6798e8d89f15331b27e Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 28 Aug 2020 15:05:06 -0700 Subject: [PATCH 275/424] Fixing merge conflict --- tom_observations/facilities/lco.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 3f76d2b51..20dce2d73 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -572,8 +572,6 @@ def layout(self): ) -<<<<<<< Updated upstream -======= class LCOSpectroscopicSequenceForm(LCOBaseObservationForm): site = forms.ChoiceField(choices=(('any', 'Any'), ('ogg', 'Hawaii'), ('coj', 'Australia'))) acquisition_radius = forms.FloatField(min_value=0) @@ -706,7 +704,6 @@ def layout(self): ) ->>>>>>> Stashed changes class LCOObservingStrategyForm(GenericStrategyForm, LCOBaseForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) From 6e53c32ee09da8a615cde5f2c524d9211f479038 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 28 Aug 2020 15:17:42 -0700 Subject: [PATCH 276/424] Adding unused import --- tom_observations/facilities/lco.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index efa9b59dc..af39d9faf 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -1,4 +1,3 @@ -from collections import OrderedDict from datetime import datetime, timedelta import requests From 8bc0e123bb49e5c04b447dbc91011b30096fcfc5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2020 13:36:06 +0000 Subject: [PATCH 277/424] Bump django from 3.1.0 to 3.1.1 Bumps [django](https://github.com/django/django) from 3.1.0 to 3.1.1. - [Release notes](https://github.com/django/django/releases) - [Commits](https://github.com/django/django/compare/3.1...3.1.1) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7229e96e5..78465e5e7 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ 'astropy==4.0.1.post1', 'beautifulsoup4==4.9.1', 'dataclasses; python_version < "3.7"', - 'django==3.1.0', # TOM Toolkit requires db math functions + 'django==3.1.1', # TOM Toolkit requires db math functions 'djangorestframework==3.11.1', 'django-bootstrap4==2.2.0', 'django-contrib-comments==1.9.2', # Earlier version are incompatible with Django >= 3.0 From 98b68212a0690e952ec0d5ada1584c0081e693f0 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2020 13:36:32 +0000 Subject: [PATCH 278/424] Bump django-extensions from 3.0.5 to 3.0.6 Bumps [django-extensions](https://github.com/django-extensions/django-extensions) from 3.0.5 to 3.0.6. - [Release notes](https://github.com/django-extensions/django-extensions/releases) - [Changelog](https://github.com/django-extensions/django-extensions/blob/master/CHANGELOG.md) - [Commits](https://github.com/django-extensions/django-extensions/compare/3.0.5...3.0.6) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7229e96e5..268f45e0e 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ 'django-bootstrap4==2.2.0', 'django-contrib-comments==1.9.2', # Earlier version are incompatible with Django >= 3.0 'django-crispy-forms==1.9.2', - 'django-extensions==3.0.5', + 'django-extensions==3.0.6', 'django-gravatar2==1.4.4', 'django-filter==2.3.0', 'django-guardian==2.3.0', From 0e0b1e6ab467fbf03050500ccd833ea6b9c5b4ef Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2020 13:44:18 +0000 Subject: [PATCH 279/424] Bump django-extensions from 3.0.6 to 3.0.7 Bumps [django-extensions](https://github.com/django-extensions/django-extensions) from 3.0.6 to 3.0.7. - [Release notes](https://github.com/django-extensions/django-extensions/releases) - [Changelog](https://github.com/django-extensions/django-extensions/blob/master/CHANGELOG.md) - [Commits](https://github.com/django-extensions/django-extensions/compare/3.0.6...3.0.7) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fbe0c2711..d177949db 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ 'django-bootstrap4==2.2.0', 'django-contrib-comments==1.9.2', # Earlier version are incompatible with Django >= 3.0 'django-crispy-forms==1.9.2', - 'django-extensions==3.0.6', + 'django-extensions==3.0.7', 'django-gravatar2==1.4.4', 'django-filter==2.3.0', 'django-guardian==2.3.0', From 845e135aa399108bf979c741c986848ddd78fdf7 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 2 Sep 2020 13:21:41 -0700 Subject: [PATCH 280/424] Updated GenericAlert.to_target to include extras and aliases --- tom_alerts/alerts.py | 11 +++++++++-- tom_alerts/tests/tests_generic.py | 2 +- tom_alerts/views.py | 4 ++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/tom_alerts/alerts.py b/tom_alerts/alerts.py index 63138695a..4cd59ec9e 100644 --- a/tom_alerts/alerts.py +++ b/tom_alerts/alerts.py @@ -81,17 +81,24 @@ class GenericAlert: def to_target(self): """ - Returns a Target instance for an object defined by an alert. + Returns a Target instance for an object defined by an alert, as well as + any TargetExtra or additional TargetNames. :returns: representation of object for an alert :rtype: `Target` + + :returns: dict of extras to be added to the new Target + :rtype: `dict` + + :returns: list of aliases to be added to the new Target + :rtype: `list` """ return Target( name=self.name, type='SIDEREAL', ra=self.ra, dec=self.dec - ) + ), {}, [] class GenericQueryForm(forms.Form): diff --git a/tom_alerts/tests/tests_generic.py b/tom_alerts/tests/tests_generic.py index 3a9ab261a..2872a97d6 100644 --- a/tom_alerts/tests/tests_generic.py +++ b/tom_alerts/tests/tests_generic.py @@ -80,7 +80,7 @@ def test_to_generic_alert(self): self.assertEqual(ga.name, test_alerts[0]['name']) def test_to_target(self): - target = TestBroker().to_generic_alert(test_alerts[0]).to_target() + target, extras, names = TestBroker().to_generic_alert(test_alerts[0]).to_target() self.assertEqual(target.name, test_alerts[0]['name']) diff --git a/tom_alerts/views.py b/tom_alerts/views.py index 64fd0c610..18753808b 100644 --- a/tom_alerts/views.py +++ b/tom_alerts/views.py @@ -234,9 +234,9 @@ def post(self, request, *args, **kwargs): messages.error(request, 'Could not create targets. Try re running the query again.') return redirect(reverse('tom_alerts:run', kwargs={'pk': query_id})) generic_alert = broker_class().to_generic_alert(json.loads(cached_alert)) - target = generic_alert.to_target() + target, extras, aliases = generic_alert.to_target() try: - target.save() + target.save(extras=extras, names=aliases) broker_class().process_reduced_data(target, json.loads(cached_alert)) for group in request.user.groups.all().exclude(name='Public'): assign_perm('tom_targets.view_target', group, target) From ce2f453293381ed69567df9a4b481558059d2da4 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 2 Sep 2020 13:34:08 -0700 Subject: [PATCH 281/424] Fixing codacy issue --- tom_alerts/tests/tests_generic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_alerts/tests/tests_generic.py b/tom_alerts/tests/tests_generic.py index 2872a97d6..1c7d7d545 100644 --- a/tom_alerts/tests/tests_generic.py +++ b/tom_alerts/tests/tests_generic.py @@ -80,7 +80,7 @@ def test_to_generic_alert(self): self.assertEqual(ga.name, test_alerts[0]['name']) def test_to_target(self): - target, extras, names = TestBroker().to_generic_alert(test_alerts[0]).to_target() + target, _, _ = TestBroker().to_generic_alert(test_alerts[0]).to_target() self.assertEqual(target.name, test_alerts[0]['name']) From f21c97944243f775936219e1114987655ea15aed Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 2 Sep 2020 14:46:53 -0700 Subject: [PATCH 282/424] Added script for setting default values for new TargetExtras --- tom_targets/management/__init__.py | 0 tom_targets/management/commands/__init__.py | 0 .../management/commands/setdefaultextras.py | 54 +++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 tom_targets/management/__init__.py create mode 100644 tom_targets/management/commands/__init__.py create mode 100644 tom_targets/management/commands/setdefaultextras.py diff --git a/tom_targets/management/__init__.py b/tom_targets/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tom_targets/management/commands/__init__.py b/tom_targets/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tom_targets/management/commands/setdefaultextras.py b/tom_targets/management/commands/setdefaultextras.py new file mode 100644 index 000000000..a1c99956a --- /dev/null +++ b/tom_targets/management/commands/setdefaultextras.py @@ -0,0 +1,54 @@ +from django.conf import settings +from django.core.management.base import BaseCommand +from django.core.exceptions import ImproperlyConfigured, ValidationError + +from tom_targets.models import Target, TargetExtra +from tom_observations import facility + + +class Command(BaseCommand): + """ + This management command should be used after adding a new `EXTRA_FIELDS` value to `settings.py`. For each given + `TargetExtra` name, the script will add a new `TargetExtra` for each `Target` that does not have one. The new + `TargetExtra` will use the default value in `settings.EXTRA_FIELDS`. + """ + + help = 'Adds the default TargetExtra value to all Targets that do not have the provided TargetExtra' + + def add_arguments(self, parser): + parser.add_argument( + '--targetextra', + nargs='+', + help='Specific TargetExtra to update for each Target. Accepts multiple TargetExtras.' + ) + + def handle(self, *args, **options): + te_names = options['targetextra'] + te_defaults = [] + + # Verify that all the specified TargetExtras are actually configured in settings.py + for te_name in te_names: + for extra_field in settings.EXTRA_FIELDS: + if te_name == extra_field['name']: + break + else: + raise ImproperlyConfigured(f'{te_name} is not configured in settings.py.') + + # Validate that settings.EXTRA_FIELDS are properly formatted + for extra_field in settings.EXTRA_FIELDS: + extra_field_name = extra_field['name'] + if extra_field_name in te_names: + if 'type' not in extra_field: + raise ValidationError(f'TargetExtra {extra_field_name} must have a type.') + if 'default' not in extra_field: + raise ValidationError(f'''TargetExtra {extra_field_name} must have a default value for this + script to function.''') + te_defaults.append(extra_field) + + # Create a TargetExtra for each Target that does not have one, and set it to the default value + for extra_field in te_defaults: + targets = Target.objects.exclude(targetextra__key=extra_field['name']) + for target in targets: + TargetExtra.objects.create(target=target, key=extra_field['name'], value=extra_field['default']) + + return From 6fb951e7b656bed1c5f20506b87349617e31160e Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 2 Sep 2020 14:53:14 -0700 Subject: [PATCH 283/424] pycodestyle --- tom_targets/management/commands/setdefaultextras.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_targets/management/commands/setdefaultextras.py b/tom_targets/management/commands/setdefaultextras.py index a1c99956a..538ed27d6 100644 --- a/tom_targets/management/commands/setdefaultextras.py +++ b/tom_targets/management/commands/setdefaultextras.py @@ -44,7 +44,7 @@ def handle(self, *args, **options): raise ValidationError(f'''TargetExtra {extra_field_name} must have a default value for this script to function.''') te_defaults.append(extra_field) - + # Create a TargetExtra for each Target that does not have one, and set it to the default value for extra_field in te_defaults: targets = Target.objects.exclude(targetextra__key=extra_field['name']) From d0893878b60fd18538338f0e788b05d3e6a0f899 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 2 Sep 2020 14:56:05 -0700 Subject: [PATCH 284/424] Removing unused import --- tom_targets/management/commands/setdefaultextras.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tom_targets/management/commands/setdefaultextras.py b/tom_targets/management/commands/setdefaultextras.py index 538ed27d6..e7ebc8efa 100644 --- a/tom_targets/management/commands/setdefaultextras.py +++ b/tom_targets/management/commands/setdefaultextras.py @@ -3,7 +3,6 @@ from django.core.exceptions import ImproperlyConfigured, ValidationError from tom_targets.models import Target, TargetExtra -from tom_observations import facility class Command(BaseCommand): From f3e814fbe39810ae8fe9768e86a8c3c54d03e9dd Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 2 Sep 2020 14:57:01 -0700 Subject: [PATCH 285/424] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bdf250c1e..7e1ce00ae 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # TOM Toolkit -[![Build Status](https://travis-ci.com/TOMToolkit/tom_base.svg?branch=master)](https://travis-ci.org/TOMToolkit/tom_base) +[![Build Status](https://travis-ci.com/TOMToolkit/tom_base.svg?branch=master)](https://travis-ci.com/TOMToolkit/tom_base) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/9846cee7c4904cae8864525101030169)](https://www.codacy.com/gh/observatorycontrolsystem/observation-portal?utm_source=github.com&utm_medium=referral&utm_content=observatorycontrolsystem/observation-portal&utm_campaign=Badge_Grade) [![Coverage Status](https://coveralls.io/repos/github/TOMToolkit/tom_base/badge.svg?branch=master)](https://coveralls.io/github/TOMToolkit/tom_base?branch=master) [![Documentation Status](https://readthedocs.org/projects/tom-toolkit/badge/?version=stable)](https://tom-toolkit.readthedocs.io/en/stable/?badge=stable) From b5f8515cf50d46b6144a39ccab24cf844197b0b7 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 3 Sep 2020 12:54:44 -0700 Subject: [PATCH 286/424] Added example command as per Lindy's suggestion --- tom_targets/management/commands/setdefaultextras.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tom_targets/management/commands/setdefaultextras.py b/tom_targets/management/commands/setdefaultextras.py index e7ebc8efa..c64bf17fd 100644 --- a/tom_targets/management/commands/setdefaultextras.py +++ b/tom_targets/management/commands/setdefaultextras.py @@ -10,6 +10,8 @@ class Command(BaseCommand): This management command should be used after adding a new `EXTRA_FIELDS` value to `settings.py`. For each given `TargetExtra` name, the script will add a new `TargetExtra` for each `Target` that does not have one. The new `TargetExtra` will use the default value in `settings.EXTRA_FIELDS`. + + Example: ./manage.py setdefaultextras --targetextra redshift discovery_date """ help = 'Adds the default TargetExtra value to all Targets that do not have the provided TargetExtra' From de5432eb146d6e964dcfa3f5cee4ffd0568ee087 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2020 13:30:08 +0000 Subject: [PATCH 287/424] Bump django-extensions from 3.0.7 to 3.0.8 Bumps [django-extensions](https://github.com/django-extensions/django-extensions) from 3.0.7 to 3.0.8. - [Release notes](https://github.com/django-extensions/django-extensions/releases) - [Changelog](https://github.com/django-extensions/django-extensions/blob/master/CHANGELOG.md) - [Commits](https://github.com/django-extensions/django-extensions/compare/3.0.7...3.0.8) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d177949db..bd1be8a09 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ 'django-bootstrap4==2.2.0', 'django-contrib-comments==1.9.2', # Earlier version are incompatible with Django >= 3.0 'django-crispy-forms==1.9.2', - 'django-extensions==3.0.7', + 'django-extensions==3.0.8', 'django-gravatar2==1.4.4', 'django-filter==2.3.0', 'django-guardian==2.3.0', From 8c4fe0756745eefa00d71150c4f3e135561a84d4 Mon Sep 17 00:00:00 2001 From: fraserw Date: Tue, 8 Sep 2020 16:45:11 -0700 Subject: [PATCH 288/424] Gemini Fixes applied. Still bugged with long notes. --- tom_observations/facilities/gemini.py | 85 +++++++++++++++++++++++++-- tom_observations/tests/factories.py | 1 + tom_observations/tests/tests.py | 3 + 3 files changed, 84 insertions(+), 5 deletions(-) diff --git a/tom_observations/facilities/gemini.py b/tom_observations/facilities/gemini.py index d90dd8c4c..3096ae103 100644 --- a/tom_observations/facilities/gemini.py +++ b/tom_observations/facilities/gemini.py @@ -3,11 +3,16 @@ from django import forms from dateutil.parser import parse from crispy_forms.layout import Layout, Div, HTML -from astropy import units as u +from astropy import units as u, time +import numpy as np from tom_observations.facility import BaseRoboticObservationFacility, BaseRoboticObservationForm from tom_common.exceptions import ImproperCredentialsException from tom_targets.models import Target +from tom_observations.facilities.utils import reconstruct_gemini_eph_note, add_month, get_hex + +import json + try: GEM_SETTINGS = settings.FACILITIES['GEM'] @@ -140,7 +145,7 @@ class GEMObservationForm(BaseRoboticObservationForm): ra - target RA [J2000], format 'HH:MM:SS.SS' dec - target Dec[J2000], format 'DD:MM:SS.SSS' mags - target magnitude information (optional) - noteTitle - title for the note, "Finding Chart" if not provided (optional) + notetitle - title for the note, "Finding Chart" if not provided (optional) note - text to include in a "Finding Chart" note (optional) posangle - position angle [degrees E of N], defaults to 0 (optional) exptime - exposure time [seconds], if not given then value in template used (optional) @@ -257,6 +262,28 @@ class GEMObservationForm(BaseRoboticObservationForm): label='Timing Window [Date Time]') window_duration = forms.IntegerField(required=False, min_value=1, label='Timing Window Duration [hr]') + + def __init__(self, *args, **kwargs): + # the ephemeris target stuff must come before super() + self.eph_target = False + if 'initial' in kwargs: + target = Target.objects.get(pk=kwargs['initial']['target_id']) + if target.type == Target.NON_SIDEREAL: + if target.scheme == 'EPHEMERIS': + self.eph_target = True + eph_json = json.loads(target.eph_json) + self.eph_GN = reconstruct_gemini_eph_note(eph_json, site='568') + self.eph_GS = reconstruct_gemini_eph_note(eph_json, site='lsc') + + super().__init__(*args, **kwargs) + self.helper.layout = Layout( + self.common_layout, + self.layout(), + #self.cadence_layout, + self.button_layout() + ) + + def layout(self): return Div( HTML('Observation Parameters'), @@ -310,9 +337,15 @@ def layout(self): css_class='col' ), css_class='form-row', - ) + ), + self.extra_layout() ) + def extra_layout(self): + # If you just want to add some fields to the end of the form, add them here. + return Div() + + def is_valid(self): super().is_valid() errors = GEMFacility.validate_observation(self.observation_payload()) @@ -355,14 +388,39 @@ def isodatetime(value): ii = obs.rfind('-') progid = obs[0:ii] obsnum = obs[ii+1:] + + if not self.eph_target: + RA, DEC = target.ra, target.dec + else: + RA, DEC = 0.0, 0.0 + if self.cleaned_data['window_start'].strip() != '': + wdate, wtime = isodatetime(self.cleaned_data['window_start']) + dtime = float(str(self.cleaned_data['window_duration']).strip()) + time_obj = time.Time(wdate+' '+wtime) + min_mjd = time_obj.mjd + max_mjd = (time_obj + dtime/24.0).mjd + + if obs[:2] == 'GN': + mjds = self.eph_GN[1] + else: + mjds = self.eph_GS[1] + mjd_k = max(0, np.sum(np.less_equal(mjds, min_mjd))-1) + mjd_K = min(len(mjds), np.sum(np.less_equal(mjds, max_mjd))+1) + + #mjd_K = mjd_k + 52 + + else: + mjd_k = 0 + mjd_K = 0 + payload = { "prog": progid, "password": GEM_SETTINGS['api_key'][get_site(obs)], "email": GEM_SETTINGS['user_email'], "obsnum": obsnum, "target": target.name, - "ra": target.ra, - "dec": target.dec, + "ra": RA, + "dec": DEC, "ready": self.cleaned_data['ready'] } @@ -370,6 +428,22 @@ def isodatetime(value): payload["noteTitle"] = self.cleaned_data['notetitle'] payload["note"] = self.cleaned_data['note'] + if self.eph_target: + if 'noteTitle' not in payload: + payload['noteTitle'] = 'Ephemeris' + payload['note'] = '' + else: + payload['noteTitle'] += ' and Ephemeris' + + if obs[:2] == 'GN': + note_text = self.eph_GN[0][0:4] + self.eph_GN[0][mjd_k:mjd_K] + self.eph_GN[0][-2:] + payload['note'] += "\n" + payload['note'] += "\n".join(note_text) + elif obs[:2] == 'GS': + note_text = self.eph_GS[0][0:4] + self.eph_GS[0][mjd_k:mjd_K] + self.eph_GS[0][-2:] + payload['note'] += "\n" + payload['note'] += "\n".join(note_text) + print(payload['note']) if self.cleaned_data['brightness'] is not None: smags = str(self.cleaned_data['brightness']).strip() + '/' + \ self.cleaned_data['brightness_band'] + '/' + \ @@ -389,6 +463,7 @@ def isodatetime(value): payload['windowTime'] = wtime payload['windowDuration'] = str(self.cleaned_data['window_duration']).strip() + # elevation/airmass if self.cleaned_data['eltype'] is not None: payload['elevationType'] = self.cleaned_data['eltype'] diff --git a/tom_observations/tests/factories.py b/tom_observations/tests/factories.py index 3499fc2df..7176b5bcd 100644 --- a/tom_observations/tests/factories.py +++ b/tom_observations/tests/factories.py @@ -22,6 +22,7 @@ class Meta: epoch = factory.Faker('pyfloat') pm_ra = factory.Faker('pyfloat') pm_dec = factory.Faker('pyfloat') + # WF: probably need to add variables here class ObservingRecordFactory(factory.django.DjangoModelFactory): diff --git a/tom_observations/tests/tests.py b/tom_observations/tests/tests.py index 183c420f4..52f76ccf1 100644 --- a/tom_observations/tests/tests.py +++ b/tom_observations/tests/tests.py @@ -130,6 +130,9 @@ def test_submit_observation_manual(self): self.assertTrue(ObservationRecord.objects.filter(observation_id='fakeid').exists()) + #def test_observation_payload(self): + + @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=False) class TestObservationViewsRowLevelPermissions(TestCase): From 54b5ebfad2044050245e79172744450286de5580 Mon Sep 17 00:00:00 2001 From: fraserw Date: Thu, 10 Sep 2020 11:43:49 -0700 Subject: [PATCH 289/424] added utils.py --- tom_observations/facilities/utils.py | 88 ++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 tom_observations/facilities/utils.py diff --git a/tom_observations/facilities/utils.py b/tom_observations/facilities/utils.py new file mode 100644 index 000000000..0b2ad9f57 --- /dev/null +++ b/tom_observations/facilities/utils.py @@ -0,0 +1,88 @@ +from astropy.time import Time +import numpy as np + +d2r = np.pi/180.0 + +#example format +""" +*************************************************************************************** + Date__(UT)__HR:MN Date_________JDUT R.A.___(ICRF/J2000.0)___DEC dRA*cosD d(DEC)/dt +*************************************************************************************** +$$SOE + 2013-Jan-01 16:00 2456294.166666667 Am 14 30 58.5670 -12 25 00.360 8.861123 -2.58933 + +$$EOE +*************************************************************************************** + """ + +def get_hex(ra, dec): + s = ra/15.0 + rh = int(s) + s -= rh + s *= 60.0 + rm = int(s) + rs = (s-rm)*60.0 + + s = abs(dec) + dh = int(dec) + s -= dh + s *= 60.0 + dm = int(s) + ds = (s-dm)*60.0 + + if dec<0: + Sign = '-' + else: + Sign = '+' + + return (rh, rm, rs, Sign, dh, dm, ds) + +def add_month(t): + T = t.replace('-01-','-Jan-').replace('-02-','-Feb-').replace('-03-','-Mar-').replace('-04-','-Apr-').replace('-05-','-May-') + T = T.replace('-06-','-Jun-').replace('-07-','-Jul-').replace('-08-','-Aug-').replace('-09-','-Sep-').replace('-10-','-Oct-') + return T.replace('-11-', '-Nov-').replace('-12-', '-Dec-') + +def reconstruct_gemini_eph_note(eph, site='568'): + mk = eph[site] + + ras = [] + decs = [] + dras = [] + ddecs = [] + mjds = [] + times = [] + for i, e in enumerate(mk): + mjds.append(float(e['t'])) + ras.append(float(e['R'])) + decs.append(float(e['D'])) + dras.append(float(e['dR'])) + ddecs.append(float(e['dD'])) + t = Time(mjds[-1], format='mjd', scale='utc') + times.append(add_month(t.iso)) + + mjds, ras, decs, dras, ddecs = np.array(mjds), np.array(ras), np.array(decs), np.array(dras), np.array(ddecs) + + rates_ra = (ras[1:]-ras[:-1])*(np.cos(decs[:-1]*d2r))/(mjds[1:]-mjds[:-1])*(3600.0/24.0) + rates_dec = (decs[1:]-decs[:-1])/(mjds[1:]-mjds[:-1])*(3600.0/24.0) + rates_ra = np.concatenate([rates_ra, rates_ra[-1:]]) + rates_dec = np.concatenate([rates_dec, rates_dec[-1:]]) + + JPL = ["***************************************************************************************", + " Date__(UT)__HR:MN Date_________JDUT R.A.___(ICRF/J2000.0)___DEC dRA*cosD d(DEC)/dt", + "***************************************************************************************", + "$$SOE", + ] + for i in range(len(mjds)): + (rh, rm, rs, S, dh, dm, ds) = get_hex(ras[i], decs[i]) + + entry = " {} {:<17f} {} {:02} {:07.4f} {}{} {:02} {:06.3f} {:8.5} {:8.5}".format(times[i][:17], + mjds[i]+2400000.5, + rh, rm, rs, + S, dh, dm, ds, + rates_ra[i], + rates_dec[i]) + JPL.append(entry) + JPL.append("$$EOE") + JPL.append("***************************************************************************************") + + return (JPL, mjds) From 2dd9bc877970480fed6c7e78183e1fe067ef7503 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2020 13:45:47 +0000 Subject: [PATCH 290/424] Bump numpy from 1.19.1 to 1.19.2 Bumps [numpy](https://github.com/numpy/numpy) from 1.19.1 to 1.19.2. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/master/doc/HOWTO_RELEASE.rst.txt) - [Commits](https://github.com/numpy/numpy/compare/v1.19.1...v1.19.2) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d177949db..f236579c9 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ 'django-guardian==2.3.0', 'fits2image==0.4.3', 'Markdown==3.2.2', # django-rest-framework doc headers require this to support Markdown - 'numpy==1.19.1', + 'numpy==1.19.2', 'pillow==7.2.0', 'plotly==4.9.0', 'python-dateutil==2.8.1', From d5393aa0884502e7fe8600ecfe4a7f9b4d89100d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2020 15:57:22 +0000 Subject: [PATCH 291/424] Bump plotly from 4.9.0 to 4.10.0 Bumps [plotly](https://github.com/plotly/plotly.py) from 4.9.0 to 4.10.0. - [Release notes](https://github.com/plotly/plotly.py/releases) - [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.py/compare/v4.9.0...v4.10.0) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f236579c9..3b8258353 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ 'Markdown==3.2.2', # django-rest-framework doc headers require this to support Markdown 'numpy==1.19.2', 'pillow==7.2.0', - 'plotly==4.9.0', + 'plotly==4.10.0', 'python-dateutil==2.8.1', 'requests==2.24.0', 'specutils==1.0', From e14df897a26e738c6faac258e952f033a85b1d02 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 11 Sep 2020 14:37:09 -0700 Subject: [PATCH 292/424] Fixed issues with cadence layout and observing template application --- tom_observations/views.py | 3 ++- tom_targets/views.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tom_observations/views.py b/tom_observations/views.py index 39950e0d2..8c2552566 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -112,7 +112,7 @@ def get(self, request, *args, **kwargs): class ObservationCreateView(LoginRequiredMixin, FormView): """ - View for creation/submission of an observation. Requries authentication. + View for creation/submission of an observation. Requires authentication. """ template_name = 'tom_observations/observation_form.html' @@ -226,6 +226,7 @@ def get_initial(self): raise Exception('Must provide target_id') initial['target_id'] = self.get_target_id() initial['facility'] = self.get_facility() + initial.update(self.request.GET.dict()) return initial def form_valid(self, form): diff --git a/tom_targets/views.py b/tom_targets/views.py index 841e9c603..554c90fc6 100644 --- a/tom_targets/views.py +++ b/tom_targets/views.py @@ -359,7 +359,10 @@ def get(self, request, *args, **kwargs): run_strategy_form = RunStrategyForm(request.GET) if run_strategy_form.is_valid(): obs_strat = ObservingStrategy.objects.get(pk=run_strategy_form.cleaned_data['observing_strategy'].id) - params = urlencode(obs_strat.parameters_as_dict) + obs_strat_params = obs_strat.parameters_as_dict + obs_strat_params['cadence_strategy'] = request.GET.get('cadence_strategy', '') + obs_strat_params['cadence_frequency'] = request.GET.get('cadence_frequency', '') + params = urlencode(obs_strat_params) return redirect( reverse('tom_observations:create', args=(obs_strat.facility,)) + f'?target_id={self.get_object().id}&' + params) From c744ea7048ab62d3441ca775073f890f071101f4 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 16 Sep 2020 19:34:27 -0700 Subject: [PATCH 293/424] create RegisteredCadence model, updated cadence workflow to support it, added custom migration to ensure data retention --- tom_base/settings.py | 2 +- tom_observations/cadence.py | 26 +++---- tom_observations/facilities/lco.py | 6 ++ .../commands/runcadencestrategies.py | 6 +- .../0010_manual_create_registered_cadence.py | 69 +++++++++++++++++++ tom_observations/models.py | 20 +++++- tom_observations/tests/test_cadence.py | 11 +-- tom_observations/views.py | 15 ++-- 8 files changed, 126 insertions(+), 29 deletions(-) create mode 100644 tom_observations/migrations/0010_manual_create_registered_cadence.py diff --git a/tom_base/settings.py b/tom_base/settings.py index 69d37ad82..886c0d24e 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -226,7 +226,7 @@ # For example: # EXTRA_FIELDS = [ # {'name': 'redshift', 'type': 'number', 'default': 0}, -# {'name': 'discoverer', 'type': 'string'} +# {'name': 'discoverer', 'type': 'string'}, # {'name': 'eligible', 'type': 'boolean', 'hidden': True}, # {'name': 'dicovery_date', 'type': 'datetime'} # ] diff --git a/tom_observations/cadence.py b/tom_observations/cadence.py index de2d8e59a..40eec61d9 100644 --- a/tom_observations/cadence.py +++ b/tom_observations/cadence.py @@ -52,9 +52,9 @@ class CadenceStrategy(ABC): In order to make use of a custom CadenceStrategy, add the path to ``TOM_CADENCE_STRATEGIES`` in your ``settings.py``. """ - def __init__(self, observation_group, *args, **kwargs): + def __init__(self, registered_cadence, *args, **kwargs): self.cadence_strategy = type(self).__name__ - self.observation_group = observation_group + self.registered_cadence = registered_cadence @abstractmethod def run(self): @@ -70,12 +70,14 @@ class RetryFailedObservationsStrategy(CadenceStrategy): description = """This strategy immediately re-submits a cadenced observation without amending any other part of the cadence.""" - def __init__(self, observation_group, advance_window_hours, *args, **kwargs): + def __init__(self, registered_cadence, advance_window_hours, *args, **kwargs): self.advance_window_hours = advance_window_hours - super().__init__(observation_group, *args, **kwargs) + super().__init__(registered_cadence, *args, **kwargs) def run(self): - failed_observations = [obsr for obsr in self.observation_group.observation_records.all() if obsr.failed] + failed_observations = [obsr for obsr + in self.registered_cadence.observation_group.observation_records.all() + if obsr.failed] new_observations = [] for obs in failed_observations: observation_payload = obs.parameters_as_dict @@ -97,8 +99,8 @@ def run(self): parameters=json.dumps(observation_payload), observation_id=observation_id ) - self.observation_group.observation_records.add(record) - self.observation_group.save() + self.registered_cadence.observation_group.observation_records.add(record) + self.registered_cadence.observation_group.save() new_observations.append(record) return new_observations @@ -124,12 +126,12 @@ class ResumeCadenceAfterFailureStrategy(CadenceStrategy): re-submits the observation until it succeeds. If it succeeds, it submits the next observation on the same cadence.""" - def __init__(self, observation_group, advance_window_hours, *args, **kwargs): + def __init__(self, registered_cadence, advance_window_hours, *args, **kwargs): self.advance_window_hours = advance_window_hours - super().__init__(observation_group, *args, **kwargs) + super().__init__(registered_cadence, *args, **kwargs) def run(self): - last_obs = self.observation_group.observation_records.order_by('-created').first() + last_obs = self.registered_cadence.observation_group.observation_records.order_by('-created').first() facility = get_service_class(last_obs.facility)() facility.update_observation_status(last_obs.observation_id) last_obs.refresh_from_db() @@ -162,8 +164,8 @@ def run(self): parameters=json.dumps(observation_payload), observation_id=observation_id ) - self.observation_group.observation_records.add(record) - self.observation_group.save() + self.registered_cadence.observation_group.observation_records.add(record) + self.registered_cadence.observation_group.save() new_observations.append(record) for obsr in new_observations: diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index af39d9faf..d614295de 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -716,6 +716,12 @@ def layout(self): class LCOObservingStrategyForm(GenericStrategyForm, LCOBaseForm): + """ + The strategy form modifies the LCOBaseForm in order to only provide fields + that make sense to stay the same for the template. For example, there is no + point to making start_time an available field, as it will change between + observations. + """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) for field_name in ['groups', 'target_id']: diff --git a/tom_observations/management/commands/runcadencestrategies.py b/tom_observations/management/commands/runcadencestrategies.py index 7eb8ce492..08b18c3a6 100644 --- a/tom_observations/management/commands/runcadencestrategies.py +++ b/tom_observations/management/commands/runcadencestrategies.py @@ -3,17 +3,17 @@ from django.core.management.base import BaseCommand from tom_observations.cadence import get_cadence_strategy -from tom_observations.models import ObservationGroup +from tom_observations.models import ObservationGroup, RegisteredCadence class Command(BaseCommand): help = 'Entry point for running cadence strategies.' def handle(self, *args, **kwargs): - cadenced_groups = ObservationGroup.objects.exclude(cadence_strategy='') + cadenced_groups = RegisteredCadence.objects.exclude(active=False) for cg in cadenced_groups: - cadence_frequency = json.loads(cg.cadence_parameters)['cadence_frequency'] + cadence_frequency = cg.cadence_parameters.get('cadence_frequency', -1) strategy = get_cadence_strategy(cg.cadence_strategy)(cg, cadence_frequency) new_observations = strategy.run() if not new_observations: diff --git a/tom_observations/migrations/0010_manual_create_registered_cadence.py b/tom_observations/migrations/0010_manual_create_registered_cadence.py new file mode 100644 index 000000000..1af8cabce --- /dev/null +++ b/tom_observations/migrations/0010_manual_create_registered_cadence.py @@ -0,0 +1,69 @@ +import json + +from django.db import migrations, models + + +def copy_cadence_fields_to_registered_cadence(apps, schema_editor): + observation_groups = apps.get_model('tom_observations', 'ObservationGroup') + for row in observation_groups.objects.exclude(cadence_strategy=''): + registered_cadence = apps.get_model('tom_observations', 'RegisteredCadence') + try: + cadence_parameters = json.loads(getattr(row, 'cadence_parameters')) + except json.decoder.JSONDecodeError: + cadence_parameters = {} + new_registered_cadence = registered_cadence( + name=getattr(row, 'name'), + observation_group=row, + cadence_strategy=getattr(row, 'cadence_strategy'), + cadence_parameters=cadence_parameters, + active=False, + created=getattr(row, 'created'), + modified=getattr(row, 'modified') + ) + new_registered_cadence.save() + + +class Migration(migrations.Migration): + dependencies = [ + ('tom_observations', '0009_observationrecord_user'), + ] + + operations = [ + migrations.CreateModel( + name='RegisteredCadence', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100, unique=True, help_text='Name of this RegisteredCadence', + verbose_name='Name of this RegisteredCadence')), + ('cadence_strategy', models.CharField(max_length=100, + verbose_name='Cadence strategy used for this RegisteredCadence')), + ('cadence_parameters', models.JSONField(verbose_name='Cadence-specific parameters')), + ('active', models.BooleanField(verbose_name='Active', + help_text='''Whether or not this RegisteredCadence should continue + to submit observations.''')), + ('created', models.DateTimeField(auto_now_add=True, + help_text='The time which this RegisteredCadence was created.')), + ('modified', models.DateTimeField(auto_now=True, + help_text='The time which this RegisteredCadence was modified.')), + ] + ), + + migrations.AddField( + model_name='registeredcadence', + name='observation_group', + field=models.ForeignKey(on_delete=models.deletion.CASCADE, null=False, default=None, + to='tom_observations.ObservationGroup'), + ), + + migrations.RunPython(copy_cadence_fields_to_registered_cadence, reverse_code=migrations.RunPython.noop), + + migrations.RemoveField( + model_name='observationgroup', + name='cadence_strategy' + ), + + migrations.RemoveField( + model_name='observationgroup', + name='cadence_parameters' + ) + ] \ No newline at end of file diff --git a/tom_observations/models.py b/tom_observations/models.py index 6cd6c0d8c..81eb005b8 100644 --- a/tom_observations/models.py +++ b/tom_observations/models.py @@ -113,13 +113,27 @@ class ObservationGroup(models.Model): """ name = models.CharField(max_length=50) observation_records = models.ManyToManyField(ObservationRecord) - cadence_strategy = models.CharField(max_length=100, blank=True, default='') - cadence_parameters = models.TextField(blank=False, default='') created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) class Meta: - ordering = ('-created',) + ordering = ('-created', 'name',) + + def __str__(self): + return self.name + + +class RegisteredCadence(models.Model): + name = models.CharField(max_length=100, unique=True, help_text='Name of this RegisteredCadence', + verbose_name='Name of this RegisteredCadence') # TODO: Does this need to exist, if so, what do with ObservationCreateView.is_valid()? + observation_group = models.ForeignKey(ObservationGroup, null=False, default=None, on_delete=models.CASCADE) + cadence_strategy = models.CharField(max_length=100, verbose_name='Cadence strategy used for this RegisteredCadence') + cadence_parameters = models.JSONField(verbose_name='Cadence-specific parameters') + active = models.BooleanField(verbose_name='Active', + help_text='''Whether or not this RegisteredCadence should + continue to submit observations.''') + created = models.DateTimeField(auto_now_add=True, help_text='The time which this RegisteredCadence was created.') + modified = models.DateTimeField(auto_now=True, help_text='The time which this RegisteredCadence was modified.') def __str__(self): return self.name diff --git a/tom_observations/tests/test_cadence.py b/tom_observations/tests/test_cadence.py index 6f4103d5b..9eb58d79b 100644 --- a/tom_observations/tests/test_cadence.py +++ b/tom_observations/tests/test_cadence.py @@ -4,7 +4,7 @@ from dateutil.parser import parse from .factories import ObservingRecordFactory, TargetFactory -from tom_observations.models import ObservationGroup +from tom_observations.models import ObservationGroup, RegisteredCadence from tom_observations.cadence import RetryFailedObservationsStrategy, ResumeCadenceAfterFailureStrategy @@ -30,6 +30,9 @@ def setUp(self): self.group = ObservationGroup.objects.create() self.group.observation_records.add(*observing_records) self.group.save() + # TODO: why doesn't this fail without cadence_strategy and name? + self.registered_cadence = RegisteredCadence.objects.create( + cadence_parameters={}, active=True, observation_group=self.group) def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4): num_records = self.group.observation_records.count() @@ -37,7 +40,7 @@ def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4): observing_record.status = 'CANCELED' observing_record.save() - strategy = RetryFailedObservationsStrategy(self.group, 72) + strategy = RetryFailedObservationsStrategy(self.registered_cadence, 72) new_records = strategy.run() self.group.refresh_from_db() # Make sure the candence run created a new observation. @@ -54,7 +57,7 @@ def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4): def test_resume_when_failed_cadence_failed_obs(self, patch1, patch2, patch3, patch4, patch5): num_records = self.group.observation_records.count() - strategy = ResumeCadenceAfterFailureStrategy(self.group, 72) + strategy = ResumeCadenceAfterFailureStrategy(self.registered_cadence, 72) new_records = strategy.run() self.group.refresh_from_db() self.assertEqual(num_records + 1, self.group.observation_records.count()) @@ -69,7 +72,7 @@ def test_resume_when_failed_cadence_successful_obs(self, patch1, patch2, patch3, num_records = self.group.observation_records.count() observing_record = self.group.observation_records.order_by('-created').first() - strategy = ResumeCadenceAfterFailureStrategy(self.group, 72) + strategy = ResumeCadenceAfterFailureStrategy(self.registered_cadence, 72) new_records = strategy.run() self.group.refresh_from_db() self.assertEqual(num_records + 1, self.group.observation_records.count()) diff --git a/tom_observations/views.py b/tom_observations/views.py index 8c2552566..d127a6352 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -27,7 +27,7 @@ from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm from tom_observations.facility import get_service_class, get_service_classes, BaseManualObservationFacility from tom_observations.forms import AddExistingObservationForm -from tom_observations.models import ObservationRecord, ObservationGroup, ObservingStrategy +from tom_observations.models import ObservationRecord, ObservationGroup, ObservingStrategy, RegisteredCadence from tom_targets.models import Target @@ -260,16 +260,19 @@ def form_valid(self, form): # TODO: redirect to observation list for multiple observations, observation detail otherwise if len(records) > 1 or form.cleaned_data.get('cadence_strategy'): - group_name = form.cleaned_data['name'] - observation_group = ObservationGroup.objects.create( - name=group_name, cadence_strategy=form.cleaned_data.get('cadence_strategy'), - cadence_parameters=json.dumps({'cadence_frequency': form.cleaned_data.get('cadence_frequency')}) - ) + observation_group = ObservationGroup.objects.create(name=form.cleaned_data['name']) observation_group.observation_records.add(*records) assign_perm('tom_observations.view_observationgroup', self.request.user, observation_group) assign_perm('tom_observations.change_observationgroup', self.request.user, observation_group) assign_perm('tom_observations.delete_observationgroup', self.request.user, observation_group) + if form.cleaned_data.get('cadence_strategy'): + registered_cadence = RegisteredCadence.objects.create( + observation_group=observation_group, + cadence_strategy=form.cleaned_data.get('cadence_strategy'), + cadence_parameters={'cadence_frequency': form.cleaned_data.get('cadence_frequency')} + ) + if not settings.TARGET_PERMISSIONS_ONLY: groups = form.cleaned_data['groups'] for record in records: From 216315469984b521c85c5ba912a834efbde1643c Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 16 Sep 2020 19:46:11 -0700 Subject: [PATCH 294/424] Added functionality to create arbitrary observation groups --- .../observationgroup_form.html | 9 +++++++ .../observationgroup_list.html | 7 ++++++ tom_observations/urls.py | 10 +++++--- tom_observations/views.py | 25 ++++++++++++++++++- 4 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 tom_observations/templates/tom_observations/observationgroup_form.html diff --git a/tom_observations/templates/tom_observations/observationgroup_form.html b/tom_observations/templates/tom_observations/observationgroup_form.html new file mode 100644 index 000000000..e21ef1af2 --- /dev/null +++ b/tom_observations/templates/tom_observations/observationgroup_form.html @@ -0,0 +1,9 @@ +{% extends 'tom_common/base.html' %} +{% load bootstrap4 %} +{% block title %}New Observation Group{% endblock %} +{% block content %} +
    {% csrf_token %} + {{ form.as_p }} + +
    +{% endblock %} diff --git a/tom_observations/templates/tom_observations/observationgroup_list.html b/tom_observations/templates/tom_observations/observationgroup_list.html index bb6d63d92..57cdf346e 100644 --- a/tom_observations/templates/tom_observations/observationgroup_list.html +++ b/tom_observations/templates/tom_observations/observationgroup_list.html @@ -3,6 +3,13 @@ {% block title %}Target Groups{% endblock %} {% block content %}

    Observation Groups

    +
    +
    +

    + Create New Grouping +

    +
    +
    {% bootstrap_pagination page_obj extra=request.GET.urlencode %}
    diff --git a/tom_observations/urls.py b/tom_observations/urls.py index 21d43afb1..1bfe7e96a 100644 --- a/tom_observations/urls.py +++ b/tom_observations/urls.py @@ -1,8 +1,8 @@ from django.urls import path from tom_observations.views import (AddExistingObservationView, ObservationCreateView, ObservationRecordUpdateView, - ObservationGroupDeleteView, ObservationGroupListView, ObservationListView, - ObservationRecordDetailView, ObservingStrategyCreateView, + ObservationGroupCreateView, ObservationGroupDeleteView, ObservationGroupListView, + ObservationListView, ObservationRecordDetailView, ObservingStrategyCreateView, ObservingStrategyDeleteView, ObservingStrategyListView, ObservingStrategyUpdateView) @@ -16,9 +16,11 @@ path('strategy//update/', ObservingStrategyUpdateView.as_view(), name='strategy-update'), path('strategy//delete/', ObservingStrategyDeleteView.as_view(), name='strategy-delete'), path('strategy//', ObservingStrategyUpdateView.as_view(), name='strategy-detail'), + # This path must be above /create + path('groups/create/', ObservationGroupCreateView.as_view(), name='group-create'), + path('groups/list/', ObservationGroupListView.as_view(), name='group-list'), + path('groups//delete/', ObservationGroupDeleteView.as_view(), name='group-delete'), path('/create/', ObservationCreateView.as_view(), name='create'), path('/update/', ObservationRecordUpdateView.as_view(), name='update'), path('/', ObservationRecordDetailView.as_view(), name='detail'), - path('groups/list/', ObservationGroupListView.as_view(), name='group-list'), - path('groups//delete/', ObservationGroupDeleteView.as_view(), name='group-delete'), ] diff --git a/tom_observations/views.py b/tom_observations/views.py index d127a6352..a7f80cb7f 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -17,7 +17,7 @@ from django.utils.safestring import mark_safe from django.views.generic import View from django.views.generic.detail import DetailView -from django.views.generic.edit import DeleteView, FormView, UpdateView +from django.views.generic.edit import CreateView, DeleteView, FormView, UpdateView from django.views.generic.list import ListView from guardian.shortcuts import get_objects_for_user, assign_perm from guardian.mixins import PermissionListMixin @@ -450,6 +450,29 @@ def get_context_data(self, *args, **kwargs): return context +class ObservationGroupCreateView(LoginRequiredMixin, CreateView): + """ + View that handles the creation of ``ObservationGroup`` objects. Requires authentication. + """ + model = ObservationGroup + fields = ['name'] + success_url = reverse_lazy('tom_observations:group-list') + + def form_valid(self, form): + """ + Runs after form validation. Saves the observation group and assigns the user's permissions to the group. + + :param form: Form data for observation group creation + :type form: django.forms.ModelForm + """ + obj = form.save(commit=False) + obj.save() + assign_perm('tom_observations.view_observationgroup', self.request.user, obj) + assign_perm('tom_observations.change_observationgroup', self.request.user, obj) + assign_perm('tom_observations.delete_observationgroup', self.request.user, obj) + return super().form_valid(form) + + class ObservationGroupListView(PermissionListMixin, ListView): """ View that handles the display of ``ObservationGroup``. From 3b72c786ecf6711d63d6e5c332dd45fbe219672c Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 16 Sep 2020 20:13:22 -0700 Subject: [PATCH 295/424] Renamed ObservingStrategy to ObservationTemplate, tests pass, no sanity checks yet --- README-dev.md | 2 +- docs/introduction/about.rst | 2 +- docs/observing/strategies.rst | 9 ++-- .../templates/tom_common/navbar_content.html | 2 +- tom_observations/facilities/lco.py | 12 ++--- .../migrations/0011_auto_20200917_0306.py | 26 ++++++++++ tom_observations/models.py | 12 ++--- ...ng_strategy.py => observation_template.py} | 35 +++++++------ .../observationrecord_detail.html | 2 +- ...> observationtemplate_confirm_delete.html} | 0 ...orm.html => observationtemplate_form.html} | 4 +- ...ist.html => observationtemplate_list.html} | 24 ++++----- .../observationtemplate_from_record.html | 1 + ..._run.html => observationtemplate_run.html} | 2 +- .../observingstrategy_from_record.html | 1 - .../templatetags/observation_extras.py | 18 +++---- tom_observations/tests/factories.py | 6 +-- tom_observations/tests/tests.py | 30 +++++------ tom_observations/tests/utils.py | 12 ++--- tom_observations/urls.py | 16 +++--- tom_observations/views.py | 52 +++++++++---------- .../templates/tom_targets/target_detail.html | 2 +- tom_targets/views.py | 26 +++++----- 23 files changed, 161 insertions(+), 135 deletions(-) create mode 100644 tom_observations/migrations/0011_auto_20200917_0306.py rename tom_observations/{observing_strategy.py => observation_template.py} (59%) rename tom_observations/templates/tom_observations/{observingstrategy_confirm_delete.html => observationtemplate_confirm_delete.html} (100%) rename tom_observations/templates/tom_observations/{observingstrategy_form.html => observationtemplate_form.html} (52%) rename tom_observations/templates/tom_observations/{observingstrategy_list.html => observationtemplate_list.html} (55%) create mode 100644 tom_observations/templates/tom_observations/partials/observationtemplate_from_record.html rename tom_observations/templates/tom_observations/partials/{observingstrategy_run.html => observationtemplate_run.html} (70%) delete mode 100644 tom_observations/templates/tom_observations/partials/observingstrategy_from_record.html diff --git a/README-dev.md b/README-dev.md index 03990fcf0..fdff3b2b0 100644 --- a/README-dev.md +++ b/README-dev.md @@ -47,7 +47,7 @@ Following deployment of a release, a Github Release is created, and this should 4. Deploy `tom-demo-dev` with new features demonstrated, pulling `tomtoolkit==x.y.z-alpha.w` from PyPI. Examples: - * Release of observing strategies should include saving an observing strategy and submitting an observation via the observing strategy + * Release of observation templates should include saving an observation template and submitting an observation via the observation_template * Release of manual facility interface should include an implementation of the new interface * Release of a new template tag should include that template tag in a template diff --git a/docs/introduction/about.rst b/docs/introduction/about.rst index 993652591..01de8d295 100644 --- a/docs/introduction/about.rst +++ b/docs/introduction/about.rst @@ -51,7 +51,7 @@ astronomers. No two TOM systems are identical, as astronomers strongly prefer to directly control the science-specific aspects of their projects such as -target selection, observing strategy and analysis techniques. At the +target selection, observation templates and analysis techniques. At the same time, while all of these systems are customized for the science goals of the projects they support, much of their underlying infrastructure and functions are very similar. diff --git a/docs/observing/strategies.rst b/docs/observing/strategies.rst index 98365530c..efdc7f12c 100644 --- a/docs/observing/strategies.rst +++ b/docs/observing/strategies.rst @@ -4,14 +4,13 @@ Cadence and Observing Strategies The TOM has a couple of unique concepts that may be unfamiliar to some at first, that will be describe here before going into detail. -The first concept is that of an observing strategy. An observing -strategy is something of a template. If an observer is consistently +The first concept is that of an observation template. If an observer is consistently submitting observations with a lot of similar parameters, it may be useful to save those as a kind of template, which can just be loaded later. The TOM Toolkit offers an interface that allows facilities to -define a strategy form, that will be saved as an observing strategy. The -strategy can then be applied to an observation, with the remaining -parameters filled in or changed. An observing strategy can also be +define a template form, that will be saved as an observation template. The +template can then be applied to an observation, with the remaining +parameters filled in or changed. An observation template can also be creating from a past observation, with a button to do so that’s available on any ObservationRecord detail page. diff --git a/tom_common/templates/tom_common/navbar_content.html b/tom_common/templates/tom_common/navbar_content.html index 08e167650..7e11994d6 100644 --- a/tom_common/templates/tom_common/navbar_content.html +++ b/tom_common/templates/tom_common/navbar_content.html @@ -31,7 +31,7 @@ Observation Groups
  • - Observation Strategies + Observation Strategies
    • diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index d614295de..8e7f1c182 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -12,7 +12,7 @@ from tom_common.exceptions import ImproperCredentialsException from tom_observations.cadence import CadenceForm from tom_observations.facility import BaseRoboticObservationFacility, BaseRoboticObservationForm, get_service_class -from tom_observations.observing_strategy import GenericStrategyForm +from tom_observations.observation_template import GenericTemplateForm from tom_observations.widgets import FilterField from tom_targets.models import Target, REQUIRED_NON_SIDEREAL_FIELDS, REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME @@ -715,9 +715,9 @@ def layout(self): ) -class LCOObservingStrategyForm(GenericStrategyForm, LCOBaseForm): +class LCOObservationTemplateForm(GenericTemplateForm, LCOBaseForm): """ - The strategy form modifies the LCOBaseForm in order to only provide fields + The template form modifies the LCOBaseForm in order to only provide fields that make sense to stay the same for the template. For example, there is no point to making start_time an available field, as it will change between observations. @@ -727,7 +727,7 @@ def __init__(self, *args, **kwargs): for field_name in ['groups', 'target_id']: self.fields.pop(field_name, None) for field in self.fields: - if field != 'strategy_name': + if field != 'template_name': self.fields[field].required = False self.helper.layout = Layout( self.common_layout, @@ -808,8 +808,8 @@ def get_form(self, observation_type): except KeyError: return LCOBaseObservationForm - def get_strategy_form(self, observation_type): - return LCOObservingStrategyForm + def get_template_form(self, observation_type): + return LCOObservationTemplateForm def submit_observation(self, observation_payload): response = make_request( diff --git a/tom_observations/migrations/0011_auto_20200917_0306.py b/tom_observations/migrations/0011_auto_20200917_0306.py new file mode 100644 index 000000000..9f0edd1bf --- /dev/null +++ b/tom_observations/migrations/0011_auto_20200917_0306.py @@ -0,0 +1,26 @@ +# Generated by Django 3.1 on 2020-09-17 03:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tom_observations', '0010_manual_create_registered_cadence'), + ] + + operations = [ + migrations.RenameModel( + old_name='ObservingStrategy', + new_name='ObservationTemplate', + ), + migrations.AlterModelOptions( + name='observationgroup', + options={'ordering': ('-created', 'name')}, + ), + migrations.AlterField( + model_name='registeredcadence', + name='active', + field=models.BooleanField(help_text='Whether or not this RegisteredCadence should\n continue to submit observations.', verbose_name='Active'), + ), + ] diff --git a/tom_observations/models.py b/tom_observations/models.py index 81eb005b8..952c919bd 100644 --- a/tom_observations/models.py +++ b/tom_observations/models.py @@ -139,23 +139,23 @@ def __str__(self): return self.name -class ObservingStrategy(models.Model): +class ObservationTemplate(models.Model): """ - Class representing an observing strategy, or template. + Class representing an observation template. - :param name: The name of the ``ObservingStrategy`` + :param name: The name of the ``ObservationTemplate`` :type name: str - :param facility: The module-specified facility name for which the strategy is valid + :param facility: The module-specified facility name for which the template is valid :type facility: str :param parameters: JSON string of observing parameters :type parameters: str - :param created: The time at which this ``ObservationGroup`` was created. + :param created: The time at which this ``ObservationTemplate`` was created. :type created: datetime - :param modified: The time at which this ``ObservationGroup`` was modified. + :param modified: The time at which this ``ObservationTemplate`` was modified. :type modified: datetime """ name = models.CharField(max_length=200) diff --git a/tom_observations/observing_strategy.py b/tom_observations/observation_template.py similarity index 59% rename from tom_observations/observing_strategy.py rename to tom_observations/observation_template.py index 81d4267c4..7ba6101cb 100644 --- a/tom_observations/observing_strategy.py +++ b/tom_observations/observation_template.py @@ -4,46 +4,47 @@ from crispy_forms.layout import Layout, Submit from django import forms -from tom_observations.models import ObservingStrategy +from tom_observations.models import ObservationTemplate from tom_observations.cadence import get_cadence_strategies from tom_targets.models import Target -class GenericStrategyForm(forms.Form): +class GenericTemplateForm(forms.Form): """ - Form used to create new observing strategy. Any facility-specific observing strategy form should inherit from + Form used to create new observation template. Any facility-specific observation template form should inherit from this form. """ facility = forms.CharField(required=True, max_length=50, widget=forms.HiddenInput()) - strategy_name = forms.CharField() + template_name = forms.CharField() def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.helper = FormHelper() self.helper.add_input(Submit('submit', 'Submit')) - self.common_layout = Layout('facility', 'strategy_name') + self.common_layout = Layout('facility', 'template_name') def serialize_parameters(self): return json.dumps(self.cleaned_data) - def save(self, strategy_id=None): - if strategy_id: - strategy = ObservingStrategy.objects.get(id=strategy_id) + def save(self, template_id=None): + if template_id: + template = ObservationTemplate.objects.get(id=template_id) else: - strategy = ObservingStrategy() - strategy.name = self.cleaned_data['strategy_name'] - strategy.facility = self.cleaned_data['facility'] - strategy.parameters = self.serialize_parameters() - strategy.save() - return strategy + template = ObservationTemplate() + template.name = self.cleaned_data['template_name'] + template.facility = self.cleaned_data['facility'] + template.parameters = self.serialize_parameters() + template.save() + return template +# TODO: should this be ApplyTemplate? RunTemplate wouldn't make sense class RunStrategyForm(forms.Form): """ - Form used for submission of parameters for pairing an observing strategy with a cadence strategy. + Form used for submission of parameters for pairing an observation template with a cadence strategy. """ target = forms.ModelChoiceField(queryset=Target.objects.all()) - observing_strategy = forms.ModelChoiceField(queryset=ObservingStrategy.objects.all()) + observation_template = forms.ModelChoiceField(queryset=ObservationTemplate.objects.all()) cadence_strategy = forms.ChoiceField( choices=[('', '')] + [(k, k) for k in get_cadence_strategies().keys()], required=False @@ -58,7 +59,7 @@ def __init__(self, *args, **kwargs): self.helper = FormHelper() self.helper.layout = Layout( 'target', - 'observing_strategy', + 'observation_template', 'cadence_strategy', 'cadence_frequency' ) diff --git a/tom_observations/templates/tom_observations/observationrecord_detail.html b/tom_observations/templates/tom_observations/observationrecord_detail.html index 0794abd80..558a6fb46 100644 --- a/tom_observations/templates/tom_observations/observationrecord_detail.html +++ b/tom_observations/templates/tom_observations/observationrecord_detail.html @@ -73,7 +73,7 @@

    Unsaved data products

    Request Parameters

    - {% observingstrategy_from_record object %} + {% observationtemplate_from_record object %}
    {% for k,v in object.parameters_as_dict.items %}
    {{ k }}
    diff --git a/tom_observations/templates/tom_observations/observingstrategy_confirm_delete.html b/tom_observations/templates/tom_observations/observationtemplate_confirm_delete.html similarity index 100% rename from tom_observations/templates/tom_observations/observingstrategy_confirm_delete.html rename to tom_observations/templates/tom_observations/observationtemplate_confirm_delete.html diff --git a/tom_observations/templates/tom_observations/observingstrategy_form.html b/tom_observations/templates/tom_observations/observationtemplate_form.html similarity index 52% rename from tom_observations/templates/tom_observations/observingstrategy_form.html rename to tom_observations/templates/tom_observations/observationtemplate_form.html index d1b624c52..2bae4d716 100644 --- a/tom_observations/templates/tom_observations/observingstrategy_form.html +++ b/tom_observations/templates/tom_observations/observationtemplate_form.html @@ -1,7 +1,7 @@ {% extends 'tom_common/base.html' %} {% load bootstrap4 crispy_forms_tags %} -{% block title %}Create an Observation Strategy{% endblock %} +{% block title %}Create an Observation Template{% endblock %} {% block content %} -

    Create a new Observation Strategy for {{ form.facility.value }}

    +

    Create a new Observation Template for {{ form.facility.value }}

    {% crispy form %} {% endblock %} \ No newline at end of file diff --git a/tom_observations/templates/tom_observations/observingstrategy_list.html b/tom_observations/templates/tom_observations/observationtemplate_list.html similarity index 55% rename from tom_observations/templates/tom_observations/observingstrategy_list.html rename to tom_observations/templates/tom_observations/observationtemplate_list.html index 09c642c5c..42ec19853 100644 --- a/tom_observations/templates/tom_observations/observingstrategy_list.html +++ b/tom_observations/templates/tom_observations/observationtemplate_list.html @@ -2,32 +2,32 @@ {% load bootstrap4 %} {% block title %}Query List{% endblock %} {% block content %} -

    Manage Observing Strategies

    +

    Manage Observation Templates

    - Create a new observing strategy using + Create a new observation template using {% for facility in installed_facilities %} - {{ facility }} + {{ facility }} {% endfor %}

    - {% for strategy in filter.qs %} + {% for template in filter.qs %} - - - + + + {% comment %} - + {% endcomment %} - + {% empty %} {% endfor %} @@ -35,14 +35,14 @@

    Manage Observing Strategies

    NameFacilityCreatedDelete
    {{ strategy.name }}{{ strategy.facility }}{{ strategy.created }}{{ template.name }}{{ template.facility }}{{ template.created }}RunRunDeleteDelete
    - No saved strategies yet, Try creating a strategy from one of the facilities listed above. + No saved templates yet, Try creating a template from one of the facilities listed above.
    -

    Filter Saved Observing Strategies

    +

    Filter Saved Observation Templates

    {% bootstrap_form filter.form %} {% buttons %} - Reset + Reset {% endbuttons %}
    diff --git a/tom_observations/templates/tom_observations/partials/observationtemplate_from_record.html b/tom_observations/templates/tom_observations/partials/observationtemplate_from_record.html new file mode 100644 index 000000000..14dd01794 --- /dev/null +++ b/tom_observations/templates/tom_observations/partials/observationtemplate_from_record.html @@ -0,0 +1 @@ +Create template from request \ No newline at end of file diff --git a/tom_observations/templates/tom_observations/partials/observingstrategy_run.html b/tom_observations/templates/tom_observations/partials/observationtemplate_run.html similarity index 70% rename from tom_observations/templates/tom_observations/partials/observingstrategy_run.html rename to tom_observations/templates/tom_observations/partials/observationtemplate_run.html index 134c1890b..2243cc8b6 100644 --- a/tom_observations/templates/tom_observations/partials/observingstrategy_run.html +++ b/tom_observations/templates/tom_observations/partials/observationtemplate_run.html @@ -1,5 +1,5 @@ {% load bootstrap4 crispy_forms_tags %} -

    Run an observing strategy

    +

    Apply an observation template

    {% csrf_token %} {% crispy form %} diff --git a/tom_observations/templates/tom_observations/partials/observingstrategy_from_record.html b/tom_observations/templates/tom_observations/partials/observingstrategy_from_record.html deleted file mode 100644 index eef660fa3..000000000 --- a/tom_observations/templates/tom_observations/partials/observingstrategy_from_record.html +++ /dev/null @@ -1 +0,0 @@ -Create strategy from request \ No newline at end of file diff --git a/tom_observations/templatetags/observation_extras.py b/tom_observations/templatetags/observation_extras.py index 318ec162e..bbd2271b0 100644 --- a/tom_observations/templatetags/observation_extras.py +++ b/tom_observations/templatetags/observation_extras.py @@ -11,7 +11,7 @@ from tom_observations.forms import AddExistingObservationForm, UpdateObservationId from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class, get_service_classes -from tom_observations.observing_strategy import RunStrategyForm +from tom_observations.observation_template import RunStrategyForm from tom_observations.utils import get_sidereal_visibility from tom_targets.models import Target @@ -130,28 +130,28 @@ def observation_list(context, target=None): return {'observations': observations} -@register.inclusion_tag('tom_observations/partials/observingstrategy_run.html') -def observingstrategy_run(target): +@register.inclusion_tag('tom_observations/partials/observationtemplate_run.html') +def observationtemplate_run(target): """ - Renders the form for running an observing strategy. + Renders the form for running an observation template. """ form = RunStrategyForm(initial={'target': target}) form.fields['target'].widget = forms.HiddenInput() return {'form': form} -@register.inclusion_tag('tom_observations/partials/observingstrategy_from_record.html') -def observingstrategy_from_record(obsr): +@register.inclusion_tag('tom_observations/partials/observationtemplate_from_record.html') +def observationtemplate_from_record(obsr): """ - Renders a button that will pre-populate and observing strategy form with parameters from the specified + Renders a button that will pre-populate and observation template form with parameters from the specified ``ObservationRecord``. """ obs_params = obsr.parameters_as_dict obs_params.pop('target_id', None) - strategy_params = urlencode(obs_params) + template_params = urlencode(obs_params) return { 'facility': obsr.facility, - 'params': strategy_params + 'params': template_params } diff --git a/tom_observations/tests/factories.py b/tom_observations/tests/factories.py index f5e4c6563..0617b034a 100644 --- a/tom_observations/tests/factories.py +++ b/tom_observations/tests/factories.py @@ -2,7 +2,7 @@ import json from tom_targets.models import Target, TargetName -from tom_observations.models import ObservationRecord, ObservingStrategy +from tom_observations.models import ObservationRecord, ObservationTemplate class TargetNameFactory(factory.django.DjangoModelFactory): @@ -50,9 +50,9 @@ class Meta: }) -class ObservingStrategyFactory(factory.django.DjangoModelFactory): +class ObservationTemplateFactory(factory.django.DjangoModelFactory): class Meta: - model = ObservingStrategy + model = ObservationTemplate facility = 'LCO' parameters = json.dumps({ diff --git a/tom_observations/tests/tests.py b/tom_observations/tests/tests.py index ad4af63ff..efd2ebf02 100644 --- a/tom_observations/tests/tests.py +++ b/tom_observations/tests/tests.py @@ -9,10 +9,10 @@ from astropy.coordinates import get_sun, SkyCoord from astropy.time import Time -from .factories import ObservingRecordFactory, ObservingStrategyFactory, TargetFactory, TargetNameFactory +from .factories import ObservingRecordFactory, ObservationTemplateFactory, TargetFactory, TargetNameFactory from tom_observations.utils import get_astroplan_sun_and_time, get_sidereal_visibility from tom_observations.tests.utils import FakeRoboticFacility -from tom_observations.models import ObservationRecord, ObservationGroup, ObservingStrategy +from tom_observations.models import ObservationRecord, ObservationGroup, ObservationTemplate from tom_targets.models import Target from guardian.shortcuts import assign_perm @@ -189,30 +189,30 @@ class TestObservationGroupViews(TestCase): @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility']) -class TestObservingStrategyViews(TestCase): +class TestObservationTemplateViews(TestCase): def setUp(self): - self.observing_strategy = ObservingStrategyFactory.create(name='Test Strategy') + self.observation_template = ObservationTemplateFactory.create(name='Test Template') self.user = User.objects.create_user(username='test', password='test') self.client.force_login(self.user) - def test_observing_strategy_list(self): - response = self.client.get(reverse('tom_observations:strategy-list')) + def test_observation_template_list(self): + response = self.client.get(reverse('tom_observations:template-list')) self.assertEqual(response.status_code, 200) self.assertContains( - response, reverse('tom_observations:strategy-update', kwargs={'pk': self.observing_strategy.id}) + response, reverse('tom_observations:template-update', kwargs={'pk': self.observation_template.id}) ) - def test_observing_strategy_create(self): - response = self.client.get(reverse('tom_observations:strategy-create', + def test_observation_template_create(self): + response = self.client.get(reverse('tom_observations:template-create', kwargs={'facility': 'FakeRoboticFacility'})) - self.assertContains(response, 'Strategy name') + self.assertContains(response, 'Template name') - def test_observing_strategy_delete(self): - response = self.client.post(reverse('tom_observations:strategy-delete', - args=(self.observing_strategy.id,)), + def test_observation_template_delete(self): + response = self.client.post(reverse('tom_observations:template-delete', + args=(self.observation_template.id,)), follow=True) - self.assertRedirects(response, reverse('tom_observations:strategy-list'), status_code=302) - self.assertFalse(ObservingStrategy.objects.filter(pk=self.observing_strategy.id).exists()) + self.assertRedirects(response, reverse('tom_observations:template-list'), status_code=302) + self.assertFalse(ObservationTemplate.objects.filter(pk=self.observation_template.id).exists()) class TestUpdatingObservations(TestCase): diff --git a/tom_observations/tests/utils.py b/tom_observations/tests/utils.py index af3203b29..2a8305d73 100644 --- a/tom_observations/tests/utils.py +++ b/tom_observations/tests/utils.py @@ -5,7 +5,7 @@ from tom_observations.facility import BaseRoboticObservationFacility, GenericObservationForm from tom_observations.facility import BaseManualObservationFacility -from tom_observations.observing_strategy import GenericStrategyForm +from tom_observations.observation_template import GenericTemplateForm # Site data matches built-in pyephem observer data for Los Angeles SITES = { @@ -27,7 +27,7 @@ class FakeFacilityForm(GenericObservationForm): test_input = forms.CharField(help_text='fake form input') -class FakeFacilityStrategyForm(GenericStrategyForm): +class FakeFacilityTemplateForm(GenericTemplateForm): pass @@ -40,8 +40,8 @@ class FakeRoboticFacility(BaseRoboticObservationFacility): def get_form(self, observation_type): return self.observation_forms[observation_type] - def get_strategy_form(self, observation_type): - return FakeFacilityStrategyForm + def get_template_form(self, observation_type): + return FakeFacilityTemplateForm def get_observing_sites(self): return SITES @@ -82,8 +82,8 @@ class FakeManualFacility(BaseManualObservationFacility): def get_form(self, observation_type): return FakeFacilityForm - def get_strategy_form(self, observation_type): - return FakeFacilityStrategyForm + def get_template_form(self, observation_type): + return FakeFacilityTemplateForm def get_observing_sites(self): return SITES diff --git a/tom_observations/urls.py b/tom_observations/urls.py index 1bfe7e96a..900452c16 100644 --- a/tom_observations/urls.py +++ b/tom_observations/urls.py @@ -2,20 +2,20 @@ from tom_observations.views import (AddExistingObservationView, ObservationCreateView, ObservationRecordUpdateView, ObservationGroupCreateView, ObservationGroupDeleteView, ObservationGroupListView, - ObservationListView, ObservationRecordDetailView, ObservingStrategyCreateView, - ObservingStrategyDeleteView, ObservingStrategyListView, - ObservingStrategyUpdateView) + ObservationListView, ObservationRecordDetailView, ObservationTemplateCreateView, + ObservationTemplateDeleteView, ObservationTemplateListView, + ObservationTemplateUpdateView) app_name = 'tom_observations' urlpatterns = [ path('add/', AddExistingObservationView.as_view(), name='add-existing'), path('list/', ObservationListView.as_view(), name='list'), - path('strategy/list/', ObservingStrategyListView.as_view(), name='strategy-list'), - path('strategy//create/', ObservingStrategyCreateView.as_view(), name='strategy-create'), - path('strategy//update/', ObservingStrategyUpdateView.as_view(), name='strategy-update'), - path('strategy//delete/', ObservingStrategyDeleteView.as_view(), name='strategy-delete'), - path('strategy//', ObservingStrategyUpdateView.as_view(), name='strategy-detail'), + path('template/list/', ObservationTemplateListView.as_view(), name='template-list'), + path('template//create/', ObservationTemplateCreateView.as_view(), name='template-create'), + path('template//update/', ObservationTemplateUpdateView.as_view(), name='template-update'), + path('template//delete/', ObservationTemplateDeleteView.as_view(), name='template-delete'), + path('template//', ObservationTemplateUpdateView.as_view(), name='template-detail'), # This path must be above /create path('groups/create/', ObservationGroupCreateView.as_view(), name='group-create'), path('groups/list/', ObservationGroupListView.as_view(), name='group-list'), diff --git a/tom_observations/views.py b/tom_observations/views.py index a7f80cb7f..666de935c 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -27,7 +27,7 @@ from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm from tom_observations.facility import get_service_class, get_service_classes, BaseManualObservationFacility from tom_observations.forms import AddExistingObservationForm -from tom_observations.models import ObservationRecord, ObservationGroup, ObservingStrategy, RegisteredCadence +from tom_observations.models import ObservationRecord, ObservationGroup, ObservationTemplate, RegisteredCadence from tom_targets.models import Target @@ -492,9 +492,9 @@ class ObservationGroupDeleteView(Raise403PermissionRequiredMixin, DeleteView): success_url = reverse_lazy('tom_observations:group-list') -class ObservingStrategyFilter(FilterSet): +class ObservationTemplateFilter(FilterSet): """ - Defines the available fields for filtering the list of ``ObservingStrategy`` objects. + Defines the available fields for filtering the list of ``ObservationTemplate`` objects. """ facility = ChoiceFilter( choices=[(k, k) for k in get_service_classes().keys()] @@ -502,17 +502,17 @@ class ObservingStrategyFilter(FilterSet): name = CharFilter(lookup_expr='icontains') class Meta: - model = ObservingStrategy + model = ObservationTemplate fields = ['name', 'facility'] -class ObservingStrategyListView(FilterView): +class ObservationTemplateListView(FilterView): """ Displays the observing strategies that exist in the TOM. """ - model = ObservingStrategy - filterset_class = ObservingStrategyFilter - template_name = 'tom_observations/observingstrategy_list.html' + model = ObservationTemplate + filterset_class = ObservationTemplateFilter + template_name = 'tom_observations/observationtemplate_list.html' def get_context_data(self, *args, **kwargs): context = super().get_context_data(*args, **kwargs) @@ -520,12 +520,12 @@ def get_context_data(self, *args, **kwargs): return context -class ObservingStrategyCreateView(FormView): +class ObservationTemplateCreateView(FormView): """ - Displays the form for creating a new observing strategy. Uses the observing strategy form specified in the + Displays the form for creating a new observation template. Uses the observation template form specified in the respective facility class. """ - template_name = 'tom_observations/observingstrategy_form.html' + template_name = 'tom_observations/observationtemplate_form.html' def get_facility_name(self): return self.kwargs['facility'] @@ -537,11 +537,11 @@ def get_form_class(self): raise ValueError('Must provide a facility name') # TODO: modify this to work with both LCO forms - return get_service_class(facility_name)().get_strategy_form(None) + return get_service_class(facility_name)().get_template_form(None) def get_form(self, form_class=None): form = super().get_form() - form.helper.form_action = reverse('tom_observations:strategy-create', + form.helper.form_action = reverse('tom_observations:template-create', kwargs={'facility': self.get_facility_name()}) return form @@ -553,26 +553,26 @@ def get_initial(self): def form_valid(self, form): form.save() - return redirect(reverse('tom_observations:strategy-list')) + return redirect(reverse('tom_observations:template-list')) -class ObservingStrategyUpdateView(LoginRequiredMixin, FormView): +class ObservationTemplateUpdateView(LoginRequiredMixin, FormView): """ - View for updating an existing observing strategy. + View for updating an existing observation template. """ - template_name = 'tom_observations/observingstrategy_form.html' + template_name = 'tom_observations/observationtemplate_form.html' def get_object(self): - return ObservingStrategy.objects.get(pk=self.kwargs['pk']) + return ObservationTemplate.objects.get(pk=self.kwargs['pk']) def get_form_class(self): self.object = self.get_object() - return get_service_class(self.object.facility)().get_strategy_form(None) + return get_service_class(self.object.facility)().get_template_form(None) def get_form(self): form = super().get_form() form.helper.form_action = reverse( - 'tom_observations:strategy-update', kwargs={'pk': self.object.id} + 'tom_observations:template-update', kwargs={'pk': self.object.id} ) return form @@ -583,13 +583,13 @@ def get_initial(self): return initial def form_valid(self, form): - form.save(strategy_id=self.object.id) - return redirect(reverse('tom_observations:strategy-list')) + form.save(template_id=self.object.id) + return redirect(reverse('tom_observations:template-list')) -class ObservingStrategyDeleteView(LoginRequiredMixin, DeleteView): +class ObservationTemplateDeleteView(LoginRequiredMixin, DeleteView): """ - Deletes an observing strategy. + Deletes an observation template. """ - model = ObservingStrategy - success_url = reverse_lazy('tom_observations:strategy-list') + model = ObservationTemplate + success_url = reverse_lazy('tom_observations:template-list') diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index 654b5959e..0af781e01 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -54,7 +54,7 @@

    Observe

    {% observing_buttons object %}
    - {% observingstrategy_run object %} + {% observationtemplate_run object %}

    Plan

    {% if object.type == 'SIDEREAL' %} diff --git a/tom_targets/views.py b/tom_targets/views.py index 554c90fc6..030eddff7 100644 --- a/tom_targets/views.py +++ b/tom_targets/views.py @@ -28,8 +28,8 @@ from tom_common.hints import add_hint from tom_common.hooks import run_hook from tom_common.mixins import Raise403PermissionRequiredMixin -from tom_observations.observing_strategy import RunStrategyForm -from tom_observations.models import ObservingStrategy +from tom_observations.observation_template import RunStrategyForm +from tom_observations.models import ObservationTemplate from tom_targets.filters import TargetFilter from tom_targets.forms import ( SiderealTargetCreateForm, NonSiderealTargetCreateForm, TargetExtraFormset, TargetNamesFormset @@ -322,15 +322,15 @@ def get_context_data(self, *args, **kwargs): :rtype: dict """ context = super().get_context_data(*args, **kwargs) - observing_strategy_form = RunStrategyForm(initial={'target': self.get_object()}) - if any(self.request.GET.get(x) for x in ['observing_strategy', 'cadence_strategy', 'cadence_frequency']): + observation_template_form = RunStrategyForm(initial={'target': self.get_object()}) + if any(self.request.GET.get(x) for x in ['observation_template', 'cadence_strategy', 'cadence_frequency']): initial = {'target': self.object} initial.update(self.request.GET) - observing_strategy_form = RunStrategyForm( + observation_template_form = RunStrategyForm( initial=initial ) - observing_strategy_form.fields['target'].widget = HiddenInput() - context['observing_strategy_form'] = observing_strategy_form + observation_template_form.fields['target'].widget = HiddenInput() + context['observation_template_form'] = observation_template_form return context def get(self, request, *args, **kwargs): @@ -358,14 +358,14 @@ def get(self, request, *args, **kwargs): run_strategy_form = RunStrategyForm(request.GET) if run_strategy_form.is_valid(): - obs_strat = ObservingStrategy.objects.get(pk=run_strategy_form.cleaned_data['observing_strategy'].id) - obs_strat_params = obs_strat.parameters_as_dict - obs_strat_params['cadence_strategy'] = request.GET.get('cadence_strategy', '') - obs_strat_params['cadence_frequency'] = request.GET.get('cadence_frequency', '') - params = urlencode(obs_strat_params) + obs_template = ObservationTemplate.objects.get(pk=run_strategy_form.cleaned_data['observation_template'].id) + obs_template_params = obs_template.parameters_as_dict + obs_template_params['cadence_strategy'] = request.GET.get('cadence_strategy', '') + obs_template_params['cadence_frequency'] = request.GET.get('cadence_frequency', '') + params = urlencode(obs_template_params) return redirect( reverse('tom_observations:create', - args=(obs_strat.facility,)) + f'?target_id={self.get_object().id}&' + params) + args=(obs_template.facility,)) + f'?target_id={self.get_object().id}&' + params) return super().get(request, *args, **kwargs) From 9a5c0d106509c75757a8cf01fe08ef0f45a05dd6 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 17 Sep 2020 13:00:33 -0700 Subject: [PATCH 296/424] RegisteredCadence -> DynamicCadence, fixed TODOS --- docs/common/refactoring_roadmap.rst | 4 ++ .../templates/tom_common/navbar_content.html | 2 +- tom_observations/cadence.py | 24 ++++++------ .../commands/runcadencestrategies.py | 4 +- ... => 0010_manual_create_dynamic_cadence.py} | 32 ++++++++-------- .../migrations/0011_auto_20200917_0306.py | 6 +-- tom_observations/models.py | 37 +++++++++++++++---- tom_observations/observation_template.py | 3 +- .../templatetags/observation_extras.py | 4 +- tom_observations/tests/test_cadence.py | 12 +++--- tom_observations/views.py | 4 +- tom_targets/views.py | 8 ++-- 12 files changed, 81 insertions(+), 59 deletions(-) create mode 100644 docs/common/refactoring_roadmap.rst rename tom_observations/migrations/{0010_manual_create_registered_cadence.py => 0010_manual_create_dynamic_cadence.py} (68%) diff --git a/docs/common/refactoring_roadmap.rst b/docs/common/refactoring_roadmap.rst new file mode 100644 index 000000000..f888a2ffd --- /dev/null +++ b/docs/common/refactoring_roadmap.rst @@ -0,0 +1,4 @@ +- Unify the format of the namespacing for TargetLists and ObservationGroups +- Rename TargetLists to TargetGroups +- Rename TargetName to Alias +- Update TextFields used for JSON to be actual JSONFields. This will require a migration script. \ No newline at end of file diff --git a/tom_common/templates/tom_common/navbar_content.html b/tom_common/templates/tom_common/navbar_content.html index 7e11994d6..2f5c831c3 100644 --- a/tom_common/templates/tom_common/navbar_content.html +++ b/tom_common/templates/tom_common/navbar_content.html @@ -31,7 +31,7 @@ Observation Groups
  • - Observation Strategies + Observation Templates
    • diff --git a/tom_observations/cadence.py b/tom_observations/cadence.py index 40eec61d9..879d6c9e4 100644 --- a/tom_observations/cadence.py +++ b/tom_observations/cadence.py @@ -52,9 +52,9 @@ class CadenceStrategy(ABC): In order to make use of a custom CadenceStrategy, add the path to ``TOM_CADENCE_STRATEGIES`` in your ``settings.py``. """ - def __init__(self, registered_cadence, *args, **kwargs): + def __init__(self, dynamic_cadence, *args, **kwargs): self.cadence_strategy = type(self).__name__ - self.registered_cadence = registered_cadence + self.dynamic_cadence = dynamic_cadence @abstractmethod def run(self): @@ -70,13 +70,13 @@ class RetryFailedObservationsStrategy(CadenceStrategy): description = """This strategy immediately re-submits a cadenced observation without amending any other part of the cadence.""" - def __init__(self, registered_cadence, advance_window_hours, *args, **kwargs): + def __init__(self, dynamic_cadence, advance_window_hours, *args, **kwargs): self.advance_window_hours = advance_window_hours - super().__init__(registered_cadence, *args, **kwargs) + super().__init__(dynamic_cadence, *args, **kwargs) def run(self): failed_observations = [obsr for obsr - in self.registered_cadence.observation_group.observation_records.all() + in self.dynamic_cadence.observation_group.observation_records.all() if obsr.failed] new_observations = [] for obs in failed_observations: @@ -99,8 +99,8 @@ def run(self): parameters=json.dumps(observation_payload), observation_id=observation_id ) - self.registered_cadence.observation_group.observation_records.add(record) - self.registered_cadence.observation_group.save() + self.dynamic_cadence.observation_group.observation_records.add(record) + self.dynamic_cadence.observation_group.save() new_observations.append(record) return new_observations @@ -126,12 +126,12 @@ class ResumeCadenceAfterFailureStrategy(CadenceStrategy): re-submits the observation until it succeeds. If it succeeds, it submits the next observation on the same cadence.""" - def __init__(self, registered_cadence, advance_window_hours, *args, **kwargs): + def __init__(self, dynamic_cadence, advance_window_hours, *args, **kwargs): self.advance_window_hours = advance_window_hours - super().__init__(registered_cadence, *args, **kwargs) + super().__init__(dynamic_cadence, *args, **kwargs) def run(self): - last_obs = self.registered_cadence.observation_group.observation_records.order_by('-created').first() + last_obs = self.dynamic_cadence.observation_group.observation_records.order_by('-created').first() facility = get_service_class(last_obs.facility)() facility.update_observation_status(last_obs.observation_id) last_obs.refresh_from_db() @@ -164,8 +164,8 @@ def run(self): parameters=json.dumps(observation_payload), observation_id=observation_id ) - self.registered_cadence.observation_group.observation_records.add(record) - self.registered_cadence.observation_group.save() + self.dynamic_cadence.observation_group.observation_records.add(record) + self.dynamic_cadence.observation_group.save() new_observations.append(record) for obsr in new_observations: diff --git a/tom_observations/management/commands/runcadencestrategies.py b/tom_observations/management/commands/runcadencestrategies.py index 08b18c3a6..f255e24df 100644 --- a/tom_observations/management/commands/runcadencestrategies.py +++ b/tom_observations/management/commands/runcadencestrategies.py @@ -3,14 +3,14 @@ from django.core.management.base import BaseCommand from tom_observations.cadence import get_cadence_strategy -from tom_observations.models import ObservationGroup, RegisteredCadence +from tom_observations.models import ObservationGroup, DynamicCadence class Command(BaseCommand): help = 'Entry point for running cadence strategies.' def handle(self, *args, **kwargs): - cadenced_groups = RegisteredCadence.objects.exclude(active=False) + cadenced_groups = DynamicCadence.objects.exclude(active=False) for cg in cadenced_groups: cadence_frequency = cg.cadence_parameters.get('cadence_frequency', -1) diff --git a/tom_observations/migrations/0010_manual_create_registered_cadence.py b/tom_observations/migrations/0010_manual_create_dynamic_cadence.py similarity index 68% rename from tom_observations/migrations/0010_manual_create_registered_cadence.py rename to tom_observations/migrations/0010_manual_create_dynamic_cadence.py index 1af8cabce..4e42d5293 100644 --- a/tom_observations/migrations/0010_manual_create_registered_cadence.py +++ b/tom_observations/migrations/0010_manual_create_dynamic_cadence.py @@ -3,24 +3,23 @@ from django.db import migrations, models -def copy_cadence_fields_to_registered_cadence(apps, schema_editor): +def copy_cadence_fields_to_dynamic_cadence(apps, schema_editor): observation_groups = apps.get_model('tom_observations', 'ObservationGroup') for row in observation_groups.objects.exclude(cadence_strategy=''): - registered_cadence = apps.get_model('tom_observations', 'RegisteredCadence') + dynamic_cadence = apps.get_model('tom_observations', 'DynamicCadence') try: cadence_parameters = json.loads(getattr(row, 'cadence_parameters')) except json.decoder.JSONDecodeError: cadence_parameters = {} - new_registered_cadence = registered_cadence( - name=getattr(row, 'name'), + new_dynamic_cadence = dynamic_cadence( observation_group=row, cadence_strategy=getattr(row, 'cadence_strategy'), cadence_parameters=cadence_parameters, - active=False, + active=True, created=getattr(row, 'created'), modified=getattr(row, 'modified') ) - new_registered_cadence.save() + new_dynamic_cadence.save() class Migration(migrations.Migration): @@ -30,32 +29,31 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name='RegisteredCadence', + name='DynamicCadence', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=100, unique=True, help_text='Name of this RegisteredCadence', - verbose_name='Name of this RegisteredCadence')), - ('cadence_strategy', models.CharField(max_length=100, - verbose_name='Cadence strategy used for this RegisteredCadence')), - ('cadence_parameters', models.JSONField(verbose_name='Cadence-specific parameters')), + ('cadence_strategy', models.CharField(max_length=100, blank=False, default=None, + verbose_name='Cadence strategy used for this DynamicCadence')), + ('cadence_parameters', models.JSONField(blank=False, null=False, + verbose_name='Cadence-specific parameters')), ('active', models.BooleanField(verbose_name='Active', - help_text='''Whether or not this RegisteredCadence should continue + help_text='''Whether or not this DynamicCadence should continue to submit observations.''')), ('created', models.DateTimeField(auto_now_add=True, - help_text='The time which this RegisteredCadence was created.')), + help_text='The time which this DynamicCadence was created.')), ('modified', models.DateTimeField(auto_now=True, - help_text='The time which this RegisteredCadence was modified.')), + help_text='The time which this DynamicCadence was modified.')), ] ), migrations.AddField( - model_name='registeredcadence', + model_name='dynamiccadence', name='observation_group', field=models.ForeignKey(on_delete=models.deletion.CASCADE, null=False, default=None, to='tom_observations.ObservationGroup'), ), - migrations.RunPython(copy_cadence_fields_to_registered_cadence, reverse_code=migrations.RunPython.noop), + migrations.RunPython(copy_cadence_fields_to_dynamic_cadence, reverse_code=migrations.RunPython.noop), migrations.RemoveField( model_name='observationgroup', diff --git a/tom_observations/migrations/0011_auto_20200917_0306.py b/tom_observations/migrations/0011_auto_20200917_0306.py index 9f0edd1bf..0f9234f70 100644 --- a/tom_observations/migrations/0011_auto_20200917_0306.py +++ b/tom_observations/migrations/0011_auto_20200917_0306.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('tom_observations', '0010_manual_create_registered_cadence'), + ('tom_observations', '0010_manual_create_dynamic_cadence'), ] operations = [ @@ -19,8 +19,8 @@ class Migration(migrations.Migration): options={'ordering': ('-created', 'name')}, ), migrations.AlterField( - model_name='registeredcadence', + model_name='dynamiccadence', name='active', - field=models.BooleanField(help_text='Whether or not this RegisteredCadence should\n continue to submit observations.', verbose_name='Active'), + field=models.BooleanField(help_text='Whether or not this DynamicCadence should\n continue to submit observations.', verbose_name='Active'), ), ] diff --git a/tom_observations/models.py b/tom_observations/models.py index 952c919bd..a1a13dd6c 100644 --- a/tom_observations/models.py +++ b/tom_observations/models.py @@ -123,17 +123,38 @@ def __str__(self): return self.name -class RegisteredCadence(models.Model): - name = models.CharField(max_length=100, unique=True, help_text='Name of this RegisteredCadence', - verbose_name='Name of this RegisteredCadence') # TODO: Does this need to exist, if so, what do with ObservationCreateView.is_valid()? +class DynamicCadence(models.Model): + """ + Class representing a dynamic cadence--that is, a cadence that follows a pattern but modifies its behavior + depending on the result of prior observations. + + :param observation_group: The ``ObservationGroup`` containing the observations that were created by this cadence. + :type observation_group: ``ObservationGroup`` + + :param cadence_strategy: The name of the cadence strategy this cadence is using. + :type cadence_strategy: str + + :param cadence_parameters: The parameters for this cadence, e.g. cadence period + :type cadence_parameters: JSON + + :param active: Whether or not this cadence should continue to submit observations + :type active: boolean + + :param created: The time at which this ``DynamicCadence`` was created. + :type created: datetime + + :param modified: The time at which this ``DynamicCadence`` was modified. + :type modified: datetime + """ observation_group = models.ForeignKey(ObservationGroup, null=False, default=None, on_delete=models.CASCADE) - cadence_strategy = models.CharField(max_length=100, verbose_name='Cadence strategy used for this RegisteredCadence') - cadence_parameters = models.JSONField(verbose_name='Cadence-specific parameters') + cadence_strategy = models.CharField(max_length=100, blank=False, default=None, + verbose_name='Cadence strategy used for this DynamicCadence') + cadence_parameters = models.JSONField(blank=False, null=False, verbose_name='Cadence-specific parameters') active = models.BooleanField(verbose_name='Active', - help_text='''Whether or not this RegisteredCadence should + help_text='''Whether or not this DynamicCadence should continue to submit observations.''') - created = models.DateTimeField(auto_now_add=True, help_text='The time which this RegisteredCadence was created.') - modified = models.DateTimeField(auto_now=True, help_text='The time which this RegisteredCadence was modified.') + created = models.DateTimeField(auto_now_add=True, help_text='The time which this DynamicCadence was created.') + modified = models.DateTimeField(auto_now=True, help_text='The time which this DynamicCadence was modified.') def __str__(self): return self.name diff --git a/tom_observations/observation_template.py b/tom_observations/observation_template.py index 7ba6101cb..9484004af 100644 --- a/tom_observations/observation_template.py +++ b/tom_observations/observation_template.py @@ -38,8 +38,7 @@ def save(self, template_id=None): return template -# TODO: should this be ApplyTemplate? RunTemplate wouldn't make sense -class RunStrategyForm(forms.Form): +class ApplyObservationTemplateForm(forms.Form): """ Form used for submission of parameters for pairing an observation template with a cadence strategy. """ diff --git a/tom_observations/templatetags/observation_extras.py b/tom_observations/templatetags/observation_extras.py index bbd2271b0..b88cbe1b6 100644 --- a/tom_observations/templatetags/observation_extras.py +++ b/tom_observations/templatetags/observation_extras.py @@ -11,7 +11,7 @@ from tom_observations.forms import AddExistingObservationForm, UpdateObservationId from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class, get_service_classes -from tom_observations.observation_template import RunStrategyForm +from tom_observations.observation_template import ApplyObservationTemplateForm from tom_observations.utils import get_sidereal_visibility from tom_targets.models import Target @@ -135,7 +135,7 @@ def observationtemplate_run(target): """ Renders the form for running an observation template. """ - form = RunStrategyForm(initial={'target': target}) + form = ApplyObservationTemplateForm(initial={'target': target}) form.fields['target'].widget = forms.HiddenInput() return {'form': form} diff --git a/tom_observations/tests/test_cadence.py b/tom_observations/tests/test_cadence.py index 9eb58d79b..5d274d150 100644 --- a/tom_observations/tests/test_cadence.py +++ b/tom_observations/tests/test_cadence.py @@ -4,7 +4,7 @@ from dateutil.parser import parse from .factories import ObservingRecordFactory, TargetFactory -from tom_observations.models import ObservationGroup, RegisteredCadence +from tom_observations.models import ObservationGroup, DynamicCadence from tom_observations.cadence import RetryFailedObservationsStrategy, ResumeCadenceAfterFailureStrategy @@ -31,8 +31,8 @@ def setUp(self): self.group.observation_records.add(*observing_records) self.group.save() # TODO: why doesn't this fail without cadence_strategy and name? - self.registered_cadence = RegisteredCadence.objects.create( - cadence_parameters={}, active=True, observation_group=self.group) + self.dynamic_cadence = DynamicCadence.objects.create( + cadence_strategy='Test Strategy', cadence_parameters={}, active=True, observation_group=self.group) def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4): num_records = self.group.observation_records.count() @@ -40,7 +40,7 @@ def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4): observing_record.status = 'CANCELED' observing_record.save() - strategy = RetryFailedObservationsStrategy(self.registered_cadence, 72) + strategy = RetryFailedObservationsStrategy(self.dynamic_cadence, 72) new_records = strategy.run() self.group.refresh_from_db() # Make sure the candence run created a new observation. @@ -57,7 +57,7 @@ def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4): def test_resume_when_failed_cadence_failed_obs(self, patch1, patch2, patch3, patch4, patch5): num_records = self.group.observation_records.count() - strategy = ResumeCadenceAfterFailureStrategy(self.registered_cadence, 72) + strategy = ResumeCadenceAfterFailureStrategy(self.dynamic_cadence, 72) new_records = strategy.run() self.group.refresh_from_db() self.assertEqual(num_records + 1, self.group.observation_records.count()) @@ -72,7 +72,7 @@ def test_resume_when_failed_cadence_successful_obs(self, patch1, patch2, patch3, num_records = self.group.observation_records.count() observing_record = self.group.observation_records.order_by('-created').first() - strategy = ResumeCadenceAfterFailureStrategy(self.registered_cadence, 72) + strategy = ResumeCadenceAfterFailureStrategy(self.dynamic_cadence, 72) new_records = strategy.run() self.group.refresh_from_db() self.assertEqual(num_records + 1, self.group.observation_records.count()) diff --git a/tom_observations/views.py b/tom_observations/views.py index 666de935c..78cb96b7a 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -27,7 +27,7 @@ from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm from tom_observations.facility import get_service_class, get_service_classes, BaseManualObservationFacility from tom_observations.forms import AddExistingObservationForm -from tom_observations.models import ObservationRecord, ObservationGroup, ObservationTemplate, RegisteredCadence +from tom_observations.models import ObservationRecord, ObservationGroup, ObservationTemplate, DynamicCadence from tom_targets.models import Target @@ -267,7 +267,7 @@ def form_valid(self, form): assign_perm('tom_observations.delete_observationgroup', self.request.user, observation_group) if form.cleaned_data.get('cadence_strategy'): - registered_cadence = RegisteredCadence.objects.create( + dynamic_cadence = DynamicCadence.objects.create( observation_group=observation_group, cadence_strategy=form.cleaned_data.get('cadence_strategy'), cadence_parameters={'cadence_frequency': form.cleaned_data.get('cadence_frequency')} diff --git a/tom_targets/views.py b/tom_targets/views.py index 030eddff7..a465f6970 100644 --- a/tom_targets/views.py +++ b/tom_targets/views.py @@ -28,7 +28,7 @@ from tom_common.hints import add_hint from tom_common.hooks import run_hook from tom_common.mixins import Raise403PermissionRequiredMixin -from tom_observations.observation_template import RunStrategyForm +from tom_observations.observation_template import ApplyObservationTemplateForm from tom_observations.models import ObservationTemplate from tom_targets.filters import TargetFilter from tom_targets.forms import ( @@ -356,9 +356,9 @@ def get(self, request, *args, **kwargs): ' the docs.')) return redirect(reverse('tom_targets:detail', args=(target_id,))) - run_strategy_form = RunStrategyForm(request.GET) - if run_strategy_form.is_valid(): - obs_template = ObservationTemplate.objects.get(pk=run_strategy_form.cleaned_data['observation_template'].id) + obs_template_form = ApplyObservationTemplateForm(request.GET) + if obs_template_form.is_valid(): + obs_template = ObservationTemplate.objects.get(pk=obs_template_form.cleaned_data['observation_template'].id) obs_template_params = obs_template.parameters_as_dict obs_template_params['cadence_strategy'] = request.GET.get('cadence_strategy', '') obs_template_params['cadence_frequency'] = request.GET.get('cadence_frequency', '') From 86b306ecb6b88ee205417eac4ead507451cc191a Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 17 Sep 2020 13:11:30 -0700 Subject: [PATCH 297/424] Updated refactoring roadmap --- docs/common/refactoring_roadmap.rst | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/docs/common/refactoring_roadmap.rst b/docs/common/refactoring_roadmap.rst index f888a2ffd..f6bb9aacd 100644 --- a/docs/common/refactoring_roadmap.rst +++ b/docs/common/refactoring_roadmap.rst @@ -1,4 +1,24 @@ -- Unify the format of the namespacing for TargetLists and ObservationGroups -- Rename TargetLists to TargetGroups -- Rename TargetName to Alias -- Update TextFields used for JSON to be actual JSONFields. This will require a migration script. \ No newline at end of file +* Unify the format of the namespacing for TargetLists and ObservationGroups + + * At present, the "names" in the urlconf entries for TargetLists are of the format -group. + The corresponding "names" in the urlconf entries for ObservationGroups are of the format group-. + We should standardize. Also, the TargetGroupingListView urlconf entry is "targetgrouping", which should also + be fixed. + +* Rename TargetLists to TargetGroups + + * The model name for the many-to-many relationship of targets is TargetList--however, this name is + avoided in documentation and displays due to the existence of the TargetListView, which lists all + targets. We should rename it to TargetGroups and ensure all related methods and references are up + to date. + +* Rename TargetName to Alias + + * TargetName is the model name for name objects that are related to a target. However, because the + target also has a "name" property, we should rename the model to "Alias". Confer with Rachel/Curtis + first. + +* Update TextFields used for JSON to be actual JSONFields. This will require a migration script. + + * When first written, Django only supported JSONField for PostgreSQL DB backends. As of 3.1, JSONField + is standard, and should replace TextField where it can. From 5de4dbb6891bdf8ee2b6f1ab60c8afd745c87a98 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 17 Sep 2020 13:12:41 -0700 Subject: [PATCH 298/424] Corrected old references to RunStrategyForm --- tom_targets/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tom_targets/views.py b/tom_targets/views.py index a465f6970..282746753 100644 --- a/tom_targets/views.py +++ b/tom_targets/views.py @@ -322,11 +322,11 @@ def get_context_data(self, *args, **kwargs): :rtype: dict """ context = super().get_context_data(*args, **kwargs) - observation_template_form = RunStrategyForm(initial={'target': self.get_object()}) + observation_template_form = ApplyObservationTemplateForm(initial={'target': self.get_object()}) if any(self.request.GET.get(x) for x in ['observation_template', 'cadence_strategy', 'cadence_frequency']): initial = {'target': self.object} initial.update(self.request.GET) - observation_template_form = RunStrategyForm( + observation_template_form = ApplyObservationTemplateForm( initial=initial ) observation_template_form.fields['target'].widget = HiddenInput() From 66eb4645e85befb8eae4b25e8c2e843fffc8a25d Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 17 Sep 2020 13:24:54 -0700 Subject: [PATCH 299/424] Updated cadencing docs to reflect new naming conventions --- docs/observing/strategies.rst | 108 +++++++++++++++++----------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/docs/observing/strategies.rst b/docs/observing/strategies.rst index efdc7f12c..49db1b572 100644 --- a/docs/observing/strategies.rst +++ b/docs/observing/strategies.rst @@ -1,4 +1,4 @@ -Cadence and Observing Strategies +Dynamic Cadences and Observation Templates ================================ The TOM has a couple of unique concepts that may be unfamiliar to some @@ -14,17 +14,17 @@ parameters filled in or changed. An observation template can also be creating from a past observation, with a button to do so that’s available on any ObservationRecord detail page. -The second concept referred to is a cadence strategy. A cadence is as it +The second concept referred to is a dynamic cadence. A cadence is as it sounds–a series of observations that are performed at regular intervals. However, most observatories don’t have built-in support for cadences, and, if they do, they may be limited to a predetermined cadence. The TOM -Toolkit, on the other hand, allows for a *reactive* cadence. Because +Toolkit, on the other hand, allows for a *dynamic* cadence. Because data is collected programmatically, and observations are submitted -programmatically, a user can write their own cadence to submit +programmatically, a user can write their own cadence strategy to submit observations depending on the success of a prior observation or the data collected from a prior observation. -Writing a custom cadence strategy +Writing a custom dynamic cadence --------------------------------- Many of the TOM modules leverage a plugin architecture that enables you @@ -91,10 +91,10 @@ through the business logic of a built-in cadence strategy. We’re going to review the ``ResumeCadenceAfterFailureStrategy``. It should also be worth mentioning at this point that the -``CadenceStrategy`` constructor takes an ``observation group``. The -``observation_group`` is the set of observations that make up the -cadence, and is created in the ``ObservationCreateView`` when the first -observation of a cadence is submitted. +``CadenceStrategy`` constructor takes a ``dynamic_cadence``. The +``dynamic_cadence`` is the association of the cadence strategy and the +observation group that make up the cadence, and is created in the +``ObservationCreateView`` when the first observation of a cadence is submitted. The ``ResumeCadenceAfterFailureStrategy`` is designed to ensure that, even after an observation fails, the cadence remains consistent. If, for @@ -106,7 +106,7 @@ Let’s look at the strategy piece by piece. .. code:: python - last_obs = self.observation_group.observation_records.order_by('-created').first() + last_obs = self.dynamic_cadence.observation_group.observation_records.order_by('-created').first() facility = get_service_class(last_obs.facility)() facility.update_observation_status(last_obs.observation_id) last_obs.refresh_from_db() @@ -173,8 +173,8 @@ observation, using a utility method that’s part of the parameters=json.dumps(observation_payload), observation_id=observation_id ) - self.observation_group.observation_records.add(record) - self.observation_group.save() + self.dynamic_cadence.observation_group.observation_records.add(record) + self.dynamic_cadence.observation_group.save() new_observations.append(record) for obsr in new_observations: @@ -195,48 +195,48 @@ Just to review, here is the strategy’s ``run()`` in its entirety: .. code:: python def run(self): - last_obs = self.observation_group.observation_records.order_by('-created').first() - facility = get_service_class(last_obs.facility)() - facility.update_observation_status(last_obs.observation_id) - last_obs.refresh_from_db() - start_keyword, end_keyword = facility.get_start_end_keywords() - observation_payload = last_obs.parameters_as_dict - new_observations = [] - if not last_obs.terminal: - return - elif last_obs.failed: - # Submit next observation to be taken as soon as possible - window_length = parse(observation_payload[end_keyword]) - parse(observation_payload[start_keyword]) - observation_payload[start_keyword] = datetime.now().isoformat() - observation_payload[end_keyword] = (parse(observation_payload[start_keyword]) + window_length).isoformat() - else: - # Advance window normally according to cadence parameters - observation_payload = self.advance_window( - observation_payload, start_keyword=start_keyword, end_keyword=end_keyword - ) - - obs_type = last_obs.parameters_as_dict.get('observation_type') - form = facility.get_form(obs_type)(observation_payload) - form.is_valid() - observation_ids = facility.submit_observation(form.observation_payload()) - - for observation_id in observation_ids: - # Create Observation record - record = ObservationRecord.objects.create( - target=last_obs.target, - facility=facility.name, - parameters=json.dumps(observation_payload), - observation_id=observation_id - ) - self.observation_group.observation_records.add(record) - self.observation_group.save() - new_observations.append(record) - - for obsr in new_observations: - facility = get_service_class(obsr.facility)() - facility.update_observation_status(obsr.observation_id) - - return new_observations + last_obs = self.dynamic_cadence.observation_group.observation_records.order_by('-created').first() + facility = get_service_class(last_obs.facility)() + facility.update_observation_status(last_obs.observation_id) + last_obs.refresh_from_db() + start_keyword, end_keyword = facility.get_start_end_keywords() + observation_payload = last_obs.parameters_as_dict + new_observations = [] + if not last_obs.terminal: + return + elif last_obs.failed: + # Submit next observation to be taken as soon as possible + window_length = parse(observation_payload[end_keyword]) - parse(observation_payload[start_keyword]) + observation_payload[start_keyword] = datetime.now().isoformat() + observation_payload[end_keyword] = (parse(observation_payload[start_keyword]) + window_length).isoformat() + else: + # Advance window normally according to cadence parameters + observation_payload = self.advance_window( + observation_payload, start_keyword=start_keyword, end_keyword=end_keyword + ) + + obs_type = last_obs.parameters_as_dict.get('observation_type') + form = facility.get_form(obs_type)(observation_payload) + form.is_valid() + observation_ids = facility.submit_observation(form.observation_payload()) + + for observation_id in observation_ids: + # Create Observation record + record = ObservationRecord.objects.create( + target=last_obs.target, + facility=facility.name, + parameters=json.dumps(observation_payload), + observation_id=observation_id + ) + self.dynamic_cadence.observation_group.observation_records.add(record) + self.dynamic_cadence.observation_group.save() + new_observations.append(record) + + for obsr in new_observations: + facility = get_service_class(obsr.facility)() + facility.update_observation_status(obsr.observation_id) + + return new_observations Configuring the cadence strategy to run automatically ----------------------------------------------------- From b91870d4fc7b89d9047141b3476e3e57b3daab98 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 17 Sep 2020 13:32:16 -0700 Subject: [PATCH 300/424] Fixed codacy issues --- tom_observations/management/commands/runcadencestrategies.py | 2 +- tom_observations/views.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tom_observations/management/commands/runcadencestrategies.py b/tom_observations/management/commands/runcadencestrategies.py index f255e24df..25d074792 100644 --- a/tom_observations/management/commands/runcadencestrategies.py +++ b/tom_observations/management/commands/runcadencestrategies.py @@ -3,7 +3,7 @@ from django.core.management.base import BaseCommand from tom_observations.cadence import get_cadence_strategy -from tom_observations.models import ObservationGroup, DynamicCadence +from tom_observations.models import DynamicCadence class Command(BaseCommand): diff --git a/tom_observations/views.py b/tom_observations/views.py index 78cb96b7a..485570686 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -267,7 +267,7 @@ def form_valid(self, form): assign_perm('tom_observations.delete_observationgroup', self.request.user, observation_group) if form.cleaned_data.get('cadence_strategy'): - dynamic_cadence = DynamicCadence.objects.create( + DynamicCadence.objects.create( observation_group=observation_group, cadence_strategy=form.cleaned_data.get('cadence_strategy'), cadence_parameters={'cadence_frequency': form.cleaned_data.get('cadence_frequency')} From bba6f1a9fe415610f74340a190c685c25dceefbd Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 17 Sep 2020 14:48:48 -0700 Subject: [PATCH 301/424] Added todos in runcadencestrategies management command --- tom_observations/management/commands/runcadencestrategies.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tom_observations/management/commands/runcadencestrategies.py b/tom_observations/management/commands/runcadencestrategies.py index 25d074792..0bf93027e 100644 --- a/tom_observations/management/commands/runcadencestrategies.py +++ b/tom_observations/management/commands/runcadencestrategies.py @@ -10,10 +10,12 @@ class Command(BaseCommand): help = 'Entry point for running cadence strategies.' def handle(self, *args, **kwargs): - cadenced_groups = DynamicCadence.objects.exclude(active=False) + cadenced_groups = DynamicCadence.objects.filter(active=True) for cg in cadenced_groups: cadence_frequency = cg.cadence_parameters.get('cadence_frequency', -1) + # TODO: pass cadence parameters in as kwargs or access them in the strategy + # TODO: make cadence form strategy-specific strategy = get_cadence_strategy(cg.cadence_strategy)(cg, cadence_frequency) new_observations = strategy.run() if not new_observations: From 37178c8544536966786e58dc5b7293732d1de916 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 17 Sep 2020 15:02:36 -0700 Subject: [PATCH 302/424] Fixing bug with missing active parameter when creating DynamicCadence --- tom_observations/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tom_observations/views.py b/tom_observations/views.py index 485570686..402da1bee 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -266,11 +266,13 @@ def form_valid(self, form): assign_perm('tom_observations.change_observationgroup', self.request.user, observation_group) assign_perm('tom_observations.delete_observationgroup', self.request.user, observation_group) + # TODO: Add a test case that includes a dynamic cadence submission if form.cleaned_data.get('cadence_strategy'): DynamicCadence.objects.create( observation_group=observation_group, cadence_strategy=form.cleaned_data.get('cadence_strategy'), - cadence_parameters={'cadence_frequency': form.cleaned_data.get('cadence_frequency')} + cadence_parameters={'cadence_frequency': form.cleaned_data.get('cadence_frequency')}, + active=True ) if not settings.TARGET_PERMISSIONS_ONLY: From dd98bfdc1c2c50f04a53d2a9ffd9ef5798c16233 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2020 13:58:32 +0000 Subject: [PATCH 303/424] Bump specutils from 1.0 to 1.1 Bumps [specutils](http://specutils.readthedocs.io/) from 1.0 to 1.1. Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 38973b8e0..355179554 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ 'plotly==4.10.0', 'python-dateutil==2.8.1', 'requests==2.24.0', - 'specutils==1.0', + 'specutils==1.1', ], extras_require={ 'test': ['factory_boy==3.0.1'] From 47712d06c3c8a07f14beb2db87f5f2a36f791415 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2020 13:58:58 +0000 Subject: [PATCH 304/424] Bump django-extensions from 3.0.8 to 3.0.9 Bumps [django-extensions](https://github.com/django-extensions/django-extensions) from 3.0.8 to 3.0.9. - [Release notes](https://github.com/django-extensions/django-extensions/releases) - [Changelog](https://github.com/django-extensions/django-extensions/blob/master/CHANGELOG.md) - [Commits](https://github.com/django-extensions/django-extensions/compare/3.0.8...3.0.9) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 38973b8e0..46fa1955c 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ 'django-bootstrap4==2.2.0', 'django-contrib-comments==1.9.2', # Earlier version are incompatible with Django >= 3.0 'django-crispy-forms==1.9.2', - 'django-extensions==3.0.8', + 'django-extensions==3.0.9', 'django-gravatar2==1.4.4', 'django-filter==2.3.0', 'django-guardian==2.3.0', From ab3c6aa4a3178ab7110d375291d9c2b23d13b431 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 22 Sep 2020 12:23:48 -0700 Subject: [PATCH 305/424] form refactor WIP --- tom_observations/cadence.py | 42 +++++++++++++++---- tom_observations/cadences/__init__.py | 0 .../cadences/resumecadenceafterfailure.py | 0 tom_observations/observation_template.py | 3 +- .../templatetags/observation_extras.py | 16 +++++-- 5 files changed, 48 insertions(+), 13 deletions(-) create mode 100644 tom_observations/cadences/__init__.py create mode 100644 tom_observations/cadences/resumecadenceafterfailure.py diff --git a/tom_observations/cadence.py b/tom_observations/cadence.py index 879d6c9e4..fad787bdf 100644 --- a/tom_observations/cadence.py +++ b/tom_observations/cadence.py @@ -70,8 +70,8 @@ class RetryFailedObservationsStrategy(CadenceStrategy): description = """This strategy immediately re-submits a cadenced observation without amending any other part of the cadence.""" - def __init__(self, dynamic_cadence, advance_window_hours, *args, **kwargs): - self.advance_window_hours = advance_window_hours + def __init__(self, dynamic_cadence, *args, **kwargs): + self.advance_window_hours = kwargs.pop('advance_window_hours') super().__init__(dynamic_cadence, *args, **kwargs) def run(self): @@ -125,11 +125,32 @@ class ResumeCadenceAfterFailureStrategy(CadenceStrategy): description = """This strategy schedules one observation in the cadence at a time. If the observation fails, it re-submits the observation until it succeeds. If it succeeds, it submits the next observation on the same cadence.""" + form_parameters = { + 'site': { + 'field': forms.ChoiceField, + 'kwargs': { + 'choices': (('cpt', 'cpt'), ('tlv', 'tlv')) + } + }, + 'period': forms.IntegerField + } + + # for key, value in form_parameters.items: + # form[key] = value['field'](**value['kwargs']) + + class ResumeCadenceForm(forms.Form): + site = forms.CharField() - def __init__(self, dynamic_cadence, advance_window_hours, *args, **kwargs): - self.advance_window_hours = advance_window_hours + def __init__(self, dynamic_cadence, *args, **kwargs): + self.advance_window_hours = kwargs.pop('advance_window_hours') super().__init__(dynamic_cadence, *args, **kwargs) + def update_observation_parameters(self): + update_observation_payload() + + def submit_next_observation(self): + submit() + def run(self): last_obs = self.dynamic_cadence.observation_group.observation_records.order_by('-created').first() facility = get_service_class(last_obs.facility)() @@ -147,9 +168,11 @@ def run(self): observation_payload[end_keyword] = (parse(observation_payload[start_keyword]) + window_length).isoformat() else: # Advance window normally according to cadence parameters - observation_payload = self.advance_window( - observation_payload, start_keyword=start_keyword, end_keyword=end_keyword - ) + self.update_observation_parameters() + self.submit_next_observation() + # observation_payload = self.advance_window( + # observation_payload, start_keyword=start_keyword, end_keyword=end_keyword + # ) obs_type = last_obs.parameters_as_dict.get('observation_type') form = facility.get_form(obs_type)(observation_payload) @@ -194,6 +217,7 @@ class CadenceForm(forms.Form): ) def __init__(self, *args, **kwargs): + print(kwargs) super().__init__(*args, **kwargs) self.fields['cadence_strategy'].widget.attrs['readonly'] = True self.fields['cadence_frequency'].widget.attrs['readonly'] = True @@ -205,9 +229,11 @@ def cadence_layout(self): if not (self.initial.get('cadence_strategy') or self.initial.get('cadence_frequency')): return Layout() else: + # TODO: Clarify dynamic vs. static cadencing in form + # TODO: Present layout for selected cadence strategy return Layout( Div( - HTML('

    Reactive cadencing parameters. Leave blank if no reactive cadencing is desired.

    '), + HTML('

    Dynamic cadencing parameters. Leave blank if no dynamic cadencing is desired.

    '), ), Div( Div( diff --git a/tom_observations/cadences/__init__.py b/tom_observations/cadences/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tom_observations/cadences/resumecadenceafterfailure.py b/tom_observations/cadences/resumecadenceafterfailure.py new file mode 100644 index 000000000..e69de29bb diff --git a/tom_observations/observation_template.py b/tom_observations/observation_template.py index 9484004af..9bd83ef46 100644 --- a/tom_observations/observation_template.py +++ b/tom_observations/observation_template.py @@ -48,6 +48,7 @@ class ApplyObservationTemplateForm(forms.Form): choices=[('', '')] + [(k, k) for k in get_cadence_strategies().keys()], required=False ) + # TODO: Remove non-cadence_strategy fields from this form cadence_frequency = forms.IntegerField( required=False, help_text='Frequency of observations, in hours' @@ -63,4 +64,4 @@ def __init__(self, *args, **kwargs): 'cadence_frequency' ) self.helper.form_method = 'GET' - self.helper.add_input(Submit('run', 'Run')) + self.helper.add_input(Submit('run', 'Apply')) diff --git a/tom_observations/templatetags/observation_extras.py b/tom_observations/templatetags/observation_extras.py index b88cbe1b6..c4ff95941 100644 --- a/tom_observations/templatetags/observation_extras.py +++ b/tom_observations/templatetags/observation_extras.py @@ -99,10 +99,18 @@ def observation_plan(target, facility, length=7, interval=60, airmass_limit=None end_time = start_time + timedelta(days=length) visibility_data = get_sidereal_visibility(target, start_time, end_time, interval, airmass_limit) - plot_data = [ - go.Scatter(x=data[0], y=data[1], mode='lines', name=site) for site, data in visibility_data.items() - ] - layout = go.Layout(yaxis=dict(autorange='reversed')) + i = 0 + plot_data = [] + for site, data in visibility_data.items(): + plot_data.append(go.Scatter(x=data[0], y=data[1], mode='markers+lines', marker={'symbol': i}, name=site)) + i += 1 + # plot_data = [ + # go.Scatter(x=data[0], y=data[1], mode='markers', marker={'symbol': 'line-ew-open'}, name=site) for site, data in visibility_data.items() + # ] + layout = go.Layout( + xaxis={'title': 'Date'}, + yaxis={'autorange': 'reversed', 'title': 'Airmass'} + ) visibility_graph = offline.plot( go.Figure(data=plot_data, layout=layout), output_type='div', show_link=False ) From e5de0193c505647f01df45de3c7aa2385a36462f Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 24 Sep 2020 15:00:28 -0700 Subject: [PATCH 306/424] Added composite form creation to ObservationCreateView --- tom_base/settings.py | 4 +- tom_observations/cadence.py | 170 ++---------------- .../cadences/resume_cadence_after_failure.py | 119 ++++++++++++ .../cadences/resumecadenceafterfailure.py | 0 .../cadences/retry_failed_observations.py | 65 +++++++ tom_observations/facilities/lco.py | 5 +- tom_observations/tests/test_cadence.py | 3 +- tom_observations/views.py | 38 +++- 8 files changed, 232 insertions(+), 172 deletions(-) create mode 100644 tom_observations/cadences/resume_cadence_after_failure.py delete mode 100644 tom_observations/cadences/resumecadenceafterfailure.py create mode 100644 tom_observations/cadences/retry_failed_observations.py diff --git a/tom_base/settings.py b/tom_base/settings.py index 886c0d24e..758f52eb0 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -217,8 +217,8 @@ ] TOM_CADENCE_STRATEGIES = [ - 'tom_observations.cadence.RetryFailedObservationsStrategy', - 'tom_observations.cadence.ResumeCadenceAfterFailureStrategy' + 'tom_observations.cadences.retry_failed_observations.RetryFailedObservationsStrategy', + 'tom_observations.cadences.resume_cadence_after_failure.ResumeCadenceAfterFailureStrategy' ] # Define extra target fields here. Types can be any of "number", "string", "boolean" or "datetime" diff --git a/tom_observations/cadence.py b/tom_observations/cadence.py index fad787bdf..3b165ef9d 100644 --- a/tom_observations/cadence.py +++ b/tom_observations/cadence.py @@ -8,13 +8,10 @@ from django import forms from django.conf import settings -from tom_observations.facility import get_service_class -from tom_observations.models import ObservationRecord - DEFAULT_CADENCE_STRATEGIES = [ - 'tom_observations.cadence.RetryFailedObservationsStrategy', - 'tom_observations.cadence.ResumeCadenceAfterFailureStrategy' + 'tom_observations.cadences.retry_failed_observations.RetryFailedObservationsStrategy', + 'tom_observations.cadences.resume_cadence_after_failure.ResumeCadenceAfterFailureStrategy' ] @@ -61,156 +58,13 @@ def run(self): pass -class RetryFailedObservationsStrategy(CadenceStrategy): - """ - The RetryFailedObservationsStrategy immediately re-submits all observations within an observation group a certain - number of hours later, as specified by ``advance_window_hours``. - """ - name = 'Retry Failed Observations' - description = """This strategy immediately re-submits a cadenced observation without amending any other part of the - cadence.""" - - def __init__(self, dynamic_cadence, *args, **kwargs): - self.advance_window_hours = kwargs.pop('advance_window_hours') - super().__init__(dynamic_cadence, *args, **kwargs) - - def run(self): - failed_observations = [obsr for obsr - in self.dynamic_cadence.observation_group.observation_records.all() - if obsr.failed] - new_observations = [] - for obs in failed_observations: - observation_payload = obs.parameters_as_dict - facility = get_service_class(obs.facility)() - start_keyword, end_keyword = facility.get_start_end_keywords() - observation_payload = self.advance_window( - observation_payload, start_keyword=start_keyword, end_keyword=end_keyword - ) - obs_type = obs.parameters_as_dict.get('observation_type', None) - form = facility.get_form(obs_type)(observation_payload) - form.is_valid() - observation_ids = facility.submit_observation(form.observation_payload()) - - for observation_id in observation_ids: - # Create Observation record - record = ObservationRecord.objects.create( - target=obs.target, - facility=facility.name, - parameters=json.dumps(observation_payload), - observation_id=observation_id - ) - self.dynamic_cadence.observation_group.observation_records.add(record) - self.dynamic_cadence.observation_group.save() - new_observations.append(record) - - return new_observations - - def advance_window(self, observation_payload, start_keyword='start', end_keyword='end'): - new_start = parse(observation_payload[start_keyword]) + timedelta(hours=self.advance_window_hours) - new_end = parse(observation_payload[end_keyword]) + timedelta(hours=self.advance_window_hours) - observation_payload[start_keyword] = new_start.isoformat() - observation_payload[end_keyword] = new_end.isoformat() - - return observation_payload - - -class ResumeCadenceAfterFailureStrategy(CadenceStrategy): - """The ResumeCadenceAfterFailureStrategy chooses when to submit the next observation based on the success of the - previous observation. If the observation is successful, it submits a new one on the same cadence--that is, if the - cadence is every three days, it will submit the next observation three days in the future. If the observations - fails, it will submit the next observation immediately, and follow the same decision tree based on the success - of the subsequent observation.""" - - name = 'Resume Cadence After Failure' - description = """This strategy schedules one observation in the cadence at a time. If the observation fails, it - re-submits the observation until it succeeds. If it succeeds, it submits the next observation on - the same cadence.""" - form_parameters = { - 'site': { - 'field': forms.ChoiceField, - 'kwargs': { - 'choices': (('cpt', 'cpt'), ('tlv', 'tlv')) - } - }, - 'period': forms.IntegerField - } - - # for key, value in form_parameters.items: - # form[key] = value['field'](**value['kwargs']) - - class ResumeCadenceForm(forms.Form): - site = forms.CharField() - - def __init__(self, dynamic_cadence, *args, **kwargs): - self.advance_window_hours = kwargs.pop('advance_window_hours') - super().__init__(dynamic_cadence, *args, **kwargs) - - def update_observation_parameters(self): - update_observation_payload() - - def submit_next_observation(self): - submit() - - def run(self): - last_obs = self.dynamic_cadence.observation_group.observation_records.order_by('-created').first() - facility = get_service_class(last_obs.facility)() - facility.update_observation_status(last_obs.observation_id) - last_obs.refresh_from_db() - start_keyword, end_keyword = facility.get_start_end_keywords() - observation_payload = last_obs.parameters_as_dict - new_observations = [] - if not last_obs.terminal: - return - elif last_obs.failed: - # Submit next observation to be taken as soon as possible - window_length = parse(observation_payload[end_keyword]) - parse(observation_payload[start_keyword]) - observation_payload[start_keyword] = datetime.now().isoformat() - observation_payload[end_keyword] = (parse(observation_payload[start_keyword]) + window_length).isoformat() - else: - # Advance window normally according to cadence parameters - self.update_observation_parameters() - self.submit_next_observation() - # observation_payload = self.advance_window( - # observation_payload, start_keyword=start_keyword, end_keyword=end_keyword - # ) - - obs_type = last_obs.parameters_as_dict.get('observation_type') - form = facility.get_form(obs_type)(observation_payload) - form.is_valid() - observation_ids = facility.submit_observation(form.observation_payload()) - - for observation_id in observation_ids: - # Create Observation record - record = ObservationRecord.objects.create( - target=last_obs.target, - facility=facility.name, - parameters=json.dumps(observation_payload), - observation_id=observation_id - ) - self.dynamic_cadence.observation_group.observation_records.add(record) - self.dynamic_cadence.observation_group.save() - new_observations.append(record) - - for obsr in new_observations: - facility = get_service_class(obsr.facility)() - facility.update_observation_status(obsr.observation_id) - - return new_observations - - def advance_window(self, observation_payload, start_keyword='start', end_keyword='end'): - new_start = parse(observation_payload[start_keyword]) + timedelta(hours=self.advance_window_hours) - new_end = parse(observation_payload[end_keyword]) + timedelta(hours=self.advance_window_hours) - observation_payload[start_keyword] = new_start.isoformat() - observation_payload[end_keyword] = new_end.isoformat() - - return observation_payload - - class CadenceForm(forms.Form): - cadence_strategy = forms.ChoiceField( - required=False, - choices=[('', '---------')] + [(k, k) for k, v in get_cadence_strategies().items()] - ) + # TODO: review this + cadence_strategy = forms.CharField(required=True, max_length=50, widget=forms.HiddenInput()) + # cadence_strategy = forms.ChoiceField( + # required=False, + # choices=[('', '---------')] + [(k, k) for k, v in get_cadence_strategies().items()] + # ) cadence_frequency = forms.IntegerField( required=False, help_text='Frequency of observations, in hours' @@ -219,7 +73,7 @@ class CadenceForm(forms.Form): def __init__(self, *args, **kwargs): print(kwargs) super().__init__(*args, **kwargs) - self.fields['cadence_strategy'].widget.attrs['readonly'] = True + # self.fields['cadence_strategy'].widget.attrs['readonly'] = True self.fields['cadence_frequency'].widget.attrs['readonly'] = True self.cadence_layout = self.cadence_layout() @@ -236,10 +90,6 @@ def cadence_layout(self): HTML('

    Dynamic cadencing parameters. Leave blank if no dynamic cadencing is desired.

    '), ), Div( - Div( - 'cadence_strategy', - css_class='col' - ), Div( 'cadence_frequency', css_class='col' @@ -247,3 +97,5 @@ def cadence_layout(self): css_class='form-row' ) ) + + extra_layout = cadence_layout diff --git a/tom_observations/cadences/resume_cadence_after_failure.py b/tom_observations/cadences/resume_cadence_after_failure.py new file mode 100644 index 000000000..df29c789b --- /dev/null +++ b/tom_observations/cadences/resume_cadence_after_failure.py @@ -0,0 +1,119 @@ +from datetime import datetime, timedelta +from dateutil.parser import parse +import json + +from crispy_forms.layout import Div, HTML, Layout +from django import forms + +from tom_observations.cadence import CadenceForm, CadenceStrategy +from tom_observations.models import ObservationRecord +from tom_observations.facility import get_service_class + + +class ResumeCadenceAfterFailureForm(CadenceForm): + site = forms.ChoiceField(choices=(('cpt', 'cpt'), ('elp', 'elp'))) + + def cadence_layout(self): + return Layout( + Div( + HTML('

    Dynamic cadencing parameters. Leave blank if no dynamic cadencing is desired.

    '), + ), + Div( + Div( + 'cadence_frequency', + css_class='col' + ), + Div( + 'site', + css_class='col' + ), + css_class='form-row' + ) + ) + + +class ResumeCadenceAfterFailureStrategy(CadenceStrategy): + """The ResumeCadenceAfterFailureStrategy chooses when to submit the next observation based on the success of the + previous observation. If the observation is successful, it submits a new one on the same cadence--that is, if the + cadence is every three days, it will submit the next observation three days in the future. If the observations + fails, it will submit the next observation immediately, and follow the same decision tree based on the success + of the subsequent observation.""" + + name = 'Resume Cadence After Failure' + description = """This strategy schedules one observation in the cadence at a time. If the observation fails, it + re-submits the observation until it succeeds. If it succeeds, it submits the next observation on + the same cadence.""" + form = ResumeCadenceAfterFailureForm + form_parameters = { + 'site': { + 'field': forms.ChoiceField, + 'kwargs': { + 'choices': (('cpt', 'cpt'), ('tlv', 'tlv')) + } + }, + 'period': forms.IntegerField + } + + # for key, value in form_parameters.items: + # form[key] = value['field'](**value['kwargs']) + + class ResumeCadenceForm(forms.Form): + site = forms.CharField() + + def __init__(self, dynamic_cadence, *args, **kwargs): + self.advance_window_hours = kwargs.pop('advance_window_hours') + super().__init__(dynamic_cadence, *args, **kwargs) + + def run(self): + last_obs = self.dynamic_cadence.observation_group.observation_records.order_by('-created').first() + facility = get_service_class(last_obs.facility)() + facility.update_observation_status(last_obs.observation_id) + last_obs.refresh_from_db() + start_keyword, end_keyword = facility.get_start_end_keywords() + observation_payload = last_obs.parameters_as_dict + new_observations = [] + if not last_obs.terminal: + return + elif last_obs.failed: + # Submit next observation to be taken as soon as possible + window_length = parse(observation_payload[end_keyword]) - parse(observation_payload[start_keyword]) + observation_payload[start_keyword] = datetime.now().isoformat() + observation_payload[end_keyword] = (parse(observation_payload[start_keyword]) + window_length).isoformat() + else: + # Advance window normally according to cadence parameters + self.update_observation_parameters() + self.submit_next_observation() + # observation_payload = self.advance_window( + # observation_payload, start_keyword=start_keyword, end_keyword=end_keyword + # ) + + obs_type = last_obs.parameters_as_dict.get('observation_type') + form = facility.get_form(obs_type)(observation_payload) + form.is_valid() + observation_ids = facility.submit_observation(form.observation_payload()) + + for observation_id in observation_ids: + # Create Observation record + record = ObservationRecord.objects.create( + target=last_obs.target, + facility=facility.name, + parameters=json.dumps(observation_payload), + observation_id=observation_id + ) + self.dynamic_cadence.observation_group.observation_records.add(record) + self.dynamic_cadence.observation_group.save() + new_observations.append(record) + + for obsr in new_observations: + facility = get_service_class(obsr.facility)() + facility.update_observation_status(obsr.observation_id) + + return new_observations + + def advance_window(self, observation_payload, start_keyword='start', end_keyword='end'): + new_start = parse(observation_payload[start_keyword]) + timedelta(hours=self.advance_window_hours) + new_end = parse(observation_payload[end_keyword]) + timedelta(hours=self.advance_window_hours) + observation_payload[start_keyword] = new_start.isoformat() + observation_payload[end_keyword] = new_end.isoformat() + + return observation_payload diff --git a/tom_observations/cadences/resumecadenceafterfailure.py b/tom_observations/cadences/resumecadenceafterfailure.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tom_observations/cadences/retry_failed_observations.py b/tom_observations/cadences/retry_failed_observations.py new file mode 100644 index 000000000..9c45f1e36 --- /dev/null +++ b/tom_observations/cadences/retry_failed_observations.py @@ -0,0 +1,65 @@ +from datetime import timedelta +from dateutil.parser import parse +import json + +from tom_observations.cadence import CadenceForm, CadenceStrategy +from tom_observations.models import ObservationRecord +from tom_observations.facility import get_service_class + + +class RetryFailedObservationsForm(CadenceForm): + pass + + +class RetryFailedObservationsStrategy(CadenceStrategy): + """ + The RetryFailedObservationsStrategy immediately re-submits all observations within an observation group a certain + number of hours later, as specified by ``advance_window_hours``. + """ + name = 'Retry Failed Observations' + description = """This strategy immediately re-submits a cadenced observation without amending any other part of the + cadence.""" + form = RetryFailedObservationsForm + + def __init__(self, dynamic_cadence, *args, **kwargs): + self.advance_window_hours = kwargs.pop('advance_window_hours') + super().__init__(dynamic_cadence, *args, **kwargs) + + def run(self): + failed_observations = [obsr for obsr + in self.dynamic_cadence.observation_group.observation_records.all() + if obsr.failed] + new_observations = [] + for obs in failed_observations: + observation_payload = obs.parameters_as_dict + facility = get_service_class(obs.facility)() + start_keyword, end_keyword = facility.get_start_end_keywords() + observation_payload = self.advance_window( + observation_payload, start_keyword=start_keyword, end_keyword=end_keyword + ) + obs_type = obs.parameters_as_dict.get('observation_type', None) + form = facility.get_form(obs_type)(observation_payload) + form.is_valid() + observation_ids = facility.submit_observation(form.observation_payload()) + + for observation_id in observation_ids: + # Create Observation record + record = ObservationRecord.objects.create( + target=obs.target, + facility=facility.name, + parameters=json.dumps(observation_payload), + observation_id=observation_id + ) + self.dynamic_cadence.observation_group.observation_records.add(record) + self.dynamic_cadence.observation_group.save() + new_observations.append(record) + + return new_observations + + def advance_window(self, observation_payload, start_keyword='start', end_keyword='end'): + new_start = parse(observation_payload[start_keyword]) + timedelta(hours=self.advance_window_hours) + new_end = parse(observation_payload[end_keyword]) + timedelta(hours=self.advance_window_hours) + observation_payload[start_keyword] = new_start.isoformat() + observation_payload[end_keyword] = new_end.isoformat() + + return observation_payload diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 8e7f1c182..069962c72 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -10,7 +10,6 @@ from django.core.cache import cache from tom_common.exceptions import ImproperCredentialsException -from tom_observations.cadence import CadenceForm from tom_observations.facility import BaseRoboticObservationFacility, BaseRoboticObservationForm, get_service_class from tom_observations.observation_template import GenericTemplateForm from tom_observations.widgets import FilterField @@ -139,7 +138,7 @@ def proposal_choices(self): return choices -class LCOBaseObservationForm(BaseRoboticObservationForm, LCOBaseForm, CadenceForm): +class LCOBaseObservationForm(BaseRoboticObservationForm, LCOBaseForm): """ The LCOBaseObservationForm provides the base set of utilities to construct an observation at Las Cumbres Observatory. While the forms that inherit from it provide a subset of instruments and filters, the @@ -802,12 +801,14 @@ class LCOFacility(BaseRoboticObservationFacility): } } + # TODO: this should be called get_form_class def get_form(self, observation_type): try: return self.observation_forms[observation_type] except KeyError: return LCOBaseObservationForm + # TODO: this should be called get_template_form_class def get_template_form(self, observation_type): return LCOObservationTemplateForm diff --git a/tom_observations/tests/test_cadence.py b/tom_observations/tests/test_cadence.py index 5d274d150..949845ba0 100644 --- a/tom_observations/tests/test_cadence.py +++ b/tom_observations/tests/test_cadence.py @@ -5,7 +5,8 @@ from .factories import ObservingRecordFactory, TargetFactory from tom_observations.models import ObservationGroup, DynamicCadence -from tom_observations.cadence import RetryFailedObservationsStrategy, ResumeCadenceAfterFailureStrategy +from tom_observations.cadences.resume_cadence_after_failure import ResumeCadenceAfterFailureStrategy +from tom_observations.cadences.retry_failed_observations import RetryFailedObservationsStrategy mock_filters = {'1M0-SCICAM-SINISTRO': { diff --git a/tom_observations/views.py b/tom_observations/views.py index 402da1bee..eeb57c254 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -1,6 +1,5 @@ from io import StringIO from urllib.parse import urlparse -import json from crispy_forms.bootstrap import FormActions from crispy_forms.layout import HTML, Layout, Submit @@ -25,6 +24,8 @@ from tom_common.hints import add_hint from tom_common.mixins import Raise403PermissionRequiredMixin from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm +from tom_observations.cadence import get_cadence_strategy +from tom_observations.cadences.resume_cadence_after_failure import ResumeCadenceAfterFailureForm from tom_observations.facility import get_service_class, get_service_classes, BaseManualObservationFacility from tom_observations.forms import AddExistingObservationForm from tom_observations.models import ObservationRecord, ObservationGroup, ObservationTemplate, DynamicCadence @@ -155,6 +156,12 @@ def get_facility_class(self): """ return get_service_class(self.get_facility()) + def get_cadence_strategy_form(self): + cadence_strategy = self.request.GET.get('cadence_strategy') + if not cadence_strategy: + return None + return get_cadence_strategy(cadence_strategy).form + def get_context_data(self, **kwargs): """ Adds the available observation types for the observing facility to the context object. @@ -168,12 +175,16 @@ def get_context_data(self, **kwargs): # reloaded due to form errors, only repopulate the form that was submitted. observation_type_choices = [] initial = self.get_initial() - for k, v in self.get_facility_class().observation_forms.items(): - form_data = {**initial, **{'observation_type': k}} + for observation_type, observation_form_class in self.get_facility_class().observation_forms.items(): + form_data = {**initial, **{'observation_type': observation_type}} # Repopulate the appropriate form with form data if the original submission was invalid - if k == self.request.POST.get('observation_type'): + if observation_type == self.request.POST.get('observation_type'): form_data.update(**self.request.POST.dict()) - observation_type_choices.append((k, v(initial=form_data))) + form_class = observation_form_class + if self.get_cadence_strategy_form(): + form_class = type(f'Composite{observation_type}Form', + (observation_form_class, self.get_cadence_strategy_form()), {}) + observation_type_choices.append((observation_type, form_class(initial=form_data))) context['observation_type_choices'] = observation_type_choices # Ensure correct tab is active if submission is unsuccessful @@ -190,12 +201,19 @@ def get_form_class(self): :returns: observation form :rtype: subclass of GenericObservationForm """ + print(self.request.GET) + # cadence_strategy = self.request.GET.get('cadence_strategy') + # cadence_form_class = get_cadence_strategy(cadence_strategy).form observation_type = None if self.request.method == 'GET': observation_type = self.request.GET.get('observation_type') elif self.request.method == 'POST': observation_type = self.request.POST.get('observation_type') - return self.get_facility_class()().get_form(observation_type) + form_class = self.get_facility_class()().get_form(observation_type) + print(form_class) + if self.get_cadence_strategy_form(): + form_class = type(f'Composite{observation_type}Form', (form_class, self.get_cadence_strategy_form()), {}) + return form_class def get_form(self): """ @@ -204,7 +222,6 @@ def get_form(self): :returns: observation form :rtype: subclass of GenericObservationForm """ - form = super().get_form() if not settings.TARGET_PERMISSIONS_ONLY: form.fields['groups'].queryset = self.request.user.groups.all() @@ -243,7 +260,8 @@ def form_valid(self, form): # Submit the observation facility = self.get_facility_class() target = self.get_target() - observation_ids = facility().submit_observation(form.observation_payload()) + print(f'form data: {form.cleaned_data}') + # observation_ids = facility().submit_observation(form.observation_payload()) records = [] for observation_id in observation_ids: @@ -268,6 +286,10 @@ def form_valid(self, form): # TODO: Add a test case that includes a dynamic cadence submission if form.cleaned_data.get('cadence_strategy'): + # cadence_parameters = {} + # cadence_form = get_cadence_strategy(form.cleaned_data.get('cadence_strategy')).form + # for field in cadence_form()['fields']: + # cadence_parameters[field] = form.cleaned_data.get(field) DynamicCadence.objects.create( observation_group=observation_group, cadence_strategy=form.cleaned_data.get('cadence_strategy'), From adc61d4b874bf5fe8ae8f16a6a0a413b93331075 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2020 13:40:02 +0000 Subject: [PATCH 307/424] Bump beautifulsoup4 from 4.9.1 to 4.9.2 Bumps [beautifulsoup4](http://www.crummy.com/software/BeautifulSoup/bs4/) from 4.9.1 to 4.9.2. Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4524b07bf..26c99d479 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ 'astroquery==0.4.1', 'astroplan==0.6', 'astropy==4.0.1.post1', - 'beautifulsoup4==4.9.1', + 'beautifulsoup4==4.9.2', 'dataclasses; python_version < "3.7"', 'django==3.1.1', # TOM Toolkit requires db math functions 'djangorestframework==3.11.1', From 6a57ebe4c6d40e00944a0cabd96766cc39f43f3e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2020 13:40:40 +0000 Subject: [PATCH 308/424] Bump django-filter from 2.3.0 to 2.4.0 Bumps [django-filter](https://github.com/carltongibson/django-filter) from 2.3.0 to 2.4.0. - [Release notes](https://github.com/carltongibson/django-filter/releases) - [Changelog](https://github.com/carltongibson/django-filter/blob/master/CHANGES.rst) - [Commits](https://github.com/carltongibson/django-filter/compare/2.3.0...2.4.0) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4524b07bf..3a133de08 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ 'django-crispy-forms==1.9.2', 'django-extensions==3.0.9', 'django-gravatar2==1.4.4', - 'django-filter==2.3.0', + 'django-filter==2.4.0', 'django-guardian==2.3.0', 'fits2image==0.4.3', 'Markdown==3.2.2', # django-rest-framework doc headers require this to support Markdown From 56a2bcc0a9e1a79779259fa34f7ceeed5857a27c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2020 16:15:08 +0000 Subject: [PATCH 309/424] Bump djangorestframework from 3.11.1 to 3.12.1 Bumps [djangorestframework](https://github.com/encode/django-rest-framework) from 3.11.1 to 3.12.1. - [Release notes](https://github.com/encode/django-rest-framework/releases) - [Commits](https://github.com/encode/django-rest-framework/compare/3.11.1...3.12.1) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e022a75bf..259f71258 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ 'beautifulsoup4==4.9.2', 'dataclasses; python_version < "3.7"', 'django==3.1.1', # TOM Toolkit requires db math functions - 'djangorestframework==3.11.1', + 'djangorestframework==3.12.1', 'django-bootstrap4==2.2.0', 'django-contrib-comments==1.9.2', # Earlier version are incompatible with Django >= 3.0 'django-crispy-forms==1.9.2', From 833be2a0ebbc5615aa08a9d14400e65f6bf29a7b Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 29 Sep 2020 10:00:15 -0700 Subject: [PATCH 310/424] WIP --- tom_observations/cadence.py | 61 +++++++------------ .../cadences/resume_cadence_after_failure.py | 47 ++++---------- .../cadences/retry_failed_observations.py | 16 ++--- tom_observations/facilities/lco.py | 21 +++++-- tom_observations/observation_template.py | 6 -- tom_observations/tests/test_cadence.py | 10 +-- tom_observations/tests/utils.py | 4 +- tom_observations/views.py | 44 ++++++------- 8 files changed, 89 insertions(+), 120 deletions(-) diff --git a/tom_observations/cadence.py b/tom_observations/cadence.py index 3b165ef9d..5d3635461 100644 --- a/tom_observations/cadence.py +++ b/tom_observations/cadence.py @@ -1,10 +1,7 @@ from abc import ABC, abstractmethod -from datetime import datetime, timedelta -from dateutil.parser import parse from importlib import import_module -import json -from crispy_forms.layout import Column, Div, HTML, Layout, Row +from crispy_forms.layout import Div, HTML, Layout, Row from django import forms from django.conf import settings @@ -50,7 +47,6 @@ class CadenceStrategy(ABC): ``settings.py``. """ def __init__(self, dynamic_cadence, *args, **kwargs): - self.cadence_strategy = type(self).__name__ self.dynamic_cadence = dynamic_cadence @abstractmethod @@ -59,43 +55,28 @@ def run(self): class CadenceForm(forms.Form): - # TODO: review this - cadence_strategy = forms.CharField(required=True, max_length=50, widget=forms.HiddenInput()) - # cadence_strategy = forms.ChoiceField( - # required=False, - # choices=[('', '---------')] + [(k, k) for k, v in get_cadence_strategies().items()] - # ) + cadence_strategy = forms.CharField(required=False, max_length=50, widget=forms.HiddenInput()) + + def cadence_layout(self): + return Layout('cadence_strategy') + + +class BaseCadenceForm(CadenceForm): cadence_frequency = forms.IntegerField( - required=False, + required=True, help_text='Frequency of observations, in hours' ) + cadence_fields = ['cadence_frequency'] - def __init__(self, *args, **kwargs): - print(kwargs) - super().__init__(*args, **kwargs) - # self.fields['cadence_strategy'].widget.attrs['readonly'] = True - self.fields['cadence_frequency'].widget.attrs['readonly'] = True - self.cadence_layout = self.cadence_layout() - + # TODO: find more elegant way of extending cadence layout def cadence_layout(self): - # If cadence strategy or cadence frequency aren't set, this is a normal observation and the widgets shouldn't - # be rendered - if not (self.initial.get('cadence_strategy') or self.initial.get('cadence_frequency')): - return Layout() - else: - # TODO: Clarify dynamic vs. static cadencing in form - # TODO: Present layout for selected cadence strategy - return Layout( - Div( - HTML('

    Dynamic cadencing parameters. Leave blank if no dynamic cadencing is desired.

    '), - ), - Div( - Div( - 'cadence_frequency', - css_class='col' - ), - css_class='form-row' - ) - ) - - extra_layout = cadence_layout + return Layout( + Div( + HTML('''

    Dynamic cadencing parameters. Leave blank if no dynamic cadencing is desired. + For more information on dynamic cadencing, see + + here.

    '''), + ), + Row('cadence_strategy'), + Row('cadence_frequency'), + ) diff --git a/tom_observations/cadences/resume_cadence_after_failure.py b/tom_observations/cadences/resume_cadence_after_failure.py index df29c789b..c150ffc2b 100644 --- a/tom_observations/cadences/resume_cadence_after_failure.py +++ b/tom_observations/cadences/resume_cadence_after_failure.py @@ -2,34 +2,15 @@ from dateutil.parser import parse import json -from crispy_forms.layout import Div, HTML, Layout from django import forms -from tom_observations.cadence import CadenceForm, CadenceStrategy +from tom_observations.cadence import BaseCadenceForm, CadenceStrategy from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class -class ResumeCadenceAfterFailureForm(CadenceForm): - site = forms.ChoiceField(choices=(('cpt', 'cpt'), ('elp', 'elp'))) - - def cadence_layout(self): - return Layout( - Div( - HTML('

    Dynamic cadencing parameters. Leave blank if no dynamic cadencing is desired.

    '), - ), - Div( - Div( - 'cadence_frequency', - css_class='col' - ), - Div( - 'site', - css_class='col' - ), - css_class='form-row' - ) - ) +class ResumeCadenceAfterFailureForm(BaseCadenceForm): + pass class ResumeCadenceAfterFailureStrategy(CadenceStrategy): @@ -37,7 +18,9 @@ class ResumeCadenceAfterFailureStrategy(CadenceStrategy): previous observation. If the observation is successful, it submits a new one on the same cadence--that is, if the cadence is every three days, it will submit the next observation three days in the future. If the observations fails, it will submit the next observation immediately, and follow the same decision tree based on the success - of the subsequent observation.""" + of the subsequent observation. + + This strategy requires the DynamicCadence to have a parameter ``cadence_frequency``.""" name = 'Resume Cadence After Failure' description = """This strategy schedules one observation in the cadence at a time. If the observation fails, it @@ -60,10 +43,6 @@ class ResumeCadenceAfterFailureStrategy(CadenceStrategy): class ResumeCadenceForm(forms.Form): site = forms.CharField() - def __init__(self, dynamic_cadence, *args, **kwargs): - self.advance_window_hours = kwargs.pop('advance_window_hours') - super().__init__(dynamic_cadence, *args, **kwargs) - def run(self): last_obs = self.dynamic_cadence.observation_group.observation_records.order_by('-created').first() facility = get_service_class(last_obs.facility)() @@ -81,11 +60,9 @@ def run(self): observation_payload[end_keyword] = (parse(observation_payload[start_keyword]) + window_length).isoformat() else: # Advance window normally according to cadence parameters - self.update_observation_parameters() - self.submit_next_observation() - # observation_payload = self.advance_window( - # observation_payload, start_keyword=start_keyword, end_keyword=end_keyword - # ) + observation_payload = self.advance_window( + observation_payload, start_keyword=start_keyword, end_keyword=end_keyword + ) obs_type = last_obs.parameters_as_dict.get('observation_type') form = facility.get_form(obs_type)(observation_payload) @@ -111,8 +88,10 @@ def run(self): return new_observations def advance_window(self, observation_payload, start_keyword='start', end_keyword='end'): - new_start = parse(observation_payload[start_keyword]) + timedelta(hours=self.advance_window_hours) - new_end = parse(observation_payload[end_keyword]) + timedelta(hours=self.advance_window_hours) + # TODO: validate that cadence frequency actually exists, throw an appropriate error + advance_window_hours = self.dynamic_cadence.cadence_parameters.get('cadence_frequency') + new_start = parse(observation_payload[start_keyword]) + timedelta(hours=advance_window_hours) + new_end = parse(observation_payload[end_keyword]) + timedelta(hours=advance_window_hours) observation_payload[start_keyword] = new_start.isoformat() observation_payload[end_keyword] = new_end.isoformat() diff --git a/tom_observations/cadences/retry_failed_observations.py b/tom_observations/cadences/retry_failed_observations.py index 9c45f1e36..793cd1042 100644 --- a/tom_observations/cadences/retry_failed_observations.py +++ b/tom_observations/cadences/retry_failed_observations.py @@ -2,12 +2,12 @@ from dateutil.parser import parse import json -from tom_observations.cadence import CadenceForm, CadenceStrategy +from tom_observations.cadence import BaseCadenceForm, CadenceStrategy from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class -class RetryFailedObservationsForm(CadenceForm): +class RetryFailedObservationsForm(BaseCadenceForm): pass @@ -15,16 +15,14 @@ class RetryFailedObservationsStrategy(CadenceStrategy): """ The RetryFailedObservationsStrategy immediately re-submits all observations within an observation group a certain number of hours later, as specified by ``advance_window_hours``. + + This strategy requires the DynamicCadence to have a parameter ``cadence_frequency``. """ name = 'Retry Failed Observations' description = """This strategy immediately re-submits a cadenced observation without amending any other part of the cadence.""" form = RetryFailedObservationsForm - def __init__(self, dynamic_cadence, *args, **kwargs): - self.advance_window_hours = kwargs.pop('advance_window_hours') - super().__init__(dynamic_cadence, *args, **kwargs) - def run(self): failed_observations = [obsr for obsr in self.dynamic_cadence.observation_group.observation_records.all() @@ -57,8 +55,10 @@ def run(self): return new_observations def advance_window(self, observation_payload, start_keyword='start', end_keyword='end'): - new_start = parse(observation_payload[start_keyword]) + timedelta(hours=self.advance_window_hours) - new_end = parse(observation_payload[end_keyword]) + timedelta(hours=self.advance_window_hours) + # TODO: validate that cadence frequency actually exists, throw an appropriate error + advance_window_hours = self.dynamic_cadence.cadence_parameters.get('cadence_frequency') + new_start = parse(observation_payload[start_keyword]) + timedelta(hours=advance_window_hours) + new_end = parse(observation_payload[end_keyword]) + timedelta(hours=advance_window_hours) observation_payload[start_keyword] = new_start.isoformat() observation_payload[end_keyword] = new_end.isoformat() diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 069962c72..77b89c154 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -81,6 +81,16 @@ """ +static_cadencing_help = """ + For information on static cadencing with LCO, + + check the Observation Portal getting started guide, starting on page 18. + +""" + def make_request(*args, **kwargs): response = requests.request(*args, **kwargs) @@ -171,7 +181,7 @@ def __init__(self, *args, **kwargs): self.helper.layout = Layout( self.common_layout, self.layout(), - self.cadence_layout, + self.cadence_layout(), # TODO: this will break when instantiating the form manually self.button_layout() ) @@ -189,7 +199,8 @@ def layout(self): css_class='form-row', ), Div( - HTML('

    Cadence parameters. Leave blank if no cadencing is desired.

    '), + HTML(f'''

    Static cadence parameters. Leave blank if no cadencing is desired. + {static_cadencing_help}

    '''), ), Div( Div( @@ -594,6 +605,7 @@ class LCOSpectroscopicSequenceForm(LCOBaseObservationForm): ) def __init__(self, *args, **kwargs): + kwargs['cadence_strategy'] = 'ResumeCadenceAfterFailure' super().__init__(*args, **kwargs) # Massage cadence form to be SNEx-styled @@ -754,8 +766,9 @@ class LCOFacility(BaseRoboticObservationFacility): observation_forms = { 'IMAGING': LCOImagingObservationForm, 'SPECTRA': LCOSpectroscopyObservationForm, - 'PHOTOMETRIC_SEQUENCE': LCOPhotometricSequenceForm, - 'SPECTROSCOPIC_SEQUENCE': LCOSpectroscopicSequenceForm + # TODO: Fix these forms + # 'PHOTOMETRIC_SEQUENCE': LCOPhotometricSequenceForm, + # 'SPECTROSCOPIC_SEQUENCE': LCOSpectroscopicSequenceForm } # The SITES dictionary is used to calculate visibility intervals in the # planning tool. All entries should contain latitude, longitude, elevation diff --git a/tom_observations/observation_template.py b/tom_observations/observation_template.py index 9bd83ef46..650081daa 100644 --- a/tom_observations/observation_template.py +++ b/tom_observations/observation_template.py @@ -48,11 +48,6 @@ class ApplyObservationTemplateForm(forms.Form): choices=[('', '')] + [(k, k) for k in get_cadence_strategies().keys()], required=False ) - # TODO: Remove non-cadence_strategy fields from this form - cadence_frequency = forms.IntegerField( - required=False, - help_text='Frequency of observations, in hours' - ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -61,7 +56,6 @@ def __init__(self, *args, **kwargs): 'target', 'observation_template', 'cadence_strategy', - 'cadence_frequency' ) self.helper.form_method = 'GET' self.helper.add_input(Submit('run', 'Apply')) diff --git a/tom_observations/tests/test_cadence.py b/tom_observations/tests/test_cadence.py index 949845ba0..98d61b8cc 100644 --- a/tom_observations/tests/test_cadence.py +++ b/tom_observations/tests/test_cadence.py @@ -31,9 +31,9 @@ def setUp(self): self.group = ObservationGroup.objects.create() self.group.observation_records.add(*observing_records) self.group.save() - # TODO: why doesn't this fail without cadence_strategy and name? self.dynamic_cadence = DynamicCadence.objects.create( - cadence_strategy='Test Strategy', cadence_parameters={}, active=True, observation_group=self.group) + cadence_strategy='Test Strategy', cadence_parameters={'cadence_frequency': 72}, active=True, + observation_group=self.group) def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4): num_records = self.group.observation_records.count() @@ -41,7 +41,7 @@ def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4): observing_record.status = 'CANCELED' observing_record.save() - strategy = RetryFailedObservationsStrategy(self.dynamic_cadence, 72) + strategy = RetryFailedObservationsStrategy(self.dynamic_cadence) new_records = strategy.run() self.group.refresh_from_db() # Make sure the candence run created a new observation. @@ -58,7 +58,7 @@ def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4): def test_resume_when_failed_cadence_failed_obs(self, patch1, patch2, patch3, patch4, patch5): num_records = self.group.observation_records.count() - strategy = ResumeCadenceAfterFailureStrategy(self.dynamic_cadence, 72) + strategy = ResumeCadenceAfterFailureStrategy(self.dynamic_cadence) new_records = strategy.run() self.group.refresh_from_db() self.assertEqual(num_records + 1, self.group.observation_records.count()) @@ -73,7 +73,7 @@ def test_resume_when_failed_cadence_successful_obs(self, patch1, patch2, patch3, num_records = self.group.observation_records.count() observing_record = self.group.observation_records.order_by('-created').first() - strategy = ResumeCadenceAfterFailureStrategy(self.dynamic_cadence, 72) + strategy = ResumeCadenceAfterFailureStrategy(self.dynamic_cadence) new_records = strategy.run() self.group.refresh_from_db() self.assertEqual(num_records + 1, self.group.observation_records.count()) diff --git a/tom_observations/tests/utils.py b/tom_observations/tests/utils.py index 2a8305d73..bb441d509 100644 --- a/tom_observations/tests/utils.py +++ b/tom_observations/tests/utils.py @@ -77,7 +77,9 @@ def validate_observation(self, observation_payload): class FakeManualFacility(BaseManualObservationFacility): name = 'FakeManualFacility' - observation_types = [('FakeManualFacility Observation', 'OBSERVATION')] + observation_forms = { + 'OBSERVATION': FakeFacilityForm + } def get_form(self, observation_type): return FakeFacilityForm diff --git a/tom_observations/views.py b/tom_observations/views.py index eeb57c254..7e32f1363 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -24,8 +24,7 @@ from tom_common.hints import add_hint from tom_common.mixins import Raise403PermissionRequiredMixin from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm -from tom_observations.cadence import get_cadence_strategy -from tom_observations.cadences.resume_cadence_after_failure import ResumeCadenceAfterFailureForm +from tom_observations.cadence import CadenceForm, get_cadence_strategy from tom_observations.facility import get_service_class, get_service_classes, BaseManualObservationFacility from tom_observations.forms import AddExistingObservationForm from tom_observations.models import ObservationRecord, ObservationGroup, ObservationTemplate, DynamicCadence @@ -111,6 +110,7 @@ def get(self, request, *args, **kwargs): return super().get(request, *args, **kwargs) +# TODO: Ensure this template includes the ApplyObservationTemplate form at the top class ObservationCreateView(LoginRequiredMixin, FormView): """ View for creation/submission of an observation. Requires authentication. @@ -159,7 +159,7 @@ def get_facility_class(self): def get_cadence_strategy_form(self): cadence_strategy = self.request.GET.get('cadence_strategy') if not cadence_strategy: - return None + return CadenceForm return get_cadence_strategy(cadence_strategy).form def get_context_data(self, **kwargs): @@ -180,10 +180,8 @@ def get_context_data(self, **kwargs): # Repopulate the appropriate form with form data if the original submission was invalid if observation_type == self.request.POST.get('observation_type'): form_data.update(**self.request.POST.dict()) - form_class = observation_form_class - if self.get_cadence_strategy_form(): - form_class = type(f'Composite{observation_type}Form', - (observation_form_class, self.get_cadence_strategy_form()), {}) + form_class = type(f'Composite{observation_type}Form', + (observation_form_class, self.get_cadence_strategy_form()), {}) observation_type_choices.append((observation_type, form_class(initial=form_data))) context['observation_type_choices'] = observation_type_choices @@ -201,18 +199,14 @@ def get_form_class(self): :returns: observation form :rtype: subclass of GenericObservationForm """ - print(self.request.GET) - # cadence_strategy = self.request.GET.get('cadence_strategy') - # cadence_form_class = get_cadence_strategy(cadence_strategy).form observation_type = None if self.request.method == 'GET': observation_type = self.request.GET.get('observation_type') elif self.request.method == 'POST': observation_type = self.request.POST.get('observation_type') - form_class = self.get_facility_class()().get_form(observation_type) - print(form_class) - if self.get_cadence_strategy_form(): - form_class = type(f'Composite{observation_type}Form', (form_class, self.get_cadence_strategy_form()), {}) + form_class = type(f'Composite{observation_type}Form', + (self.get_facility_class()().get_form(observation_type), self.get_cadence_strategy_form()), + {}) return form_class def get_form(self): @@ -246,6 +240,13 @@ def get_initial(self): initial.update(self.request.GET.dict()) return initial + def post(self, request, *args, **kwargs): + form = self.get_form() + if form.is_valid(): + return self.form_valid(form) + else: + return self.form_invalid(form) + def form_valid(self, form): """ Runs after form validation. Submits the observation to the desired facility and creates an associated @@ -260,8 +261,7 @@ def form_valid(self, form): # Submit the observation facility = self.get_facility_class() target = self.get_target() - print(f'form data: {form.cleaned_data}') - # observation_ids = facility().submit_observation(form.observation_payload()) + observation_ids = facility().submit_observation(form.observation_payload()) records = [] for observation_id in observation_ids: @@ -286,14 +286,14 @@ def form_valid(self, form): # TODO: Add a test case that includes a dynamic cadence submission if form.cleaned_data.get('cadence_strategy'): - # cadence_parameters = {} - # cadence_form = get_cadence_strategy(form.cleaned_data.get('cadence_strategy')).form - # for field in cadence_form()['fields']: - # cadence_parameters[field] = form.cleaned_data.get(field) + cadence_parameters = {} + cadence_form = get_cadence_strategy(form.cleaned_data.get('cadence_strategy')).form + for field in cadence_form().cadence_fields: + cadence_parameters[field] = form.cleaned_data.get(field) DynamicCadence.objects.create( observation_group=observation_group, cadence_strategy=form.cleaned_data.get('cadence_strategy'), - cadence_parameters={'cadence_frequency': form.cleaned_data.get('cadence_frequency')}, + cadence_parameters=cadence_parameters, active=True ) @@ -560,7 +560,7 @@ def get_form_class(self): if not facility_name: raise ValueError('Must provide a facility name') - # TODO: modify this to work with both LCO forms + # TODO: modify this to work with all LCO forms return get_service_class(facility_name)().get_template_form(None) def get_form(self, form_class=None): From ccac0ba73b7ae0f54780d7972b89e00deea7058f Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 30 Sep 2020 13:08:50 -0700 Subject: [PATCH 311/424] Got cadence layouts working properly --- tom_observations/cadence.py | 6 +++--- .../cadences/resume_cadence_after_failure.py | 12 ------------ tom_observations/facilities/lco.py | 6 ++++-- tom_observations/facility.py | 3 --- tom_observations/views.py | 9 +++++---- 5 files changed, 12 insertions(+), 24 deletions(-) diff --git a/tom_observations/cadence.py b/tom_observations/cadence.py index 5d3635461..d23612113 100644 --- a/tom_observations/cadence.py +++ b/tom_observations/cadence.py @@ -35,7 +35,8 @@ def get_cadence_strategy(name): try: return available_classes[name] except KeyError: - raise ImportError('Could not a find a facility with that name. Did you add it to TOM_FACILITY_CLASSES?') + raise ImportError('''Could not a find a cadence strategy with that name. + Did you add it to TOM_CADENCE_STRATEGIES?''') class CadenceStrategy(ABC): @@ -58,7 +59,7 @@ class CadenceForm(forms.Form): cadence_strategy = forms.CharField(required=False, max_length=50, widget=forms.HiddenInput()) def cadence_layout(self): - return Layout('cadence_strategy') + return Layout() class BaseCadenceForm(CadenceForm): @@ -68,7 +69,6 @@ class BaseCadenceForm(CadenceForm): ) cadence_fields = ['cadence_frequency'] - # TODO: find more elegant way of extending cadence layout def cadence_layout(self): return Layout( Div( diff --git a/tom_observations/cadences/resume_cadence_after_failure.py b/tom_observations/cadences/resume_cadence_after_failure.py index c150ffc2b..32b505a36 100644 --- a/tom_observations/cadences/resume_cadence_after_failure.py +++ b/tom_observations/cadences/resume_cadence_after_failure.py @@ -27,18 +27,6 @@ class ResumeCadenceAfterFailureStrategy(CadenceStrategy): re-submits the observation until it succeeds. If it succeeds, it submits the next observation on the same cadence.""" form = ResumeCadenceAfterFailureForm - form_parameters = { - 'site': { - 'field': forms.ChoiceField, - 'kwargs': { - 'choices': (('cpt', 'cpt'), ('tlv', 'tlv')) - } - }, - 'period': forms.IntegerField - } - - # for key, value in form_parameters.items: - # form[key] = value['field'](**value['kwargs']) class ResumeCadenceForm(forms.Form): site = forms.CharField() diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 77b89c154..3d1f1c095 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -10,6 +10,7 @@ from django.core.cache import cache from tom_common.exceptions import ImproperCredentialsException +from tom_observations.cadence import CadenceForm from tom_observations.facility import BaseRoboticObservationFacility, BaseRoboticObservationForm, get_service_class from tom_observations.observation_template import GenericTemplateForm from tom_observations.widgets import FilterField @@ -174,16 +175,17 @@ class LCOBaseObservationForm(BaseRoboticObservationForm, LCOBaseForm): observation_mode = forms.ChoiceField( choices=(('NORMAL', 'Normal'), ('TARGET_OF_OPPORTUNITY', 'Rapid Response')), help_text=observation_mode_help - ) + ) # TODO: Update this to support current observation modes def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.helper.layout = Layout( self.common_layout, self.layout(), - self.cadence_layout(), # TODO: this will break when instantiating the form manually self.button_layout() ) + if isinstance(self, CadenceForm): + self.helper.layout.insert(2, self.cadence_layout()) def layout(self): return Div( diff --git a/tom_observations/facility.py b/tom_observations/facility.py index d32ece7a3..4309a9a9c 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -55,9 +55,6 @@ def get_service_class(name): raise ImportError('Could not a find a facility with that name. Did you add it to TOM_FACILITY_CLASSES?') -# TODO: Ensure docstrings are up to date - - class BaseObservationForm(forms.Form): """ This is the class that is responsible for displaying the observation request form. diff --git a/tom_observations/views.py b/tom_observations/views.py index 7e32f1363..db7e6b4ef 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -25,7 +25,8 @@ from tom_common.mixins import Raise403PermissionRequiredMixin from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm from tom_observations.cadence import CadenceForm, get_cadence_strategy -from tom_observations.facility import get_service_class, get_service_classes, BaseManualObservationFacility +from tom_observations.facility import get_service_class, get_service_classes +from tom_observations.facility import BaseManualObservationFacility from tom_observations.forms import AddExistingObservationForm from tom_observations.models import ObservationRecord, ObservationGroup, ObservationTemplate, DynamicCadence from tom_targets.models import Target @@ -180,9 +181,9 @@ def get_context_data(self, **kwargs): # Repopulate the appropriate form with form data if the original submission was invalid if observation_type == self.request.POST.get('observation_type'): form_data.update(**self.request.POST.dict()) - form_class = type(f'Composite{observation_type}Form', - (observation_form_class, self.get_cadence_strategy_form()), {}) - observation_type_choices.append((observation_type, form_class(initial=form_data))) + observation_form_class = type(f'Composite{observation_type}Form', + (self.get_cadence_strategy_form(), observation_form_class), {}) + observation_type_choices.append((observation_type, observation_form_class(initial=form_data))) context['observation_type_choices'] = observation_type_choices # Ensure correct tab is active if submission is unsuccessful From ce86c0b1e7e7568a3c1ae2931e23361dbfd13543 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 30 Sep 2020 13:15:02 -0700 Subject: [PATCH 312/424] Updated observation modes --- tom_observations/facilities/lco.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 8e7f1c182..deefb76b5 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -163,7 +163,7 @@ class LCOBaseObservationForm(BaseRoboticObservationForm, LCOBaseForm, CadenceFor period = forms.FloatField(required=False) jitter = forms.FloatField(required=False) observation_mode = forms.ChoiceField( - choices=(('NORMAL', 'Normal'), ('TARGET_OF_OPPORTUNITY', 'Rapid Response')), + choices=(('NORMAL', 'Normal'), ('RAPID_RESPONSE', 'Rapid-Response'), ('TIME_CRITICAL', 'Time-Critical')), help_text=observation_mode_help ) From 87208cf6b8ec6871c9c31c82cd0ffb6d272a86fe Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 30 Sep 2020 13:15:13 -0700 Subject: [PATCH 313/424] Switched to flake8 in travis --- .travis.yml | 4 ++-- tom_observations/cadence.py | 2 +- tom_observations/management/commands/runcadencestrategies.py | 2 -- tom_observations/views.py | 1 - tom_publications/forms.py | 2 -- tom_publications/templatetags/publication_extras.py | 2 -- tom_targets/groups.py | 2 +- 7 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1dd125000..4ba2e57fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,8 +26,8 @@ stages: jobs: include: - stage: "Style Checks" - install: pip install -I pycodestyle - script: pycodestyle tom_* --exclude=*/migrations/* --max-line-length=120 + install: pip install -I flake8 + script: flake8 tom_* --exclude=*/migrations/* --max-line-length=120 - stage: "test" os: osx diff --git a/tom_observations/cadence.py b/tom_observations/cadence.py index 879d6c9e4..d8fd7c1af 100644 --- a/tom_observations/cadence.py +++ b/tom_observations/cadence.py @@ -4,7 +4,7 @@ from importlib import import_module import json -from crispy_forms.layout import Column, Div, HTML, Layout, Row +from crispy_forms.layout import Div, HTML, Layout from django import forms from django.conf import settings diff --git a/tom_observations/management/commands/runcadencestrategies.py b/tom_observations/management/commands/runcadencestrategies.py index 0bf93027e..a8d5af64d 100644 --- a/tom_observations/management/commands/runcadencestrategies.py +++ b/tom_observations/management/commands/runcadencestrategies.py @@ -1,5 +1,3 @@ -import json - from django.core.management.base import BaseCommand from tom_observations.cadence import get_cadence_strategy diff --git a/tom_observations/views.py b/tom_observations/views.py index 402da1bee..f00e2310b 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -1,6 +1,5 @@ from io import StringIO from urllib.parse import urlparse -import json from crispy_forms.bootstrap import FormActions from crispy_forms.layout import HTML, Layout, Submit diff --git a/tom_publications/forms.py b/tom_publications/forms.py index 02c836b0c..1322da272 100644 --- a/tom_publications/forms.py +++ b/tom_publications/forms.py @@ -1,7 +1,5 @@ from django import forms -from tom_publications.models import LatexConfiguration - class LatexTableForm(forms.Form): diff --git a/tom_publications/templatetags/publication_extras.py b/tom_publications/templatetags/publication_extras.py index 9242a6676..b99785015 100644 --- a/tom_publications/templatetags/publication_extras.py +++ b/tom_publications/templatetags/publication_extras.py @@ -1,7 +1,5 @@ from django import template -from tom_publications.forms import LatexTableForm - register = template.Library() diff --git a/tom_targets/groups.py b/tom_targets/groups.py index ec0dc0b13..12409b189 100644 --- a/tom_targets/groups.py +++ b/tom_targets/groups.py @@ -105,7 +105,7 @@ def remove_all_from_grouping(filter_data, grouping_object, request): failure_targets = [] try: target_queryset = TargetFilter(request=request, data=filter_data, queryset=Target.objects.all()).qs - except Exception as e: + except Exception: messages.error(request, "Error with filter parameters. No target(s) were removed from group '{}'." .format(grouping_object.name)) return From abd086dc2b7b8f77f09afc7129b83886c479c214 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 30 Sep 2020 13:27:39 -0700 Subject: [PATCH 314/424] Fixed management command for cadence interface changes and shortened too-long line --- tom_observations/management/commands/runcadencestrategies.py | 5 +---- tom_observations/templatetags/observation_extras.py | 3 --- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/tom_observations/management/commands/runcadencestrategies.py b/tom_observations/management/commands/runcadencestrategies.py index a8d5af64d..1d34ffc08 100644 --- a/tom_observations/management/commands/runcadencestrategies.py +++ b/tom_observations/management/commands/runcadencestrategies.py @@ -11,10 +11,7 @@ def handle(self, *args, **kwargs): cadenced_groups = DynamicCadence.objects.filter(active=True) for cg in cadenced_groups: - cadence_frequency = cg.cadence_parameters.get('cadence_frequency', -1) - # TODO: pass cadence parameters in as kwargs or access them in the strategy - # TODO: make cadence form strategy-specific - strategy = get_cadence_strategy(cg.cadence_strategy)(cg, cadence_frequency) + strategy = get_cadence_strategy(cg.cadence_strategy)(cg) new_observations = strategy.run() if not new_observations: return 'No changes from cadence strategy.' diff --git a/tom_observations/templatetags/observation_extras.py b/tom_observations/templatetags/observation_extras.py index c4ff95941..15a221a09 100644 --- a/tom_observations/templatetags/observation_extras.py +++ b/tom_observations/templatetags/observation_extras.py @@ -104,9 +104,6 @@ def observation_plan(target, facility, length=7, interval=60, airmass_limit=None for site, data in visibility_data.items(): plot_data.append(go.Scatter(x=data[0], y=data[1], mode='markers+lines', marker={'symbol': i}, name=site)) i += 1 - # plot_data = [ - # go.Scatter(x=data[0], y=data[1], mode='markers', marker={'symbol': 'line-ew-open'}, name=site) for site, data in visibility_data.items() - # ] layout = go.Layout( xaxis={'title': 'Date'}, yaxis={'autorange': 'reversed', 'title': 'Airmass'} From 39b3cf194fd12b44d0eccbd97a02ec86206bfcbb Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 30 Sep 2020 14:00:33 -0700 Subject: [PATCH 315/424] Attempting to fix coveralls --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4ba2e57fc..11e3ce14a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ before_install: install: pip install -r requirements.txt coverage coveralls -script: coverage run manage.py test +script: coverage run manage.py test --include=tom_* after_success: coveralls From 3c7161c55c95fd89e9c3d0fff6560adf4486302e Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 30 Sep 2020 14:09:16 -0700 Subject: [PATCH 316/424] Fixing coverage command --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 11e3ce14a..bb120792a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ before_install: install: pip install -r requirements.txt coverage coveralls -script: coverage run manage.py test --include=tom_* +script: coverage run --include=tom_* manage.py test after_success: coveralls From 3b06de197222a14d3b04d7e094178d6907def930 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 30 Sep 2020 15:16:44 -0700 Subject: [PATCH 317/424] Fixed snex observation sequence forms --- tom_observations/facilities/lco.py | 47 ++++++++++-------------------- 1 file changed, 16 insertions(+), 31 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 5f4845bce..438cc782a 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -469,10 +469,7 @@ class LCOPhotometricSequenceForm(LCOBaseObservationForm): """ valid_instruments = ['1M0-SCICAM-SINISTRO', '0M4-SCICAM-SBIG', '2M0-SPECTRAL-AG'] filters = ['U', 'B', 'V', 'R', 'I', 'u', 'g', 'r', 'i', 'z', 'w'] - cadence_type = forms.ChoiceField( - choices=[('once', 'Once in the next'), ('repeat', 'Repeating every')], - required=True - ) + cadence_frequency = forms.IntegerField(required=True, help_text='in hours') def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -482,12 +479,10 @@ def __init__(self, *args, **kwargs): self.fields[filter_name] = FilterField(label=filter_name, required=False) # Massage cadence form to be SNEx-styled - self.fields['cadence_strategy'].widget = forms.HiddenInput() - self.fields['cadence_strategy'].required = False - self.fields['cadence_frequency'].required = True - self.fields['cadence_frequency'].widget.attrs['readonly'] = False - self.fields['cadence_frequency'].widget.attrs['help_text'] = 'in hours' - + self.fields['cadence_strategy'] = forms.ChoiceField( + choices=[('', 'Once in the next'), ('ResumeCadenceAfterFailureStrategy', 'Repeating every')], + required=False, + ) for field_name in ['exposure_time', 'exposure_count', 'start', 'end', 'filter']: self.fields.pop(field_name) if self.fields.get('groups'): @@ -496,7 +491,7 @@ def __init__(self, *args, **kwargs): self.helper.layout = Layout( Div( Column('name'), - Column('cadence_type'), + Column('cadence_strategy'), Column('cadence_frequency'), css_class='form-row' ), @@ -538,8 +533,6 @@ def clean(self): cleaned_data['start'] = datetime.strftime(datetime.now(), '%Y-%m-%dT%H:%M:%S') cleaned_data['end'] = datetime.strftime(now + timedelta(hours=cleaned_data['cadence_frequency']), '%Y-%m-%dT%H:%M:%S') - if cleaned_data['cadence_type'] == 'repeat': - cleaned_data['cadence_strategy'] = 'Resume Cadence After Failure' return cleaned_data @@ -600,27 +593,22 @@ class LCOSpectroscopicSequenceForm(LCOBaseObservationForm): acquisition_radius = forms.FloatField(min_value=0) guider_mode = forms.ChoiceField(choices=[('on', 'On'), ('off', 'Off'), ('optional', 'Optional')], required=True) guider_exposure_time = forms.IntegerField(min_value=0) - cadence_type = forms.ChoiceField( - choices=[('once', 'Once in the next'), ('repeat', 'Repeating every')], - required=True, - label='' - ) + cadence_frequency = forms.IntegerField(required=True, + widget=forms.NumberInput(attrs={'placeholder': 'Hours'})) def __init__(self, *args, **kwargs): - kwargs['cadence_strategy'] = 'ResumeCadenceAfterFailure' super().__init__(*args, **kwargs) # Massage cadence form to be SNEx-styled self.fields['name'].label = '' self.fields['name'].widget.attrs['placeholder'] = 'Name' self.fields['min_lunar_distance'].widget.attrs['placeholder'] = 'Degrees' - self.fields['cadence_strategy'].widget = forms.HiddenInput() - self.fields['cadence_strategy'].required = False - self.fields['cadence_frequency'].required = True + self.fields['cadence_strategy'] = forms.ChoiceField( + choices=[('', 'Once in the next'), ('ResumeCadenceAfterFailureStrategy', 'Repeating every')], + required=False, + label='' + ) self.fields['cadence_frequency'].label = '' - self.fields['cadence_frequency'].widget.attrs['readonly'] = False - self.fields['cadence_frequency'].widget.attrs['placeholder'] = 'Hours' - self.fields['cadence_frequency'].help_text = None # Remove start and end because those are determined by the cadence for field_name in ['start', 'end']: @@ -631,7 +619,7 @@ def __init__(self, *args, **kwargs): self.helper.layout = Layout( Div( Column('name'), - Column('cadence_type'), + Column('cadence_strategy'), Column(AppendedText('cadence_frequency', 'Hours')), css_class='form-row' ), @@ -689,8 +677,6 @@ def clean(self): cleaned_data['start'] = datetime.strftime(datetime.now(), '%Y-%m-%dT%H:%M:%S') cleaned_data['end'] = datetime.strftime(now + timedelta(hours=cleaned_data['cadence_frequency']), '%Y-%m-%dT%H:%M:%S') - if cleaned_data['cadence_type'] == 'repeat': - cleaned_data['cadence_strategy'] = 'Resume Cadence After Failure' return cleaned_data @@ -768,9 +754,8 @@ class LCOFacility(BaseRoboticObservationFacility): observation_forms = { 'IMAGING': LCOImagingObservationForm, 'SPECTRA': LCOSpectroscopyObservationForm, - # TODO: Fix these forms - # 'PHOTOMETRIC_SEQUENCE': LCOPhotometricSequenceForm, - # 'SPECTROSCOPIC_SEQUENCE': LCOSpectroscopicSequenceForm + 'PHOTOMETRIC_SEQUENCE': LCOPhotometricSequenceForm, + 'SPECTROSCOPIC_SEQUENCE': LCOSpectroscopicSequenceForm } # The SITES dictionary is used to calculate visibility intervals in the # planning tool. All entries should contain latitude, longitude, elevation From 62930f9dc244ec5f9da9a40c373fced4f9b34f5e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2020 13:29:16 +0000 Subject: [PATCH 318/424] Bump django from 3.1.1 to 3.1.2 Bumps [django](https://github.com/django/django) from 3.1.1 to 3.1.2. - [Release notes](https://github.com/django/django/releases) - [Commits](https://github.com/django/django/compare/3.1.1...3.1.2) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 259f71258..3f8624d77 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ 'astropy==4.0.1.post1', 'beautifulsoup4==4.9.2', 'dataclasses; python_version < "3.7"', - 'django==3.1.1', # TOM Toolkit requires db math functions + 'django==3.1.2', # TOM Toolkit requires db math functions 'djangorestframework==3.12.1', 'django-bootstrap4==2.2.0', 'django-contrib-comments==1.9.2', # Earlier version are incompatible with Django >= 3.0 From 4a24fb5b13818e41a4285567559aa8bdd3aed6ae Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2020 13:48:40 +0000 Subject: [PATCH 319/424] Bump plotly from 4.10.0 to 4.11.0 Bumps [plotly](https://github.com/plotly/plotly.py) from 4.10.0 to 4.11.0. - [Release notes](https://github.com/plotly/plotly.py/releases) - [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.py/compare/v4.10.0...v4.11.0) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 259f71258..3f8736d3d 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ 'Markdown==3.2.2', # django-rest-framework doc headers require this to support Markdown 'numpy==1.19.2', 'pillow==7.2.0', - 'plotly==4.10.0', + 'plotly==4.11.0', 'python-dateutil==2.8.1', 'requests==2.24.0', 'specutils==1.1', From 7d13672ba4d480e4f842983d9765042cd8eeb8e5 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 2 Oct 2020 10:41:19 -0700 Subject: [PATCH 320/424] Added notes to the resume_cadence_after_failure strategy --- .../cadences/resume_cadence_after_failure.py | 35 +++++++++++++++---- tom_observations/facility.py | 1 + .../commands/runcadencestrategies.py | 6 ++++ 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/tom_observations/cadences/resume_cadence_after_failure.py b/tom_observations/cadences/resume_cadence_after_failure.py index 32b505a36..8542a3df1 100644 --- a/tom_observations/cadences/resume_cadence_after_failure.py +++ b/tom_observations/cadences/resume_cadence_after_failure.py @@ -20,6 +20,8 @@ class ResumeCadenceAfterFailureStrategy(CadenceStrategy): fails, it will submit the next observation immediately, and follow the same decision tree based on the success of the subsequent observation. + In order to properly subclass this CadenceStrategy, one should override ``update_observation_payload``. + This strategy requires the DynamicCadence to have a parameter ``cadence_frequency``.""" name = 'Resume Cadence After Failure' @@ -31,32 +33,51 @@ class ResumeCadenceAfterFailureStrategy(CadenceStrategy): class ResumeCadenceForm(forms.Form): site = forms.CharField() + def update_observation_payload(self, observation_payload): + """ + :param observation_payload: Payload + :type observation_payload: dict + """ + return observation_payload + def run(self): + # gets the most recent observation because the next observation is just going to modify these parameters last_obs = self.dynamic_cadence.observation_group.observation_records.order_by('-created').first() + + # Make a call to the facility to get the current status of the observation facility = get_service_class(last_obs.facility)() - facility.update_observation_status(last_obs.observation_id) - last_obs.refresh_from_db() + facility.update_observation_status(last_obs.observation_id) # Updates the DB record + last_obs.refresh_from_db() # Gets the record updates + + # Boilerplate to get necessary properties for future calls start_keyword, end_keyword = facility.get_start_end_keywords() observation_payload = last_obs.parameters_as_dict - new_observations = [] + + # Cadence logic + # If the observation hasn't finished, do nothing if not last_obs.terminal: return - elif last_obs.failed: - # Submit next observation to be taken as soon as possible + elif last_obs.failed: # If the observation failed + # Submit next observation to be taken as soon as possible with the same window length window_length = parse(observation_payload[end_keyword]) - parse(observation_payload[start_keyword]) observation_payload[start_keyword] = datetime.now().isoformat() observation_payload[end_keyword] = (parse(observation_payload[start_keyword]) + window_length).isoformat() - else: + else: # If the observation succeeded # Advance window normally according to cadence parameters observation_payload = self.advance_window( observation_payload, start_keyword=start_keyword, end_keyword=end_keyword ) + observation_payload = self.update_observation_payload(observation_payload) + + # Submission of the new observation to the facility obs_type = last_obs.parameters_as_dict.get('observation_type') form = facility.get_form(obs_type)(observation_payload) form.is_valid() observation_ids = facility.submit_observation(form.observation_payload()) + # Creation of corresponding ObservationRecord objects for the observations + new_observations = [] for observation_id in observation_ids: # Create Observation record record = ObservationRecord.objects.create( @@ -65,10 +86,12 @@ def run(self): parameters=json.dumps(observation_payload), observation_id=observation_id ) + # Add ObservationRecords to the DynamicCadence self.dynamic_cadence.observation_group.observation_records.add(record) self.dynamic_cadence.observation_group.save() new_observations.append(record) + # Update the status of the ObservationRecords in the DB for obsr in new_observations: facility = get_service_class(obsr.facility)() facility.update_observation_status(obsr.observation_id) diff --git a/tom_observations/facility.py b/tom_observations/facility.py index 4309a9a9c..ea5142cc9 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -213,6 +213,7 @@ def get_form(self, observation_type): """ pass + # TODO: consider making submit_observation create ObservationRecords as well @abstractmethod def submit_observation(self, observation_payload): """ diff --git a/tom_observations/management/commands/runcadencestrategies.py b/tom_observations/management/commands/runcadencestrategies.py index 1d34ffc08..738cf12bb 100644 --- a/tom_observations/management/commands/runcadencestrategies.py +++ b/tom_observations/management/commands/runcadencestrategies.py @@ -5,6 +5,12 @@ class Command(BaseCommand): + """ + This management command ensures that all cadences are kept up to date. It is intended to be run + by a cron job, and the frequency should be whatever is determined to be the desired frequency + by the PI. + """ + help = 'Entry point for running cadence strategies.' def handle(self, *args, **kwargs): From 86828849d3e909129a683962b62d838ab550b0aa Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 2 Oct 2020 10:51:02 -0700 Subject: [PATCH 321/424] Added validation on cadence_frequency for existing cadence strategies --- tom_observations/cadences/resume_cadence_after_failure.py | 6 ++++-- tom_observations/cadences/retry_failed_observations.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tom_observations/cadences/resume_cadence_after_failure.py b/tom_observations/cadences/resume_cadence_after_failure.py index 8542a3df1..d07b404cd 100644 --- a/tom_observations/cadences/resume_cadence_after_failure.py +++ b/tom_observations/cadences/resume_cadence_after_failure.py @@ -99,8 +99,10 @@ def run(self): return new_observations def advance_window(self, observation_payload, start_keyword='start', end_keyword='end'): - # TODO: validate that cadence frequency actually exists, throw an appropriate error - advance_window_hours = self.dynamic_cadence.cadence_parameters.get('cadence_frequency') + cadence_frequency = self.dynamic_cadence.cadence_parameters.get('cadence_frequency') + if not cadence_frequency: + raise Exception(f'The {self.name} strategy requires a cadence_frequency cadence_parameter.') + advance_window_hours = cadence_frequency new_start = parse(observation_payload[start_keyword]) + timedelta(hours=advance_window_hours) new_end = parse(observation_payload[end_keyword]) + timedelta(hours=advance_window_hours) observation_payload[start_keyword] = new_start.isoformat() diff --git a/tom_observations/cadences/retry_failed_observations.py b/tom_observations/cadences/retry_failed_observations.py index 793cd1042..d2cb547c1 100644 --- a/tom_observations/cadences/retry_failed_observations.py +++ b/tom_observations/cadences/retry_failed_observations.py @@ -55,8 +55,10 @@ def run(self): return new_observations def advance_window(self, observation_payload, start_keyword='start', end_keyword='end'): - # TODO: validate that cadence frequency actually exists, throw an appropriate error - advance_window_hours = self.dynamic_cadence.cadence_parameters.get('cadence_frequency') + cadence_frequency = self.dynamic_cadence.cadence_parameters.get('cadence_frequency') + if not cadence_frequency: + raise Exception(f'The {self.name} strategy requires a cadence_frequency cadence_parameter.') + advance_window_hours = cadence_frequency new_start = parse(observation_payload[start_keyword]) + timedelta(hours=advance_window_hours) new_end = parse(observation_payload[end_keyword]) + timedelta(hours=advance_window_hours) observation_payload[start_keyword] = new_start.isoformat() From c64cfe6ddd82cd37424e0a10d9188ef2c18da81c Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 2 Oct 2020 10:54:41 -0700 Subject: [PATCH 322/424] flake8 fix --- tom_observations/cadences/resume_cadence_after_failure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_observations/cadences/resume_cadence_after_failure.py b/tom_observations/cadences/resume_cadence_after_failure.py index d07b404cd..e1335495d 100644 --- a/tom_observations/cadences/resume_cadence_after_failure.py +++ b/tom_observations/cadences/resume_cadence_after_failure.py @@ -35,7 +35,7 @@ class ResumeCadenceForm(forms.Form): def update_observation_payload(self, observation_payload): """ - :param observation_payload: Payload + :param observation_payload: form parameters for facility observation form :type observation_payload: dict """ return observation_payload From a75222b61a80dce17205c6f440c16c17b3364d22 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2020 13:29:38 +0000 Subject: [PATCH 323/424] Bump factory-boy from 3.0.1 to 3.1.0 Bumps [factory-boy](https://github.com/FactoryBoy/factory_boy) from 3.0.1 to 3.1.0. - [Release notes](https://github.com/FactoryBoy/factory_boy/releases) - [Changelog](https://github.com/FactoryBoy/factory_boy/blob/master/docs/changelog.rst) - [Commits](https://github.com/FactoryBoy/factory_boy/compare/3.0.1...3.1.0) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3f8624d77..d8859a860 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ 'specutils==1.1', ], extras_require={ - 'test': ['factory_boy==3.0.1'] + 'test': ['factory_boy==3.1.0'] }, include_package_data=True, ) From 07ff8ef4f4d6bfc54ca7cf5d1a1e8bd0c3a522cd Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2020 13:30:01 +0000 Subject: [PATCH 324/424] Bump beautifulsoup4 from 4.9.2 to 4.9.3 Bumps [beautifulsoup4](http://www.crummy.com/software/BeautifulSoup/bs4/) from 4.9.2 to 4.9.3. Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3f8624d77..e2e43e9f8 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ 'astroquery==0.4.1', 'astroplan==0.6', 'astropy==4.0.1.post1', - 'beautifulsoup4==4.9.2', + 'beautifulsoup4==4.9.3', 'dataclasses; python_version < "3.7"', 'django==3.1.2', # TOM Toolkit requires db math functions 'djangorestframework==3.12.1', From 59a3857db60eae89ae046a6b06e4598676783754 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 5 Oct 2020 16:35:40 -0700 Subject: [PATCH 325/424] Fixed bug in comments filtering for postgres, added test for homepage loading, fixed cadence tests in postgres --- tom_common/templatetags/tom_common_extras.py | 15 ++++++++++++- tom_common/tests.py | 12 ++++++++++ tom_observations/tests/test_cadence.py | 23 +++++++++++++++++++- 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/tom_common/templatetags/tom_common_extras.py b/tom_common/templatetags/tom_common_extras.py index 8d0179d94..e8d932f16 100644 --- a/tom_common/templatetags/tom_common_extras.py +++ b/tom_common/templatetags/tom_common_extras.py @@ -1,4 +1,6 @@ from django import template +from django.db.models import IntegerField +from django.db.models.functions import Cast from django.conf import settings from django_comments.models import Comment from guardian.shortcuts import get_objects_for_user @@ -34,8 +36,19 @@ def recent_comments(context, limit=10): """ user = context['request'].user targets_for_user = get_objects_for_user(user, 'tom_targets.view_target') + + # In django-contrib-comments, the Comment model has a field ``object_pk`` which refers to the primary key + # of the object it is related to, i.e., a comment on a ``Target`` has an ``object_pk`` corresponding with the + # ``Target`` id. The ``object_pk`` is stored as a TextField. + + # In order to filter on ``object_pk`` with an iterable of ``IntegerFields`` using the ``in`` comparator, + # we have to cast the ``object_pk`` to an int and annotate it as ``object_pk_as_int``. return { - 'comment_list': Comment.objects.filter(object_pk__in=targets_for_user).order_by('-submit_date')[:limit] + 'comment_list': Comment.objects.annotate( + object_pk_as_int=Cast('object_pk', output_field=IntegerField()) + ).filter( + object_pk_as_int__in=targets_for_user + ).order_by('-submit_date')[:limit] } diff --git a/tom_common/tests.py b/tom_common/tests.py index d47bae175..36bc33703 100644 --- a/tom_common/tests.py +++ b/tom_common/tests.py @@ -4,6 +4,18 @@ from django.urls import reverse +class TestCommonViews(TestCase): + def setUp(self): + pass + + def test_index(self): + self.admin = User.objects.create_superuser(username='admin', password='admin', email='test@example.com') + self.client.force_login(self.admin) + + response = self.client.get(reverse('home')) + self.assertEqual(response.status_code, 200) + + class TestUserManagement(TestCase): def setUp(self): self.admin = User.objects.create_superuser(username='admin', password='admin', email='test@example.com') diff --git a/tom_observations/tests/test_cadence.py b/tom_observations/tests/test_cadence.py index 5d274d150..e401c89e3 100644 --- a/tom_observations/tests/test_cadence.py +++ b/tom_observations/tests/test_cadence.py @@ -1,3 +1,5 @@ +import json + from django.test import TestCase from unittest.mock import patch from datetime import datetime, timedelta @@ -17,6 +19,22 @@ } } +obs_params = { + 'facility': 'LCO', + 'observation_type': 'IMAGING', + 'name': 'With Perms', + 'ipp_value': 1.05, + 'start': '2020-01-01T00:00:00', + 'end': '2020-01-02T00:00:00', + 'exposure_count': 1, + 'exposure_time': 2.0, + 'max_airmass': 4.0, + 'observation_mode': 'NORMAL', + 'proposal': 'LCOSchedulerTest', + 'filter': 'I', + 'instrument_type': '1M0-SCICAM-SINISTRO' + } + @patch('tom_observations.facilities.lco.LCOBaseForm._get_instruments', return_value=mock_filters) @patch('tom_observations.facilities.lco.LCOBaseForm.proposal_choices', @@ -26,7 +44,10 @@ class TestReactiveCadencing(TestCase): def setUp(self): target = TargetFactory.create() - observing_records = ObservingRecordFactory.create_batch(5, target_id=target.id) + obs_params['target_id'] = target.id + observing_records = ObservingRecordFactory.create_batch(5, + target_id=target.id, + parameters=json.dumps(obs_params)) self.group = ObservationGroup.objects.create() self.group.observation_records.add(*observing_records) self.group.save() From d78799fbf818501f2e059c8a2c5ded4325e2c463 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 5 Oct 2020 16:42:17 -0700 Subject: [PATCH 326/424] Added TODO --- tom_common/tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tom_common/tests.py b/tom_common/tests.py index 36bc33703..bc30a291d 100644 --- a/tom_common/tests.py +++ b/tom_common/tests.py @@ -13,6 +13,9 @@ def test_index(self): self.client.force_login(self.admin) response = self.client.get(reverse('home')) + # TODO: Use python http status enumerator in place of magic number everywhere + # from http import HTTPStatus + # assertEqual(response.status_code, HTTPStatus.OK) self.assertEqual(response.status_code, 200) From 671918492eacc62b7e5bb2e399daa78b991b2685 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 6 Oct 2020 17:47:53 -0700 Subject: [PATCH 327/424] WIP --- tom_observations/facilities/lco.py | 5 +++-- tom_observations/widgets.py | 9 +++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 438cc782a..77672ed51 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -168,7 +168,7 @@ class LCOBaseObservationForm(BaseRoboticObservationForm, LCOBaseForm): exposure_time = forms.FloatField(min_value=0.1, widget=forms.TextInput(attrs={'placeholder': 'Seconds'}), help_text=exposure_time_help) - max_airmass = forms.FloatField(help_text=max_airmass_help) + max_airmass = forms.FloatField(help_text=max_airmass_help, min_value=0) min_lunar_distance = forms.IntegerField(min_value=0, label='Minimum Lunar Distance', required=False) period = forms.FloatField(required=False) jitter = forms.FloatField(required=False) @@ -476,7 +476,7 @@ def __init__(self, *args, **kwargs): # Add fields for each available filter as specified in the filters property for filter_name in self.filters: - self.fields[filter_name] = FilterField(label=filter_name, required=False) + self.fields[filter_name] = FilterField(label=filter_name, required=False, min_value=0) # Massage cadence form to be SNEx-styled self.fields['cadence_strategy'] = forms.ChoiceField( @@ -751,6 +751,7 @@ class LCOFacility(BaseRoboticObservationFacility): """ name = 'LCO' + # TODO: make the keys the display values instead observation_forms = { 'IMAGING': LCOImagingObservationForm, 'SPECTRA': LCOSpectroscopyObservationForm, diff --git a/tom_observations/widgets.py b/tom_observations/widgets.py index 8c3a31abb..7cd28aa54 100644 --- a/tom_observations/widgets.py +++ b/tom_observations/widgets.py @@ -1,9 +1,11 @@ from django import forms +from django.core.validators import MinValueValidator class FilterConfigurationWidget(forms.widgets.MultiWidget): def __init__(self, attrs=None): + print(attrs) if not attrs: attrs = {} _default_attrs = {'class': 'form-control col-md-3', 'style': 'margin-right: 10px; display: inline-block'} @@ -24,8 +26,11 @@ class FilterField(forms.MultiValueField): widget = FilterConfigurationWidget def __init__(self, *args, **kwargs): - fields = (forms.IntegerField(), forms.IntegerField(), forms.IntegerField()) - super().__init__(fields, *args, **kwargs) + min_value = kwargs.pop('min_value', 0) + fields = (forms.IntegerField(validators=[MinValueValidator(min_value)]), + forms.IntegerField(validators=[MinValueValidator(min_value)]), + forms.IntegerField(validators=[MinValueValidator(min_value)])) + super().__init__(fields=fields, *args, **kwargs) def compress(self, data_list): return data_list From 077d5942b24fb214f9b2637e96b456e9233e15d7 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 6 Oct 2020 23:26:14 -0700 Subject: [PATCH 328/424] Cleaned up a few very minor things related to cadencing and observations --- tom_observations/cadence.py | 2 +- tom_observations/facilities/lco.py | 2 +- tom_observations/models.py | 2 +- .../templates/tom_observations/partials/observing_buttons.html | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tom_observations/cadence.py b/tom_observations/cadence.py index d23612113..86153baa1 100644 --- a/tom_observations/cadence.py +++ b/tom_observations/cadence.py @@ -67,7 +67,7 @@ class BaseCadenceForm(CadenceForm): required=True, help_text='Frequency of observations, in hours' ) - cadence_fields = ['cadence_frequency'] + cadence_fields = set(['cadence_frequency']) def cadence_layout(self): return Layout( diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 77672ed51..fa44922aa 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -530,7 +530,7 @@ def clean(self): """ cleaned_data = super().clean() now = datetime.now() - cleaned_data['start'] = datetime.strftime(datetime.now(), '%Y-%m-%dT%H:%M:%S') + cleaned_data['start'] = datetime.strftime(now, '%Y-%m-%dT%H:%M:%S') cleaned_data['end'] = datetime.strftime(now + timedelta(hours=cleaned_data['cadence_frequency']), '%Y-%m-%dT%H:%M:%S') diff --git a/tom_observations/models.py b/tom_observations/models.py index a1a13dd6c..7ad2885ee 100644 --- a/tom_observations/models.py +++ b/tom_observations/models.py @@ -157,7 +157,7 @@ class DynamicCadence(models.Model): modified = models.DateTimeField(auto_now=True, help_text='The time which this DynamicCadence was modified.') def __str__(self): - return self.name + return f'{self.cadence_strategy} with parameters {self.cadence_parameters}' class ObservationTemplate(models.Model): diff --git a/tom_observations/templates/tom_observations/partials/observing_buttons.html b/tom_observations/templates/tom_observations/partials/observing_buttons.html index a43bc6803..81b6adac6 100644 --- a/tom_observations/templates/tom_observations/partials/observing_buttons.html +++ b/tom_observations/templates/tom_observations/partials/observing_buttons.html @@ -1,3 +1,3 @@ {% for facility in facilities %} -{{ facility }} +{{ facility }} {% endfor %} \ No newline at end of file From 967531a35508dfb33e0e7a446fce9d5a66f8a378 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 6 Oct 2020 23:30:01 -0700 Subject: [PATCH 329/424] Removing superfluous code --- tom_observations/facilities/lco.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index fa44922aa..ce91e8150 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -476,7 +476,7 @@ def __init__(self, *args, **kwargs): # Add fields for each available filter as specified in the filters property for filter_name in self.filters: - self.fields[filter_name] = FilterField(label=filter_name, required=False, min_value=0) + self.fields[filter_name] = FilterField(label=filter_name, required=False) # Massage cadence form to be SNEx-styled self.fields['cadence_strategy'] = forms.ChoiceField( From 27f5f7efa0e403b743aa0bf5333b165270a7f922 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 6 Oct 2020 23:32:40 -0700 Subject: [PATCH 330/424] Removing code that doesn't belong --- tom_observations/widgets.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tom_observations/widgets.py b/tom_observations/widgets.py index 7cd28aa54..7e16bd29c 100644 --- a/tom_observations/widgets.py +++ b/tom_observations/widgets.py @@ -5,7 +5,6 @@ class FilterConfigurationWidget(forms.widgets.MultiWidget): def __init__(self, attrs=None): - print(attrs) if not attrs: attrs = {} _default_attrs = {'class': 'form-control col-md-3', 'style': 'margin-right: 10px; display: inline-block'} @@ -26,10 +25,7 @@ class FilterField(forms.MultiValueField): widget = FilterConfigurationWidget def __init__(self, *args, **kwargs): - min_value = kwargs.pop('min_value', 0) - fields = (forms.IntegerField(validators=[MinValueValidator(min_value)]), - forms.IntegerField(validators=[MinValueValidator(min_value)]), - forms.IntegerField(validators=[MinValueValidator(min_value)])) + fields = (forms.IntegerField(), forms.IntegerField(), forms.IntegerField()) super().__init__(fields=fields, *args, **kwargs) def compress(self, data_list): From ab350a932b84e6183c21318b00ff987cdc7e2ec8 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 6 Oct 2020 23:33:03 -0700 Subject: [PATCH 331/424] Removing code that doesn't belong --- tom_observations/widgets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_observations/widgets.py b/tom_observations/widgets.py index 7e16bd29c..e11e92872 100644 --- a/tom_observations/widgets.py +++ b/tom_observations/widgets.py @@ -26,7 +26,7 @@ class FilterField(forms.MultiValueField): def __init__(self, *args, **kwargs): fields = (forms.IntegerField(), forms.IntegerField(), forms.IntegerField()) - super().__init__(fields=fields, *args, **kwargs) + super().__init__(fields, *args, **kwargs) def compress(self, data_list): return data_list From 26b64639de5290e0668bbf159c8bfef7a6adaa6f Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 6 Oct 2020 23:33:19 -0700 Subject: [PATCH 332/424] Removing unused import --- tom_observations/widgets.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tom_observations/widgets.py b/tom_observations/widgets.py index e11e92872..8c3a31abb 100644 --- a/tom_observations/widgets.py +++ b/tom_observations/widgets.py @@ -1,5 +1,4 @@ from django import forms -from django.core.validators import MinValueValidator class FilterConfigurationWidget(forms.widgets.MultiWidget): From f75797a1b6f718426731a0db98714ff3e4c39ead Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2020 13:57:26 +0000 Subject: [PATCH 333/424] Bump markdown from 3.2.2 to 3.3 Bumps [markdown](https://github.com/Python-Markdown/markdown) from 3.2.2 to 3.3. - [Release notes](https://github.com/Python-Markdown/markdown/releases) - [Commits](https://github.com/Python-Markdown/markdown/compare/3.2.2...3.3) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 50b05f5a1..c774de010 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ 'django-filter==2.4.0', 'django-guardian==2.3.0', 'fits2image==0.4.3', - 'Markdown==3.2.2', # django-rest-framework doc headers require this to support Markdown + 'Markdown==3.3', # django-rest-framework doc headers require this to support Markdown 'numpy==1.19.2', 'pillow==7.2.0', 'plotly==4.11.0', From 907192c2b55dd387a25ee1fc5bd186db7c786d64 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 8 Oct 2020 11:59:43 -0700 Subject: [PATCH 334/424] Added logging to runcadencestrategies management commands --- tom_base/settings.py | 1 + .../commands/runcadencestrategies.py | 20 +++++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index 758f52eb0..5b347a928 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -182,6 +182,7 @@ } } } +logging.config.dictConfig(LOGGING) TARGET_TYPE = 'SIDEREAL' FACILITIES = { diff --git a/tom_observations/management/commands/runcadencestrategies.py b/tom_observations/management/commands/runcadencestrategies.py index 738cf12bb..eefa35a59 100644 --- a/tom_observations/management/commands/runcadencestrategies.py +++ b/tom_observations/management/commands/runcadencestrategies.py @@ -1,9 +1,14 @@ +import logging + from django.core.management.base import BaseCommand from tom_observations.cadence import get_cadence_strategy from tom_observations.models import DynamicCadence +logger = logging.getLogger(__name__) + + class Command(BaseCommand): """ This management command ensures that all cadences are kept up to date. It is intended to be run @@ -16,10 +21,21 @@ class Command(BaseCommand): def handle(self, *args, **kwargs): cadenced_groups = DynamicCadence.objects.filter(active=True) + updated_cadences = [] + for cg in cadenced_groups: strategy = get_cadence_strategy(cg.cadence_strategy)(cg) new_observations = strategy.run() if not new_observations: - return 'No changes from cadence strategy.' + logger.log(msg=f'No changes from dynamic cadence {cg}', level=logging.INFO) else: - return 'Cadence update completed, {0} new observations created.'.format(len(new_observations)) + logger.log(msg=f'''Cadence update completed for dynamic cadence {cg}, + {len(new_observations)} new observations created.''', + level=logging.INFO) + updated_cadences.append(cg.observation_group) + + if updated_cadences: + msg = 'Created new observations for dynamic cadences with observation groups: {0}.' + return msg.format(', '.join([str(cg) for cg in updated_cadences])) + else: + return 'No new observations for any dynamic cadences.' From c3cfae11feb7650b1c50334d1dc8b542bda9f9db Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 8 Oct 2020 12:04:47 -0700 Subject: [PATCH 335/424] Added logging import in settings --- tom_base/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index 5b347a928..b9d0476bc 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -9,7 +9,7 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/2.0/ref/settings/ """ - +import logging import os import tempfile From 4914511d6ea97e693a7adfb608b8bb2d339904db Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 8 Oct 2020 12:17:06 -0700 Subject: [PATCH 336/424] fixed logging import --- tom_base/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index b9d0476bc..319ecb151 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -9,7 +9,7 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/2.0/ref/settings/ """ -import logging +import logging.config import os import tempfile From bfd45d9fa7372aef08e37ce3eed5cc9ef46a26a8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 12 Oct 2020 13:44:14 +0000 Subject: [PATCH 337/424] Bump django-bootstrap4 from 2.2.0 to 2.3.0 Bumps [django-bootstrap4](https://github.com/zostera/django-bootstrap4) from 2.2.0 to 2.3.0. - [Release notes](https://github.com/zostera/django-bootstrap4/releases) - [Changelog](https://github.com/zostera/django-bootstrap4/blob/main/CHANGELOG.md) - [Commits](https://github.com/zostera/django-bootstrap4/compare/v2.2.0...v2.3.0) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c774de010..6648ef0cf 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ 'dataclasses; python_version < "3.7"', 'django==3.1.2', # TOM Toolkit requires db math functions 'djangorestframework==3.12.1', - 'django-bootstrap4==2.2.0', + 'django-bootstrap4==2.3.0', 'django-contrib-comments==1.9.2', # Earlier version are incompatible with Django >= 3.0 'django-crispy-forms==1.9.2', 'django-extensions==3.0.9', From 37922ac4ac965c0e0398de7de1aba8050cfaca30 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 13 Oct 2020 13:42:04 +0000 Subject: [PATCH 338/424] Bump markdown from 3.3 to 3.3.1 Bumps [markdown](https://github.com/Python-Markdown/markdown) from 3.3 to 3.3.1. - [Release notes](https://github.com/Python-Markdown/markdown/releases) - [Commits](https://github.com/Python-Markdown/markdown/compare/3.3...3.3.1) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6648ef0cf..75cd7eb2f 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ 'django-filter==2.4.0', 'django-guardian==2.3.0', 'fits2image==0.4.3', - 'Markdown==3.3', # django-rest-framework doc headers require this to support Markdown + 'Markdown==3.3.1', # django-rest-framework doc headers require this to support Markdown 'numpy==1.19.2', 'pillow==7.2.0', 'plotly==4.11.0', From 58c889dd08c66c63e0cfc7bdc04183125e28645d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2020 13:52:00 +0000 Subject: [PATCH 339/424] Bump pillow from 7.2.0 to 8.0.0 Bumps [pillow](https://github.com/python-pillow/Pillow) from 7.2.0 to 8.0.0. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/master/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/7.2.0...8.0.0) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 75cd7eb2f..16d9ae081 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ 'fits2image==0.4.3', 'Markdown==3.3.1', # django-rest-framework doc headers require this to support Markdown 'numpy==1.19.2', - 'pillow==7.2.0', + 'pillow==8.0.0', 'plotly==4.11.0', 'python-dateutil==2.8.1', 'requests==2.24.0', From 4515ea5fd7863e050914bfe78286c6a545730c6a Mon Sep 17 00:00:00 2001 From: fraserw Date: Thu, 15 Oct 2020 14:27:06 -0700 Subject: [PATCH 340/424] big commit before merge with latest tom_base --- tom_observations/facilities/gemini.py | 4 +- tom_observations/facilities/lco.py | 10 +- tom_observations/facilities/utils.py | 2 +- tom_observations/forms.py | 37 +++- .../tom_observations/partials/tile_plan.html | 11 ++ .../templatetags/observation_extras.py | 72 +++++++- tom_observations/tiler.py | 167 ++++++++++++++++++ tom_observations/utils.py | 53 ++++++ tom_targets/forms.py | 2 + .../templates/tom_targets/target_detail.html | 11 +- tom_targets/templatetags/targets_extras.py | 15 ++ tom_targets/utils.py | 3 +- 12 files changed, 375 insertions(+), 12 deletions(-) create mode 100644 tom_observations/templates/tom_observations/partials/tile_plan.html create mode 100644 tom_observations/tiler.py diff --git a/tom_observations/facilities/gemini.py b/tom_observations/facilities/gemini.py index 3096ae103..f1119d93b 100644 --- a/tom_observations/facilities/gemini.py +++ b/tom_observations/facilities/gemini.py @@ -437,11 +437,11 @@ def isodatetime(value): if obs[:2] == 'GN': note_text = self.eph_GN[0][0:4] + self.eph_GN[0][mjd_k:mjd_K] + self.eph_GN[0][-2:] - payload['note'] += "\n" + payload['note'] += "\n\n" payload['note'] += "\n".join(note_text) elif obs[:2] == 'GS': note_text = self.eph_GS[0][0:4] + self.eph_GS[0][mjd_k:mjd_K] + self.eph_GS[0][-2:] - payload['note'] += "\n" + payload['note'] += "\n\n" payload['note'] += "\n".join(note_text) print(payload['note']) if self.cleaned_data['brightness'] is not None: diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 31947338c..732bafeed 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -342,15 +342,21 @@ def _build_target_fields(self): self.cleaned_data['imaging_interval'], 'LCO', site) + if mjd_vals is not None: for i in range(len(ra_vals)-1): + print((air_vals[i] < float(self.cleaned_data['max_airmass']), + air_vals[i+1] < float(self.cleaned_data['max_airmass']), + air_vals[i] > 1.0, + air_vals[i+1] > 1.0, + sun_alt_vals[i] < -30.0, + sun_alt_vals[i+1] < -30.0)) if (air_vals[i] < float(self.cleaned_data['max_airmass']) and air_vals[i+1] < float(self.cleaned_data['max_airmass']) and air_vals[i] > 1.0 and air_vals[i+1] > 1.0 and sun_alt_vals[i] < -30.0 and sun_alt_vals[i+1] < -30.0): - new_target_fields = {} new_target_fields['type'] = 'ICRS' new_target_fields['ra'] = (ra_vals[i]+ra_vals[i+1])/2.0 @@ -371,7 +377,6 @@ def _build_target_fields(self): ephemeris_windows[site].append([start.isot, end.isot]) elif mjd_vals is None and sun_alt_vals == -2: self.add_error(None, 'Date range outside range available in the provided ephemeris.') - return (ephemeris_targets, ephemeris_windows) else: @@ -434,6 +439,7 @@ def _build_ephemeris_request_parts(self): locations = [] for site in sites: for i in range(len(new_targets[site])): + print(self._build_instrument_config()) single_obs_config = { 'type': self.instrument_to_type(self.cleaned_data['instrument_type']), 'instrument_type': self.cleaned_data['instrument_type'], diff --git a/tom_observations/facilities/utils.py b/tom_observations/facilities/utils.py index 0b2ad9f57..ab1b0c12b 100644 --- a/tom_observations/facilities/utils.py +++ b/tom_observations/facilities/utils.py @@ -24,7 +24,7 @@ def get_hex(ra, dec): rs = (s-rm)*60.0 s = abs(dec) - dh = int(dec) + dh = int(s) s -= dh s *= 60.0 dm = int(s) diff --git a/tom_observations/forms.py b/tom_observations/forms.py index db1e434cc..d45597c55 100644 --- a/tom_observations/forms.py +++ b/tom_observations/forms.py @@ -1,7 +1,7 @@ from django import forms from django.urls import reverse from crispy_forms.helper import FormHelper -from crispy_forms.layout import ButtonHolder, Column, Layout, Row, Submit +from crispy_forms.layout import ButtonHolder, Column, Layout, Row, Submit, Div from tom_observations.facility import get_service_classes @@ -68,3 +68,38 @@ def __init__(self, *args, **kwargs): ) ) ) + +class TileForm(forms.Form): + field_overlap = forms.DecimalField(required=True, label='Field Overlap', initial=0.3) + min_fill_fraction = forms.DecimalField(required=True, label='Minimum Fill Fraction', initial=0.5) + shimmy_factor = forms.DecimalField(required=True, label='Shimmy Factor', initial=0.0) + ra_uncertainty = forms.DecimalField(required=False, label='R.A. Uncertainty (")') + dec_uncertainty = forms.DecimalField(required=False, label='Dec. Uncertainty (")') + selected_date = forms.DateTimeField(required=False, label='Date', widget=forms.TextInput(attrs={'type': 'date'})) + selected_time = forms.DateTimeField(required=False, label='Time', widget=forms.TextInput(attrs={'type': 'time'})) + + def clean(self): + cleaned_data = super().clean() + field_overlap = cleaned_data.get('field_overlap') + min_fill_fraction = cleaned_data.get('min_fill_fraction') + target = self.data['target'] + + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.helper = FormHelper() + self.helper.layout = Layout( + self.layout() + ) + + def layout(self): + return Div( + Div( + Div('field_overlap', css_class='col'), + Div('ra_uncertainty', css_class='col'), + Div('dec_uncertainty', css_class='col'), + Div('min_fill_fraction', css_class='col'), + Div('shimmy_factor', css_class='col'), + css_class='form-row'), + Div('selected_date', 'selected_time'), + ) diff --git a/tom_observations/templates/tom_observations/partials/tile_plan.html b/tom_observations/templates/tom_observations/partials/tile_plan.html new file mode 100644 index 000000000..b982a2b61 --- /dev/null +++ b/tom_observations/templates/tom_observations/partials/tile_plan.html @@ -0,0 +1,11 @@ +{% load bootstrap4 %} +
    +
    + {% csrf_token %} + {% bootstrap_form form %} + {% buttons %} + + {% endbuttons %} +
    + {{ tile_graph|safe }} +
    diff --git a/tom_observations/templatetags/observation_extras.py b/tom_observations/templatetags/observation_extras.py index 0237d9fa2..930099ce5 100644 --- a/tom_observations/templatetags/observation_extras.py +++ b/tom_observations/templatetags/observation_extras.py @@ -8,11 +8,12 @@ from plotly import offline import plotly.graph_objs as go -from tom_observations.forms import AddExistingObservationForm, UpdateObservationId +from tom_observations.forms import AddExistingObservationForm, UpdateObservationId, TileForm from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class, get_service_classes from tom_observations.observing_strategy import RunStrategyForm -from tom_observations.utils import get_sidereal_visibility +from tom_observations.utils import get_sidereal_visibility, get_ellipse, get_astrom_uncert_ephemeris +from tom_observations.tiler import make_tiles from tom_targets.models import Target @@ -261,3 +262,70 @@ def facility_status(): facility_statuses.append(status) return {'facilities': facility_statuses} + + +@register.inclusion_tag('tom_observations/partials/tile_plan.html', takes_context=True) +def tile_plan(context): + """ + Displays a figure showing the uncertainty ellipse, and the tiled observation sequence + on the ellipse. + """ + request = context['request'] + tile_form = TileForm() + + tile_graph = '' + + if all(request.GET.get(x) for x in ['field_overlap']): + tile_form = TileForm({ + 'field_overlap': request.GET.get('field_overlap'), + 'min_fill_fraction': request.GET.get('min_fill_fraction'), + 'shimmy_factor': request.GET.get('shimmy_factor'), + 'target': context['object'] + }) + if tile_form.is_valid(): + field_overlap = float(request.GET['field_overlap']) + min_fill_fraction = float(request.GET.get('min_fill_fraction')) + shimmy_factor = float(request.GET.get('shimmy_factor')) + if request.GET.get('ra_uncertainty') and request.GET.get('dec_uncertainty'): + ra_uncertainty = float(request.GET.get('ra_uncertainty'))/3600.0 + dec_uncertainty = float(request.GET.get('dec_uncertainty'))/3600.0 + else: + selected_date = request.GET['selected_date'] + selected_time = request.GET['selected_time'] + if selected_date != '' and selected_time != '': + date_str = selected_date+'T'+selected_time+':00' + else: + date_str = '' + (ra, dec, ra_uncertainty, dec_uncertainty) = get_astrom_uncert_ephemeris(context['object'], date_str) + + + fov = 6.0/60.0 + if shimmy_factor>0: + allowShimmy = True + n_shimmy = int(shimmy_factor) + else: + allowShimmy = False + n_shimmy = 0 + tiles = make_tiles(fov, ra_uncertainty, dec_uncertainty, + overlap = field_overlap, min_fill_fraction = min_fill_fraction, + allowShimmy = allowShimmy, n_shimmy = n_shimmy ) + + plot_data = [] + for i, tile in enumerate(tiles): + x = [tile[0]-fov/2, tile[0]-fov/2, tile[0]+fov/2, tile[0]+fov/2, tile[0]-fov/2] + y = [tile[1]-fov/2, tile[1]+fov/2, tile[1]+fov/2, tile[1]-fov/2, tile[1]-fov/2] + plot_data.append(go.Scatter(x=x, y=y, mode='lines', line_color='red', name=str(i))) + (ellip_x, ellip_y) = get_ellipse(ra_uncertainty, dec_uncertainty) + plot_data.append(go.Scatter(x=ellip_x, y=ellip_y, mode='lines', line_color='black', name='Uncertainty Ellipse')) + layout = go.Layout(title='{} tiles in mosaic'.format(len(tiles)), xaxis=dict(title="RA"), yaxis=dict(title='Dec.'), showlegend=False) + tile_graph = offline.plot({ + "data": plot_data, + "layout": layout + }, + output_type='div', show_link=False) + + return { + 'form': tile_form, + 'target': context['object'], + 'tile_graph': tile_graph + } diff --git a/tom_observations/tiler.py b/tom_observations/tiler.py new file mode 100644 index 000000000..36f4e712a --- /dev/null +++ b/tom_observations/tiler.py @@ -0,0 +1,167 @@ +# import pylab as pyl +import numpy as np +from plotly import offline +import plotly.graph_objs as go +import time + +def checkThoseCorners(cx, cy, fov, a, b, min_fraction = 0.9): + # determine if enough of the fov is filled by the error ellipse at this point + # to include it in the final tiled grid + fov2 = fov/2.0 + + X = np.linspace(cx-fov2, cx+fov2, 10) + Y = np.linspace(cy-fov2, cy+fov2, 10) + (xv, yv) = np.meshgrid(X, Y) + + r2 = (xv/a)**2 + (yv/b)**2 + w = np.where(r2 < 1) + if len(w[0]) > min_fraction*len(X)*len(Y): + return True + else: + return False + +def get_ellipse(a, b): + ang = np.linspace(0, 2*np.pi, 200) + return (a*np.cos(ang), b*np.sin(ang)) + + +def make_tiles(fov, a, b, overlap = 0.3, min_fill_fraction = 0.3, + allowShimmy = True, n_shimmy = 20, + drawPlot = False): + """ + Make a tile layout to cover an ellipse descibed by the + RA (a) and Dec (b) uncertainties. Units assumed to be degrees. + """ + + if 2*a <= fov and 2*b <= fov: + cent_a = np.array([0.0]) + cent_b = np.array([0.0]) + frames = np.array([[0.0, 0.0]]) + + else: + n_a = int(2*a/(fov*(1 - overlap)))+1 + n_b = int(2*b/(fov*(1 - overlap)))+1 + if n_a%2>0: + a_offset = (min_fill_fraction - 1.0)*fov/2 + else: + a_offset = 0.0 + if n_b%2>0: + b_offset = (min_fill_fraction - 1.0)*fov/2 + else: + b_offset = 0.0 + + if 2*a=b: + for i in range(len(cent_a)): + strip = [] + for j in range(len(cent_b)): + if checkThoseCorners(cent_a[i], cent_b[j], fov, a, b, min_fill_fraction) or len(cent_b)==1: + strip.append([cent_a[i], cent_b[j]]) + + if len(strip) > 0: + strip = np.array(strip) + strip[:, 1] -= np.mean(strip[:, 1]) + for j in strip: + frames.append(j) + + elif b>a: + for j in range(len(cent_b)): + strip = [] + for i in range(len(cent_a)): + if checkThoseCorners(cent_a[i], cent_b[j], fov, a, b, min_fill_fraction) or len(cent_a)==1: + strip.append([cent_a[i], cent_b[j]]) + + if len(strip) > 0: + strip = np.array(strip) + strip[:, 0] -= np.mean(strip[:, 0]) + + for i in strip: + frames.append(i) + frames = np.array(frames) + + if allowShimmy and len(frames)>1: + # make a map that is fov/n pixel scale + scale = fov/float(n_shimmy) + nx = int(2*a/scale)+1 + ny = int(2*b/scale)+1 + + x = np.linspace(0, 2*a, nx) + y = np.linspace(0, 2*b, ny) + + gx, gy = np.meshgrid(x, y) + orig_map = np.zeros((ny, nx), dtype = 'float64') + + r2 = ((gx-a)/a)**2 + ((gy-b)/b)**2 + w = np.where(r2 < 1) + orig_map[w]=1 + n_ellipse = len(np.where(orig_map>0)[0]) + + shimmy = [] + for i in range(int(-n_shimmy/2), int(n_shimmy/2)+1): + for j in range(int(-n_shimmy/2), int(n_shimmy/2)+1): + map = np.copy(orig_map) + for f in frames: + dx = scale*i + dy = scale*j + w = np.where((gx>=f[0]+a+dx-fov/2.0) & (gx<=f[0]+a+dx+fov/2.0) & \ + (gy>=f[1]+b+dy-fov/2.0) & (gy<=f[1]+b+dy+fov/2.0) & (map>0)) + map[w]+=1 + + w_missed = np.where(map==1) + shimmy.append([len(w_missed[0]), dx, dy]) + shimmy = np.array(shimmy) + argmin = np.argmin(shimmy[:, 0]) + frames[:, 0] += shimmy[argmin][1] + frames[:, 1] += shimmy[argmin][2] + print('Shimmied by {}, {}.'.format(shimmy[argmin][1], shimmy[argmin][2])) + + + if drawPlot: + fig = pyl.figure(1) + sp = fig.add_subplot(111) + for i in frames: + pyl.scatter(i[0], i[1]) + rekt = pyl.Rectangle([i[0]-fov/2.0, i[1]-fov/2.0], + fov, fov, + facecolor = 'none', edgecolor='g') + sp.add_patch(rekt) + (x, y) = get_ellipse(a, b) + pyl.plot(x, y) + pyl.show() + + return frames + + + + + +if __name__ == "__main__": + #print(make_tiles(6.0/60.0, 5.5/60.0, 3.5/60.0, drawPlot=True)) + #print(make_tiles(6.0/60.0, 3.5/60.0, 5.5/60.0, drawPlot=True)) + fov = 6.0/60.0 + a, b = 1300.0/3600.0, 950.0/3600.0 + tiles = make_tiles(fov, a, b, min_fill_fraction = 0.3, allowShimmy = False, n_shimmy = 20, drawPlot=False) + + #plot_data = [go.Scatter(x=tiles[:, 0], y=tiles[:, 1], mode='markers')] + plot_data = [] + for i, f in enumerate(tiles): + x = [f[0]-fov/2, f[0]-fov/2, f[0]+fov/2, f[0]+fov/2, f[0]-fov/2] + y = [f[1]-fov/2, f[1]+fov/2, f[1]+fov/2, f[1]-fov/2, f[1]-fov/2] + plot_data.append(go.Scatter(x=x, y=y, mode='lines', line_color='red', name=str(i))) + (x, y) = get_ellipse(a, b) + plot_data.append(go.Scatter(x=x, y=y, mode='lines', line_color='black', name='Uncertainty Ellipse')) + layout = go.Layout(title=None, xaxis=dict(title="RA"), yaxis=dict(title='Dec.')) + offline.plot({ + "data": plot_data, + "layout": layout, + + }) diff --git a/tom_observations/utils.py b/tom_observations/utils.py index 2111ea990..22f591f2e 100644 --- a/tom_observations/utils.py +++ b/tom_observations/utils.py @@ -4,6 +4,7 @@ from astroplan import Observer, FixedTarget, time_grid_from_range import numpy as np from scipy import interpolate as interp +import json import logging from tom_observations import facility @@ -12,6 +13,58 @@ logger = logging.getLogger(__name__) +def get_ellipse(a, b): + ang = np.linspace(0, 2*np.pi, 200) + return (a*np.cos(ang), b*np.sin(ang)) + +def get_astrom_uncert_ephemeris(target, selected_time): + """ + Get the astrometric uncertainty of a EPHEMERIS target. + """ + if target.type == target.NON_SIDEREAL: + if target.scheme == 'EPHEMERIS': + eph_json = json.loads(target.eph_json) + sites = list(eph_json) + + mk = eph_json[sites[0]] + + ras = [] + decs = [] + dras = [] + ddecs = [] + mjds = [] + times = [] + for i, e in enumerate(mk): + mjds.append(float(e['t'])) + ras.append(float(e['R'])) + decs.append(float(e['D'])) + dras.append(float(e['dR'])) + ddecs.append(float(e['dD'])) + + mjds, ras, decs, dras, ddecs = np.array(mjds), np.array(ras), np.array(decs), np.array(dras), np.array(ddecs) + + fra = interp.interp1d(mjds, ras) + fdec = interp.interp1d(mjds, decs) + fdra = interp.interp1d(mjds, dras) + fddec = interp.interp1d(mjds, ddecs) + + if selected_time == '': + selected_mjd = Time.now().mjd + else: + selected_mjd = Time(selected_time).mjd + try: + out = (fra(selected_mjd), + fdec(selected_mjd), + fdra(selected_mjd), + fddec(selected_mjd)) + return out + except: + raise('Selected time outside ephemeris range.') + + raise Exception("Target type does not contain astrometric uncertainty information. Please specify.") + else: + raise Exception("Target type does not contain astrometric uncertainty information. Please specify.") + def get_radec_ephemeris(eph_json_single, start_time, end_time, interval, observing_facility, observing_site): observing_facility_class = facility.get_service_class(observing_facility) sites = observing_facility_class().get_observing_sites() diff --git a/tom_targets/forms.py b/tom_targets/forms.py index 149ec5cdb..bd1d2e41b 100644 --- a/tom_targets/forms.py +++ b/tom_targets/forms.py @@ -162,6 +162,8 @@ def clean(self): raise forms.ValidationError('Airmass plotting is only supported for sidereal targets') + + TargetExtraFormset = inlineformset_factory(Target, TargetExtra, fields=('key', 'value'), widgets={'value': forms.TextInput()}) TargetNamesFormset = inlineformset_factory(Target, TargetName, fields=('name',), validate_min=False, can_delete=True, diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index 569fc95d6..76b88d70d 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -1,5 +1,5 @@ {% extends 'tom_common/base.html' %} -{% load comments bootstrap4 tom_common_extras targets_extras observation_extras dataproduct_extras publication_extras static cache %} +{% load comments bootstrap4 tom_common_extras targets_extras observation_extras dataproduct_extras publication_extras static cache nonsidereal_airmass_extras%} {% block title %}Target {{ object.name }}{% endblock %} {% block additional_css %} @@ -21,7 +21,7 @@ {% if object.type == 'SIDEREAL' %} {% aladin object %} {% endif %} - +
    @@ -55,12 +55,17 @@

    Observe

    {% observingstrategy_run object %}
    + {% if object.type == 'NON_SIDEREAL' %} +

    Tile

    + {% tile_plan %} + {% endif %}

    Plan

    {% if object.type == 'SIDEREAL' %} {% target_plan %} {% moon_distance object %} {% elif target.type == 'NON_SIDEREAL' %} -

    Airmass plotting for non-sidereal targets is not currently supported. If you would like to add this functionality, please check out the non-sidereal airmass plugin.

    + {% nonsidereal_target_plan %} + Airmass plotting for non-sidereal targets is not currently supported. If you would like to add this functionality, please check out the non-sidereal airmass plugin.

    > {% endif %}
    diff --git a/tom_targets/templatetags/targets_extras.py b/tom_targets/templatetags/targets_extras.py index 42a1fd16f..0aaf010f8 100644 --- a/tom_targets/templatetags/targets_extras.py +++ b/tom_targets/templatetags/targets_extras.py @@ -267,6 +267,11 @@ def eph_json_to_value_ra(value): eph_json = json.loads(value) keys = list(eph_json.keys()) k = keys[0] + + # bug catch for truly empty ephemerides, which can happen if a user provides a poorly formatted ephemeris file + if len(eph_json[k]) == 0: + return -32768.0 + eph_len = len(eph_json[k][0]) return deg_to_sexigesimal(float(eph_json[k][int(eph_len/2)]['R']), 'hms') else: @@ -282,6 +287,11 @@ def eph_json_to_value_dec(value): eph_json = json.loads(value) keys = list(eph_json.keys()) k = keys[0] + + # bug catch for truly empty ephemerides, which can happen if a user provides a poorly formatted ephemeris file + if len(eph_json[k]) == 0: + return -32768.0 + eph_len = len(eph_json[k][0]) return deg_to_sexigesimal(float(eph_json[k][int(eph_len/2)]['D']), 'dms') else: @@ -297,6 +307,11 @@ def eph_json_to_value_mjd(value): eph_json = json.loads(value) keys = list(eph_json.keys()) k = keys[0] + + # bug catch for truly empty ephemerides, which can happen if a user provides a poorly formatted ephemeris file + if len(eph_json[k]) == 0: + return -32768.0 + eph_len = len(eph_json[k][0]) return round(float(eph_json[k][int(eph_len/2)]['t']), 5) else: diff --git a/tom_targets/utils.py b/tom_targets/utils.py index f5e3b0068..cdce056ae 100644 --- a/tom_targets/utils.py +++ b/tom_targets/utils.py @@ -138,7 +138,7 @@ def import_ephemeris_target(stream): num_sites += 1 if num_sites != 8: - errors.append(Warning('WARNING: Provided file does not have ephemerides for all 7 LCO sites.')) + errors.append(Warning('WARNING: Provided file does not have ephemerides for all 8 LCO sites.')) eph_json = {} end_ind = 0 @@ -189,6 +189,7 @@ def import_ephemeris_target(stream): # ephemerides. TO-DO: put a better error check and correctly thrown # warning for now being lazy if loop_inds == [-1, -1] or ra_inds is None or jd_inds is None: + print(name, loop_inds , ra_inds , jd_inds) errors.append(Exception('We were not able to understand that ephemeris file.')) mjds = [] From 4427425828b3585d50c42794542bb236552a2ee5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2020 13:47:14 +0000 Subject: [PATCH 341/424] Bump django-bootstrap4 from 2.3.0 to 2.3.1 Bumps [django-bootstrap4](https://github.com/zostera/django-bootstrap4) from 2.3.0 to 2.3.1. - [Release notes](https://github.com/zostera/django-bootstrap4/releases) - [Changelog](https://github.com/zostera/django-bootstrap4/blob/main/CHANGELOG.md) - [Commits](https://github.com/zostera/django-bootstrap4/compare/v2.3.0...v2.3.1) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 16d9ae081..948fb31ed 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ 'dataclasses; python_version < "3.7"', 'django==3.1.2', # TOM Toolkit requires db math functions 'djangorestframework==3.12.1', - 'django-bootstrap4==2.3.0', + 'django-bootstrap4==2.3.1', 'django-contrib-comments==1.9.2', # Earlier version are incompatible with Django >= 3.0 'django-crispy-forms==1.9.2', 'django-extensions==3.0.9', From e96811f2638ad860054fe8159c412e1abf8fc0b7 Mon Sep 17 00:00:00 2001 From: fraserw Date: Mon, 19 Oct 2020 11:30:05 -0700 Subject: [PATCH 342/424] -- many updates related to non_sidereal functions --- tom_observations/facilities/lco.py | 22 ++++++------- .../templatetags/observation_extras.py | 2 +- .../tom_targets/partials/target_buttons.html | 2 +- .../tom_targets/partials/target_data.html | 19 ++++++++---- .../target_distribution_nonsidereal.html | 1 + .../tom_targets/partials/target_ssois.html | 1 + .../templates/tom_targets/target_detail.html | 3 +- .../templates/tom_targets/target_list.html | 3 +- tom_targets/templatetags/targets_extras.py | 31 +++++++++++++++++++ tom_targets/urls.py | 1 + tom_targets/views.py | 2 ++ 11 files changed, 64 insertions(+), 23 deletions(-) create mode 100644 tom_targets/templates/tom_targets/partials/target_distribution_nonsidereal.html create mode 100644 tom_targets/templates/tom_targets/partials/target_ssois.html diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 807cd4fe3..a16ee491b 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -20,7 +20,6 @@ ) from tom_observations.observation_template import GenericTemplateForm from tom_observations.widgets import FilterField -from tom_observations.observing_strategy import GenericStrategyForm from tom_targets.models import ( Target, REQUIRED_NON_SIDEREAL_FIELDS, REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME @@ -354,12 +353,6 @@ def _build_target_fields(self): if mjd_vals is not None: for i in range(len(ra_vals)-1): - print((air_vals[i] < float(self.cleaned_data['max_airmass']), - air_vals[i+1] < float(self.cleaned_data['max_airmass']), - air_vals[i] > 1.0, - air_vals[i+1] > 1.0, - sun_alt_vals[i] < -30.0, - sun_alt_vals[i+1] < -30.0)) if (air_vals[i] < float(self.cleaned_data['max_airmass']) and air_vals[i+1] < float(self.cleaned_data['max_airmass']) and air_vals[i] > 1.0 and @@ -385,7 +378,7 @@ def _build_target_fields(self): ephemeris_targets[site].append(new_target_fields) ephemeris_windows[site].append([start.isot, end.isot]) elif mjd_vals is None and sun_alt_vals == -2: - self.add_error(None, 'Date range outside range available in the provided ephemeris.') + self.add_error(None, 'Date range outside range available in the stored ephemeris.') return (ephemeris_targets, ephemeris_windows) else: @@ -454,12 +447,11 @@ def _build_ephemeris_request_parts(self): locations = [] for site in sites: for i in range(len(new_targets[site])): - print(self._build_instrument_config()) single_obs_config = { 'type': self.instrument_to_type(self.cleaned_data['instrument_type']), 'instrument_type': self.cleaned_data['instrument_type'], 'target': new_targets[site][i], - 'instrument_configs': [self._build_instrument_config()], + 'instrument_configs': self._build_instrument_config(), 'acquisition_config': { }, @@ -538,7 +530,7 @@ def observation_payload(self): # this is inefficient as the request validation is done to check for site+scope # configuration errors, and then is done again later to check for other errors. # - # This could be used to estiamte airmass windows instead of using astropy as + # This could be used to estimate airmass windows instead of using astropy as # is done in tom_base/utils.py. obs_module = get_service_class(self.cleaned_data['facility']) requests = self._build_ephemeris_requests() @@ -546,10 +538,15 @@ def observation_payload(self): for j in range(len(requests)): if requests[j]['location'] not in locations: locations.append(requests[j]['location']) + """ + # I dont understand why the following is inappropriate when selecting a single + # telescope location, but MANY seems to always be required now. if len(locations) > 1: operator = "MANY" else: - operator = "SINGLE" + operator = "MANY"#"SINGLE" + """ + operator = "MANY" errors = obs_module().validate_observation({ "name": self.cleaned_data['name'], @@ -560,6 +557,7 @@ def observation_payload(self): "requests": requests }) + if len(errors) > 0: valid_requests = [] for i, e in enumerate(errors['requests']): diff --git a/tom_observations/templatetags/observation_extras.py b/tom_observations/templatetags/observation_extras.py index 43323d385..95b834347 100644 --- a/tom_observations/templatetags/observation_extras.py +++ b/tom_observations/templatetags/observation_extras.py @@ -11,7 +11,7 @@ from tom_observations.forms import AddExistingObservationForm, UpdateObservationId, TileForm from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class, get_service_classes -from tom_observations.observing_strategy import RunStrategyForm +#from tom_observations.observing_strategy import RunStrategyForm from tom_observations.observation_template import ApplyObservationTemplateForm from tom_observations.utils import get_sidereal_visibility, get_ellipse, get_astrom_uncert_ephemeris from tom_observations.tiler import make_tiles diff --git a/tom_targets/templates/tom_targets/partials/target_buttons.html b/tom_targets/templates/tom_targets/partials/target_buttons.html index 5cd86e4f4..4d7649adf 100644 --- a/tom_targets/templates/tom_targets/partials/target_buttons.html +++ b/tom_targets/templates/tom_targets/partials/target_buttons.html @@ -1,2 +1,2 @@ Update Target -Delete Target \ No newline at end of file +Delete Target diff --git a/tom_targets/templates/tom_targets/partials/target_data.html b/tom_targets/templates/tom_targets/partials/target_data.html index a0c4d3095..6abf42a6f 100644 --- a/tom_targets/templates/tom_targets/partials/target_data.html +++ b/tom_targets/templates/tom_targets/partials/target_data.html @@ -11,12 +11,19 @@ {% for key, value in target.as_dict.items %} {% if value and key != 'name' %} {% if key == 'eph_json' %} -
    Typical RA
    -
    {{ value|eph_json_to_value_ra }}
    -
    Typical Dec
    -
    {{ value|eph_json_to_value_dec }}
    -
    At MJD
    -
    {{ value|eph_json_to_value_mjd }}
    + {% if value != 'None' %} +
    Typical RA
    +
    {{ value|eph_json_to_value_ra }}
    +
    Typical Dec
    +
    {{ value|eph_json_to_value_dec }}
    +
    At MJD
    +
    {{ value|eph_json_to_value_mjd }}
    + {% else %} +
    Today's RA
    +
    {{ target.names|non_sidereal_ra }}
    +
    Today's Dec
    +
    {{ target.names|non_sidereal_dec }}
    + {% endif %} {% else %}
    {% verbose_name target key %}
    {{ value|truncate_number }}
    diff --git a/tom_targets/templates/tom_targets/partials/target_distribution_nonsidereal.html b/tom_targets/templates/tom_targets/partials/target_distribution_nonsidereal.html new file mode 100644 index 000000000..f6d076c77 --- /dev/null +++ b/tom_targets/templates/tom_targets/partials/target_distribution_nonsidereal.html @@ -0,0 +1 @@ +{{ figure|safe }} diff --git a/tom_targets/templates/tom_targets/partials/target_ssois.html b/tom_targets/templates/tom_targets/partials/target_ssois.html new file mode 100644 index 000000000..761f06aad --- /dev/null +++ b/tom_targets/templates/tom_targets/partials/target_ssois.html @@ -0,0 +1 @@ +SSOIS diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index 38cb3998f..387ad99cb 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -22,7 +22,7 @@ {% if object.type == 'SIDEREAL' %} {% aladin object %} {% endif %} - + {% target_ssois object %}
    @@ -66,7 +66,6 @@

    Plan

    {% moon_distance object %} {% elif target.type == 'NON_SIDEREAL' %} {% nonsidereal_target_plan %} - Airmass plotting for non-sidereal targets is not currently supported. If you would like to add this functionality, please check out the non-sidereal airmass plugin.

    > {% endif %}
    diff --git a/tom_targets/templates/tom_targets/target_list.html b/tom_targets/templates/tom_targets/target_list.html index 1a92e6645..087df1a73 100644 --- a/tom_targets/templates/tom_targets/target_list.html +++ b/tom_targets/templates/tom_targets/target_list.html @@ -1,5 +1,6 @@ {% extends 'tom_common/base.html' %} {% load bootstrap4 targets_extras %} +{% load nonsidereal_airmass_extras %} {% block title %}Targets{% endblock %} {% block content %}
    @@ -24,7 +25,7 @@
    {% select_target_js %} - {% target_distribution filter.qs %} + {% target_distribution_nonsidereal filter.qs %} {% bootstrap_pagination page_obj extra=request.GET.urlencode %} diff --git a/tom_targets/templatetags/targets_extras.py b/tom_targets/templatetags/targets_extras.py index feea8280b..f39317be8 100644 --- a/tom_targets/templatetags/targets_extras.py +++ b/tom_targets/templatetags/targets_extras.py @@ -18,6 +18,11 @@ import json +from astroquery.jplhorizons import Horizons + +# global ephemeris object such that the horizons query doesn't happen twice +eph_obj_coords = None + register = template.Library() @@ -318,3 +323,29 @@ def eph_json_to_value_mjd(value): return round(float(eph_json[k][int(eph_len/2)]['t']), 5) else: return -32768.0 + + +@register.filter +def non_sidereal_ra(target_name): + global eph_obj_coords + + if eph_obj_coords is None: + try: + # if there is a space in the nane, assume the first string is an acceptable name + obj = Horizons(id=target_name[0].split()[0], epochs=Time.now().jd) + eph_obj_coords = [obj.ephemerides()['RA'][0], obj.ephemerides()['DEC'][0]] + return eph_obj_coords[0] + except: + pass + return None + + +@register.filter +def non_sidereal_dec(target_name): + global eph_obj_coords + + if eph_obj_coords is not None: + dec = eph_obj_coords[1] + eph_obj_coords = None + return dec + return None diff --git a/tom_targets/urls.py b/tom_targets/urls.py index b898195fb..ab652989e 100644 --- a/tom_targets/urls.py +++ b/tom_targets/urls.py @@ -16,6 +16,7 @@ path('add-remove-grouping/', TargetAddRemoveGroupingView.as_view(), name='add-remove-grouping'), path('/update/', TargetUpdateView.as_view(), name='update'), path('/delete/', TargetDeleteView.as_view(), name='delete'), + path('/ssois/', TargetUpdateView.as_view(), name='ssois'), path('/', TargetDetailView.as_view(), name='detail'), path('targetgrouping//delete/', TargetGroupingDeleteView.as_view(), name='delete-group'), path('targetgrouping/create/', TargetGroupingCreateView.as_view(), name='create-group') diff --git a/tom_targets/views.py b/tom_targets/views.py index 298eb8584..200404ef9 100644 --- a/tom_targets/views.py +++ b/tom_targets/views.py @@ -307,6 +307,8 @@ class TargetDeleteView(Raise403PermissionRequiredMixin, DeleteView): success_url = reverse_lazy('targets:list') model = Target +# class TargetSSOISView(ListView): +# model = Target class TargetDetailView(Raise403PermissionRequiredMixin, DetailView): """ From 597f01cf4cce44f7a8cd2919732e40358a5cb299 Mon Sep 17 00:00:00 2001 From: fraserw Date: Mon, 19 Oct 2020 14:10:49 -0700 Subject: [PATCH 343/424] --added SSOIS query button + changes to allow EPHEMERIS target updates --- tom_targets/models.py | 8 +++---- .../tom_targets/partials/target_ssois.html | 2 +- .../templates/tom_targets/target_detail.html | 4 +++- tom_targets/templatetags/targets_extras.py | 8 +++++++ tom_targets/urls.py | 4 ++-- tom_targets/views.py | 21 ++++++++++++++++--- 6 files changed, 36 insertions(+), 11 deletions(-) diff --git a/tom_targets/models.py b/tom_targets/models.py index eb820a679..0c72609f6 100644 --- a/tom_targets/models.py +++ b/tom_targets/models.py @@ -22,18 +22,18 @@ REQUIRED_SIDEREAL_FIELDS = ['ra', 'dec'] REQUIRED_NON_SIDEREAL_FIELDS = [ - 'scheme', 'epoch_of_elements', + 'scheme', ] # Additional non-sidereal fields that are required for specific orbital element # schemes REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME = { 'MPC_COMET': ['perihdist', 'epoch_of_perihelion', 'inclination', - 'lng_asc_node', 'arg_of_perihelion', 'eccentricity'], + 'lng_asc_node', 'arg_of_perihelion', 'eccentricity', 'epoch_of_elements'], 'MPC_MINOR_PLANET': ['mean_anomaly', 'semimajor_axis', 'inclination', - 'lng_asc_node', 'arg_of_perihelion', 'eccentricity'], + 'lng_asc_node', 'arg_of_perihelion', 'eccentricity', 'epoch_of_elements'], 'JPL_MAJOR_PLANET': ['mean_daily_motion', 'mean_anomaly', 'semimajor_axis', 'inclination', 'lng_asc_node', 'arg_of_perihelion', - 'eccentricity'], + 'eccentricity', 'epoch_of_elements'], 'EPHEMERIS': ['eph_json'] } diff --git a/tom_targets/templates/tom_targets/partials/target_ssois.html b/tom_targets/templates/tom_targets/partials/target_ssois.html index 761f06aad..1cdfd708d 100644 --- a/tom_targets/templates/tom_targets/partials/target_ssois.html +++ b/tom_targets/templates/tom_targets/partials/target_ssois.html @@ -1 +1 @@ -SSOIS +SSOIS diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index 387ad99cb..f1b1e6b11 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -17,12 +17,14 @@
    {% endif %} {% target_buttons object %} + {% if object.type == 'NON_SIDEREAL' %} + {% target_ssois object %} + {% endif %} {% target_data object %} {% recent_photometry object limit=3 %} {% if object.type == 'SIDEREAL' %} {% aladin object %} {% endif %} - {% target_ssois object %}
    diff --git a/tom_targets/templatetags/targets_extras.py b/tom_targets/templatetags/targets_extras.py index f39317be8..e0e10f974 100644 --- a/tom_targets/templatetags/targets_extras.py +++ b/tom_targets/templatetags/targets_extras.py @@ -60,6 +60,14 @@ def target_buttons(target): return {'target': target} +@register.inclusion_tag('tom_targets/partials/target_ssois.html') +def target_ssois(target): + """ + Displays the ssois query button. + """ + return {'target': target} + + @register.inclusion_tag('tom_targets/partials/target_data.html') def target_data(target): """ diff --git a/tom_targets/urls.py b/tom_targets/urls.py index ab652989e..afb6c51e7 100644 --- a/tom_targets/urls.py +++ b/tom_targets/urls.py @@ -1,6 +1,6 @@ from django.urls import path -from .views import TargetCreateView, TargetUpdateView, TargetDetailView +from .views import TargetCreateView, TargetUpdateView, TargetDetailView, TargetSSOISView from .views import TargetDeleteView, TargetListView, TargetImportView, TargetImportEphemerisView, TargetExportView from .views import TargetGroupingView, TargetGroupingDeleteView, TargetGroupingCreateView, TargetAddRemoveGroupingView @@ -16,7 +16,7 @@ path('add-remove-grouping/', TargetAddRemoveGroupingView.as_view(), name='add-remove-grouping'), path('/update/', TargetUpdateView.as_view(), name='update'), path('/delete/', TargetDeleteView.as_view(), name='delete'), - path('/ssois/', TargetUpdateView.as_view(), name='ssois'), + path('/ssois/', TargetSSOISView.as_view(), name='ssois'), path('/', TargetDetailView.as_view(), name='detail'), path('targetgrouping//delete/', TargetGroupingDeleteView.as_view(), name='delete-group'), path('targetgrouping/create/', TargetGroupingCreateView.as_view(), name='create-group') diff --git a/tom_targets/views.py b/tom_targets/views.py index 200404ef9..8c6b43bd9 100644 --- a/tom_targets/views.py +++ b/tom_targets/views.py @@ -12,7 +12,7 @@ from django.db import transaction from django.http import QueryDict, StreamingHttpResponse from django.forms import HiddenInput -from django.shortcuts import redirect +from django.shortcuts import redirect, redirect from django.urls import reverse_lazy, reverse from django.utils.text import slugify from django.utils.safestring import mark_safe @@ -20,6 +20,7 @@ from django.views.generic.detail import DetailView from django.views.generic.list import ListView from django.views.generic import TemplateView, View +from django.views.generic.base import RedirectView from django_filters.views import FilterView from guardian.mixins import PermissionListMixin @@ -307,8 +308,22 @@ class TargetDeleteView(Raise403PermissionRequiredMixin, DeleteView): success_url = reverse_lazy('targets:list') model = Target -# class TargetSSOISView(ListView): -# model = Target + +class TargetSSOISView(RedirectView): + """ + View that redirect to SSOIS + """ + + model = Target + + def get_redirect_url(*args, **kwargs): + + now = datetime.now() + targ_name_guess = kwargs['pk'].split()[0].split('-')[0] + url = 'http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/cadcbin/ssos/ssosclf.pl?lang=en&object={}'.format(targ_name_guess.split()[0]) + url += '%0D%0A&search=bynameall&epoch1=1990+01+01&epoch2={}+{}+{}'.format(now.year, now.month, now.day) + url += '&eellipse=&eunits=arcseconds&extres=no&xyres=no' + return url class TargetDetailView(Raise403PermissionRequiredMixin, DetailView): """ From 525c1bc6f0222f1d6f5fc9267cc18e9d338b7ebd Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2020 13:28:28 +0000 Subject: [PATCH 344/424] Bump markdown from 3.3.1 to 3.3.2 Bumps [markdown](https://github.com/Python-Markdown/markdown) from 3.3.1 to 3.3.2. - [Release notes](https://github.com/Python-Markdown/markdown/releases) - [Commits](https://github.com/Python-Markdown/markdown/compare/3.3.1...3.3.2) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 948fb31ed..418705c8e 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ 'django-filter==2.4.0', 'django-guardian==2.3.0', 'fits2image==0.4.3', - 'Markdown==3.3.1', # django-rest-framework doc headers require this to support Markdown + 'Markdown==3.3.2', # django-rest-framework doc headers require this to support Markdown 'numpy==1.19.2', 'pillow==8.0.0', 'plotly==4.11.0', From c61feb266f179372223d83e0e8c1edf8aa2b98f6 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 20 Oct 2020 13:06:06 -0700 Subject: [PATCH 345/424] Updated soar for interface changes --- tom_observations/facilities/soar.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tom_observations/facilities/soar.py b/tom_observations/facilities/soar.py index 947c840c4..f8d8bea7a 100644 --- a/tom_observations/facilities/soar.py +++ b/tom_observations/facilities/soar.py @@ -96,6 +96,10 @@ class SOARFacility(LCOFacility): name = 'SOAR' observation_types = [('IMAGING', 'Imaging'), ('SPECTRA', 'Spectroscopy')] + observation_forms = { + 'IMAGING': SOARImagingObservationForm, + 'SPECTRA': SOARSpectroscopyObservationForm + } # The SITES dictionary is used to calculate visibility intervals in the # planning tool. All entries should contain latitude, longitude, elevation # and a code. @@ -109,9 +113,7 @@ class SOARFacility(LCOFacility): } def get_form(self, observation_type): - if observation_type == 'IMAGING': - return SOARImagingObservationForm - elif observation_type == 'SPECTRA': - return SOARSpectroscopyObservationForm - else: + try: + return self.observation_forms[observation_type] + except KeyError: return SOARBaseObservationForm From 3a7e7e308ffabc74c51389649123937dccbdfddb Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 20 Oct 2020 13:19:55 -0700 Subject: [PATCH 346/424] Switched to using .get --- tom_observations/facilities/lco.py | 5 +---- tom_observations/facilities/soar.py | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index deefb76b5..cca7457e0 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -803,10 +803,7 @@ class LCOFacility(BaseRoboticObservationFacility): } def get_form(self, observation_type): - try: - return self.observation_forms[observation_type] - except KeyError: - return LCOBaseObservationForm + return self.observation_forms.get('observation_type', LCOBaseObservationForm) def get_template_form(self, observation_type): return LCOObservationTemplateForm diff --git a/tom_observations/facilities/soar.py b/tom_observations/facilities/soar.py index f8d8bea7a..34e1244f1 100644 --- a/tom_observations/facilities/soar.py +++ b/tom_observations/facilities/soar.py @@ -113,7 +113,4 @@ class SOARFacility(LCOFacility): } def get_form(self, observation_type): - try: - return self.observation_forms[observation_type] - except KeyError: - return SOARBaseObservationForm + return self.observation_forms.get('observation_type', SOARBaseObservationForm) From 3188014aed5b86882b454f691f825e3688990929 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 20 Oct 2020 13:25:28 -0700 Subject: [PATCH 347/424] replacing string literal with observation_type argument --- tom_observations/facilities/lco.py | 2 +- tom_observations/facilities/soar.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index cca7457e0..a932ad693 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -803,7 +803,7 @@ class LCOFacility(BaseRoboticObservationFacility): } def get_form(self, observation_type): - return self.observation_forms.get('observation_type', LCOBaseObservationForm) + return self.observation_forms.get(observation_type, LCOBaseObservationForm) def get_template_form(self, observation_type): return LCOObservationTemplateForm diff --git a/tom_observations/facilities/soar.py b/tom_observations/facilities/soar.py index 34e1244f1..0334e60ca 100644 --- a/tom_observations/facilities/soar.py +++ b/tom_observations/facilities/soar.py @@ -113,4 +113,4 @@ class SOARFacility(LCOFacility): } def get_form(self, observation_type): - return self.observation_forms.get('observation_type', SOARBaseObservationForm) + return self.observation_forms.get(observation_type, SOARBaseObservationForm) From 46e0dfe9835e229736f356e445161d513e335aa4 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 20 Oct 2020 13:44:00 -0700 Subject: [PATCH 348/424] Removing unnecessary line --- tom_observations/facilities/soar.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tom_observations/facilities/soar.py b/tom_observations/facilities/soar.py index 0334e60ca..e42a84ff1 100644 --- a/tom_observations/facilities/soar.py +++ b/tom_observations/facilities/soar.py @@ -95,7 +95,6 @@ class SOARFacility(LCOFacility): """ name = 'SOAR' - observation_types = [('IMAGING', 'Imaging'), ('SPECTRA', 'Spectroscopy')] observation_forms = { 'IMAGING': SOARImagingObservationForm, 'SPECTRA': SOARSpectroscopyObservationForm From 5ae9b397ca8becd62a09a92aa735802e3c99b3f1 Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 21 Oct 2020 14:54:25 -0700 Subject: [PATCH 349/424] Added additional message box notifying users of unknown observation statuses --- tom_observations/facilities/gemini.py | 1 + tom_observations/facilities/lco.py | 3 +++ .../tom_targets/partials/target_unknown_statuses.html | 3 +++ tom_targets/templates/tom_targets/target_detail.html | 1 + tom_targets/templatetags/targets_extras.py | 8 ++++++++ 5 files changed, 16 insertions(+) create mode 100644 tom_targets/templates/tom_targets/partials/target_unknown_statuses.html diff --git a/tom_observations/facilities/gemini.py b/tom_observations/facilities/gemini.py index 4b496e749..6507f626a 100644 --- a/tom_observations/facilities/gemini.py +++ b/tom_observations/facilities/gemini.py @@ -35,6 +35,7 @@ } PORTAL_URL = GEM_SETTINGS['portal_url'] +VALID_OBSERVING_STATES = ['TRIGGERED', 'ON_HOLD'] TERMINAL_OBSERVING_STATES = ['TRIGGERED', 'ON_HOLD'] # Units of flux and wavelength for converting to Specutils Spectrum1D objects diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index a932ad693..0439824de 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -27,6 +27,9 @@ # Module specific settings. PORTAL_URL = LCO_SETTINGS['portal_url'] +# Valid observing states at LCO are defined here: https://developers.lco.global/#data-format-definition +VALID_OBSERVING_STATES = ['PENDING', 'COMPLETED', 'WINDOW_EXPIRED', 'CANCELED'] +PENDING_OBSERVING_STATES = ['PENDING'] SUCCESSFUL_OBSERVING_STATES = ['COMPLETED'] FAILED_OBSERVING_STATES = ['WINDOW_EXPIRED', 'CANCELED'] TERMINAL_OBSERVING_STATES = SUCCESSFUL_OBSERVING_STATES + FAILED_OBSERVING_STATES diff --git a/tom_targets/templates/tom_targets/partials/target_unknown_statuses.html b/tom_targets/templates/tom_targets/partials/target_unknown_statuses.html new file mode 100644 index 000000000..9c86b26a7 --- /dev/null +++ b/tom_targets/templates/tom_targets/partials/target_unknown_statuses.html @@ -0,0 +1,3 @@ +
    + There are {{ num_unknown_statuses }} observations with unknown status. +
    \ No newline at end of file diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index 0af781e01..22dd01a8a 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -15,6 +15,7 @@
    {{ object.future_observations|length }} upcoming observation{{ object.future_observations|pluralize }}
    + {% target_unknown_statuses object %} {% endif %} {% target_buttons object %} {% target_data object %} diff --git a/tom_targets/templatetags/targets_extras.py b/tom_targets/templatetags/targets_extras.py index c9b9f8cbc..72df73883 100644 --- a/tom_targets/templatetags/targets_extras.py +++ b/tom_targets/templatetags/targets_extras.py @@ -7,6 +7,7 @@ from dateutil.parser import parse from django import template from django.conf import settings +from django.db.models import Q from guardian.shortcuts import get_objects_for_user import numpy as np from plotly import offline @@ -65,6 +66,13 @@ def target_data(target): } +@register.inclusion_tag('tom_targets/partials/target_unknown_statuses.html') +def target_unknown_statuses(target): + return { + 'num_unknown_statuses': len(target.observationrecord_set.filter(Q(status='') | Q(status=None))) + } + + @register.inclusion_tag('tom_targets/partials/target_groups.html') def target_groups(target): """ From 566d7b93164d7dd613529093147e10c4caa446ee Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 22 Oct 2020 13:42:00 +0000 Subject: [PATCH 350/424] Bump astropy from 4.0.1.post1 to 4.1 Bumps [astropy](https://github.com/astropy/astropy) from 4.0.1.post1 to 4.1. - [Release notes](https://github.com/astropy/astropy/releases) - [Changelog](https://github.com/astropy/astropy/blob/master/CHANGES.rst) - [Commits](https://github.com/astropy/astropy/compare/v4.0.1.post1...v4.1) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 418705c8e..192754733 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ install_requires=[ 'astroquery==0.4.1', 'astroplan==0.6', - 'astropy==4.0.1.post1', + 'astropy==4.1', 'beautifulsoup4==4.9.3', 'dataclasses; python_version < "3.7"', 'django==3.1.2', # TOM Toolkit requires db math functions From a336946e30db8ebfd9e770559941384acf6b3e05 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 22 Oct 2020 08:45:47 -0700 Subject: [PATCH 351/424] Excluding empty statuses from future_observations property on Target --- tom_targets/models.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tom_targets/models.py b/tom_targets/models.py index dd98a98a8..8d40fe5ea 100644 --- a/tom_targets/models.py +++ b/tom_targets/models.py @@ -1,10 +1,12 @@ -from django.db import models, transaction +from datetime import datetime +from dateutil.parser import parse + +from django.conf import settings from django.core.exceptions import ValidationError -from django.urls import reverse +from django.db import models, transaction +from django.db.models import Q from django.forms.models import model_to_dict -from django.conf import settings -from dateutil.parser import parse -from datetime import datetime +from django.urls import reverse from tom_common.hooks import run_hook @@ -306,7 +308,7 @@ def future_observations(self): :rtype: list """ return [ - obs for obs in self.observationrecord_set.all().order_by('scheduled_start') if not obs.terminal + obs for obs in self.observationrecord_set.exclude(status='').order_by('scheduled_start') if not obs.terminal ] @property From f137c36d0d5df82e513114839a94647266cde339 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 22 Oct 2020 08:53:59 -0700 Subject: [PATCH 352/424] Removing unused import --- tom_targets/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tom_targets/models.py b/tom_targets/models.py index 8d40fe5ea..fea8a19e8 100644 --- a/tom_targets/models.py +++ b/tom_targets/models.py @@ -4,7 +4,6 @@ from django.conf import settings from django.core.exceptions import ValidationError from django.db import models, transaction -from django.db.models import Q from django.forms.models import model_to_dict from django.urls import reverse From 1285ae9738263e96101c98eebe52db58e101e47a Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 22 Oct 2020 10:11:49 -0700 Subject: [PATCH 353/424] Updating SOAR to work with upstream changes --- tom_observations/facilities/soar.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/tom_observations/facilities/soar.py b/tom_observations/facilities/soar.py index e42a84ff1..5177775ea 100644 --- a/tom_observations/facilities/soar.py +++ b/tom_observations/facilities/soar.py @@ -69,20 +69,15 @@ def filter_choices(self): ]) def _build_instrument_config(self): - instrument_config = { - 'exposure_count': self.cleaned_data['exposure_count'], - 'exposure_time': self.cleaned_data['exposure_time'], - 'rotator_mode': 'SKY', - 'extra_params': { - 'rotator_angle': self.cleaned_data['rotator_angle'] - }, - 'optical_elements': { - 'slit': self.cleaned_data['filter'], - 'grating': SPECTRAL_GRATING - } + instrument_configs = super()._build_instrument_config() + + instrument_configs[0]['optical_elements'] = { + 'slit': self.cleaned_data['filter'], + 'grating': SPECTRAL_GRATING } + instrument_configs[0]['rotator_mode'] = 'SKY' - return instrument_config + return instrument_configs class SOARFacility(LCOFacility): From 66cbea932bb9e957e25e9c77affd99ba4b801df9 Mon Sep 17 00:00:00 2001 From: Doug Arnold Date: Fri, 23 Oct 2020 14:38:09 +0100 Subject: [PATCH 354/424] fix-key-issue-tns-alerts --- tom_alerts/brokers/tns.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tom_alerts/brokers/tns.py b/tom_alerts/brokers/tns.py index ab0b785c3..0cbc9c027 100644 --- a/tom_alerts/brokers/tns.py +++ b/tom_alerts/brokers/tns.py @@ -91,7 +91,7 @@ def fetch_alerts(cls, parameters): else: public_timestamp = '' data = { - 'api_key': settings.BROKER_CREDENTIALS['TNS_APIKEY'], + 'api_key': settings.ALERT_CREDENTIALS['TNS']['api_key'], 'data': json.dumps({ 'name': parameters['target_name'], 'internal_name': parameters['internal_name'], @@ -102,13 +102,16 @@ def fetch_alerts(cls, parameters): 'public_timestamp': public_timestamp, }) } + print("About to send payload to TNS", data) response = requests.post(tns_search_url, data) + print("Response received", response) response.raise_for_status() transients = response.json() + print("Contents: ", transients) alerts = [] for transient in transients['data']['reply']: data = { - 'api_key': settings.BROKER_CREDENTIALS['TNS_APIKEY'], + 'api_key': settings.ALERT_CREDENTIALS['TNS']['api_key'], 'data': json.dumps({ 'objname': transient['objname'], 'photometry': 1, @@ -131,15 +134,16 @@ def fetch_alerts(cls, parameters): alerts.append(alert) else: alerts.append(alert) + print("\n\n\n", alert) return iter(alerts) @classmethod def to_generic_alert(cls, alert): return GenericAlert( timestamp=alert['discoverydate'], - url='https://wis-tns.weizmann.ac.il/object/' + alert['name'], - id=alert['name'], - name=alert['name_prefix'] + alert['name'], + url='https://wis-tns.weizmann.ac.il/object/' + alert['objname'], + id=alert['objname'], + name=alert['name_prefix'] + alert['objname'], ra=alert['radeg'], dec=alert['decdeg'], mag=alert['discoverymag'], From 28e967eccf8cde445f789c46b83ce90fb306128c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2020 13:48:11 +0000 Subject: [PATCH 355/424] Bump plotly from 4.11.0 to 4.12.0 Bumps [plotly](https://github.com/plotly/plotly.py) from 4.11.0 to 4.12.0. - [Release notes](https://github.com/plotly/plotly.py/releases) - [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.py/compare/v4.11.0...v4.12.0) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 418705c8e..8c29d65e5 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ 'Markdown==3.3.2', # django-rest-framework doc headers require this to support Markdown 'numpy==1.19.2', 'pillow==8.0.0', - 'plotly==4.11.0', + 'plotly==4.12.0', 'python-dateutil==2.8.1', 'requests==2.24.0', 'specutils==1.1', From 2f577fb8f16212f17e82dfdc5923d27f3c707f33 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2020 13:48:35 +0000 Subject: [PATCH 356/424] Bump pillow from 8.0.0 to 8.0.1 Bumps [pillow](https://github.com/python-pillow/Pillow) from 8.0.0 to 8.0.1. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/master/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/8.0.0...8.0.1) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 418705c8e..46cc8d1d0 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ 'fits2image==0.4.3', 'Markdown==3.3.2', # django-rest-framework doc headers require this to support Markdown 'numpy==1.19.2', - 'pillow==8.0.0', + 'pillow==8.0.1', 'plotly==4.11.0', 'python-dateutil==2.8.1', 'requests==2.24.0', From 12afdf11d6a57d1be1426c947a86ac8950e7a0c1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2020 13:30:44 +0000 Subject: [PATCH 357/424] Bump markdown from 3.3.2 to 3.3.3 Bumps [markdown](https://github.com/Python-Markdown/markdown) from 3.3.2 to 3.3.3. - [Release notes](https://github.com/Python-Markdown/markdown/releases) - [Commits](https://github.com/Python-Markdown/markdown/compare/3.3.2...3.3.3) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e14edad46..5ceb8647f 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ 'django-filter==2.4.0', 'django-guardian==2.3.0', 'fits2image==0.4.3', - 'Markdown==3.3.2', # django-rest-framework doc headers require this to support Markdown + 'Markdown==3.3.3', # django-rest-framework doc headers require this to support Markdown 'numpy==1.19.2', 'pillow==8.0.1', 'plotly==4.12.0', From 981f3dcf047c2617a841627e71a02f36428bcbc3 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2020 13:29:07 +0000 Subject: [PATCH 358/424] Bump astroplan from 0.6 to 0.7 Bumps [astroplan](https://github.com/astropy/astroplan) from 0.6 to 0.7. - [Release notes](https://github.com/astropy/astroplan/releases) - [Changelog](https://github.com/astropy/astroplan/blob/master/CHANGES.rst) - [Commits](https://github.com/astropy/astroplan/compare/v0.6...v0.7) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5ceb8647f..a38b4ced0 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup_requires=['setuptools_scm', 'wheel'], install_requires=[ 'astroquery==0.4.1', - 'astroplan==0.6', + 'astroplan==0.7', 'astropy==4.0.1.post1', 'beautifulsoup4==4.9.3', 'dataclasses; python_version < "3.7"', From fb0575f427bf784466720bb18837ae2f984796bf Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 27 Oct 2020 16:59:39 -0700 Subject: [PATCH 359/424] Expanded alert interface to allow upstream alert submission and added corresponding view --- docs/api/tom_alerts/brokers.rst | 6 +++ docs/api/tom_alerts/exceptions.rst | 5 +++ docs/api/tom_alerts/index.rst | 1 + tom_alerts/alerts.py | 19 ++++++++++ tom_alerts/exceptions.py | 5 +++ .../partials/submit_upstream_button.html | 1 + tom_alerts/templatetags/alerts_extras.py | 30 +++++++++++++++ tom_alerts/urls.py | 7 ++-- tom_alerts/views.py | 37 ++++++++++++++++++- 9 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 docs/api/tom_alerts/exceptions.rst create mode 100644 tom_alerts/exceptions.py create mode 100644 tom_alerts/templates/tom_alerts/partials/submit_upstream_button.html create mode 100644 tom_alerts/templatetags/alerts_extras.py diff --git a/docs/api/tom_alerts/brokers.rst b/docs/api/tom_alerts/brokers.rst index 3f64946ca..5fa260da4 100644 --- a/docs/api/tom_alerts/brokers.rst +++ b/docs/api/tom_alerts/brokers.rst @@ -39,6 +39,12 @@ MARS :members: +****** +SCIMMA +****** + + + ***** Scout ***** diff --git a/docs/api/tom_alerts/exceptions.rst b/docs/api/tom_alerts/exceptions.rst new file mode 100644 index 000000000..14519d68f --- /dev/null +++ b/docs/api/tom_alerts/exceptions.rst @@ -0,0 +1,5 @@ +Exceptions +========== + +.. automodule:: tom_alerts.exceptions + :members: \ No newline at end of file diff --git a/docs/api/tom_alerts/index.rst b/docs/api/tom_alerts/index.rst index b53ad7dc2..b780475fd 100644 --- a/docs/api/tom_alerts/index.rst +++ b/docs/api/tom_alerts/index.rst @@ -5,5 +5,6 @@ Alerts :maxdepth: 2 brokers + exceptions models views diff --git a/tom_alerts/alerts.py b/tom_alerts/alerts.py index 4cd59ec9e..f54c98ef1 100644 --- a/tom_alerts/alerts.py +++ b/tom_alerts/alerts.py @@ -8,6 +8,7 @@ import json from abc import ABC, abstractmethod +from tom_alerts.exceptions import AlertSubmissionException from tom_alerts.models import BrokerQuery from tom_targets.models import Target @@ -199,6 +200,24 @@ def to_target(self, alert): """ pass + def submit_upstream_alert(self, target=None, observation_record=None, **kwargs): + """ + Submits an alert upstream back to the broker. At least one of a target or an + observation record must be provided. + + :param target: ``Target`` object to be converted to an alert and submitted upstream + :type target: ``Target`` + + :param observation_record: ``ObservationRecord`` object to be converted to an alert and submitted upstream + :type observation_record: ``ObservationRecord`` + + :returns: True or False depending on success of message submission + :rtype: bool + """ + if not (target or observation_record): + raise AlertSubmissionException('Must provide either Target or ObservationRecord to be submitted upstream.') + return + @abstractmethod def to_generic_alert(self, alert): """ diff --git a/tom_alerts/exceptions.py b/tom_alerts/exceptions.py new file mode 100644 index 000000000..1af865dfa --- /dev/null +++ b/tom_alerts/exceptions.py @@ -0,0 +1,5 @@ +class AlertSubmissionException(Exception): + """ + The AlertSubmissionException should be used when an alert fails to be submitted to an upstream broker. + """ + pass diff --git a/tom_alerts/templates/tom_alerts/partials/submit_upstream_button.html b/tom_alerts/templates/tom_alerts/partials/submit_upstream_button.html new file mode 100644 index 000000000..2b5362bf7 --- /dev/null +++ b/tom_alerts/templates/tom_alerts/partials/submit_upstream_button.html @@ -0,0 +1 @@ +Submit to {{ broker }} \ No newline at end of file diff --git a/tom_alerts/templatetags/alerts_extras.py b/tom_alerts/templatetags/alerts_extras.py new file mode 100644 index 000000000..fe9fc22f9 --- /dev/null +++ b/tom_alerts/templatetags/alerts_extras.py @@ -0,0 +1,30 @@ +from django import template + + +register = template.Library() + + +@register.inclusion_tag('tom_alerts/partials/submit_upstream_button.html') +def submit_upstream_button(broker, target=None, observation_record=None, redirect_url=None): + """ + Renders a button to submit an alert upstream to a broker. At least one of target/obs record should be given. + + :param broker: The name of the broker to which the button will lead, as in the name field of the broker module. + :type broker: str + + :param target: The target to be submitted as an alert, if any. + :type target: ``Target`` + + :param observation_record: The observation record to be submitted as an alert, if any. + :type observation_record: ``ObservationRecord`` + + :param redirect_url: + :type redirect_url: str + """ + # TODO: document this function somewhere prominent, as it likely won't live in the default templates + return { + 'broker': broker, + 'target': target, + 'observation_record': observation_record, + 'redirect_url': redirect_url + } diff --git a/tom_alerts/urls.py b/tom_alerts/urls.py index 67c5e5baa..aa00993e1 100644 --- a/tom_alerts/urls.py +++ b/tom_alerts/urls.py @@ -1,7 +1,7 @@ from django.urls import path -from tom_alerts.views import BrokerQueryCreateView, BrokerQueryListView, BrokerQueryUpdateView, RunQueryView -from tom_alerts.views import CreateTargetFromAlertView, BrokerQueryDeleteView +from tom_alerts.views import BrokerQueryCreateView, BrokerQueryListView, BrokerQueryUpdateView, BrokerQueryDeleteView +from tom_alerts.views import CreateTargetFromAlertView, RunQueryView, SubmitAlertUpstreamView app_name = 'tom_alerts' @@ -11,5 +11,6 @@ path('query//update/', BrokerQueryUpdateView.as_view(), name='update'), path('query//run/', RunQueryView.as_view(), name='run'), path('query//delete/', BrokerQueryDeleteView.as_view(), name='delete'), - path('alert/create/', CreateTargetFromAlertView.as_view(), name='create-target') + path('alert/create/', CreateTargetFromAlertView.as_view(), name='create-target'), + path('alert//submit/', SubmitAlertUpstreamView.as_view(), name='submit-alert') ] diff --git a/tom_alerts/views.py b/tom_alerts/views.py index 18753808b..996837eec 100644 --- a/tom_alerts/views.py +++ b/tom_alerts/views.py @@ -1,7 +1,8 @@ import json +import logging from django.views.generic.edit import FormView, DeleteView -from django.views.generic.base import TemplateView, View +from django.views.generic.base import TemplateView, View, RedirectView from django.db import IntegrityError from django.shortcuts import redirect, get_object_or_404 from django.utils import timezone @@ -13,8 +14,13 @@ from django_filters.views import FilterView from django_filters import FilterSet, ChoiceFilter, CharFilter -from tom_alerts.models import BrokerQuery from tom_alerts.alerts import get_service_class, get_service_classes +from tom_alerts.models import BrokerQuery +from tom_alerts.exceptions import AlertSubmissionException +from tom_observations.models import ObservationRecord +from tom_targets.models import Target + +logger = logging.getLogger(__name__) class BrokerQueryCreateView(LoginRequiredMixin, FormView): @@ -255,3 +261,30 @@ def post(self, request, *args, **kwargs): return redirect(reverse( 'tom_targets:list') ) + + +class SubmitAlertUpstreamView(LoginRequiredMixin, RedirectView): + + def get(self, request, *args, **kwargs): + target_id = request.GET.get('target_id') + target = Target.objects.get(pk=target_id) if target_id else None + + observation_id = request.GET.get('observation_id') + obsr = ObservationRecord.objects.get(pk=observation_id) if observation_id else None + + topic = request.GET.get('topic') + + broker_name = kwargs.get('broker') + broker_class = get_service_class(kwargs.get('broker'))() + try: + broker_class.submit_upstream_alert(target=target, observation_record=obsr, topic=topic) + except AlertSubmissionException as e: + logger.log(msg=f'Failed to submit alert: {e}', level=logging.WARN) + messages.warning(request, f'Unable to submit one or more alerts to {broker_name}') + super().get(request, *args, **kwargs) + + def get_redirect_url(self, *args, **kwargs): + # TODO: more elegant redirection + redirect_url = self.request.GET.get('next') + if not redirect_url: + redirect_url = self.request.META.get('HTTP_REFERER') From 8c91c4c9cce94a768d032f3efa4a81d7ef6b97e5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2020 13:51:51 +0000 Subject: [PATCH 360/424] Bump numpy from 1.19.2 to 1.19.3 Bumps [numpy](https://github.com/numpy/numpy) from 1.19.2 to 1.19.3. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/master/doc/HOWTO_RELEASE.rst.txt) - [Commits](https://github.com/numpy/numpy/compare/v1.19.2...v1.19.3) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ed8c7ef2e..65acedf8a 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ 'django-guardian==2.3.0', 'fits2image==0.4.3', 'Markdown==3.3.3', # django-rest-framework doc headers require this to support Markdown - 'numpy==1.19.2', + 'numpy==1.19.3', 'pillow==8.0.1', 'plotly==4.12.0', 'python-dateutil==2.8.1', From b707eeee842fc073773d37c8f63cb1d76030afc1 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 29 Oct 2020 14:21:57 -0700 Subject: [PATCH 361/424] Allowed upstream submission to include arbitrary query params --- .../partials/submit_upstream_button.html | 2 +- tom_alerts/tests/tests_generic.py | 43 ++++++++++++++++--- tom_alerts/urls.py | 2 +- tom_alerts/views.py | 37 ++++++++++------ 4 files changed, 65 insertions(+), 19 deletions(-) diff --git a/tom_alerts/templates/tom_alerts/partials/submit_upstream_button.html b/tom_alerts/templates/tom_alerts/partials/submit_upstream_button.html index 2b5362bf7..af7ae28ad 100644 --- a/tom_alerts/templates/tom_alerts/partials/submit_upstream_button.html +++ b/tom_alerts/templates/tom_alerts/partials/submit_upstream_button.html @@ -1 +1 @@ -Submit to {{ broker }} \ No newline at end of file +Submit to {{ broker }} \ No newline at end of file diff --git a/tom_alerts/tests/tests_generic.py b/tom_alerts/tests/tests_generic.py index 1c7d7d545..b8238092a 100644 --- a/tom_alerts/tests/tests_generic.py +++ b/tom_alerts/tests/tests_generic.py @@ -1,12 +1,15 @@ -from django.test import TestCase, override_settings +import json + from django import forms from django.contrib.auth.models import User, Group -from django.urls import reverse +from django.contrib.messages import get_messages from django.core.cache import cache -import json +from django.test import TestCase, override_settings +from django.urls import reverse -from tom_alerts.alerts import GenericQueryForm, GenericAlert, get_service_class +from tom_alerts.alerts import GenericBroker, GenericQueryForm, GenericAlert, get_service_class from tom_alerts.models import BrokerQuery +from tom_observations.models import ObservationRecord from tom_targets.models import Target # Test alert data. Normally this would come from a remote source. @@ -25,7 +28,7 @@ class TestBrokerForm(GenericQueryForm): name = forms.CharField(required=True) -class TestBroker: +class TestBroker(GenericBroker): """ The broker class encapsulates the logic for querying remote brokers and transforming the returned data into TOM Toolkit Targets so they can be used elsewhere in the system. The following methods and attributes are all required, but a broker can be as complex as needed. @@ -58,6 +61,9 @@ def to_generic_alert(self, alert): score=alert['score'] ) + def submit_upstream_alert(self, **kwargs): + return super().submit_upstream_alert(**kwargs) + @override_settings(TOM_ALERT_CLASSES=['tom_alerts.tests.tests_generic.TestBroker']) class TestBrokerClass(TestCase): @@ -217,3 +223,30 @@ def test_create_no_targets(self): response = self.client.post(reverse('tom_alerts:create-target'), data=post_data, follow=True) self.assertEqual(Target.objects.count(), 0) self.assertRedirects(response, reverse('tom_alerts:run', kwargs={'pk': query.id})) + + def test_submit_alert_success(self): + """Test submission of an alert to a broker.""" + target = Target.objects.create(name='test_target', ra=1, dec=2) + response = self.client.get('{0}?{1}'.format( + reverse('tom_alerts:submit-alert', kwargs={'broker': 'TEST'}), f'target_id={target.id}' + )) + + self.assertEqual(response.status_code, 302) + self.assertRedirects(response, reverse('home')) + + obsr = ObservationRecord.objects.create(target=target, facility='Test', parameters={}, observation_id=1) + response = self.client.get('{0}?{1}&{2}'.format( + reverse('tom_alerts:submit-alert', kwargs={'broker': 'TEST'}), + f'observation_record_id={obsr.id}', + f'next={reverse("tom_targets:list")}' + )) + + self.assertEqual(response.status_code, 302) + self.assertRedirects(response, reverse('tom_targets:list')) + + def test_submit_alert_failure(self): + """Test that an alert submission fails when neither target_id nor observation_id are included.""" + response = self.client.get(reverse('tom_alerts:submit-alert', kwargs={'broker': 'TEST'})) + messages = [(m.message, m.level) for m in get_messages(response.wsgi_request)] + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0][0], 'Unable to submit one or more alerts to TEST') diff --git a/tom_alerts/urls.py b/tom_alerts/urls.py index aa00993e1..77c720517 100644 --- a/tom_alerts/urls.py +++ b/tom_alerts/urls.py @@ -12,5 +12,5 @@ path('query//run/', RunQueryView.as_view(), name='run'), path('query//delete/', BrokerQueryDeleteView.as_view(), name='delete'), path('alert/create/', CreateTargetFromAlertView.as_view(), name='create-target'), - path('alert//submit/', SubmitAlertUpstreamView.as_view(), name='submit-alert') + path('/submit/', SubmitAlertUpstreamView.as_view(), name='submit-alert') ] diff --git a/tom_alerts/views.py b/tom_alerts/views.py index 996837eec..738e226bd 100644 --- a/tom_alerts/views.py +++ b/tom_alerts/views.py @@ -1,3 +1,4 @@ +from copy import copy import json import logging @@ -266,25 +267,37 @@ def post(self, request, *args, **kwargs): class SubmitAlertUpstreamView(LoginRequiredMixin, RedirectView): def get(self, request, *args, **kwargs): - target_id = request.GET.get('target_id') - target = Target.objects.get(pk=target_id) if target_id else None + # TODO: This should be a FormView and a POST + query_params = request.GET.dict() - observation_id = request.GET.get('observation_id') - obsr = ObservationRecord.objects.get(pk=observation_id) if observation_id else None + target_id = query_params.pop('target_id', None) + target = Target.objects.get(pk=target_id) if target_id else None - topic = request.GET.get('topic') + obsr_id = query_params.pop('observation_record_id', None) + obsr = ObservationRecord.objects.get(pk=obsr_id) if obsr_id else None - broker_name = kwargs.get('broker') - broker_class = get_service_class(kwargs.get('broker'))() + broker_name = kwargs['broker'] + broker_class = get_service_class(broker_name)() try: - broker_class.submit_upstream_alert(target=target, observation_record=obsr, topic=topic) + # Pass non-standard fields from query parameters as kwargs + broker_class.submit_upstream_alert(target=target, observation_record=obsr, **query_params) except AlertSubmissionException as e: logger.log(msg=f'Failed to submit alert: {e}', level=logging.WARN) messages.warning(request, f'Unable to submit one or more alerts to {broker_name}') - super().get(request, *args, **kwargs) + + return super().get(request, *args, **kwargs) def get_redirect_url(self, *args, **kwargs): - # TODO: more elegant redirection - redirect_url = self.request.GET.get('next') + """ + If ``next`` is provided in the query params, redirects to ``next``. If ``HTTP_REFERER`` is present on the + ``META`` property of the request, redirects to ``HTTP_REFERER``. Else redirects to /. + + :returns: url to redirect to + :rtype: str + """ + next_url = self.request.GET.get('next') + redirect_url = next_url if next_url else self.request.META.get('HTTP_REFERER') if not redirect_url: - redirect_url = self.request.META.get('HTTP_REFERER') + redirect_url = reverse('home') + + return redirect_url From c03ff27a6bec6abfb0ce91a2598299a941c27b90 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 29 Oct 2020 14:50:12 -0700 Subject: [PATCH 362/424] Removing unused import --- tom_alerts/views.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tom_alerts/views.py b/tom_alerts/views.py index 738e226bd..7bfa31743 100644 --- a/tom_alerts/views.py +++ b/tom_alerts/views.py @@ -1,4 +1,3 @@ -from copy import copy import json import logging @@ -267,7 +266,6 @@ def post(self, request, *args, **kwargs): class SubmitAlertUpstreamView(LoginRequiredMixin, RedirectView): def get(self, request, *args, **kwargs): - # TODO: This should be a FormView and a POST query_params = request.GET.dict() target_id = query_params.pop('target_id', None) From 6af1eca4881f67bbf37c2a6f579fd6368864f212 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 29 Oct 2020 15:01:30 -0700 Subject: [PATCH 363/424] Added a bit of documentation --- tom_alerts/views.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tom_alerts/views.py b/tom_alerts/views.py index 7bfa31743..3b189f9f2 100644 --- a/tom_alerts/views.py +++ b/tom_alerts/views.py @@ -264,6 +264,12 @@ def post(self, request, *args, **kwargs): class SubmitAlertUpstreamView(LoginRequiredMixin, RedirectView): + """ + View used to submit alerts to an upstream broker, such as SCIMMA's Hopskotch or the Transient Name Server. + + While this view handles the query parameters for target_id and observation_record_id by default, it will + send any additional query parameters to the broker, allowing a broker to use any arbitrary parameters. + """ def get(self, request, *args, **kwargs): query_params = request.GET.dict() From 3f944cf3378268ca4e6c9566e804824ee9e08ddd Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 29 Oct 2020 15:37:46 -0700 Subject: [PATCH 364/424] Refactored to be a form view --- tom_alerts/alerts.py | 33 +++++++-- tom_alerts/exceptions.py | 1 - .../partials/submit_upstream_form.html | 2 + tom_alerts/templatetags/alerts_extras.py | 16 +++++ tom_alerts/tests/tests_generic.py | 4 +- tom_alerts/views.py | 68 ++++++++++++++----- 6 files changed, 97 insertions(+), 27 deletions(-) create mode 100644 tom_alerts/templates/tom_alerts/partials/submit_upstream_form.html diff --git a/tom_alerts/alerts.py b/tom_alerts/alerts.py index f54c98ef1..51474c886 100644 --- a/tom_alerts/alerts.py +++ b/tom_alerts/alerts.py @@ -1,15 +1,18 @@ -from django.conf import settings -from django import forms -from importlib import import_module -from datetime import datetime +from abc import ABC, abstractmethod from dataclasses import dataclass +from datetime import datetime +from importlib import import_module +import json + +from django import forms +from django.conf import settings +from django.shortcuts import reverse from crispy_forms.helper import FormHelper from crispy_forms.layout import Submit, Layout -import json -from abc import ABC, abstractmethod from tom_alerts.exceptions import AlertSubmissionException from tom_alerts.models import BrokerQuery +from tom_observations.models import ObservationRecord from tom_targets.models import Target @@ -147,6 +150,23 @@ def save(self, query_id=None): return query +class GenericUpstreamSubmissionForm(forms.Form): + target = forms.ModelChoiceField(required=False, queryset=Target.objects.all(), widget=forms.HiddenInput()) + observation_record = forms.ModelChoiceField(required=False, queryset=ObservationRecord.objects.all(), + widget=forms.HiddenInput()) + redirect_url = forms.CharField(required=False, max_length=100, widget=forms.HiddenInput()) + + def __init__(self, *args, **kwargs): + broker_name = kwargs.pop('broker') + super().__init__(*args, **kwargs) + self.helper = FormHelper() + # TODO: this needs to look not like a submit button + # TODO: for some reason at present the css class affects the text and not the button + self.helper.add_input(Submit('submit', f'Submit to {broker_name}', css_class='btn btn-outline-primary')) + self.helper.form_action = reverse('tom_alerts:submit-alert', kwargs={'broker': broker_name}) + self.common_layout = Layout('broker', 'target', 'observation_record') + + class GenericBroker(ABC): """ The ``GenericBroker`` provides an interface for implementing a broker module. It contains a number of methods to be @@ -156,6 +176,7 @@ class GenericBroker(ABC): For an implementation example, please see https://github.com/TOMToolkit/tom_base/blob/master/tom_alerts/brokers/mars.py """ + alert_submission_form = GenericUpstreamSubmissionForm @abstractmethod def fetch_alerts(self, parameters): diff --git a/tom_alerts/exceptions.py b/tom_alerts/exceptions.py index 1af865dfa..5f4e83087 100644 --- a/tom_alerts/exceptions.py +++ b/tom_alerts/exceptions.py @@ -2,4 +2,3 @@ class AlertSubmissionException(Exception): """ The AlertSubmissionException should be used when an alert fails to be submitted to an upstream broker. """ - pass diff --git a/tom_alerts/templates/tom_alerts/partials/submit_upstream_form.html b/tom_alerts/templates/tom_alerts/partials/submit_upstream_form.html new file mode 100644 index 000000000..f0101d994 --- /dev/null +++ b/tom_alerts/templates/tom_alerts/partials/submit_upstream_form.html @@ -0,0 +1,2 @@ +{% load crispy_forms_tags %} +{% crispy submit_upstream_form %} \ No newline at end of file diff --git a/tom_alerts/templatetags/alerts_extras.py b/tom_alerts/templatetags/alerts_extras.py index fe9fc22f9..90247f312 100644 --- a/tom_alerts/templatetags/alerts_extras.py +++ b/tom_alerts/templatetags/alerts_extras.py @@ -1,5 +1,6 @@ from django import template +from tom_alerts.alerts import get_service_class register = template.Library() @@ -28,3 +29,18 @@ def submit_upstream_button(broker, target=None, observation_record=None, redirec 'observation_record': observation_record, 'redirect_url': redirect_url } + + +@register.inclusion_tag('tom_alerts/partials/submit_upstream_form.html') +def submit_upstream_form(broker, target=None, observation_record=None, redirect_url=None): + broker_class = get_service_class(broker) + form_class = broker_class.alert_submission_form + form = form_class(broker=broker, initial={ + 'target': target, + 'observation_record': observation_record, + 'redirect_url': redirect_url + }) + + return { + 'submit_upstream_form': form + } diff --git a/tom_alerts/tests/tests_generic.py b/tom_alerts/tests/tests_generic.py index b8238092a..b5d5d326b 100644 --- a/tom_alerts/tests/tests_generic.py +++ b/tom_alerts/tests/tests_generic.py @@ -61,8 +61,8 @@ def to_generic_alert(self, alert): score=alert['score'] ) - def submit_upstream_alert(self, **kwargs): - return super().submit_upstream_alert(**kwargs) + def submit_upstream_alert(self, target=None, observation_record=None): + return super().submit_upstream_alert(target=target, observation_record=observation_record) @override_settings(TOM_ALERT_CLASSES=['tom_alerts.tests.tests_generic.TestBroker']) diff --git a/tom_alerts/views.py b/tom_alerts/views.py index 3b189f9f2..a3b4177bd 100644 --- a/tom_alerts/views.py +++ b/tom_alerts/views.py @@ -2,7 +2,7 @@ import logging from django.views.generic.edit import FormView, DeleteView -from django.views.generic.base import TemplateView, View, RedirectView +from django.views.generic.base import TemplateView, View from django.db import IntegrityError from django.shortcuts import redirect, get_object_or_404 from django.utils import timezone @@ -263,7 +263,7 @@ def post(self, request, *args, **kwargs): ) -class SubmitAlertUpstreamView(LoginRequiredMixin, RedirectView): +class SubmitAlertUpstreamView(LoginRequiredMixin, FormView): """ View used to submit alerts to an upstream broker, such as SCIMMA's Hopskotch or the Transient Name Server. @@ -271,25 +271,20 @@ class SubmitAlertUpstreamView(LoginRequiredMixin, RedirectView): send any additional query parameters to the broker, allowing a broker to use any arbitrary parameters. """ - def get(self, request, *args, **kwargs): - query_params = request.GET.dict() - - target_id = query_params.pop('target_id', None) - target = Target.objects.get(pk=target_id) if target_id else None + def get_broker_name(self): + return self.kwargs['broker'] - obsr_id = query_params.pop('observation_record_id', None) - obsr = ObservationRecord.objects.get(pk=obsr_id) if obsr_id else None + def get_form_class(self): + broker_name = self.get_broker_name() + return get_service_class(broker_name).alert_submission_form - broker_name = kwargs['broker'] - broker_class = get_service_class(broker_name)() - try: - # Pass non-standard fields from query parameters as kwargs - broker_class.submit_upstream_alert(target=target, observation_record=obsr, **query_params) - except AlertSubmissionException as e: - logger.log(msg=f'Failed to submit alert: {e}', level=logging.WARN) - messages.warning(request, f'Unable to submit one or more alerts to {broker_name}') + def get_form_kwargs(self): + kwargs = super().get_form_kwargs() + kwargs.update({ + 'broker': self.get_broker_name() + }) - return super().get(request, *args, **kwargs) + return kwargs def get_redirect_url(self, *args, **kwargs): """ @@ -305,3 +300,40 @@ def get_redirect_url(self, *args, **kwargs): redirect_url = reverse('home') return redirect_url + + def form_valid(self, form): + broker_name = self.get_broker_name() + broker = get_service_class(broker_name)() + + target = form.cleaned_data.pop('target') + obsr = form.cleaned_data.pop('observation_record') + redirect_url = form.cleaned_data.pop('redirect_url') + + try: + # Pass non-standard fields from query parameters as kwargs + broker.submit_upstream_alert(target=target, observation_record=obsr, **form.cleaned_data) + except AlertSubmissionException as e: + logger.log(msg=f'Failed to submit alert: {e}', level=logging.WARN) + messages.warning(self.request, f'Unable to submit one or more alerts to {broker_name}') + + return redirect(self.get_redirect_url(redirect_url)) + + # def get(self, request, *args, **kwargs): + # query_params = request.GET.dict() + + # target_id = query_params.pop('target_id', None) + # target = Target.objects.get(pk=target_id) if target_id else None + + # obsr_id = query_params.pop('observation_record_id', None) + # obsr = ObservationRecord.objects.get(pk=obsr_id) if obsr_id else None + + # broker_name = kwargs['broker'] + # broker_class = get_service_class(broker_name)() + # try: + # # Pass non-standard fields from query parameters as kwargs + # broker_class.submit_upstream_alert(target=target, observation_record=obsr, **query_params) + # except AlertSubmissionException as e: + # logger.log(msg=f'Failed to submit alert: {e}', level=logging.WARN) + # messages.warning(request, f'Unable to submit one or more alerts to {broker_name}') + + # return super().get(request, *args, **kwargs) From d649c14373f622f8562cdaf6f3de5f2e58346b8c Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 29 Oct 2020 16:58:42 -0700 Subject: [PATCH 365/424] Refactored to be a post and a formview --- tom_alerts/alerts.py | 13 +++++-- tom_alerts/tests/tests_generic.py | 61 +++++++++++++++++++++++-------- tom_alerts/views.py | 34 +++++------------ 3 files changed, 64 insertions(+), 44 deletions(-) diff --git a/tom_alerts/alerts.py b/tom_alerts/alerts.py index 51474c886..55473e5c1 100644 --- a/tom_alerts/alerts.py +++ b/tom_alerts/alerts.py @@ -10,7 +10,6 @@ from crispy_forms.helper import FormHelper from crispy_forms.layout import Submit, Layout -from tom_alerts.exceptions import AlertSubmissionException from tom_alerts.models import BrokerQuery from tom_observations.models import ObservationRecord from tom_targets.models import Target @@ -166,6 +165,14 @@ def __init__(self, *args, **kwargs): self.helper.form_action = reverse('tom_alerts:submit-alert', kwargs={'broker': broker_name}) self.common_layout = Layout('broker', 'target', 'observation_record') + def clean(self): + cleaned_data = super().clean() + + if not (cleaned_data.get('target') or cleaned_data.get('observation_record')): + raise forms.ValidationError('Must provide either Target or ObservationRecord to be submitted upstream.') + + return cleaned_data + class GenericBroker(ABC): """ @@ -235,9 +242,7 @@ def submit_upstream_alert(self, target=None, observation_record=None, **kwargs): :returns: True or False depending on success of message submission :rtype: bool """ - if not (target or observation_record): - raise AlertSubmissionException('Must provide either Target or ObservationRecord to be submitted upstream.') - return + pass @abstractmethod def to_generic_alert(self, alert): diff --git a/tom_alerts/tests/tests_generic.py b/tom_alerts/tests/tests_generic.py index b5d5d326b..a89c8f6ed 100644 --- a/tom_alerts/tests/tests_generic.py +++ b/tom_alerts/tests/tests_generic.py @@ -1,4 +1,5 @@ import json +from unittest.mock import patch from django import forms from django.contrib.auth.models import User, Group @@ -7,7 +8,9 @@ from django.test import TestCase, override_settings from django.urls import reverse -from tom_alerts.alerts import GenericBroker, GenericQueryForm, GenericAlert, get_service_class +from tom_alerts.alerts import GenericBroker, GenericQueryForm, GenericUpstreamSubmissionForm, GenericAlert +from tom_alerts.alerts import get_service_class +from tom_alerts.exceptions import AlertSubmissionException from tom_alerts.models import BrokerQuery from tom_observations.models import ObservationRecord from tom_targets.models import Target @@ -21,13 +24,22 @@ class TestBrokerForm(GenericQueryForm): """ All brokers must have a form which will be used to construct and save queries - to the broker. They should sublcass `GenericQueryForm` which includes some required + to the broker. They should subclass `GenericQueryForm` which includes some required fields and contains logic for serializing and persisting the query parameters to the database. This test form will only have one field. """ name = forms.CharField(required=True) +class TestUpstreamSubmissionForm(GenericUpstreamSubmissionForm): + """ + Brokers supporting upstream submission can have a form used for constructing the submission. If should subclass + GenericUpstreamSubmissionForm. This test form will have only one additional field in order to test that the + additional field value is submitted to the broker correctly. + """ + topic = forms.CharField(required=False) + + class TestBroker(GenericBroker): """ The broker class encapsulates the logic for querying remote brokers and transforming the returned data into TOM Toolkit Targets so they can be used elsewhere in the system. The @@ -35,6 +47,7 @@ class TestBroker(GenericBroker): """ name = 'TEST' # The name of this broker. form = TestBrokerForm # The form that will be used to write and save queries. + alert_submission_form = TestUpstreamSubmissionForm def fetch_alerts(self, parameters): """ All brokers must implement this method. It must return a list of alerts. @@ -61,7 +74,7 @@ def to_generic_alert(self, alert): score=alert['score'] ) - def submit_upstream_alert(self, target=None, observation_record=None): + def submit_upstream_alert(self, target=None, observation_record=None, **kwargs): return super().submit_upstream_alert(target=target, observation_record=observation_record) @@ -224,29 +237,47 @@ def test_create_no_targets(self): self.assertEqual(Target.objects.count(), 0) self.assertRedirects(response, reverse('tom_alerts:run', kwargs={'pk': query.id})) - def test_submit_alert_success(self): + @patch('tom_alerts.tests.tests_generic.TestBroker.submit_upstream_alert') + def test_submit_alert_success(self, mock_submit_upstream_alert): """Test submission of an alert to a broker.""" + + # Tests that an alert is submitted with just a target, and that no redirect_url results in redirect to home target = Target.objects.create(name='test_target', ra=1, dec=2) - response = self.client.get('{0}?{1}'.format( - reverse('tom_alerts:submit-alert', kwargs={'broker': 'TEST'}), f'target_id={target.id}' - )) + response = self.client.post(reverse('tom_alerts:submit-alert', kwargs={'broker': 'TEST'}), + data={'target': target.id}) + mock_submit_upstream_alert.assert_called_with(target=target, observation_record=None, topic='') self.assertEqual(response.status_code, 302) self.assertRedirects(response, reverse('home')) + # Tests that an alert is submitted with just an observation, and that redirect is to redirect_url obsr = ObservationRecord.objects.create(target=target, facility='Test', parameters={}, observation_id=1) - response = self.client.get('{0}?{1}&{2}'.format( - reverse('tom_alerts:submit-alert', kwargs={'broker': 'TEST'}), - f'observation_record_id={obsr.id}', - f'next={reverse("tom_targets:list")}' - )) + response = self.client.post(reverse('tom_alerts:submit-alert', kwargs={'broker': 'TEST'}), + data={'observation_record': obsr.id, 'redirect_url': reverse('tom_targets:list')}) + mock_submit_upstream_alert.assert_called_with(target=None, observation_record=obsr, topic='') self.assertEqual(response.status_code, 302) self.assertRedirects(response, reverse('tom_targets:list')) - def test_submit_alert_failure(self): - """Test that an alert submission fails when neither target_id nor observation_id are included.""" - response = self.client.get(reverse('tom_alerts:submit-alert', kwargs={'broker': 'TEST'})) + # Tests that an alert submitted with additional parameters calls submit_upstream_alert correctly. + response = self.client.post(reverse('tom_alerts:submit-alert', kwargs={'broker': 'TEST'}), + data={'observation_record': obsr.id, 'topic': 'test topic'}) + mock_submit_upstream_alert.assert_called_with(target=None, observation_record=obsr, topic='test topic') + + @patch('tom_alerts.tests.tests_generic.TestBroker.submit_upstream_alert') + def test_submit_alert_failure(self, mock_submit_upstream_alert): + """Test that an alert submission returns an appropriate message when alert submission fails.""" + mock_submit_upstream_alert.side_effect = AlertSubmissionException() + + response = self.client.post(reverse('tom_alerts:submit-alert', kwargs={'broker': 'TEST'}), data={}) + messages = [(m.message, m.level) for m in get_messages(response.wsgi_request)] + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0][0], 'Unable to submit one or more alerts to TEST') + + def test_submit_alert_invalid_form(self): + """Test that an alert submission failed when form is invalid.""" + response = self.client.post(reverse('tom_alerts:submit-alert', kwargs={'broker': 'TEST'}), data={}) messages = [(m.message, m.level) for m in get_messages(response.wsgi_request)] self.assertEqual(len(messages), 1) self.assertEqual(messages[0][0], 'Unable to submit one or more alerts to TEST') + self.assertRedirects(response, reverse('home')) diff --git a/tom_alerts/views.py b/tom_alerts/views.py index a3b4177bd..819b2c551 100644 --- a/tom_alerts/views.py +++ b/tom_alerts/views.py @@ -1,7 +1,7 @@ import json import logging -from django.views.generic.edit import FormView, DeleteView +from django.views.generic.edit import DeleteView, FormMixin, FormView, ProcessFormView from django.views.generic.base import TemplateView, View from django.db import IntegrityError from django.shortcuts import redirect, get_object_or_404 @@ -17,8 +17,6 @@ from tom_alerts.alerts import get_service_class, get_service_classes from tom_alerts.models import BrokerQuery from tom_alerts.exceptions import AlertSubmissionException -from tom_observations.models import ObservationRecord -from tom_targets.models import Target logger = logging.getLogger(__name__) @@ -263,7 +261,7 @@ def post(self, request, *args, **kwargs): ) -class SubmitAlertUpstreamView(LoginRequiredMixin, FormView): +class SubmitAlertUpstreamView(LoginRequiredMixin, FormMixin, ProcessFormView, View): """ View used to submit alerts to an upstream broker, such as SCIMMA's Hopskotch or the Transient Name Server. @@ -294,13 +292,19 @@ def get_redirect_url(self, *args, **kwargs): :returns: url to redirect to :rtype: str """ - next_url = self.request.GET.get('next') + # TODO: this needs to work with the new POST flow + next_url = self.request.POST.get('redirect_url') redirect_url = next_url if next_url else self.request.META.get('HTTP_REFERER') if not redirect_url: redirect_url = reverse('home') return redirect_url + def form_invalid(self, form): + logger.log(msg=f'Form invalid: {form.errors}', level=logging.WARN) + messages.warning(self.request, f'Unable to submit one or more alerts to {self.get_broker_name()}') + return redirect(self.get_redirect_url()) # TODO: fix this + def form_valid(self, form): broker_name = self.get_broker_name() broker = get_service_class(broker_name)() @@ -317,23 +321,3 @@ def form_valid(self, form): messages.warning(self.request, f'Unable to submit one or more alerts to {broker_name}') return redirect(self.get_redirect_url(redirect_url)) - - # def get(self, request, *args, **kwargs): - # query_params = request.GET.dict() - - # target_id = query_params.pop('target_id', None) - # target = Target.objects.get(pk=target_id) if target_id else None - - # obsr_id = query_params.pop('observation_record_id', None) - # obsr = ObservationRecord.objects.get(pk=obsr_id) if obsr_id else None - - # broker_name = kwargs['broker'] - # broker_class = get_service_class(broker_name)() - # try: - # # Pass non-standard fields from query parameters as kwargs - # broker_class.submit_upstream_alert(target=target, observation_record=obsr, **query_params) - # except AlertSubmissionException as e: - # logger.log(msg=f'Failed to submit alert: {e}', level=logging.WARN) - # messages.warning(request, f'Unable to submit one or more alerts to {broker_name}') - - # return super().get(request, *args, **kwargs) From b243ba792c8af7d5777e7e373c8e17803750a046 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 29 Oct 2020 17:00:27 -0700 Subject: [PATCH 366/424] Removing unnecessary code --- tom_alerts/views.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tom_alerts/views.py b/tom_alerts/views.py index 819b2c551..f920271e4 100644 --- a/tom_alerts/views.py +++ b/tom_alerts/views.py @@ -284,7 +284,7 @@ def get_form_kwargs(self): return kwargs - def get_redirect_url(self, *args, **kwargs): + def get_redirect_url(self): """ If ``next`` is provided in the query params, redirects to ``next``. If ``HTTP_REFERER`` is present on the ``META`` property of the request, redirects to ``HTTP_REFERER``. Else redirects to /. @@ -292,7 +292,6 @@ def get_redirect_url(self, *args, **kwargs): :returns: url to redirect to :rtype: str """ - # TODO: this needs to work with the new POST flow next_url = self.request.POST.get('redirect_url') redirect_url = next_url if next_url else self.request.META.get('HTTP_REFERER') if not redirect_url: @@ -311,7 +310,6 @@ def form_valid(self, form): target = form.cleaned_data.pop('target') obsr = form.cleaned_data.pop('observation_record') - redirect_url = form.cleaned_data.pop('redirect_url') try: # Pass non-standard fields from query parameters as kwargs @@ -320,4 +318,4 @@ def form_valid(self, form): logger.log(msg=f'Failed to submit alert: {e}', level=logging.WARN) messages.warning(self.request, f'Unable to submit one or more alerts to {broker_name}') - return redirect(self.get_redirect_url(redirect_url)) + return redirect(self.get_redirect_url()) From f58378676654be22c0f26cccc2e4a8329cec0f61 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 29 Oct 2020 17:16:29 -0700 Subject: [PATCH 367/424] Fixed button styling --- tom_alerts/alerts.py | 12 +++++++----- tom_alerts/views.py | 8 ++++++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tom_alerts/alerts.py b/tom_alerts/alerts.py index 55473e5c1..0fdf96ca5 100644 --- a/tom_alerts/alerts.py +++ b/tom_alerts/alerts.py @@ -7,8 +7,9 @@ from django import forms from django.conf import settings from django.shortcuts import reverse +from crispy_forms.bootstrap import StrictButton from crispy_forms.helper import FormHelper -from crispy_forms.layout import Submit, Layout +from crispy_forms.layout import Layout, Submit from tom_alerts.models import BrokerQuery from tom_observations.models import ObservationRecord @@ -159,11 +160,12 @@ def __init__(self, *args, **kwargs): broker_name = kwargs.pop('broker') super().__init__(*args, **kwargs) self.helper = FormHelper() - # TODO: this needs to look not like a submit button - # TODO: for some reason at present the css class affects the text and not the button - self.helper.add_input(Submit('submit', f'Submit to {broker_name}', css_class='btn btn-outline-primary')) self.helper.form_action = reverse('tom_alerts:submit-alert', kwargs={'broker': broker_name}) - self.common_layout = Layout('broker', 'target', 'observation_record') + self.helper.layout = Layout( + 'target', + 'observation_record', + 'redirect_url', + StrictButton(f'Submit to {broker_name}', type='submit', css_class='btn-outline-primary')) def clean(self): cleaned_data = super().clean() diff --git a/tom_alerts/views.py b/tom_alerts/views.py index f920271e4..b91a15ce2 100644 --- a/tom_alerts/views.py +++ b/tom_alerts/views.py @@ -310,12 +310,16 @@ def form_valid(self, form): target = form.cleaned_data.pop('target') obsr = form.cleaned_data.pop('observation_record') + form.cleaned_data.pop('redirect_url') # redirect_url doesn't need to be passed to submit_upstream_alert try: # Pass non-standard fields from query parameters as kwargs - broker.submit_upstream_alert(target=target, observation_record=obsr, **form.cleaned_data) + success = broker.submit_upstream_alert(target=target, observation_record=obsr, **form.cleaned_data) except AlertSubmissionException as e: logger.log(msg=f'Failed to submit alert: {e}', level=logging.WARN) - messages.warning(self.request, f'Unable to submit one or more alerts to {broker_name}') + messages.warning(self.request, f'Unable to submit one or more alerts to {broker_name}.') + + if success: + messages.success(self.request, f'Successfully submitted alerts to {broker_name}!') return redirect(self.get_redirect_url()) From 6a0a8075e2336369a12e7a90b921696da476bfe8 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 29 Oct 2020 20:44:43 -0700 Subject: [PATCH 368/424] Adding case for if message submission fails --- tom_alerts/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tom_alerts/views.py b/tom_alerts/views.py index b91a15ce2..93506aeb8 100644 --- a/tom_alerts/views.py +++ b/tom_alerts/views.py @@ -321,5 +321,7 @@ def form_valid(self, form): if success: messages.success(self.request, f'Successfully submitted alerts to {broker_name}!') + else: + messages.warning(self.request, f'Unable to submit one or more alerts to {broker_name}.') return redirect(self.get_redirect_url()) From 5152a7d67a3305abb377089c34bab1e121652366 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 30 Oct 2020 10:24:29 -0700 Subject: [PATCH 369/424] Attempting to improve test coverage --- tom_alerts/tests/tests_generic.py | 18 +++++++++++++++--- tom_alerts/views.py | 2 +- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tom_alerts/tests/tests_generic.py b/tom_alerts/tests/tests_generic.py index a89c8f6ed..4562f7b17 100644 --- a/tom_alerts/tests/tests_generic.py +++ b/tom_alerts/tests/tests_generic.py @@ -266,18 +266,30 @@ def test_submit_alert_success(self, mock_submit_upstream_alert): @patch('tom_alerts.tests.tests_generic.TestBroker.submit_upstream_alert') def test_submit_alert_failure(self, mock_submit_upstream_alert): - """Test that an alert submission returns an appropriate message when alert submission fails.""" + """Test that a failed alert submission returns an appropriate message.""" + target = Target.objects.create(name='test_target', ra=1, dec=2) + mock_submit_upstream_alert.return_value = False + response = self.client.post(reverse('tom_alerts:submit-alert', kwargs={'broker': 'TEST'}), + data={'target': target.id}) + + messages = [(m.message, m.level) for m in get_messages(response.wsgi_request)] + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0][0], 'Unable to submit one or more alerts to TEST.') + + @patch('tom_alerts.tests.tests_generic.TestBroker.submit_upstream_alert') + def test_submit_alert_exception(self, mock_submit_upstream_alert): + """Test that an alert submission returns an appropriate message when alert submission raises an exception.""" mock_submit_upstream_alert.side_effect = AlertSubmissionException() response = self.client.post(reverse('tom_alerts:submit-alert', kwargs={'broker': 'TEST'}), data={}) messages = [(m.message, m.level) for m in get_messages(response.wsgi_request)] self.assertEqual(len(messages), 1) - self.assertEqual(messages[0][0], 'Unable to submit one or more alerts to TEST') + self.assertEqual(messages[0][0], 'Unable to submit one or more alerts to TEST.') def test_submit_alert_invalid_form(self): """Test that an alert submission failed when form is invalid.""" response = self.client.post(reverse('tom_alerts:submit-alert', kwargs={'broker': 'TEST'}), data={}) messages = [(m.message, m.level) for m in get_messages(response.wsgi_request)] self.assertEqual(len(messages), 1) - self.assertEqual(messages[0][0], 'Unable to submit one or more alerts to TEST') + self.assertEqual(messages[0][0], 'Unable to submit one or more alerts to TEST.') self.assertRedirects(response, reverse('home')) diff --git a/tom_alerts/views.py b/tom_alerts/views.py index 93506aeb8..1423ca63c 100644 --- a/tom_alerts/views.py +++ b/tom_alerts/views.py @@ -301,7 +301,7 @@ def get_redirect_url(self): def form_invalid(self, form): logger.log(msg=f'Form invalid: {form.errors}', level=logging.WARN) - messages.warning(self.request, f'Unable to submit one or more alerts to {self.get_broker_name()}') + messages.warning(self.request, f'Unable to submit one or more alerts to {self.get_broker_name()}.') return redirect(self.get_redirect_url()) # TODO: fix this def form_valid(self, form): From 5ed5d7e321f309c5630c4e8f774c072c1b947856 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 30 Oct 2020 10:27:32 -0700 Subject: [PATCH 370/424] Added SCIMMA broker stub --- tom_alerts/alerts.py | 3 +- tom_alerts/brokers/scimma.py | 32 +++++++++++++++++++++ tom_setup/templates/tom_setup/settings.tmpl | 3 +- 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 tom_alerts/brokers/scimma.py diff --git a/tom_alerts/alerts.py b/tom_alerts/alerts.py index 0fdf96ca5..ed37fbacc 100644 --- a/tom_alerts/alerts.py +++ b/tom_alerts/alerts.py @@ -22,7 +22,8 @@ 'tom_alerts.brokers.scout.ScoutBroker', 'tom_alerts.brokers.alerce.ALeRCEBroker', 'tom_alerts.brokers.antares.ANTARESBroker', - 'tom_alerts.brokers.gaia.GaiaBroker' + 'tom_alerts.brokers.gaia.GaiaBroker', + 'tom_alerts.brokers.scimma.SCIMMABroker', ] diff --git a/tom_alerts/brokers/scimma.py b/tom_alerts/brokers/scimma.py new file mode 100644 index 000000000..01dfaab43 --- /dev/null +++ b/tom_alerts/brokers/scimma.py @@ -0,0 +1,32 @@ +from crispy_forms.layout import Layout, HTML + +from tom_alerts.alerts import GenericBroker, GenericQueryForm, GenericAlert + + +class SCIMMAQueryForm(GenericQueryForm): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.helper.inputs.pop() + self.helper.layout = Layout( + HTML(''' +

    + This plugin is a stub for the SCIMMA plugin. In order to install the full plugin, please see the + instructions here. +

    + '''), + HTML('''Back''') + ) + + +class SCIMMABroker(GenericBroker): + name = 'SCIMMA' + form = SCIMMAQueryForm + + def fetch_alerts(self, parameters): + return iter([]) + + def process_reduced_data(self, target, alert=None): + return + + def to_generic_alert(self, alert): + return GenericAlert() diff --git a/tom_setup/templates/tom_setup/settings.tmpl b/tom_setup/templates/tom_setup/settings.tmpl index c05526688..530b8a05b 100644 --- a/tom_setup/templates/tom_setup/settings.tmpl +++ b/tom_setup/templates/tom_setup/settings.tmpl @@ -248,7 +248,8 @@ TOM_ALERT_CLASSES = [ 'tom_alerts.brokers.scout.ScoutBroker', 'tom_alerts.brokers.tns.TNSBroker', 'tom_alerts.brokers.antares.ANTARESBroker', - 'tom_alerts.brokers.gaia.GaiaBroker' + 'tom_alerts.brokers.gaia.GaiaBroker', + 'tom_alerts.brokers.scimma.SCIMMABroker', ] ALERT_CREDENTIALS = { From d8844b97ad8e67c802a779db2e7ae2fe3df75efa Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 30 Oct 2020 11:05:32 -0700 Subject: [PATCH 371/424] trying out this caching business --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index bb120792a..05b28a25a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ python: os: - linux dist: bionic +cache: pip before_install: - sudo apt-get install -y gfortran From 6b62c8c63e040e7c52c1c6b9f49df5cbb79208cb Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 30 Oct 2020 11:12:30 -0700 Subject: [PATCH 372/424] Removing old code --- .../partials/submit_upstream_button.html | 1 - tom_alerts/templatetags/alerts_extras.py | 18 ++++-------------- 2 files changed, 4 insertions(+), 15 deletions(-) delete mode 100644 tom_alerts/templates/tom_alerts/partials/submit_upstream_button.html diff --git a/tom_alerts/templates/tom_alerts/partials/submit_upstream_button.html b/tom_alerts/templates/tom_alerts/partials/submit_upstream_button.html deleted file mode 100644 index af7ae28ad..000000000 --- a/tom_alerts/templates/tom_alerts/partials/submit_upstream_button.html +++ /dev/null @@ -1 +0,0 @@ -Submit to {{ broker }} \ No newline at end of file diff --git a/tom_alerts/templatetags/alerts_extras.py b/tom_alerts/templatetags/alerts_extras.py index 90247f312..b93e94c8e 100644 --- a/tom_alerts/templatetags/alerts_extras.py +++ b/tom_alerts/templatetags/alerts_extras.py @@ -5,10 +5,11 @@ register = template.Library() -@register.inclusion_tag('tom_alerts/partials/submit_upstream_button.html') -def submit_upstream_button(broker, target=None, observation_record=None, redirect_url=None): +@register.inclusion_tag('tom_alerts/partials/submit_upstream_form.html') +def submit_upstream_form(broker, target=None, observation_record=None, redirect_url=None): """ - Renders a button to submit an alert upstream to a broker. At least one of target/obs record should be given. + Renders a form to submit an alert upstream to a broker. + At least one of target/obs record should be given. :param broker: The name of the broker to which the button will lead, as in the name field of the broker module. :type broker: str @@ -22,17 +23,6 @@ def submit_upstream_button(broker, target=None, observation_record=None, redirec :param redirect_url: :type redirect_url: str """ - # TODO: document this function somewhere prominent, as it likely won't live in the default templates - return { - 'broker': broker, - 'target': target, - 'observation_record': observation_record, - 'redirect_url': redirect_url - } - - -@register.inclusion_tag('tom_alerts/partials/submit_upstream_form.html') -def submit_upstream_form(broker, target=None, observation_record=None, redirect_url=None): broker_class = get_service_class(broker) form_class = broker_class.alert_submission_form form = form_class(broker=broker, initial={ From 4491fe083b9c2aa9b55d9211056c1c08ad77b278 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 30 Oct 2020 11:37:59 -0700 Subject: [PATCH 373/424] A few review improvements --- tom_alerts/alerts.py | 2 +- tom_alerts/views.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tom_alerts/alerts.py b/tom_alerts/alerts.py index ed37fbacc..ccd2825f0 100644 --- a/tom_alerts/alerts.py +++ b/tom_alerts/alerts.py @@ -158,7 +158,7 @@ class GenericUpstreamSubmissionForm(forms.Form): redirect_url = forms.CharField(required=False, max_length=100, widget=forms.HiddenInput()) def __init__(self, *args, **kwargs): - broker_name = kwargs.pop('broker') + broker_name = kwargs.pop('broker') # NOTE: parent constructor is not expecting broker and will fail super().__init__(*args, **kwargs) self.helper = FormHelper() self.helper.form_action = reverse('tom_alerts:submit-alert', kwargs={'broker': broker_name}) diff --git a/tom_alerts/views.py b/tom_alerts/views.py index 1423ca63c..0bb871a5e 100644 --- a/tom_alerts/views.py +++ b/tom_alerts/views.py @@ -301,8 +301,9 @@ def get_redirect_url(self): def form_invalid(self, form): logger.log(msg=f'Form invalid: {form.errors}', level=logging.WARN) - messages.warning(self.request, f'Unable to submit one or more alerts to {self.get_broker_name()}.') - return redirect(self.get_redirect_url()) # TODO: fix this + messages.warning(self.request, + f'Unable to submit one or more alerts to {self.get_broker_name()}. See logs for details.') + return redirect(self.get_redirect_url()) def form_valid(self, form): broker_name = self.get_broker_name() @@ -317,11 +318,12 @@ def form_valid(self, form): success = broker.submit_upstream_alert(target=target, observation_record=obsr, **form.cleaned_data) except AlertSubmissionException as e: logger.log(msg=f'Failed to submit alert: {e}', level=logging.WARN) - messages.warning(self.request, f'Unable to submit one or more alerts to {broker_name}.') + success = False if success: messages.success(self.request, f'Successfully submitted alerts to {broker_name}!') else: - messages.warning(self.request, f'Unable to submit one or more alerts to {broker_name}.') + messages.warning(self.request, + f'Unable to submit one or more alerts to {self.get_broker_name()}. See logs for details.') return redirect(self.get_redirect_url()) From 80f9dedef2018e97fa0dfb17b684cf247e997743 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 30 Oct 2020 11:54:22 -0700 Subject: [PATCH 374/424] Fixing tests for new error messages --- tom_alerts/tests/tests_generic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tom_alerts/tests/tests_generic.py b/tom_alerts/tests/tests_generic.py index 4562f7b17..1f890fb15 100644 --- a/tom_alerts/tests/tests_generic.py +++ b/tom_alerts/tests/tests_generic.py @@ -274,7 +274,7 @@ def test_submit_alert_failure(self, mock_submit_upstream_alert): messages = [(m.message, m.level) for m in get_messages(response.wsgi_request)] self.assertEqual(len(messages), 1) - self.assertEqual(messages[0][0], 'Unable to submit one or more alerts to TEST.') + self.assertEqual(messages[0][0], 'Unable to submit one or more alerts to TEST. See logs for details.') @patch('tom_alerts.tests.tests_generic.TestBroker.submit_upstream_alert') def test_submit_alert_exception(self, mock_submit_upstream_alert): @@ -284,12 +284,12 @@ def test_submit_alert_exception(self, mock_submit_upstream_alert): response = self.client.post(reverse('tom_alerts:submit-alert', kwargs={'broker': 'TEST'}), data={}) messages = [(m.message, m.level) for m in get_messages(response.wsgi_request)] self.assertEqual(len(messages), 1) - self.assertEqual(messages[0][0], 'Unable to submit one or more alerts to TEST.') + self.assertEqual(messages[0][0], 'Unable to submit one or more alerts to TEST. See logs for details.') def test_submit_alert_invalid_form(self): """Test that an alert submission failed when form is invalid.""" response = self.client.post(reverse('tom_alerts:submit-alert', kwargs={'broker': 'TEST'}), data={}) messages = [(m.message, m.level) for m in get_messages(response.wsgi_request)] self.assertEqual(len(messages), 1) - self.assertEqual(messages[0][0], 'Unable to submit one or more alerts to TEST.') + self.assertEqual(messages[0][0], 'Unable to submit one or more alerts to TEST. See logs for details.') self.assertRedirects(response, reverse('home')) From 2655647e86f166780a9b5ad1b7f555362bb1e2e0 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2020 13:23:58 +0000 Subject: [PATCH 375/424] Bump django from 3.1.2 to 3.1.3 Bumps [django](https://github.com/django/django) from 3.1.2 to 3.1.3. - [Release notes](https://github.com/django/django/releases) - [Commits](https://github.com/django/django/compare/3.1.2...3.1.3) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 65acedf8a..4627a12f4 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ 'astropy==4.1', 'beautifulsoup4==4.9.3', 'dataclasses; python_version < "3.7"', - 'django==3.1.2', # TOM Toolkit requires db math functions + 'django==3.1.3', # TOM Toolkit requires db math functions 'djangorestframework==3.12.1', 'django-bootstrap4==2.3.1', 'django-contrib-comments==1.9.2', # Earlier version are incompatible with Django >= 3.0 From a96c525cef209a14e497c616eca5e09a14e1a9bc Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 2 Nov 2020 08:08:40 -0800 Subject: [PATCH 376/424] Moved around the tests --- tom_alerts/tests/brokers/__init__.py | 0 tom_alerts/tests/brokers/test_alerce.py | 22 +++++++++++++++++++ .../{tests_gaia.py => brokers/test_gaia.py} | 0 .../{tests_mars.py => brokers/test_mars.py} | 17 +++++++++++++- .../tests/{tests_generic.py => tests.py} | 10 ++++----- 5 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 tom_alerts/tests/brokers/__init__.py create mode 100644 tom_alerts/tests/brokers/test_alerce.py rename tom_alerts/tests/{tests_gaia.py => brokers/test_gaia.py} (100%) rename tom_alerts/tests/{tests_mars.py => brokers/test_mars.py} (93%) rename tom_alerts/tests/{tests_generic.py => tests.py} (97%) diff --git a/tom_alerts/tests/brokers/__init__.py b/tom_alerts/tests/brokers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tom_alerts/tests/brokers/test_alerce.py b/tom_alerts/tests/brokers/test_alerce.py new file mode 100644 index 000000000..5c3059188 --- /dev/null +++ b/tom_alerts/tests/brokers/test_alerce.py @@ -0,0 +1,22 @@ +from django.tests import override_settings, tag, TestCase + +class TestALeRCEBrokerClass(TestCase): + def setUp(self): + pass + + # def test_fetch_alerts_payload + + +@tag('canary') +class TestALeRCEModuleCanary(TestCase): + def setUp(self): + pass + + def test_fetch_alerts(self): + pass + + def test_fetch_alert(self): + pass + + def test_process_reduced_data(self): + pass diff --git a/tom_alerts/tests/tests_gaia.py b/tom_alerts/tests/brokers/test_gaia.py similarity index 100% rename from tom_alerts/tests/tests_gaia.py rename to tom_alerts/tests/brokers/test_gaia.py diff --git a/tom_alerts/tests/tests_mars.py b/tom_alerts/tests/brokers/test_mars.py similarity index 93% rename from tom_alerts/tests/tests_mars.py rename to tom_alerts/tests/brokers/test_mars.py index 3598ad4d6..4a788c322 100644 --- a/tom_alerts/tests/tests_mars.py +++ b/tom_alerts/tests/brokers/test_mars.py @@ -2,7 +2,7 @@ from requests import Response from django.utils import timezone -from django.test import TestCase, override_settings +from django.test import override_settings, tag, TestCase from unittest import mock from tom_alerts.brokers.mars import MARSBroker @@ -115,3 +115,18 @@ def test_to_generic_alert(self): created_alert = MARSBroker().to_generic_alert(test_alert) self.assertEqual(created_alert.name, 'ZTF18aberpsh') + + +@tag('canary') +class TestMARSModuleCanary(TestCase): + def setUp(self): + pass + + def test_fetch_alerts(self): + pass + + def test_fetch_alert(self): + pass + + def test_process_reduced_data(self): + pass diff --git a/tom_alerts/tests/tests_generic.py b/tom_alerts/tests/tests.py similarity index 97% rename from tom_alerts/tests/tests_generic.py rename to tom_alerts/tests/tests.py index 1f890fb15..c00d0ec50 100644 --- a/tom_alerts/tests/tests_generic.py +++ b/tom_alerts/tests/tests.py @@ -78,7 +78,7 @@ def submit_upstream_alert(self, target=None, observation_record=None, **kwargs): return super().submit_upstream_alert(target=target, observation_record=observation_record) -@override_settings(TOM_ALERT_CLASSES=['tom_alerts.tests.tests_generic.TestBroker']) +@override_settings(TOM_ALERT_CLASSES=['tom_alerts.tests.tests.TestBroker']) class TestBrokerClass(TestCase): """ Test the functionality of the TestBroker, we modify the django settings to make sure it is the only installed broker. @@ -103,7 +103,7 @@ def test_to_target(self): self.assertEqual(target.name, test_alerts[0]['name']) -@override_settings(TOM_ALERT_CLASSES=['tom_alerts.tests.tests_generic.TestBroker']) +@override_settings(TOM_ALERT_CLASSES=['tom_alerts.tests.tests.TestBroker']) class TestBrokerViews(TestCase): """ Test the views that use the broker classes """ @@ -237,7 +237,7 @@ def test_create_no_targets(self): self.assertEqual(Target.objects.count(), 0) self.assertRedirects(response, reverse('tom_alerts:run', kwargs={'pk': query.id})) - @patch('tom_alerts.tests.tests_generic.TestBroker.submit_upstream_alert') + @patch('tom_alerts.tests.tests.TestBroker.submit_upstream_alert') def test_submit_alert_success(self, mock_submit_upstream_alert): """Test submission of an alert to a broker.""" @@ -264,7 +264,7 @@ def test_submit_alert_success(self, mock_submit_upstream_alert): data={'observation_record': obsr.id, 'topic': 'test topic'}) mock_submit_upstream_alert.assert_called_with(target=None, observation_record=obsr, topic='test topic') - @patch('tom_alerts.tests.tests_generic.TestBroker.submit_upstream_alert') + @patch('tom_alerts.tests.tests.TestBroker.submit_upstream_alert') def test_submit_alert_failure(self, mock_submit_upstream_alert): """Test that a failed alert submission returns an appropriate message.""" target = Target.objects.create(name='test_target', ra=1, dec=2) @@ -276,7 +276,7 @@ def test_submit_alert_failure(self, mock_submit_upstream_alert): self.assertEqual(len(messages), 1) self.assertEqual(messages[0][0], 'Unable to submit one or more alerts to TEST. See logs for details.') - @patch('tom_alerts.tests.tests_generic.TestBroker.submit_upstream_alert') + @patch('tom_alerts.tests.tests.TestBroker.submit_upstream_alert') def test_submit_alert_exception(self, mock_submit_upstream_alert): """Test that an alert submission returns an appropriate message when alert submission raises an exception.""" mock_submit_upstream_alert.side_effect = AlertSubmissionException() From b66b9c8ec2095dbfef808c9bd4d2a741960817a6 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 2 Nov 2020 08:46:20 -0800 Subject: [PATCH 377/424] Attempting a first cron workflow --- .travis.yml | 26 +++++++++++++--- tom_alerts/tests/brokers/test_mars.py | 45 ++++++++++++++++++++++++--- 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 05b28a25a..2db3cc0fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,30 +14,40 @@ before_install: install: pip install -r requirements.txt coverage coveralls -script: coverage run --include=tom_* manage.py test +script: coverage run --include=tom_* manage.py test --exclude-tag=canary after_success: coveralls stages: - "Style Checks" - "test" + - "Canary Tests" - "Deploy Development" - "Deploy Master" jobs: include: - stage: "Style Checks" + if: type != cron install: pip install -I flake8 script: flake8 tom_* --exclude=*/migrations/* --max-line-length=120 - stage: "test" + if: type != cron os: osx language: shell before_install: pip3 install -U pip + - stage: "Canary Tests" + python: 3.8 + if: type = cron + script: manage.py test --tag=canary + # Deploy alpha releases to PyPi and generate draft release on Github Releases (incomplete) - stage: "Deploy Development" - if: tag IS present + if: + - tag IS present + - type != cron script: skip deploy: provider: pypi @@ -53,7 +63,9 @@ jobs: distributions: "sdist bdist_wheel" - stage: "Deploy Development" - if: tag IS present + if: + - tag IS present + - type != cron python: 3.8 script: skip deploy: @@ -71,7 +83,9 @@ jobs: # Deploy full releases to PyPi and generate draft release on Github Releases (incomplete) - stage: "Deploy Master" - if: tag IS present + if: + - tag IS present + - type != cron python: 3.8 script: skip deploy: @@ -88,7 +102,9 @@ jobs: distributions: "sdist bdist_wheel" - stage: "Deploy Master" - if: tag IS present + if: + - tag IS present + - type != cron python: 3.8 script: skip deploy: diff --git a/tom_alerts/tests/brokers/test_mars.py b/tom_alerts/tests/brokers/test_mars.py index 4a788c322..e71a2ecfb 100644 --- a/tom_alerts/tests/brokers/test_mars.py +++ b/tom_alerts/tests/brokers/test_mars.py @@ -1,3 +1,5 @@ +from datetime import datetime +from itertools import islice import json from requests import Response @@ -120,13 +122,48 @@ def test_to_generic_alert(self): @tag('canary') class TestMARSModuleCanary(TestCase): def setUp(self): - pass + self.broker = MARSBroker() + self.expected_keys = ['avro', 'candid', 'candidate', 'lco_id', 'objectId', 'publisher'] + self.expected_candidate_keys = ['aimage', 'aimagerat', 'b', 'bimage', 'bimagerat', 'candid', 'chinr', 'chipsf', + 'classtar', 'clrcoeff', 'clrcounc', 'clrmed', 'clrrms', 'dec', 'decnr', + 'deltamaglatest', 'deltamagref', 'diffmaglim', 'distnr', 'distpsnr1', + 'distpsnr2', 'distpsnr3', 'drb', 'drbversion', 'dsdiff', 'dsnrms', 'elong', + 'exptime', 'fid', 'field', 'filter', 'fwhm', 'isdiffpos', 'jd', 'jdendhist', + 'jdendref', 'jdstarthist', 'jdstartref', 'l', 'magap', 'magapbig', 'magdiff', + 'magfromlim', 'maggaia', 'maggaiabright', 'magnr', 'magpsf', 'magzpsci', + 'magzpscirms', 'magzpsciunc', 'mindtoedge', 'nbad', 'ncovhist', 'ndethist', + 'neargaia', 'neargaiabright', 'nframesref', 'nid', 'nmatches', 'nmtchps', + 'nneg', 'objectidps1', 'objectidps2', 'objectidps3', 'pdiffimfilename', 'pid', + 'programid', 'programpi', 'ra', 'ranr', 'rb', 'rbversion', 'rcid', 'rfid', + 'scorr', 'seeratio', 'sgmag1', 'sgmag2', 'sgmag3', 'sgscore1', 'sgscore2', + 'sgscore3', 'sharpnr', 'sigmagap', 'sigmagapbig', 'sigmagnr', 'sigmapsf', + 'simag1', 'simag2', 'simag3', 'sky', 'srmag1', 'srmag2', 'srmag3', 'ssdistnr', + 'ssmagnr', 'ssnamenr', 'ssnrms', 'sumrat', 'szmag1', 'szmag2', 'szmag3', + 'tblid', 'tooflag', 'wall_time', 'xpos', 'ypos', 'zpclrcov', 'zpmed'] def test_fetch_alerts(self): - pass + response = self.broker.fetch_alerts({'time__gt': '2018-06-01', 'time__lt': '2018-06-30'}) + + alerts = [] + for alert in islice(response, 10): + alerts.append(alert) + self.assertEqual(len(alerts), 10) + + for key in self.expected_keys: + self.assertTrue(key in alerts[0].keys()) + for key in self.expected_candidate_keys: + self.assertTrue(key in alerts[0]['candidate'].keys()) def test_fetch_alert(self): - pass + alert = self.broker.fetch_alert(1065519) + + for key in self.expected_keys: + self.assertTrue(key in alert.keys()) + for key in self.expected_candidate_keys: + self.assertTrue(key in alert['candidate'].keys()) def test_process_reduced_data(self): - pass + alert = self.broker.fetch_alert(1065519) + t = Target.objects.create(name='test target', ra=1, dec=2) + self.broker.process_reduced_data(t, alert=alert) + self.assertGreaterEqual(ReducedDatum.objects.filter(target=t, timestamp__lte=datetime(2020, 11, 3)).count(), 526) From f68447fc0f476050ce62226d1afc3df336e672af Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 2 Nov 2020 08:58:52 -0800 Subject: [PATCH 378/424] Made a few code style improvements to alerce --- tom_alerts/brokers/alerce.py | 90 ++++++++++++------------- tom_alerts/tests/brokers/test_alerce.py | 3 +- tom_alerts/tests/brokers/test_mars.py | 3 +- 3 files changed, 48 insertions(+), 48 deletions(-) diff --git a/tom_alerts/brokers/alerce.py b/tom_alerts/brokers/alerce.py index 57200398b..e31c114d6 100644 --- a/tom_alerts/brokers/alerce.py +++ b/tom_alerts/brokers/alerce.py @@ -1,6 +1,7 @@ import requests from django import forms +from django.core.cache import cache from crispy_forms.layout import Layout, Div, Fieldset from astropy.time import Time, TimezoneInfo import datetime @@ -12,10 +13,10 @@ ALERCE_SEARCH_URL = 'https://ztf.alerce.online/query' ALERCE_CLASSES_URL = 'https://ztf.alerce.online/get_current_classes' -SORT_CHOICES = [("nobs", "Number Of Epochs"), - ("lastmjd", "Last Detection"), - ("pclassrf", "Late Probability"), - ("pclassearly", "Early Probability")] +SORT_CHOICES = [('nobs', 'Number Of Epochs'), + ('lastmjd', 'Last Detection'), + ('pclassrf', 'Late Probability'), + ('pclassearly', 'Early Probability')] PAGES_CHOICES = [ (i, i) for i in [1, 5, 10, 15] @@ -108,17 +109,8 @@ class ALeRCEQueryForm(GenericQueryForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - response = requests.post(ALERCE_CLASSES_URL) - response.raise_for_status() - parsed = response.json() - - EARLY_CHOICES = [(c["id"], c["name"]) for c in parsed["early"]] - EARLY_CHOICES.insert(0, (None, "")) - LATE_CHOICES = [(c["id"], c["name"]) for c in parsed["late"]] - LATE_CHOICES.insert(0, (None, "")) - - self.fields["classearly"].choices = EARLY_CHOICES - self.fields["classrf"].choices = LATE_CHOICES + self.fields['classearly'].choices = self.early_classifier_choices() + self.fields['classrf'].choices = self.late_classifier_choices() self.helper.layout = Layout( self.common_layout, @@ -133,7 +125,7 @@ def __init__(self, *args, **kwargs): 'nobs__lt', css_class='col', ), - css_class="form-row", + css_class='form-row', ) ), Fieldset( @@ -149,7 +141,7 @@ def __init__(self, *args, **kwargs): 'pclassearly', css_class='col', ), - css_class="form-row", + css_class='form-row', ) ), Fieldset( @@ -157,7 +149,7 @@ def __init__(self, *args, **kwargs): Div( Div( 'ra', - css_class="col" + css_class='col' ), Div( 'dec', @@ -167,14 +159,14 @@ def __init__(self, *args, **kwargs): 'sr', css_class='col' ), - css_class="form-row" + css_class='form-row' ) ), Fieldset( 'Time Filters', Div( Fieldset( - "Relative time", + 'Relative time', Div( 'relative_mjd__gt', css_class='col', @@ -182,7 +174,7 @@ def __init__(self, *args, **kwargs): css_class='col' ), Fieldset( - "Absolute time", + 'Absolute time', Div( Div( 'mjd__gt', @@ -192,32 +184,48 @@ def __init__(self, *args, **kwargs): 'mjd__lt', css_class='col', ), - css_class="form-row" + css_class='form-row' ) ), - css_class="form-row" + css_class='form-row' ) ), Fieldset( 'General Parameters', Div( Div( - "sort_by", - css_class="col" + 'sort_by', + css_class='col' ), Div( - "records", - css_class="col" + 'records', + css_class='col' ), Div( - "max_pages", - css_class="col" + 'max_pages', + css_class='col' ), - css_class="form-row" + css_class='form-row' ) ), ) + def _get_classifiers(self): + cached_classifiers = cache.get('alerce_classifiers') + + if not cached_classifiers: + response = requests.get(ALERCE_CLASSES_URL) + response.raise_for_status() + cached_classifiers = response.json() + + return cached_classifiers + + def early_classifier_choices(self): + return [(None, '')] + [(c['id'], c['name']) for c in self._get_classifiers()['early']] + + def late_classifier_choices(self): + return [(None, '')] + [(c['id'], c['name']) for c in self._get_classifiers()['late']] + class ALeRCEBroker(GenericBroker): name = 'ALeRCE' @@ -234,15 +242,9 @@ def _fetch_alerts_payload(self, parameters): if parameters.get('total'): payload['total'] = parameters.get('total') - if any([parameters['nobs__gt'], - parameters['nobs__lt'], - parameters['classrf'], - parameters['pclassrf'], - parameters['classearly'], - parameters['pclassearly']]): + if any(parameters[k] for k in ['nobs__gt', 'nobs__lt', 'classrf', 'pclassrf', 'classearly', 'pclassearly']): filters = {} - if any([parameters['nobs__gt'], - parameters['nobs__lt']]): + if any(parameters[k] for k in ['nobs__gt', 'nobs__lt']): filters['nobs'] = {} if parameters['nobs__gt']: filters['nobs']['min'] = parameters['nobs__gt'] @@ -258,9 +260,7 @@ def _fetch_alerts_payload(self, parameters): filters['pclassearly'] = parameters['pclassearly'] payload['query_parameters']['filters'] = filters - if all([parameters['ra'], - parameters['dec'], - parameters['sr']]): + if all([parameters['ra'], parameters['dec'], parameters['sr']]): coordinates = {} if parameters['ra']: coordinates['ra'] = parameters['ra'] @@ -270,9 +270,7 @@ def _fetch_alerts_payload(self, parameters): coordinates['sr'] = parameters['sr'] payload['query_parameters']['coordinates'] = coordinates - if any([parameters['mjd__gt'], - parameters['mjd__lt'], - parameters['relative_mjd__gt']]): + if any(parameters[k] for k in ['mjd__gt', 'mjd__lt', 'relative_mjd__gt']): dates = {'firstmjd': {}} if parameters['mjd__gt']: dates['firstmjd']['min'] = parameters['mjd__gt'] @@ -294,7 +292,7 @@ def fetch_alerts(self, parameters): response.raise_for_status() parsed = response.json() alerts = [alert_data for alert, alert_data in parsed['result'].items()] - if parsed['page'] < parsed['num_pages'] and parsed['page'] != int(parameters["max_pages"]): + if parsed['page'] < parsed['num_pages'] and parsed['page'] != int(parameters['max_pages']): parameters['page'] = parameters.get('page', 1) + 1 parameters['total'] = parsed.get('total') alerts += self.fetch_alerts(parameters) @@ -335,7 +333,7 @@ def to_generic_alert(self, alert): max_mag = alert['mean_magpsf_r'] if is_r else alert['mean_magpsf_g'] if alert['pclassrf']: - score = alert["pclassrf"] + score = alert['pclassrf'] elif alert['pclassearly']: score = alert['pclassearly'] else: diff --git a/tom_alerts/tests/brokers/test_alerce.py b/tom_alerts/tests/brokers/test_alerce.py index 5c3059188..d9db37322 100644 --- a/tom_alerts/tests/brokers/test_alerce.py +++ b/tom_alerts/tests/brokers/test_alerce.py @@ -1,4 +1,5 @@ -from django.tests import override_settings, tag, TestCase +from django.test import tag, TestCase + class TestALeRCEBrokerClass(TestCase): def setUp(self): diff --git a/tom_alerts/tests/brokers/test_mars.py b/tom_alerts/tests/brokers/test_mars.py index e71a2ecfb..cc612bb81 100644 --- a/tom_alerts/tests/brokers/test_mars.py +++ b/tom_alerts/tests/brokers/test_mars.py @@ -166,4 +166,5 @@ def test_process_reduced_data(self): alert = self.broker.fetch_alert(1065519) t = Target.objects.create(name='test target', ra=1, dec=2) self.broker.process_reduced_data(t, alert=alert) - self.assertGreaterEqual(ReducedDatum.objects.filter(target=t, timestamp__lte=datetime(2020, 11, 3)).count(), 526) + self.assertGreaterEqual(ReducedDatum.objects.filter(target=t, timestamp__lte=datetime(2020, 11, 3)).count(), + 526) From 27edba380e9fb98b89ecddea5cdf42825d3e2208 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 2 Nov 2020 10:01:07 -0800 Subject: [PATCH 379/424] Fixing cron command --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2db3cc0fa..bf6501936 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,7 +41,7 @@ jobs: - stage: "Canary Tests" python: 3.8 if: type = cron - script: manage.py test --tag=canary + script: python manage.py test --tag=canary # Deploy alpha releases to PyPi and generate draft release on Github Releases (incomplete) - stage: "Deploy Development" From a881992d8ec6479b12c80d3981642da8deb8f0d2 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 2 Nov 2020 15:31:12 -0800 Subject: [PATCH 380/424] Added a bunch of alerce tests --- tom_alerts/brokers/alerce.py | 159 ++++++++++++++--------- tom_alerts/tests/brokers/test_alerce.py | 164 +++++++++++++++++++++++- 2 files changed, 261 insertions(+), 62 deletions(-) diff --git a/tom_alerts/brokers/alerce.py b/tom_alerts/brokers/alerce.py index e31c114d6..4244b3ed5 100644 --- a/tom_alerts/brokers/alerce.py +++ b/tom_alerts/brokers/alerce.py @@ -1,10 +1,10 @@ +from datetime import datetime, timedelta import requests +from astropy.time import Time, TimezoneInfo +from crispy_forms.layout import Layout, Div, Fieldset from django import forms from django.core.cache import cache -from crispy_forms.layout import Layout, Div, Fieldset -from astropy.time import Time, TimezoneInfo -import datetime from tom_alerts.alerts import GenericQueryForm, GenericBroker, GenericAlert from tom_targets.models import Target @@ -29,9 +29,6 @@ class ALeRCEQueryForm(GenericQueryForm): - RF_CLASSIFIERS = [] - STAMP_CLASSIFIERS = [] - nobs__gt = forms.IntegerField( required=False, label='Detections Lower', @@ -42,19 +39,21 @@ class ALeRCEQueryForm(GenericQueryForm): label='Detections Upper', widget=forms.TextInput(attrs={'placeholder': 'Max number of epochs'}) ) - classrf = forms.ChoiceField( + classrf = forms.TypedChoiceField( required=False, label='Late Classifier (Random Forest)', - choices=RF_CLASSIFIERS + choices=[], # Choices are populated dynamically in the constructor + coerce=int ) pclassrf = forms.FloatField( required=False, label='Classifier Probability (Random Forest)' ) - classearly = forms.ChoiceField( + classearly = forms.TypedChoiceField( required=False, label='Early Classifier (Stamp Classifier)', - choices=STAMP_CLASSIFIERS + choices=[], # Choices are populated dynamically in the constructor + coerce=int ) pclassearly = forms.FloatField( required=False, @@ -78,17 +77,20 @@ class ALeRCEQueryForm(GenericQueryForm): mjd__gt = forms.FloatField( required=False, label='Min date of first detection ', - widget=forms.TextInput(attrs={'placeholder': 'Date (MJD)'}) + widget=forms.TextInput(attrs={'placeholder': 'Date (MJD)'}), + min_value=0.0 ) mjd__lt = forms.FloatField( required=False, label='Max date of first detection', - widget=forms.TextInput(attrs={'placeholder': 'Date (MJD)'}) + widget=forms.TextInput(attrs={'placeholder': 'Date (MJD)'}), + min_value=0.0 ) relative_mjd__gt = forms.FloatField( required=False, label='Relative date of object discovery.', - widget=forms.TextInput(attrs={'placeholder': 'Hours'}) + widget=forms.TextInput(attrs={'placeholder': 'Hours'}), + min_value=0.0 ) sort_by = forms.ChoiceField( choices=SORT_CHOICES, @@ -98,12 +100,14 @@ class ALeRCEQueryForm(GenericQueryForm): max_pages = forms.TypedChoiceField( choices=PAGES_CHOICES, required=False, - label='Max Number of Pages' + label='Max Number of Pages', + coerce=int ) - records = forms.ChoiceField( + records = forms.TypedChoiceField( choices=RECORDS_CHOICES, required=False, - label='Records per page' + label='Records per page', + coerce=int ) def __init__(self, *args, **kwargs): @@ -220,74 +224,107 @@ def _get_classifiers(self): return cached_classifiers + def clean_relative_mjd__gt(self): + if self.cleaned_data['relative_mjd__gt']: + return Time(datetime.now() - timedelta(hours=self.cleaned_data['relative_mjd__gt'])).mjd + return None + + def clean(self): + cleaned_data = super().clean() + + print(cleaned_data) + + # Ensure that all cone search fields are present + if any(cleaned_data[k] for k in ['ra', 'dec', 'sr']) and not all(cleaned_data[k] for k in ['ra', 'dec', 'sr']): + raise forms.ValidationError('All of RA, Dec, and Search Radius must be included to execute a cone search.') + + # Ensure that both relative and absolute time filters are not present + if any(cleaned_data[k] for k in ['mjd__lt', 'mjd__gt']) and cleaned_data.get('relative_mjd__gt'): + raise forms.ValidationError('Cannot filter by both relative and absolute time.') + + # Ensure that absolute time filters have sensible values + if all(cleaned_data[k] for k in ['mjd__lt', 'mjd__gt']) and cleaned_data['mjd__lt'] <= cleaned_data['mjd__gt']: + raise forms.ValidationError('Min date of first detection must be earlier than max date of first detection.') + + return cleaned_data + def early_classifier_choices(self): - return [(None, '')] + [(c['id'], c['name']) for c in self._get_classifiers()['early']] + return [(None, '')] + sorted([(c['id'], c['name']) for c in self._get_classifiers()['early']], + key=lambda classifier: classifier[1]) def late_classifier_choices(self): - return [(None, '')] + [(c['id'], c['name']) for c in self._get_classifiers()['late']] + return [(None, '')] + sorted([(c['id'], c['name']) for c in self._get_classifiers()['late']], + key=lambda classifier: classifier[1]) class ALeRCEBroker(GenericBroker): name = 'ALeRCE' form = ALeRCEQueryForm - def _fetch_alerts_payload(self, parameters): + def _clean_coordinate_parameters(self, parameters): + if all([parameters['ra'], parameters['dec'], parameters['sr']]): + return { + 'ra': parameters['ra'], + 'dec': parameters['dec'], + 'sr': parameters['sr'] + } + else: + return None + + def _clean_date_parameters(self, parameters): + dates = {} + + if any(parameters[k] for k in ['mjd__gt', 'mjd__lt']): + dates = {'firstmjd': {}} + if parameters['mjd__gt']: + dates['firstmjd']['min'] = parameters['mjd__gt'] + if parameters['mjd__lt']: + dates['firstmjd']['max'] = parameters['mjd__lt'] + elif parameters['relative_mjd__gt']: + dates = {'firstmjd': {'min': parameters['relative_mjd__gt']}} + + return dates + + def _clean_filter_parameters(self, parameters): + filters = {} + + if any(parameters[k] is not None for k in ['nobs__gt', 'nobs__lt']): + filters['nobs'] = {} + if parameters['nobs__gt']: + filters['nobs']['min'] = parameters['nobs__gt'] + if parameters['nobs__lt']: + filters['nobs']['max'] = parameters['nobs__lt'] + filters.update({k: parameters[k] + for k in ['classrf', 'pclassrf', 'classearly', 'pclassearly'] + if parameters[k]}) + + return filters + + def _clean_parameters(self, parameters): payload = { 'page': parameters.get('page', 1), - 'records_per_pages': int(parameters.get('records', 20)), + 'records_per_pages': parameters.get('records', 20), 'sortBy': parameters.get('sort_by'), - 'query_parameters': { - } + 'query_parameters': {} } + if parameters.get('total'): payload['total'] = parameters.get('total') - if any(parameters[k] for k in ['nobs__gt', 'nobs__lt', 'classrf', 'pclassrf', 'classearly', 'pclassearly']): - filters = {} - if any(parameters[k] for k in ['nobs__gt', 'nobs__lt']): - filters['nobs'] = {} - if parameters['nobs__gt']: - filters['nobs']['min'] = parameters['nobs__gt'] - if parameters['nobs__lt']: - filters['nobs']['max'] = parameters['nobs__lt'] - if parameters['classrf']: - filters['classrf'] = int(parameters['classrf']) - if parameters['pclassrf']: - filters['pclassrf'] = parameters['pclassrf'] - if parameters['classearly']: - filters['classearly'] = int(parameters['classearly']) - if parameters['pclassearly']: - filters['pclassearly'] = parameters['pclassearly'] - payload['query_parameters']['filters'] = filters + payload['query_parameters']['filters'] = self._clean_filter_parameters(parameters) - if all([parameters['ra'], parameters['dec'], parameters['sr']]): - coordinates = {} - if parameters['ra']: - coordinates['ra'] = parameters['ra'] - if parameters['dec']: - coordinates['dec'] = parameters['dec'] - if parameters['sr']: - coordinates['sr'] = parameters['sr'] + coordinates = self._clean_coordinate_parameters(parameters) + if coordinates: payload['query_parameters']['coordinates'] = coordinates - if any(parameters[k] for k in ['mjd__gt', 'mjd__lt', 'relative_mjd__gt']): - dates = {'firstmjd': {}} - if parameters['mjd__gt']: - dates['firstmjd']['min'] = parameters['mjd__gt'] - elif parameters['relative_mjd__gt']: - now = datetime.datetime.utcnow() - relative = now - datetime.timedelta(hours=parameters['relative_mjd__gt']) - relative_astro = Time(relative) - dates['firstmjd']['min'] = relative_astro.mjd - - if parameters['mjd__lt']: - dates['firstmjd']['max'] = parameters['mjd__lt'] - payload['query_parameters']['dates'] = dates + payload['query_parameters']['dates'] = self._clean_date_parameters(parameters) return payload def fetch_alerts(self, parameters): - payload = self._fetch_alerts_payload(parameters) + print(parameters) + payload = self._clean_parameters(parameters) + print(payload) response = requests.post(ALERCE_SEARCH_URL, json=payload) response.raise_for_status() parsed = response.json() diff --git a/tom_alerts/tests/brokers/test_alerce.py b/tom_alerts/tests/brokers/test_alerce.py index d9db37322..dce97c61b 100644 --- a/tom_alerts/tests/brokers/test_alerce.py +++ b/tom_alerts/tests/brokers/test_alerce.py @@ -1,11 +1,167 @@ +from datetime import datetime + +from astropy.time import Time from django.test import tag, TestCase +from tom_alerts.brokers.alerce import ALeRCEBroker, ALeRCEQueryForm + + +class TestALeRCEBrokerForm(TestCase): + def setUp(self): + self.base_form_data = { + 'query_name': 'Test Query', + 'broker': 'ALeRCE' + } + + def test_cone_search_validation(self): + """Test cross-field validation for cone search filters.""" + + # Test that validation fails if not all fields are present + parameters_list = [ + {'ra': 10, 'dec': 10}, {'dec': 10, 'sr': 10}, {'ra': 10, 'sr': 10} + ] + for parameters in parameters_list: + with self.subTest(): + parameters.update(self.base_form_data) + form = ALeRCEQueryForm(parameters) + self.assertFalse(form.is_valid()) + self.assertIn('All of RA, Dec, and Search Radius must be included to execute a cone search.', + form.non_field_errors()) + + # Test that validation passes when all three fields are present + self.base_form_data.update({'ra': 10, 'dec': 10, 'sr': 10}) + form = ALeRCEQueryForm(self.base_form_data) + self.assertTrue(form.is_valid()) + + def test_time_filters_validation(self): + """Test validation for time filters.""" + + # Test that validation fails when either absolute time filter is paired with relative time filter + parameters_list = [ + {'mjd__lt': 58000, 'relative_mjd__gt': 168}, + {'mjd__gt': 58000, 'relative_mjd__gt': 168} + ] + for parameters in parameters_list: + with self.subTest(): + parameters.update(self.base_form_data) + form = ALeRCEQueryForm(parameters) + self.assertFalse(form.is_valid()) + self.assertIn('Cannot filter by both relative and absolute time.', form.non_field_errors()) + + # Test that mjd__lt and mjd__gt fail when mjd__lt is less than mjd__gt + with self.subTest(): + parameters = {'mjd__lt': 57000, 'mjd__gt': 57001} + parameters.update(self.base_form_data) + form = ALeRCEQueryForm(parameters) + self.assertFalse(form.is_valid()) + self.assertIn('Min date of first detection must be earlier than max date of first detection.', + form.non_field_errors()) + + # Test that form validation succeeds when relative time fields make sense and absolute time field is used alone. + parameters_list = [ + {'mjd__gt': 57000, 'mjd__lt': 58000}, + {'relative_mjd__gt': 168} + ] + for parameters in parameters_list: + with self.subTest(): + parameters.update(self.base_form_data) + print(parameters) + form = ALeRCEQueryForm(parameters) + print(form.errors) + self.assertTrue(form.is_valid()) + + # Test that form validation succeeds when absolute time field is used on its own. + with self.subTest(): + parameters = {'relative_mjd__gt': 168} + parameters.update(self.base_form_data) + form = ALeRCEQueryForm(parameters) + self.assertTrue(form.is_valid()) + # Test that clean_relative_mjd__gt works as expected + expected_mjd = Time(datetime.now()).mjd - parameters['relative_mjd__gt']/24 + self.assertAlmostEqual(form.cleaned_data['relative_mjd__gt'], expected_mjd) + class TestALeRCEBrokerClass(TestCase): def setUp(self): + self.base_form_data = { + 'query_name': 'Test ALeRCE', + 'broker': 'ALeRCE', + } + self.broker = ALeRCEBroker() + + def test_clean_coordinate_parameters(self): + """Test that _clean_date_parameters results in the correct dict structure.""" + parameters_list = [ + ({'ra': 10, 'dec': 10, 'sr': None}, None), + ({'ra': 10, 'dec': 10, 'sr': 10}, {'ra': 10, 'dec': 10, 'sr': 10}) + ] + for parameters, expected in parameters_list: + with self.subTest(): + self.assertEqual(self.broker._clean_coordinate_parameters(parameters), expected) + + def test_clean_date_parameters(self): + """Test that _clean_date_parameters results in the correct dict structure.""" + parameters_list = [ + ({'mjd__gt': 57000, 'mjd__lt': 58000, 'relative_mjd__gt': None}, + {'firstmjd': {'min': 57000, 'max': 58000}}), + ({'mjd__gt': 57000, 'mjd__lt': None, 'relative_mjd__gt': None}, {'firstmjd': {'min': 57000}}), + ({'mjd__gt': None, 'mjd__lt': None, 'relative_mjd__gt': 57000}, {'firstmjd': {'min': 57000}}) + ] + for parameters, expected in parameters_list: + with self.subTest(): + self.assertDictEqual(self.broker._clean_date_parameters(parameters), expected) + + def test_clean_filter_parameters(self): + """Test that _clean_filter_parameters results in the correct dict structure.""" + # Test that number of observations is populated correctly + parameters_list = [ + ({'nobs__gt': 1, 'nobs__lt': 10}, {'nobs': {'min': 1, 'max': 10}}), + ({'nobs__gt': 1, 'nobs__lt': None}, {'nobs': {'min': 1}}), + ({'nobs__gt': None, 'nobs__lt': 10}, {'nobs': {'max': 10}}) + ] + for parameters, expected in parameters_list: + with self.subTest(): + parameters.update({k: None for k in ['classrf', 'pclassrf', 'classearly', 'pclassearly']}) + self.assertDictContainsSubset(expected, self.broker._clean_filter_parameters(parameters)) + + # Test that classifiers are populated correctly + parameters_list = [ + ({'classrf': 19, 'pclassrf': 0.7, 'classearly': 10, 'pclassearly': 0.5}), + ({'classrf': 19, 'pclassrf': 0.7, 'classearly': None, 'pclassearly': None}), + ({'classrf': None, 'pclassrf': None, 'classearly': 10, 'pclassearly': 0.5}) + ] + for parameters in parameters_list: + with self.subTest(): + parameters.update({k: None for k in ['nobs__gt', 'nobs__lt']}) + filters = self.broker._clean_filter_parameters(parameters) + for key, value in parameters.items(): + if value is not None: + self.assertIn(key, filters) + else: + self.assertNotIn(key, filters) + + # def test_clean_parameters(self): + # form = ALeRCEQueryForm(self.base_form_data) + # form.is_valid() + # print(form.cleaned_data) + # query = form.save() + # print(query.parameters_as_dict) + # cleaned_params = self.broker._clean_parameters(query.parameters_as_dict) + # payload = self.broker._fetch_alerts_payload(query.parameters_as_dict) + # print(cleaned_params) + # print(payload) + + def test_fetch_alerts(self): pass - # def test_fetch_alerts_payload + def test_fetch_alert(self): + pass + + def test_to_target(self, alert): + pass + + def test_to_generic_alert(self, alert): + pass @tag('canary') @@ -13,6 +169,12 @@ class TestALeRCEModuleCanary(TestCase): def setUp(self): pass + def test_early_classifier_choices(self): + pass + + def test_late_classifier_choices(self): + pass + def test_fetch_alerts(self): pass From db487c425b759e21c1263ab9268170e5bd388a53 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 2 Nov 2020 21:24:57 -0800 Subject: [PATCH 381/424] Finished alerce unit tests --- tom_alerts/brokers/alerce.py | 39 +++++--- tom_alerts/tests/brokers/test_alerce.py | 126 ++++++++++++++++++++---- 2 files changed, 134 insertions(+), 31 deletions(-) diff --git a/tom_alerts/brokers/alerce.py b/tom_alerts/brokers/alerce.py index 4244b3ed5..d25391d1f 100644 --- a/tom_alerts/brokers/alerce.py +++ b/tom_alerts/brokers/alerce.py @@ -322,20 +322,33 @@ def _clean_parameters(self, parameters): return payload def fetch_alerts(self, parameters): - print(parameters) payload = self._clean_parameters(parameters) - print(payload) response = requests.post(ALERCE_SEARCH_URL, json=payload) response.raise_for_status() parsed = response.json() alerts = [alert_data for alert, alert_data in parsed['result'].items()] - if parsed['page'] < parsed['num_pages'] and parsed['page'] != int(parameters['max_pages']): + if parsed['page'] < parsed['num_pages'] and parsed['page'] != parameters['max_pages']: parameters['page'] = parameters.get('page', 1) + 1 parameters['total'] = parsed.get('total') alerts += self.fetch_alerts(parameters) return iter(alerts) def fetch_alert(self, id): + """ + The response for a single alert is as follows: + + { + "total": 1, + "num_pages": 1, + "page": 1, + "result": { + "ZTF20acnsdjd": { + "oid": "ZTF20acnsdjd", + other alert values + } + } + } + """ payload = { 'query_parameters': { 'filters': { @@ -345,7 +358,7 @@ def fetch_alert(self, id): } response = requests.post(ALERCE_SEARCH_URL, json=payload) response.raise_for_status() - return response.json()['result'][0] + return list(response.json()['result'].items())[0][1] def to_target(self, alert): return Target.objects.create( @@ -360,14 +373,16 @@ def to_generic_alert(self, alert): timestamp = Time(alert['lastmjd'], format='mjd', scale='utc').to_datetime(timezone=TimezoneInfo()) else: timestamp = '' - url = '{0}/{1}/{2}'.format(ALERCE_URL, 'object', alert['oid']) - - exits = (alert['mean_magpsf_g'] is None and alert['mean_magpsf_r'] is not None) - both_exists = (alert['mean_magpsf_g'] is not None and alert['mean_magpsf_r'] is not None) - bigger = (both_exists and (alert['mean_magpsf_r'] < alert['mean_magpsf_g'] is not None)) - is_r = any([exits, bigger]) + url = f'{ALERCE_URL}/object/{alert["oid"]}' - max_mag = alert['mean_magpsf_r'] if is_r else alert['mean_magpsf_g'] + # Use the smaller value between r and g if both are present, else use the value that is present + mag = None + if alert['mean_magpsf_r'] is not None and alert['mean_magpsf_g'] is not None: + mag = alert['mean_magpsf_g'] if alert['mean_magpsf_r'] > alert['mean_magpsf_g'] else alert['mean_magpsf_r'] + elif alert['mean_magpsf_r'] is not None: + mag = alert['mean_magpsf_r'] + elif alert['mean_magpsf_g'] is not None: + mag = alert['mean_magpsf_g'] if alert['pclassrf']: score = alert['pclassrf'] @@ -383,6 +398,6 @@ def to_generic_alert(self, alert): name=alert['oid'], ra=alert['meanra'], dec=alert['meandec'], - mag=max_mag, + mag=mag, score=score ) diff --git a/tom_alerts/tests/brokers/test_alerce.py b/tom_alerts/tests/brokers/test_alerce.py index dce97c61b..5af32ca23 100644 --- a/tom_alerts/tests/brokers/test_alerce.py +++ b/tom_alerts/tests/brokers/test_alerce.py @@ -1,9 +1,36 @@ -from datetime import datetime +from datetime import datetime, timezone +import json +from requests import Response +from unittest.mock import patch from astropy.time import Time from django.test import tag, TestCase +from faker import Faker from tom_alerts.brokers.alerce import ALeRCEBroker, ALeRCEQueryForm +from tom_targets.models import Target + + +def create_alerce_alert(lastmjd=None, mean_magpsf_g=None, mean_magpsf_r=None, pclassrf=None, pclassearly=None): + fake = Faker() + + return {'oid': fake.pystr_format(string_format='ZTF##???????', letters='abcdefghijklmnopqrstuvwxyz'), + 'meanra': fake.pyfloat(min_value=0, max_value=360), + 'meandec': fake.pyfloat(min_value=0, max_value=360), + 'lastmjd': lastmjd if lastmjd else fake.pyfloat(min_value=56000, max_value=59000, right_digits=1), + 'mean_magpsf_g': mean_magpsf_g if mean_magpsf_g else fake.pyfloat(min_value=16, max_value=25), + 'mean_magpsf_r': mean_magpsf_r if mean_magpsf_r else fake.pyfloat(min_value=16, max_value=25), + 'pclassrf': pclassrf if pclassrf else fake.pyfloat(min_value=0, max_value=1), + 'pclassearly': pclassearly if pclassearly else fake.pyfloat(min_value=0, max_value=1)} + + +def create_alerce_query_response(num_alerts): + alerts = [create_alerce_alert() for i in range(0, num_alerts)] + + return { + 'total': num_alerts, 'num_pages': 1, 'page': 1, + 'result': {alert['oid']: alert for alert in alerts} + } class TestALeRCEBrokerForm(TestCase): @@ -140,28 +167,89 @@ def test_clean_filter_parameters(self): else: self.assertNotIn(key, filters) - # def test_clean_parameters(self): - # form = ALeRCEQueryForm(self.base_form_data) - # form.is_valid() - # print(form.cleaned_data) - # query = form.save() - # print(query.parameters_as_dict) - # cleaned_params = self.broker._clean_parameters(query.parameters_as_dict) - # payload = self.broker._fetch_alerts_payload(query.parameters_as_dict) - # print(cleaned_params) - # print(payload) + @patch('tom_alerts.brokers.alerce.requests.post') + @patch('tom_alerts.brokers.alerce.ALeRCEBroker._clean_parameters') + def test_fetch_alerts(self, mock_clean_parameters, mock_requests_post): + """Test fetch_alerts broker method.""" + mock_response = Response() + mock_response_content = create_alerce_query_response(25) + mock_response._content = str.encode(json.dumps(mock_response_content)) + mock_response.status_code = 200 + mock_requests_post.return_value = mock_response - def test_fetch_alerts(self): - pass + response = self.broker.fetch_alerts({}) + alerts = [] + for alert in response: + alerts.append(alert) + self.assertDictEqual(alert, mock_response_content['result'][alert['oid']]) + self.assertEqual(25, len(alerts)) - def test_fetch_alert(self): - pass + @patch('tom_alerts.brokers.alerce.requests.post') + def test_fetch_alert(self, mock_requests_post): + """Test fetch_alert broker method.""" + mock_response = Response() + mock_response_content = create_alerce_query_response(1) + mock_response._content = str.encode(json.dumps(mock_response_content)) + mock_response.status_code = 200 + mock_requests_post.return_value = mock_response - def test_to_target(self, alert): - pass + alert = self.broker.fetch_alert(list(mock_response_content['result'])[0]) + self.assertDictEqual(list(mock_response_content['result'].items())[0][1], alert) - def test_to_generic_alert(self, alert): - pass + def test_to_target(self): + """Test to_target broker method.""" + mock_alert = create_alerce_alert() + self.broker.to_target(mock_alert) + t = Target.objects.first() + + self.assertEqual(mock_alert['oid'], t.name) + self.assertEqual(mock_alert['meanra'], t.ra) + self.assertEqual(mock_alert['meandec'], t.dec) + + def test_to_generic_alert(self): + """Test to_generic_alert broker method.""" + + # Test that timestamp is populated correctly. + mock_alert = create_alerce_alert() + mock_alert['lastmjd'] = None + self.assertEqual('', self.broker.to_generic_alert(mock_alert).timestamp) + + mock_alert = create_alerce_alert(lastmjd=59155) + self.assertEqual(datetime(2020, 11, 2, tzinfo=timezone.utc), + self.broker.to_generic_alert(mock_alert).timestamp) + + # Test that the url is created properly. + mock_alert = create_alerce_alert() + self.assertEqual(f'https://alerce.online/object/{mock_alert["oid"]}', + self.broker.to_generic_alert(mock_alert).url) + + # Test that the magnitude is selected correctly + mock_alert = create_alerce_alert(mean_magpsf_g=20, mean_magpsf_r=18) + self.assertEqual(mock_alert['mean_magpsf_r'], self.broker.to_generic_alert(mock_alert).mag) + + mock_alert = create_alerce_alert(mean_magpsf_g=18, mean_magpsf_r=20) + self.assertEqual(mock_alert['mean_magpsf_g'], self.broker.to_generic_alert(mock_alert).mag) + + mock_alert = create_alerce_alert(mean_magpsf_r=18) + mock_alert['mean_magpsf_g'] = None + self.assertEqual(mock_alert['mean_magpsf_r'], self.broker.to_generic_alert(mock_alert).mag) + + mock_alert = create_alerce_alert(mean_magpsf_g=18, mean_magpsf_r=20) + mock_alert['mean_magpsf_r'] = None + self.assertEqual(mock_alert['mean_magpsf_g'], self.broker.to_generic_alert(mock_alert).mag) + + # Test that the classification is selected correctly + mock_alert = create_alerce_alert() + self.assertEqual(mock_alert['pclassrf'], self.broker.to_generic_alert(mock_alert).score) + + mock_alert = create_alerce_alert() + mock_alert['pclassrf'] = None + self.assertEqual(mock_alert['pclassearly'], self.broker.to_generic_alert(mock_alert).score) + + mock_alert = create_alerce_alert() + mock_alert['pclassrf'] = None + mock_alert['pclassearly'] = None + self.assertEqual(None, self.broker.to_generic_alert(mock_alert).score) @tag('canary') From e0887d4e0e621fb5da9ed4b477c69012936a1272 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 2 Nov 2020 21:56:09 -0800 Subject: [PATCH 382/424] Added canary tests to alerce --- tom_alerts/brokers/alerce.py | 5 ++- tom_alerts/tests/brokers/test_alerce.py | 44 +++++++++++++++++-------- tom_observations/facilities/gemini.py | 6 +++- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/tom_alerts/brokers/alerce.py b/tom_alerts/brokers/alerce.py index d25391d1f..e815cb1ae 100644 --- a/tom_alerts/brokers/alerce.py +++ b/tom_alerts/brokers/alerce.py @@ -214,7 +214,8 @@ def __init__(self, *args, **kwargs): ), ) - def _get_classifiers(self): + @staticmethod + def _get_classifiers(): cached_classifiers = cache.get('alerce_classifiers') if not cached_classifiers: @@ -232,8 +233,6 @@ def clean_relative_mjd__gt(self): def clean(self): cleaned_data = super().clean() - print(cleaned_data) - # Ensure that all cone search fields are present if any(cleaned_data[k] for k in ['ra', 'dec', 'sr']) and not all(cleaned_data[k] for k in ['ra', 'dec', 'sr']): raise forms.ValidationError('All of RA, Dec, and Search Radius must be included to execute a cone search.') diff --git a/tom_alerts/tests/brokers/test_alerce.py b/tom_alerts/tests/brokers/test_alerce.py index 5af32ca23..ce6a4551d 100644 --- a/tom_alerts/tests/brokers/test_alerce.py +++ b/tom_alerts/tests/brokers/test_alerce.py @@ -92,9 +92,7 @@ def test_time_filters_validation(self): for parameters in parameters_list: with self.subTest(): parameters.update(self.base_form_data) - print(parameters) form = ALeRCEQueryForm(parameters) - print(form.errors) self.assertTrue(form.is_valid()) # Test that form validation succeeds when absolute time field is used on its own. @@ -215,7 +213,7 @@ def test_to_generic_alert(self): self.assertEqual('', self.broker.to_generic_alert(mock_alert).timestamp) mock_alert = create_alerce_alert(lastmjd=59155) - self.assertEqual(datetime(2020, 11, 2, tzinfo=timezone.utc), + self.assertEqual(datetime(2020, 11, 2, tzinfo=timezone.utc), self.broker.to_generic_alert(mock_alert).timestamp) # Test that the url is created properly. @@ -255,19 +253,39 @@ def test_to_generic_alert(self): @tag('canary') class TestALeRCEModuleCanary(TestCase): def setUp(self): - pass + self.broker = ALeRCEBroker() - def test_early_classifier_choices(self): - pass + @patch('tom_alerts.brokers.alerce.cache.get') + def test_get_classifiers(self, mock_cache_get): + mock_cache_get.return_value = None # Ensure cache is not used - def test_late_classifier_choices(self): - pass + classifiers = ALeRCEQueryForm._get_classifiers() + self.assertIn('early', classifiers.keys()) + self.assertIn('late', classifiers.keys()) + for classifier in classifiers['early'] + classifiers['late']: + self.assertIn('name', classifier.keys()) + self.assertIn('id', classifier.keys()) + # TODO: form should be okay without including records or sort_by def test_fetch_alerts(self): - pass + form = ALeRCEQueryForm({'query_name': 'Test', 'broker': 'ALeRCE', 'records': 20, 'sort_by': 'nobs', + 'nobs__gt': 1, 'classearly': 19, 'pclassearly': 0.7, + 'mjd__gt': 59148.78219219812}) + form.is_valid() + query = form.save() - def test_fetch_alert(self): - pass + alerts = [alert for alert in self.broker.fetch_alerts(query.parameters_as_dict)] - def test_process_reduced_data(self): - pass + self.assertGreaterEqual(len(alerts), 6) + for k in ['oid', 'lastmjd', 'mean_magpsf_r', 'mean_magpsf_g', 'pclassrf', 'pclassearly', 'meanra', 'meandec']: + self.assertIn(k, alerts[0]) + + def test_fetch_alert(self): + alert = self.broker.fetch_alert('ZTF20acnsdjd') + + self.assertDictContainsSubset({ + 'oid': 'ZTF20acnsdjd', + 'last_magpsf_r': 17.8492107391357, + 'first_magpsf_r': 17.0198993682861, + 'firstmjd': 59149.1119328998, + }, alert) diff --git a/tom_observations/facilities/gemini.py b/tom_observations/facilities/gemini.py index 4b496e749..bf5e752dd 100644 --- a/tom_observations/facilities/gemini.py +++ b/tom_observations/facilities/gemini.py @@ -1,4 +1,6 @@ +import logging import requests + from django.conf import settings from django import forms from dateutil.parser import parse @@ -9,6 +11,8 @@ from tom_common.exceptions import ImproperCredentialsException from tom_targets.models import Target +logger = logging.getLogger(__name__) + try: GEM_SETTINGS = settings.FACILITIES['GEM'] except KeyError: @@ -60,7 +64,7 @@ def make_request(*args, **kwargs): response = requests.request(*args, **kwargs) if 400 <= response.status_code < 500: - print('Request failed: {}'.format(response.content)) + logger.log(msg=f'Gemini request failed: {response.content}', level=logging.WARN) raise ImproperCredentialsException('GEM') response.raise_for_status() return response From 40bcfa5e89dfffe427082096020ff9a90300bda1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2020 13:49:33 +0000 Subject: [PATCH 383/424] Bump numpy from 1.19.3 to 1.19.4 Bumps [numpy](https://github.com/numpy/numpy) from 1.19.3 to 1.19.4. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/master/doc/HOWTO_RELEASE.rst.txt) - [Commits](https://github.com/numpy/numpy/compare/v1.19.3...v1.19.4) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4627a12f4..cdbfe5366 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ 'django-guardian==2.3.0', 'fits2image==0.4.3', 'Markdown==3.3.3', # django-rest-framework doc headers require this to support Markdown - 'numpy==1.19.3', + 'numpy==1.19.4', 'pillow==8.0.1', 'plotly==4.12.0', 'python-dateutil==2.8.1', From 1ae3423a7fe5dc3dfb078e450e0e6dad0d6abf8b Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 3 Nov 2020 07:43:16 -0800 Subject: [PATCH 384/424] Quick improvement to ALeRCE test coverage --- tom_alerts/tests/brokers/test_alerce.py | 30 +++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tom_alerts/tests/brokers/test_alerce.py b/tom_alerts/tests/brokers/test_alerce.py index ce6a4551d..aba14ca10 100644 --- a/tom_alerts/tests/brokers/test_alerce.py +++ b/tom_alerts/tests/brokers/test_alerce.py @@ -165,6 +165,36 @@ def test_clean_filter_parameters(self): else: self.assertNotIn(key, filters) + @patch('tom_alerts.brokers.alerce.ALeRCEBroker._clean_filter_parameters') + @patch('tom_alerts.brokers.alerce.ALeRCEBroker._clean_date_parameters') + @patch('tom_alerts.brokers.alerce.ALeRCEBroker._clean_coordinate_parameters') + def test_clean_parameters(self, mock_coordinate, mock_date, mock_filter): + mock_coordinate.return_value = {'ra': 10, 'dec': 10, 'sr': 10} + mock_date.return_value = {'firstmjd': {'min': 58000}} + mock_filter.return_value = {'nobs__gt': 1} + + # Ensure that passed in values are used to populate the payload + parameters = {'page': 2, 'records': 25, 'sort_by': 'nobs', 'total': 30} + payload = self.broker._clean_parameters(parameters) + with self.subTest(): + self.assertEqual(payload['page'], parameters['page']) + self.assertEqual(payload['records_per_pages'], parameters['records']) + self.assertEqual(payload['sortBy'], parameters['sort_by']) + self.assertEqual(payload['total'], parameters['total']) + self.assertIn('firstmjd', payload['query_parameters']['dates']) + self.assertIn('nobs__gt', payload['query_parameters']['filters']) + self.assertIn('coordinates', payload['query_parameters']) + + # Ensure that missing values result in default values being used to populate the payload + mock_coordinate.return_value = None + payload = self.broker._clean_parameters({}) + with self.subTest(): + self.assertEqual(payload['page'], 1) + self.assertEqual(payload['records_per_pages'], 20) + self.assertEqual(payload['sortBy'], None) + self.assertNotIn('total', payload) + self.assertNotIn('coordinates', payload['query_parameters']) + @patch('tom_alerts.brokers.alerce.requests.post') @patch('tom_alerts.brokers.alerce.ALeRCEBroker._clean_parameters') def test_fetch_alerts(self, mock_clean_parameters, mock_requests_post): From 0343ab88e66283629df3abf5afd53c4baebb6100 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 3 Nov 2020 08:00:50 -0800 Subject: [PATCH 385/424] Fixed up form processing --- tom_alerts/brokers/alerce.py | 11 ++++++++++- tom_alerts/tests/brokers/test_alerce.py | 10 ++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tom_alerts/brokers/alerce.py b/tom_alerts/brokers/alerce.py index e815cb1ae..7c533fef8 100644 --- a/tom_alerts/brokers/alerce.py +++ b/tom_alerts/brokers/alerce.py @@ -115,6 +115,8 @@ def __init__(self, *args, **kwargs): self.fields['classearly'].choices = self.early_classifier_choices() self.fields['classrf'].choices = self.late_classifier_choices() + # self.fields['records'].initial = 20 + # self.fields['sort_by'].initial = 'nobs' self.helper.layout = Layout( self.common_layout, @@ -225,6 +227,13 @@ def _get_classifiers(): return cached_classifiers + def clean_sort_by(self): + return self.cleaned_data['sort_by'] if self.cleaned_data['sort_by'] else 'nobs' + + + def clean_records(self): + return self.cleaned_data['records'] if self.cleaned_data['records'] else 20 + def clean_relative_mjd__gt(self): if self.cleaned_data['relative_mjd__gt']: return Time(datetime.now() - timedelta(hours=self.cleaned_data['relative_mjd__gt'])).mjd @@ -303,7 +312,7 @@ def _clean_parameters(self, parameters): payload = { 'page': parameters.get('page', 1), 'records_per_pages': parameters.get('records', 20), - 'sortBy': parameters.get('sort_by'), + 'sortBy': parameters.get('sort_by', 'nobs'), 'query_parameters': {} } diff --git a/tom_alerts/tests/brokers/test_alerce.py b/tom_alerts/tests/brokers/test_alerce.py index aba14ca10..ef52d498d 100644 --- a/tom_alerts/tests/brokers/test_alerce.py +++ b/tom_alerts/tests/brokers/test_alerce.py @@ -174,7 +174,7 @@ def test_clean_parameters(self, mock_coordinate, mock_date, mock_filter): mock_filter.return_value = {'nobs__gt': 1} # Ensure that passed in values are used to populate the payload - parameters = {'page': 2, 'records': 25, 'sort_by': 'nobs', 'total': 30} + parameters = {'page': 2, 'records': 25, 'sort_by': 'lastmjd', 'total': 30} payload = self.broker._clean_parameters(parameters) with self.subTest(): self.assertEqual(payload['page'], parameters['page']) @@ -191,7 +191,7 @@ def test_clean_parameters(self, mock_coordinate, mock_date, mock_filter): with self.subTest(): self.assertEqual(payload['page'], 1) self.assertEqual(payload['records_per_pages'], 20) - self.assertEqual(payload['sortBy'], None) + self.assertEqual(payload['sortBy'], 'nobs') self.assertNotIn('total', payload) self.assertNotIn('coordinates', payload['query_parameters']) @@ -296,11 +296,9 @@ def test_get_classifiers(self, mock_cache_get): self.assertIn('name', classifier.keys()) self.assertIn('id', classifier.keys()) - # TODO: form should be okay without including records or sort_by def test_fetch_alerts(self): - form = ALeRCEQueryForm({'query_name': 'Test', 'broker': 'ALeRCE', 'records': 20, 'sort_by': 'nobs', - 'nobs__gt': 1, 'classearly': 19, 'pclassearly': 0.7, - 'mjd__gt': 59148.78219219812}) + form = ALeRCEQueryForm({'query_name': 'Test', 'broker': 'ALeRCE', 'nobs__gt': 1, 'classearly': 19, + 'pclassearly': 0.7, 'mjd__gt': 59148.78219219812}) form.is_valid() query = form.save() From 3adbb5ffb531537082048b8976f0a8bb61ee9c92 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 3 Nov 2020 08:05:35 -0800 Subject: [PATCH 386/424] Fixing flake8 issues --- tom_alerts/brokers/alerce.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tom_alerts/brokers/alerce.py b/tom_alerts/brokers/alerce.py index 7c533fef8..282e32c01 100644 --- a/tom_alerts/brokers/alerce.py +++ b/tom_alerts/brokers/alerce.py @@ -229,7 +229,6 @@ def _get_classifiers(): def clean_sort_by(self): return self.cleaned_data['sort_by'] if self.cleaned_data['sort_by'] else 'nobs' - def clean_records(self): return self.cleaned_data['records'] if self.cleaned_data['records'] else 20 From 1067e4f941a67697eff11b8fae237e593531e059 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 3 Nov 2020 08:18:16 -0800 Subject: [PATCH 387/424] Removing commented code --- tom_alerts/brokers/alerce.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tom_alerts/brokers/alerce.py b/tom_alerts/brokers/alerce.py index 282e32c01..730ba9f81 100644 --- a/tom_alerts/brokers/alerce.py +++ b/tom_alerts/brokers/alerce.py @@ -115,8 +115,6 @@ def __init__(self, *args, **kwargs): self.fields['classearly'].choices = self.early_classifier_choices() self.fields['classrf'].choices = self.late_classifier_choices() - # self.fields['records'].initial = 20 - # self.fields['sort_by'].initial = 'nobs' self.helper.layout = Layout( self.common_layout, From c7c28cf9e820b5be21e474a49393c4b76fee1db0 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 3 Nov 2020 08:32:01 -0800 Subject: [PATCH 388/424] Attempting to fix test --- tom_alerts/brokers/alerce.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tom_alerts/brokers/alerce.py b/tom_alerts/brokers/alerce.py index 730ba9f81..461bed072 100644 --- a/tom_alerts/brokers/alerce.py +++ b/tom_alerts/brokers/alerce.py @@ -389,9 +389,9 @@ def to_generic_alert(self, alert): elif alert['mean_magpsf_g'] is not None: mag = alert['mean_magpsf_g'] - if alert['pclassrf']: + if alert['pclassrf'] is not None: score = alert['pclassrf'] - elif alert['pclassearly']: + elif alert['pclassearly'] is not None: score = alert['pclassearly'] else: score = None From a3ad7e3635e5ac59a2ddbb50e1410c51b2689d38 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 3 Nov 2020 09:40:57 -0800 Subject: [PATCH 389/424] Made change to broker settings key --- .gitignore | 1 + docs/common/customsettings.rst | 57 +++++++++++++++------ tom_alerts/brokers/tns.py | 10 ++-- tom_catalogs/harvesters/tns.py | 2 +- tom_setup/templates/tom_setup/settings.tmpl | 17 ++++-- 5 files changed, 60 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index a5e7ba9ed..bd8990047 100644 --- a/.gitignore +++ b/.gitignore @@ -84,6 +84,7 @@ docs/_build/ # VS Code .vscode/ +*.code-workspace # PyBuilder target/ diff --git a/docs/common/customsettings.rst b/docs/common/customsettings.rst index f9d84412a..296a0e190 100644 --- a/docs/common/customsettings.rst +++ b/docs/common/customsettings.rst @@ -6,22 +6,6 @@ your project’s ``settings.py``. For explanations of Django specific settings, see the `official documentation `__. -`ALERT_CREDENTIALS <#alert_credentials>`__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Default: - -.. code-block:: - - { - 'TNS': { - 'api_key': '' - } - } - -Credentials for any brokers that require them. At the moment, the only -built-in TOM Toolkit broker module that requires credentials is the TNS. - `AUTH_STRATEGY <#auth_strategy>`__ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -33,6 +17,31 @@ TOM, but not to change anything. A value of **LOCKED** requires all users to login before viewing any page. Use the `OPEN_URLS <#open_urls>`__ setting for adding exemptions. +`BROKERS <#brokers>`__ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: + +.. code-block:: + + { + 'TNS': { + 'api_key': '' + }, + 'SCIMMA': { + 'url': 'http://skip.dev.hop.scimma.org', + 'api_key': os.getenv('SKIP_API_KEY', ''), + 'hopskotch_url': 'dev.hop.scimma.org', + 'hopskotch_username': os.getenv('HOPSKOTCH_USERNAME', ''), + 'hopskotch_password': os.getenv('HOPSKOTCH_PASSWORD', ''), + 'default_hopskotch_topic': '' + } + } + +Credentials and settings for any brokers that require them. At the moment, the only +built-in TOM Toolkit broker module that requires credentials is the TNS. SCIMMA and +ANTARES, which are available as add-on modules, also use this setting. + `DATA_PROCESSORS <#data_processors>`__ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -97,6 +106,22 @@ one you’ll probably have to configure it here first. For example the LCO facility requires you to provide a value for the ``api_key`` configuration value. +`HARVESTERS <#harvesters>`__ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: + +.. code-block:: + + { + 'TNS': { + 'api_key': '' + }, + } + +Credentials and settings for any harvesters that require them. At the moment, the only +built-in TOM Toolkit broker module that requires credentials is the TNS. + `HINTS <#hints>`__ ~~~~~~~~~~~~~~~~~~ diff --git a/tom_alerts/brokers/tns.py b/tom_alerts/brokers/tns.py index ab0b785c3..11c74236c 100644 --- a/tom_alerts/brokers/tns.py +++ b/tom_alerts/brokers/tns.py @@ -91,7 +91,7 @@ def fetch_alerts(cls, parameters): else: public_timestamp = '' data = { - 'api_key': settings.BROKER_CREDENTIALS['TNS_APIKEY'], + 'api_key': settings.BROKERS['TNS']['api_key'], 'data': json.dumps({ 'name': parameters['target_name'], 'internal_name': parameters['internal_name'], @@ -108,7 +108,7 @@ def fetch_alerts(cls, parameters): alerts = [] for transient in transients['data']['reply']: data = { - 'api_key': settings.BROKER_CREDENTIALS['TNS_APIKEY'], + 'api_key': settings.BROKERS['TNS']['api_key'], 'data': json.dumps({ 'objname': transient['objname'], 'photometry': 1, @@ -137,9 +137,9 @@ def fetch_alerts(cls, parameters): def to_generic_alert(cls, alert): return GenericAlert( timestamp=alert['discoverydate'], - url='https://wis-tns.weizmann.ac.il/object/' + alert['name'], - id=alert['name'], - name=alert['name_prefix'] + alert['name'], + url='https://wis-tns.weizmann.ac.il/object/' + alert['objname'], + id=alert['objname'], + name=alert['name_prefix'] + alert['objname'], ra=alert['radeg'], dec=alert['decdeg'], mag=alert['discoverymag'], diff --git a/tom_catalogs/harvesters/tns.py b/tom_catalogs/harvesters/tns.py index 4e9d3ffa6..3716b90c0 100644 --- a/tom_catalogs/harvesters/tns.py +++ b/tom_catalogs/harvesters/tns.py @@ -12,7 +12,7 @@ TNS_URL = 'https://wis-tns.weizmann.ac.il' try: - TNS_CREDENTIALS = settings.ALERT_CREDENTIALS['TNS'] + TNS_CREDENTIALS = settings.HARVESTERS['TNS'] except (AttributeError, KeyError): TNS_CREDENTIALS = { 'api_key': '' diff --git a/tom_setup/templates/tom_setup/settings.tmpl b/tom_setup/templates/tom_setup/settings.tmpl index 530b8a05b..ef8d8c111 100644 --- a/tom_setup/templates/tom_setup/settings.tmpl +++ b/tom_setup/templates/tom_setup/settings.tmpl @@ -243,16 +243,23 @@ TOM_FACILITY_CLASSES = [ ] TOM_ALERT_CLASSES = [ - 'tom_alerts.brokers.mars.MARSBroker', - 'tom_alerts.brokers.lasair.LasairBroker', - 'tom_alerts.brokers.scout.ScoutBroker', - 'tom_alerts.brokers.tns.TNSBroker', + 'tom_alerts.brokers.alerce.ALeRCEBroker', 'tom_alerts.brokers.antares.ANTARESBroker', 'tom_alerts.brokers.gaia.GaiaBroker', + 'tom_alerts.brokers.lasair.LasairBroker', + 'tom_alerts.brokers.mars.MARSBroker', 'tom_alerts.brokers.scimma.SCIMMABroker', + 'tom_alerts.brokers.scout.ScoutBroker', + 'tom_alerts.brokers.tns.TNSBroker', ] -ALERT_CREDENTIALS = { +BROKERS = { + 'TNS': { + 'api_key': '' + } +} + +HARVESTERS = { 'TNS': { 'api_key': '' } From e2acaf7cfdb01e0162724b89ee80bf3252e51ee9 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 3 Nov 2020 14:33:52 -0800 Subject: [PATCH 390/424] Added test for base form --- tom_observations/facilities/lco.py | 30 ++-- tom_observations/tests/facilities/__init__.py | 0 tom_observations/tests/facilities/test_lco.py | 141 ++++++++++++++++++ 3 files changed, 160 insertions(+), 11 deletions(-) create mode 100644 tom_observations/tests/facilities/__init__.py create mode 100644 tom_observations/tests/facilities/test_lco.py diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index f409a1242..e33ab8734 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -113,7 +113,8 @@ def __init__(self, *args, **kwargs): self.fields['filter'] = forms.ChoiceField(choices=self.filter_choices()) self.fields['instrument_type'] = forms.ChoiceField(choices=self.instrument_choices()) - def _get_instruments(self): + @staticmethod + def _get_instruments(): cached_instruments = cache.get('lco_instruments') if not cached_instruments: @@ -127,16 +128,19 @@ def _get_instruments(self): return cached_instruments - def instrument_choices(self): - return sorted([(k, v['name']) for k, v in self._get_instruments().items()], key=lambda inst: inst[1]) + @staticmethod + def instrument_choices(): + return sorted([(k, v['name']) for k, v in LCOBaseForm._get_instruments().items()], key=lambda inst: inst[1]) - def filter_choices(self): + @staticmethod + def filter_choices(): return sorted(set([ - (f['code'], f['name']) for ins in self._get_instruments().values() for f in + (f['code'], f['name']) for ins in LCOBaseForm._get_instruments().values() for f in ins['optical_elements'].get('filters', []) + ins['optical_elements'].get('slits', []) ]), key=lambda filter_tuple: filter_tuple[1]) - def proposal_choices(self): + @staticmethod + def proposal_choices(): response = make_request( 'GET', PORTAL_URL + '/api/profile/', @@ -383,13 +387,17 @@ class LCOImagingObservationForm(LCOBaseObservationForm): The LCOImagingObservationForm allows the selection of parameters for observing using LCO's Imagers. The list of Imagers and their details can be found here: https://lco.global/observatory/instruments/ """ - def instrument_choices(self): - return sorted([(k, v['name']) for k, v in self._get_instruments().items() if 'IMAGE' in v['type']], - key=lambda inst: inst[1]) + @staticmethod + def instrument_choices(): + return sorted( + [(k, v['name']) for k, v in LCOImagingObservationForm._get_instruments().items() if 'IMAGE' in v['type']], + key=lambda inst: inst[1] + ) - def filter_choices(self): + @staticmethod + def filter_choices(): return sorted(set([ - (f['code'], f['name']) for ins in self._get_instruments().values() for f in + (f['code'], f['name']) for ins in LCOImagingObservationForm._get_instruments().values() for f in ins['optical_elements'].get('filters', []) ]), key=lambda filter_tuple: filter_tuple[1]) diff --git a/tom_observations/tests/facilities/__init__.py b/tom_observations/tests/facilities/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tom_observations/tests/facilities/test_lco.py b/tom_observations/tests/facilities/test_lco.py new file mode 100644 index 000000000..d2963b4b6 --- /dev/null +++ b/tom_observations/tests/facilities/test_lco.py @@ -0,0 +1,141 @@ +import json +from requests import Response +from unittest.mock import patch + +from django.test import TestCase + +from tom_observations.facilities.lco import make_request +from tom_observations.facilities.lco import LCOBaseForm, LCOBaseObservationForm + + +instrument_response = { + '2M0-FLOYDS-SCICAM': { + 'type': 'SPECTRA', 'class': '2m0', 'name': '2.0 meter FLOYDS', 'optical_elements': { + 'slits': [ + {'name': '6.0 arcsec slit', 'code': 'slit_6.0as', 'schedulable': True, 'default': False}, + {'name': '1.6 arcsec slit', 'code': 'slit_1.6as', 'schedulable': True, 'default': False}, + {'name': '2.0 arcsec slit', 'code': 'slit_2.0as', 'schedulable': True, 'default': False}, + {'name': '1.2 arcsec slit', 'code': 'slit_1.2as', 'schedulable': True, 'default': False} + ] + } + }, + '0M4-SCICAM-SBIG': { + 'type': 'IMAGE', 'class': '0m4', 'name': '0.4 meter SBIG', 'optical_elements': { + 'filters': [ + {'name': 'Opaque', 'code': 'opaque', 'schedulable': False, 'default': False}, + {'name': '100um Pinhole', 'code': '100um-Pinhole', 'schedulable': False, 'default': False}, + ] + }, + }, + 'SOAR_GHTS_REDCAM': { + 'type': 'SPECTRA', 'class': '4m0', 'name': 'Goodman Spectrograph RedCam', 'optical_elements': { + 'gratings': [ + {'name': '400 line grating', 'code': 'SYZY_400', 'schedulable': True, 'default': True}, + ], + 'slits': [ + {'name': '1.0 arcsec slit', 'code': 'slit_1.0as', 'schedulable': True, 'default': True} + ] + }, + } +} + + +class TestMakeRequest(TestCase): + + @patch('tom_observations.facilities.lco.requests.request') + def test_make_request(self, mock_request): + mock_response = Response() + mock_response._content = str.encode(json.dumps({'test': 'test'})) + mock_response.status_code = 200 + mock_request.return_value = mock_response + + self.assertDictEqual({'test': 'test'}, make_request('GET', 'google.com', headers={'test': 'test'}).json()) + + +class TestLCOBaseForm(TestCase): + + @patch('tom_observations.facilities.lco.make_request') + @patch('tom_observations.facilities.lco.cache') + def test_get_instruments(self, mock_cache, mock_make_request): + mock_response = Response() + mock_response._content = str.encode(json.dumps(instrument_response)) + mock_response.status_code = 200 + mock_make_request.return_value = mock_response + + # Test that cached value is returned + with self.subTest(): + test_instruments = {'test instrument': {'type': 'IMAGE'}} + mock_cache.get.return_value = test_instruments + + instruments = LCOBaseForm._get_instruments() + self.assertDictContainsSubset({'test instrument': {'type': 'IMAGE'}}, instruments) + self.assertNotIn('0M4-SCICAM-SBIG', instruments) + + # Test that empty cache results in mock_instruments, and cache.set is called + with self.subTest(): + mock_cache.get.return_value = None + + instruments = LCOBaseForm._get_instruments() + self.assertIn('0M4-SCICAM-SBIG', instruments) + self.assertDictContainsSubset({'type': 'IMAGE'}, instruments['0M4-SCICAM-SBIG']) + self.assertNotIn('SOAR_GHTS_REDCAM', instruments) + mock_cache.set.assert_called() + + @patch('tom_observations.facilities.lco.LCOBaseForm._get_instruments') + def test_instrument_choices(self, mock_get_instruments): + mock_get_instruments.return_value = {k: v for k, v in instrument_response.items() if 'SOAR' not in k} + + inst_choices = LCOBaseForm.instrument_choices() + self.assertIn(('2M0-FLOYDS-SCICAM', '2.0 meter FLOYDS'), inst_choices) + self.assertIn(('0M4-SCICAM-SBIG', '0.4 meter SBIG'), inst_choices) + + @patch('tom_observations.facilities.lco.LCOBaseForm._get_instruments') + def test_filter_choices(self, mock_get_instruments): + mock_get_instruments.return_value = {k: v for k, v in instrument_response.items() if 'SOAR' not in k} + + filter_choices = LCOBaseForm.filter_choices() + for expected in [('opaque', 'Opaque'), ('100um-Pinhole', '100um Pinhole'), ('slit_6.0as', '6.0 arcsec slit')]: + self.assertIn(expected, filter_choices) + self.assertEqual(len(filter_choices), 6) + + @patch('tom_observations.facilities.lco.make_request') + def test_proposal_choices(self, mock_make_request): + mock_response = Response() + mock_response._content = str.encode(json.dumps({'proposals': + [{'id': 'ActiveProposal', 'title': 'Active', 'current': True}, + {'id': 'InactiveProposal', 'title': 'Inactive', 'current': False}] + })) + mock_response.status_code = 200 + mock_make_request.return_value = mock_response + + proposal_choices = LCOBaseForm.proposal_choices() + self.assertIn(('ActiveProposal', 'Active (ActiveProposal)'), proposal_choices) + self.assertNotIn(('InactiveProposal', 'Inactive (InactiveProposal)'), proposal_choices) + + +class TestLCOBaseObservationForm(TestCase): + pass + + +class TestLCOImagingObservationForm(TestCase): + pass + + +class TestLCOSpectroscopyObservationForm(TestCase): + pass + + +class TestLCOPhotometricSequenceForm(TestCase): + pass + + +class TestLCOSpectroscopicSequenceForm(TestCase): + pass + + +class TestLCOObservationTemplateForm(TestCase): + pass + + +class TestLCOFacility(TestCase): + pass From 192ba2ab790c38eff77babe87b686284346a33eb Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 3 Nov 2020 17:17:22 -0800 Subject: [PATCH 391/424] Added more tests --- tom_dataproducts/tests/tests.py | 14 +- tom_observations/facilities/lco.py | 20 ++- tom_observations/tests/facilities/test_lco.py | 167 +++++++++++++++++- tom_observations/tests/factories.py | 24 ++- tom_observations/tests/test_cadence.py | 4 +- tom_observations/tests/tests.py | 10 +- 6 files changed, 209 insertions(+), 30 deletions(-) diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index 2edcef18c..e7c60d494 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -16,7 +16,7 @@ import numpy as np from tom_observations.tests.utils import FakeRoboticFacility -from tom_observations.tests.factories import TargetFactory, ObservingRecordFactory +from tom_observations.tests.factories import SiderealTargetFactory, ObservingRecordFactory from tom_dataproducts.models import DataProduct, is_fits_image_file from tom_dataproducts.forms import DataProductUploadForm from tom_dataproducts.processors.photometry_processor import PhotometryProcessor @@ -44,7 +44,7 @@ def mock_is_fits_image_file(filename): @patch('tom_dataproducts.models.DataProduct.get_preview', return_value='/no-image.jpg') class Views(TestCase): def setUp(self): - self.target = TargetFactory.create() + self.target = SiderealTargetFactory.create() self.observation_record = ObservingRecordFactory.create( target_id=self.target.id, facility=FakeRoboticFacility.name, @@ -152,7 +152,7 @@ def test_create_jpeg(self, dp_mock): @patch('tom_dataproducts.models.DataProduct.get_preview', return_value='/no-image.jpg') class TestViewsWithPermissions(TestCase): def setUp(self): - self.target = TargetFactory.create() + self.target = SiderealTargetFactory.create() self.observation_record = ObservingRecordFactory.create( target_id=self.target.id, facility=FakeRoboticFacility.name, @@ -220,7 +220,7 @@ def test_upload_data_extended_permissions(self, dp_mock): @patch('tom_dataproducts.views.run_data_processor') class TestUploadDataProducts(TestCase): def setUp(self): - self.target = TargetFactory.create() + self.target = SiderealTargetFactory.create() self.observation_record = ObservingRecordFactory.create( target_id=self.target.id, facility=FakeRoboticFacility.name, @@ -273,7 +273,7 @@ def test_upload_data_for_observation(self, run_data_processor_mock): class TestDeleteDataProducts(TestCase): def setUp(self): - self.target = TargetFactory.create() + self.target = SiderealTargetFactory.create() self.data_product = DataProduct.objects.create( product_id='testproductid', target=self.target, @@ -309,7 +309,7 @@ def test_delete_data_product_authorized(self): class TestDataUploadForms(TestCase): def setUp(self): - self.target = TargetFactory.create() + self.target = SiderealTargetFactory.create() self.observation_record = ObservingRecordFactory.create( target_id=self.target.id, facility=FakeRoboticFacility.name, @@ -383,7 +383,7 @@ def test_deserialize_spectrum_invalid(self): @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility']) class TestDataProcessor(TestCase): def setUp(self): - self.target = TargetFactory.create() + self.target = SiderealTargetFactory.create() self.data_product = DataProduct.objects.create( target=self.target ) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index e33ab8734..d90b1de4b 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -84,10 +84,7 @@ static_cadencing_help = """ For information on static cadencing with LCO, - + check the Observation Portal getting started guide, starting on page 18. """ @@ -141,6 +138,7 @@ def filter_choices(): @staticmethod def proposal_choices(): + print('here') response = make_request( 'GET', PORTAL_URL + '/api/profile/', @@ -179,7 +177,7 @@ class LCOBaseObservationForm(BaseRoboticObservationForm, LCOBaseForm): observation_mode = forms.ChoiceField( choices=(('NORMAL', 'Normal'), ('RAPID_RESPONSE', 'Rapid-Response'), ('TIME_CRITICAL', 'Time-Critical')), help_text=observation_mode_help - ) # TODO: Update this to support current observation modes + ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -229,13 +227,16 @@ def clean_end(self): end = self.cleaned_data['end'] return parse(end).isoformat() - def is_valid(self): - super().is_valid() + def validate_at_facility(self): obs_module = get_service_class(self.cleaned_data['facility']) errors = obs_module().validate_observation(self.observation_payload()) if errors: self.add_error(None, self._flatten_error_dict(errors)) - return not errors + + def is_valid(self): + super().is_valid() + self.validate_at_facility() + return not self._errors def _flatten_error_dict(self, error_dict): non_field_errors = [] @@ -259,7 +260,8 @@ def _flatten_error_dict(self, error_dict): return non_field_errors - def instrument_to_type(self, instrument_type): + @staticmethod + def instrument_to_type(instrument_type): if 'FLOYDS' in instrument_type: return 'SPECTRUM' elif 'NRES' in instrument_type: diff --git a/tom_observations/tests/facilities/test_lco.py b/tom_observations/tests/facilities/test_lco.py index d2963b4b6..7505a9e0b 100644 --- a/tom_observations/tests/facilities/test_lco.py +++ b/tom_observations/tests/facilities/test_lco.py @@ -4,8 +4,10 @@ from django.test import TestCase +from tom_common.exceptions import ImproperCredentialsException from tom_observations.facilities.lco import make_request from tom_observations.facilities.lco import LCOBaseForm, LCOBaseObservationForm +from tom_observations.tests.factories import SiderealTargetFactory, NonSiderealTargetFactory instrument_response = { @@ -51,6 +53,11 @@ def test_make_request(self, mock_request): self.assertDictEqual({'test': 'test'}, make_request('GET', 'google.com', headers={'test': 'test'}).json()) + mock_response.status_code = 403 + mock_request.return_value = mock_response + with self.assertRaises(ImproperCredentialsException): + make_request('GET', 'google.com', headers={'test': 'test'}) + class TestLCOBaseForm(TestCase): @@ -101,9 +108,9 @@ def test_filter_choices(self, mock_get_instruments): @patch('tom_observations.facilities.lco.make_request') def test_proposal_choices(self, mock_make_request): mock_response = Response() - mock_response._content = str.encode(json.dumps({'proposals': - [{'id': 'ActiveProposal', 'title': 'Active', 'current': True}, - {'id': 'InactiveProposal', 'title': 'Inactive', 'current': False}] + mock_response._content = str.encode(json.dumps({'proposals': [ + {'id': 'ActiveProposal', 'title': 'Active', 'current': True}, + {'id': 'InactiveProposal', 'title': 'Inactive', 'current': False}] })) mock_response.status_code = 200 mock_make_request.return_value = mock_response @@ -113,8 +120,158 @@ def test_proposal_choices(self, mock_make_request): self.assertNotIn(('InactiveProposal', 'Inactive (InactiveProposal)'), proposal_choices) -class TestLCOBaseObservationForm(TestCase): - pass +@patch('tom_observations.facilities.lco.LCOBaseForm.proposal_choices') +@patch('tom_observations.facilities.lco.LCOBaseForm.filter_choices') +@patch('tom_observations.facilities.lco.LCOBaseForm.instrument_choices') +@patch('tom_observations.facilities.lco.LCOBaseObservationForm.validate_at_facility') +class TestLCOBaseObservationFormPayload(TestCase): + + def setUp(self): + self.st = SiderealTargetFactory.create() + self.nst = NonSiderealTargetFactory.create(scheme='MPC_MINOR_PLANET') + self.valid_form_data = { + 'name': 'test', 'facility': 'LCO', 'target_id': self.st.id, 'ipp_value': 0.5, 'start': '2020-11-03', + 'end': '2020-11-04', 'exposure_count': 1, 'exposure_time': 30, 'max_airmass': 3, + 'min_lunar_distance': 20, 'period': 60, 'jitter': 15, 'observation_mode': 'NORMAL', + 'proposal': 'sampleproposal', 'filter': 'opaque', 'instrument_type': '0M4-SCICAM-SBIG' + } + self.instrument_choices = [(k, v['name']) for k, v in instrument_response.items() if 'SOAR' not in k] + self.filter_choices = set([ + (f['code'], f['name']) for ins in instrument_response.values() for f in + ins['optical_elements'].get('filters', []) + ins['optical_elements'].get('slits', []) + ]) + self.proposal_choices = [('sampleproposal', 'Sample Proposal')] + + def test_clean_and_validate(self, mock_validate, mock_insts, mock_filters, mock_proposals): + """Test clean_start, clean_end, and is_valid()""" + mock_validate.return_value = [] + mock_insts.return_value = self.instrument_choices + mock_filters.return_value = self.filter_choices + mock_proposals.return_value = self.proposal_choices + + # Test that a valid form returns True, and that start and end are cleaned properly + form = LCOBaseObservationForm(self.valid_form_data) + self.assertTrue(form.is_valid()) + self.assertEqual('2020-11-03T00:00:00', form.cleaned_data['start']) + self.assertEqual('2020-11-04T00:00:00', form.cleaned_data['end']) + + # Test that an invalid form returns False + self.valid_form_data.pop('target_id') + form = LCOBaseObservationForm(self.valid_form_data) + self.assertFalse(form.is_valid()) + + # TODO: Add test for when validate_at_facility returns errors + + def test_flatten_error_dict(self, mock_validate, mock_insts, mock_filters, mock_proposals): + pass + + def test_instrument_to_type(self, mock_validate, mock_insts, mock_filters, mock_proposals): + """Test instrument_to_type method.""" + self.assertEqual('SPECTRUM', LCOBaseObservationForm.instrument_to_type('2M0-FLOYDS-SCICAM')) + self.assertEqual('NRES_SPECTRUM', LCOBaseObservationForm.instrument_to_type('1M0-NRES-SCICAM')) + self.assertEqual('EXPOSE', LCOBaseObservationForm.instrument_to_type('0M4-SCICAM-SBIG')) + + def test_build_target_fields(self, mock_validate, mock_insts, mock_filters, mock_proposals): + """Test _build_target_fields method.""" + mock_validate.return_value = [] + mock_insts.return_value = self.instrument_choices + mock_filters.return_value = self.filter_choices + mock_proposals.return_value = self.proposal_choices + + # Test correct population of target fields for a sidereal target + with self.subTest(): + form = LCOBaseObservationForm(self.valid_form_data) + self.assertTrue(form.is_valid()) + self.assertDictEqual({ + 'name': self.st.name, 'type': 'ICRS', 'ra': self.st.ra, 'dec': self.st.dec, + 'proper_motion_ra': self.st.pm_ra, 'proper_motion_dec': self.st.pm_dec, 'epoch': self.st.epoch + }, form._build_target_fields()) + + # Test correct population of target fields for a non-sidereal target + with self.subTest(): + self.valid_form_data['target_id'] = self.nst.id + form = LCOBaseObservationForm(self.valid_form_data) + self.assertTrue(form.is_valid()) + self.assertDictContainsSubset({ + 'name': self.nst.name, 'type': 'ORBITAL_ELEMENTS', 'epochofel': self.nst.epoch_of_elements, + 'orbinc': self.nst.inclination, 'longascnode': self.nst.lng_asc_node, + 'argofperih': self.nst.arg_of_perihelion, 'meananom': self.nst.mean_anomaly, + 'meandist': self.nst.semimajor_axis + }, form._build_target_fields()) + + def test_build_instrument_config(self, mock_validate, mock_insts, mock_filters, mock_proposals): + """Test _build_instrument_config method.""" + mock_validate.return_value = [] + mock_insts.return_value = self.instrument_choices + mock_filters.return_value = self.filter_choices + mock_proposals.return_value = self.proposal_choices + + form = LCOBaseObservationForm(self.valid_form_data) + self.assertTrue(form.is_valid()) + self.assertEqual( + [{'exposure_count': self.valid_form_data['exposure_count'], + 'exposure_time': self.valid_form_data['exposure_time'], + 'optical_elements': {'filter': self.valid_form_data['filter']} + }], + form._build_instrument_config() + ) + + def test_build_acquisition_config(self, mock_validate, mock_insts, mock_filters, mock_proposals): + """Test _build_acquisition_config method.""" + mock_validate.return_value = [] + mock_insts.return_value = self.instrument_choices + mock_filters.return_value = self.filter_choices + mock_proposals.return_value = self.proposal_choices + + form = LCOBaseObservationForm(self.valid_form_data) + self.assertTrue(form.is_valid()) + self.assertDictEqual({}, form._build_acquisition_config()) + + def test_build_guiding_config(self, mock_validate, mock_insts, mock_filters, mock_proposals): + """Test _build_guiding_config method.""" + mock_validate.return_value = [] + mock_insts.return_value = self.instrument_choices + mock_filters.return_value = self.filter_choices + mock_proposals.return_value = self.proposal_choices + + form = LCOBaseObservationForm(self.valid_form_data) + self.assertTrue(form.is_valid()) + self.assertDictEqual({}, form._build_guiding_config()) + + # This should but does not mock instrument_to_type, _build_target_fields, _build_instrument_config, + # _build_acquisition_config, and _build_guiding_config + def test_build_configuration(self, mock_validate, mock_insts, mock_filters, mock_proposals): + """Test _build_configuration method.""" + mock_validate.return_value = [] + mock_insts.return_value = self.instrument_choices + mock_filters.return_value = self.filter_choices + mock_proposals.return_value = self.proposal_choices + + form = LCOBaseObservationForm(self.valid_form_data) + self.assertTrue(form.is_valid()) + configuration = form._build_configuration() + self.assertDictContainsSubset( + {'type': 'EXPOSE', 'instrument_type': '0M4-SCICAM-SBIG', 'constraints': {'max_airmass': 3}}, + configuration) + for key in ['target', 'instrument_configs', 'acquisition_config', 'guiding_config']: + self.assertIn(key, configuration) + + def test_build_location(self, mock_validate, mock_insts, mock_filters, mock_proposals): + """Test _build_location method.""" + mock_validate.return_value = [] + mock_insts.return_value = self.instrument_choices + mock_filters.return_value = self.filter_choices + mock_proposals.return_value = self.proposal_choices + + form = LCOBaseObservationForm(self.valid_form_data) + self.assertTrue(form.is_valid()) + self.assertDictEqual({'telescope_class': '0m4'}, form._build_location()) + + def test_expand_cadence_request(self, mock_validate, mock_insts, mock_filters, mock_proposals): + pass + + def test_observation_payload(self, mock_validate, mock_insts, mock_filters, mock_proposals): + pass class TestLCOImagingObservationForm(TestCase): diff --git a/tom_observations/tests/factories.py b/tom_observations/tests/factories.py index 0617b034a..75abf165b 100644 --- a/tom_observations/tests/factories.py +++ b/tom_observations/tests/factories.py @@ -12,11 +12,12 @@ class Meta: name = factory.Faker('pystr') -class TargetFactory(factory.django.DjangoModelFactory): +class SiderealTargetFactory(factory.django.DjangoModelFactory): class Meta: model = Target name = factory.Faker('pystr') + type = Target.SIDEREAL ra = factory.Faker('pyfloat', min_value=-90, max_value=90) dec = factory.Faker('pyfloat', min_value=-90, max_value=90) epoch = factory.Faker('pyfloat') @@ -24,11 +25,30 @@ class Meta: pm_dec = factory.Faker('pyfloat') +class NonSiderealTargetFactory(factory.django.DjangoModelFactory): + class Meta: + model = Target + + name = factory.Faker('pystr') + type = Target.NON_SIDEREAL + scheme = factory.Faker('random_element', elements=[s[0] for s in Target.TARGET_SCHEMES]) + mean_anomaly = factory.Faker('pyfloat') + arg_of_perihelion = factory.Faker('pyfloat') + lng_asc_node = factory.Faker('pyfloat') + inclination = factory.Faker('pyfloat') + mean_daily_motion = factory.Faker('pyfloat') + semimajor_axis = factory.Faker('pyfloat') + ephemeris_period = factory.Faker('pyfloat') + ephemeris_period_err = factory.Faker('pyfloat') + ephemeris_epoch = factory.Faker('pyfloat') + ephemeris_epoch_err = factory.Faker('pyfloat') + + class ObservingRecordFactory(factory.django.DjangoModelFactory): class Meta: model = ObservationRecord - target = factory.RelatedFactory(TargetFactory) + target = factory.RelatedFactory(SiderealTargetFactory) facility = 'LCO' observation_id = factory.Faker('pydecimal', right_digits=0, left_digits=7) status = 'PENDING' diff --git a/tom_observations/tests/test_cadence.py b/tom_observations/tests/test_cadence.py index c6f6f6d44..ce13a592e 100644 --- a/tom_observations/tests/test_cadence.py +++ b/tom_observations/tests/test_cadence.py @@ -5,7 +5,7 @@ from datetime import datetime, timedelta from dateutil.parser import parse -from .factories import ObservingRecordFactory, TargetFactory +from .factories import ObservingRecordFactory, SiderealTargetFactory from tom_observations.models import ObservationGroup, DynamicCadence from tom_observations.cadences.resume_cadence_after_failure import ResumeCadenceAfterFailureStrategy from tom_observations.cadences.retry_failed_observations import RetryFailedObservationsStrategy @@ -44,7 +44,7 @@ @patch('tom_observations.facilities.lco.LCOFacility.validate_observation') class TestReactiveCadencing(TestCase): def setUp(self): - target = TargetFactory.create() + target = SiderealTargetFactory.create() obs_params['target_id'] = target.id observing_records = ObservingRecordFactory.create_batch(5, target_id=target.id, diff --git a/tom_observations/tests/tests.py b/tom_observations/tests/tests.py index efd2ebf02..f0c89d77c 100644 --- a/tom_observations/tests/tests.py +++ b/tom_observations/tests/tests.py @@ -9,7 +9,7 @@ from astropy.coordinates import get_sun, SkyCoord from astropy.time import Time -from .factories import ObservingRecordFactory, ObservationTemplateFactory, TargetFactory, TargetNameFactory +from .factories import ObservingRecordFactory, ObservationTemplateFactory, SiderealTargetFactory, TargetNameFactory from tom_observations.utils import get_astroplan_sun_and_time, get_sidereal_visibility from tom_observations.tests.utils import FakeRoboticFacility from tom_observations.models import ObservationRecord, ObservationGroup, ObservationTemplate @@ -22,7 +22,7 @@ TARGET_PERMISSIONS_ONLY=True) class TestObservationViews(TestCase): def setUp(self): - self.target = TargetFactory.create() + self.target = SiderealTargetFactory.create() self.target_name = TargetNameFactory.create(target=self.target) self.observation_record = ObservingRecordFactory.create( target_id=self.target.id, @@ -137,7 +137,7 @@ def test_submit_observation_manual(self): TARGET_PERMISSIONS_ONLY=False) class TestObservationViewsRowLevelPermissions(TestCase): def setUp(self): - self.target = TargetFactory.create() + self.target = SiderealTargetFactory.create() self.target_name = TargetNameFactory.create(target=self.target) self.observation_record = ObservingRecordFactory.create( target_id=self.target.id, @@ -217,11 +217,11 @@ def test_observation_template_delete(self): class TestUpdatingObservations(TestCase): def setUp(self): - self.t1 = TargetFactory.create() + self.t1 = SiderealTargetFactory.create() self.or1 = ObservingRecordFactory.create(target_id=self.t1.id, facility='FakeRoboticFacility', status='PENDING') self.or2 = ObservingRecordFactory.create(target_id=self.t1.id, status='COMPLETED') self.or3 = ObservingRecordFactory.create(target_id=self.t1.id, facility='FakeRoboticFacility', status='PENDING') - self.t2 = TargetFactory.create() + self.t2 = SiderealTargetFactory.create() self.or4 = ObservingRecordFactory.create(target_id=self.t2.id, status='PENDING') # Tests that only 2 of the three created observing records are updated, as From cfae0f0700b3f3b381b52eb1d3c46700901406aa Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 3 Nov 2020 17:30:10 -0800 Subject: [PATCH 392/424] Mocking out a method --- tom_observations/tests/facilities/test_lco.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tom_observations/tests/facilities/test_lco.py b/tom_observations/tests/facilities/test_lco.py index 7505a9e0b..636d664bf 100644 --- a/tom_observations/tests/facilities/test_lco.py +++ b/tom_observations/tests/facilities/test_lco.py @@ -256,8 +256,10 @@ def test_build_configuration(self, mock_validate, mock_insts, mock_filters, mock for key in ['target', 'instrument_configs', 'acquisition_config', 'guiding_config']: self.assertIn(key, configuration) - def test_build_location(self, mock_validate, mock_insts, mock_filters, mock_proposals): + @patch('tom_observations.facilities.lco.LCOBaseForm._get_instruments') + def test_build_location(self, mock_get_instruments, mock_validate, mock_insts, mock_filters, mock_proposals): """Test _build_location method.""" + mock_get_instruments.return_value = {k: v for k, v in instrument_response.items() if 'SOAR' not in k} mock_validate.return_value = [] mock_insts.return_value = self.instrument_choices mock_filters.return_value = self.filter_choices From 846cd60a5c23537946d0110981071f2c29402fbf Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 3 Nov 2020 17:30:34 -0800 Subject: [PATCH 393/424] Fixing a small indentation discrepancy --- tom_common/templates/tom_common/base.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tom_common/templates/tom_common/base.html b/tom_common/templates/tom_common/base.html index deedcf3f3..29d871c80 100644 --- a/tom_common/templates/tom_common/base.html +++ b/tom_common/templates/tom_common/base.html @@ -75,8 +75,8 @@ {% block javascript %} - {% endblock %} - {% block extra_javascript %} - {% endblock %} + {% endblock %} + {% block extra_javascript %} + {% endblock %} From becae091e93b9ead184a23d7e9f71d63e726ea2c Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 3 Nov 2020 18:13:56 -0800 Subject: [PATCH 394/424] Added more lco tests --- tom_observations/facilities/lco.py | 23 +++++------ tom_observations/tests/facilities/test_lco.py | 40 +++++++++++++++++-- 2 files changed, 48 insertions(+), 15 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index d90b1de4b..db2982171 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -138,7 +138,6 @@ def filter_choices(): @staticmethod def proposal_choices(): - print('here') response = make_request( 'GET', PORTAL_URL + '/api/profile/', @@ -360,21 +359,21 @@ def _expand_cadence_request(self, payload): def observation_payload(self): payload = { - "name": self.cleaned_data['name'], - "proposal": self.cleaned_data['proposal'], - "ipp_value": self.cleaned_data['ipp_value'], - "operator": "SINGLE", - "observation_type": self.cleaned_data['observation_mode'], - "requests": [ + 'name': self.cleaned_data['name'], + 'proposal': self.cleaned_data['proposal'], + 'ipp_value': self.cleaned_data['ipp_value'], + 'operator': 'SINGLE', + 'observation_type': self.cleaned_data['observation_mode'], + 'requests': [ { - "configurations": [self._build_configuration()], - "windows": [ + 'configurations': [self._build_configuration()], + 'windows': [ { - "start": self.cleaned_data['start'], - "end": self.cleaned_data['end'] + 'start': self.cleaned_data['start'], + 'end': self.cleaned_data['end'] } ], - "location": self._build_location() + 'location': self._build_location() } ] } diff --git a/tom_observations/tests/facilities/test_lco.py b/tom_observations/tests/facilities/test_lco.py index 636d664bf..a887dff54 100644 --- a/tom_observations/tests/facilities/test_lco.py +++ b/tom_observations/tests/facilities/test_lco.py @@ -132,7 +132,7 @@ def setUp(self): self.valid_form_data = { 'name': 'test', 'facility': 'LCO', 'target_id': self.st.id, 'ipp_value': 0.5, 'start': '2020-11-03', 'end': '2020-11-04', 'exposure_count': 1, 'exposure_time': 30, 'max_airmass': 3, - 'min_lunar_distance': 20, 'period': 60, 'jitter': 15, 'observation_mode': 'NORMAL', + 'min_lunar_distance': 20, 'observation_mode': 'NORMAL', 'proposal': 'sampleproposal', 'filter': 'opaque', 'instrument_type': '0M4-SCICAM-SBIG' } self.instrument_choices = [(k, v['name']) for k, v in instrument_response.items() if 'SOAR' not in k] @@ -272,8 +272,42 @@ def test_build_location(self, mock_get_instruments, mock_validate, mock_insts, m def test_expand_cadence_request(self, mock_validate, mock_insts, mock_filters, mock_proposals): pass - def test_observation_payload(self, mock_validate, mock_insts, mock_filters, mock_proposals): - pass + @patch('tom_observations.facilities.lco.LCOBaseObservationForm._build_location') + @patch('tom_observations.facilities.lco.LCOBaseObservationForm._build_configuration') + @patch('tom_observations.facilities.lco.make_request') + def test_observation_payload(self, mock_make_request, mock_build_configuration, mock_build_location, mock_validate, + mock_insts, mock_filters, mock_proposals): + """Test observation_payload method.""" + mock_build_configuration.return_value = {} + mock_build_location.return_value = {} + mock_validate.return_value = [] + mock_insts.return_value = self.instrument_choices + mock_filters.return_value = self.filter_choices + mock_proposals.return_value = self.proposal_choices + + # Test a non-static cadenced form + with self.subTest(): + form = LCOBaseObservationForm(self.valid_form_data) + self.assertTrue(form.is_valid()) + obs_payload = form.observation_payload() + self.assertDictContainsSubset( + {'name': 'test', 'proposal': 'sampleproposal', 'ipp_value': 0.5, 'operator': 'SINGLE', + 'observation_type': 'NORMAL'}, obs_payload + ) + self.assertNotIn('cadence', obs_payload['requests'][0]) + + # Test a static cadence form + with self.subTest(): + mock_response = Response() + mock_response._content = str.encode(json.dumps({'test': 'test_static_cadence'})) + mock_response.status_code = 200 + mock_make_request.return_value = mock_response + + self.valid_form_data['period'] = 60 + self.valid_form_data['jitter'] = 15 + form = LCOBaseObservationForm(self.valid_form_data) + self.assertTrue(form.is_valid()) + self.assertDictEqual({'test': 'test_static_cadence'}, form.observation_payload()) class TestLCOImagingObservationForm(TestCase): From ebe13f6770ca67f5e32869258db4ccd9db0080dd Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 3 Nov 2020 19:43:05 -0800 Subject: [PATCH 395/424] Added tests for two more LCO forms --- tom_observations/facilities/lco.py | 16 ++- tom_observations/tests/facilities/test_lco.py | 100 +++++++++++++++++- 2 files changed, 107 insertions(+), 9 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index db2982171..69eb89750 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -112,6 +112,7 @@ def __init__(self, *args, **kwargs): @staticmethod def _get_instruments(): + print('bad') cached_instruments = cache.get('lco_instruments') if not cached_instruments: @@ -442,14 +443,19 @@ def layout(self): ) ) - def instrument_choices(self): - return sorted([(k, v['name']) for k, v in self._get_instruments().items() if 'SPECTRA' in v['type']], - key=lambda inst: inst[1]) + @staticmethod + def instrument_choices(): + return sorted( + [(k, v['name']) + for k, v in LCOSpectroscopyObservationForm._get_instruments().items() + if 'SPECTRA' in v['type']], + key=lambda inst: inst[1]) # NRES does not take a slit, and therefore needs an option of None - def filter_choices(self): + @staticmethod + def filter_choices(): return sorted(set([ - (f['code'], f['name']) for ins in self._get_instruments().values() for f in + (f['code'], f['name']) for ins in LCOSpectroscopyObservationForm._get_instruments().values() for f in ins['optical_elements'].get('slits', []) ] + [('None', 'None')]), key=lambda filter_tuple: filter_tuple[1]) diff --git a/tom_observations/tests/facilities/test_lco.py b/tom_observations/tests/facilities/test_lco.py index a887dff54..17a80e6e8 100644 --- a/tom_observations/tests/facilities/test_lco.py +++ b/tom_observations/tests/facilities/test_lco.py @@ -6,7 +6,8 @@ from tom_common.exceptions import ImproperCredentialsException from tom_observations.facilities.lco import make_request -from tom_observations.facilities.lco import LCOBaseForm, LCOBaseObservationForm +from tom_observations.facilities.lco import LCOBaseForm, LCOBaseObservationForm, LCOImagingObservationForm +from tom_observations.facilities.lco import LCOSpectroscopyObservationForm from tom_observations.tests.factories import SiderealTargetFactory, NonSiderealTargetFactory @@ -95,6 +96,7 @@ def test_instrument_choices(self, mock_get_instruments): inst_choices = LCOBaseForm.instrument_choices() self.assertIn(('2M0-FLOYDS-SCICAM', '2.0 meter FLOYDS'), inst_choices) self.assertIn(('0M4-SCICAM-SBIG', '0.4 meter SBIG'), inst_choices) + self.assertEqual(len(inst_choices), 2) @patch('tom_observations.facilities.lco.LCOBaseForm._get_instruments') def test_filter_choices(self, mock_get_instruments): @@ -124,7 +126,7 @@ def test_proposal_choices(self, mock_make_request): @patch('tom_observations.facilities.lco.LCOBaseForm.filter_choices') @patch('tom_observations.facilities.lco.LCOBaseForm.instrument_choices') @patch('tom_observations.facilities.lco.LCOBaseObservationForm.validate_at_facility') -class TestLCOBaseObservationFormPayload(TestCase): +class TestLCOBaseObservationForm(TestCase): def setUp(self): self.st = SiderealTargetFactory.create() @@ -142,6 +144,9 @@ def setUp(self): ]) self.proposal_choices = [('sampleproposal', 'Sample Proposal')] + def test_validate_at_facility(self, mock_validate, mock_insts, mock_filters, mock_proposals): + pass + def test_clean_and_validate(self, mock_validate, mock_insts, mock_filters, mock_proposals): """Test clean_start, clean_end, and is_valid()""" mock_validate.return_value = [] @@ -310,12 +315,99 @@ def test_observation_payload(self, mock_make_request, mock_build_configuration, self.assertDictEqual({'test': 'test_static_cadence'}, form.observation_payload()) +@patch('tom_observations.facilities.lco.LCOImagingObservationForm._get_instruments') class TestLCOImagingObservationForm(TestCase): - pass + def test_instrument_choices(self, mock_get_instruments): + """Test LCOImagingObservationForm._instrument_choices.""" + mock_get_instruments.return_value = {k: v for k, v in instrument_response.items() if 'SOAR' not in k} + + inst_choices = LCOImagingObservationForm.instrument_choices() + self.assertIn(('0M4-SCICAM-SBIG', '0.4 meter SBIG'), inst_choices) + self.assertNotIn(('2M0-FLOYDS-SCICAM', '2.0 meter FLOYDS'), inst_choices) + self.assertEqual(len(inst_choices), 1) + + def test_filter_choices(self, mock_get_instruments): + """Test LCOImagingObservationForm._filter_choices.""" + mock_get_instruments.return_value = {k: v for k, v in instrument_response.items() if 'SOAR' not in k} + + filter_choices = LCOImagingObservationForm.filter_choices() + for expected in [('opaque', 'Opaque'), ('100um-Pinhole', '100um Pinhole')]: + self.assertIn(expected, filter_choices) + for not_expected in [('slit_6.0as', '6.0 arcsec slit'), ('slit_1.6as', '1.6 arcsec slit'), + ('slit_2.0as', '2.0 arcsec slit'), ('slit_1.2as', '1.2 arcsec slit')]: + self.assertNotIn(not_expected, filter_choices) + self.assertEqual(len(filter_choices), 2) class TestLCOSpectroscopyObservationForm(TestCase): - pass + @patch('tom_observations.facilities.lco.LCOSpectroscopyObservationForm._get_instruments') + def test_instrument_choices(self, mock_get_instruments): + """Test LCOSpectroscopyObservationForm._instrument_choices.""" + mock_get_instruments.return_value = {k: v for k, v in instrument_response.items() if 'SOAR' not in k} + + inst_choices = LCOSpectroscopyObservationForm.instrument_choices() + self.assertIn(('2M0-FLOYDS-SCICAM', '2.0 meter FLOYDS'), inst_choices) + self.assertNotIn(('0M4-SCICAM-SBIG', '0.4 meter SBIG'), inst_choices) + self.assertEqual(len(inst_choices), 1) + + @patch('tom_observations.facilities.lco.LCOSpectroscopyObservationForm._get_instruments') + def test_filter_choices(self, mock_get_instruments): + """Test LCOSpectroscopyObservationForm._filter_choices.""" + mock_get_instruments.return_value = {k: v for k, v in instrument_response.items() if 'SOAR' not in k} + + filter_choices = LCOSpectroscopyObservationForm.filter_choices() + for expected in [('slit_6.0as', '6.0 arcsec slit'), ('slit_1.6as', '1.6 arcsec slit'), + ('slit_2.0as', '2.0 arcsec slit'), ('slit_1.2as', '1.2 arcsec slit'), ('None', 'None')]: + self.assertIn(expected, filter_choices) + for not_expected in [('opaque', 'Opaque'), ('100um-Pinhole', '100um Pinhole')]: + self.assertNotIn(not_expected, filter_choices) + self.assertEqual(len(filter_choices), 5) + + @patch('tom_observations.facilities.lco.LCOSpectroscopyObservationForm.proposal_choices') + @patch('tom_observations.facilities.lco.LCOSpectroscopyObservationForm.filter_choices') + @patch('tom_observations.facilities.lco.LCOSpectroscopyObservationForm.instrument_choices') + @patch('tom_observations.facilities.lco.LCOSpectroscopyObservationForm.validate_at_facility') + def test_build_instrument_config(self, mock_validate, mock_insts, mock_filters, mock_proposals): + mock_validate.return_value = [] + mock_insts.return_value = [(k, v['name']) for k, v in instrument_response.items() if 'SPECTRA' in v['type']] + mock_filters.return_value = set([ + (f['code'], f['name']) for ins in instrument_response.values() for f in + ins['optical_elements'].get('slits', []) + ] + [('None', 'None')]) + mock_proposals.return_value = [('sampleproposal', 'Sample Proposal')] + + st = SiderealTargetFactory.create() + valid_form_data = { + 'name': 'test', 'facility': 'LCO', 'target_id': st.id, 'ipp_value': 0.5, 'start': '2020-11-03', + 'end': '2020-11-04', 'exposure_count': 1, 'exposure_time': 30.0, 'max_airmass': 3, + 'min_lunar_distance': 20, 'observation_mode': 'NORMAL', 'proposal': 'sampleproposal', + 'filter': 'slit_2.0as', 'instrument_type': '2M0-FLOYDS-SCICAM', 'rotator_angle': 1.0 + } + + # Test that optical_elements['slit'] is populated when filter is included + with self.subTest(): + form = LCOSpectroscopyObservationForm(valid_form_data) + self.assertTrue(form.is_valid()) + self.assertEqual( + [{'exposure_count': valid_form_data['exposure_count'], + 'exposure_time': valid_form_data['exposure_time'], + 'optical_elements': {'slit': valid_form_data['filter']}, + 'rotator_mode': 'VFLOAT', + 'extra_params': {'rotator_angle': valid_form_data['rotator_angle']} + }], form._build_instrument_config() + ) + + # Test that optical elements is removed when filter is excluded + with self.subTest(): + valid_form_data['filter'] = 'None' + form = LCOSpectroscopyObservationForm(valid_form_data) + self.assertTrue(form.is_valid()) + self.assertEqual( + [{'exposure_count': valid_form_data['exposure_count'], + 'exposure_time': valid_form_data['exposure_time'], + 'rotator_mode': 'VFLOAT', 'extra_params': {'rotator_angle': valid_form_data['rotator_angle']} + }], form._build_instrument_config() + ) class TestLCOPhotometricSequenceForm(TestCase): From c7c1cf151b04ba3102358cc3d9ccc63454822ff2 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 3 Nov 2020 19:48:40 -0800 Subject: [PATCH 396/424] Fixing style check --- tom_observations/tests/facilities/test_lco.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_observations/tests/facilities/test_lco.py b/tom_observations/tests/facilities/test_lco.py index 17a80e6e8..b5141d3e2 100644 --- a/tom_observations/tests/facilities/test_lco.py +++ b/tom_observations/tests/facilities/test_lco.py @@ -406,7 +406,7 @@ def test_build_instrument_config(self, mock_validate, mock_insts, mock_filters, [{'exposure_count': valid_form_data['exposure_count'], 'exposure_time': valid_form_data['exposure_time'], 'rotator_mode': 'VFLOAT', 'extra_params': {'rotator_angle': valid_form_data['rotator_angle']} - }], form._build_instrument_config() + }], form._build_instrument_config() ) From 5afcdb3a7cbadd2a3fc96a6e69627fb566ab9580 Mon Sep 17 00:00:00 2001 From: David Collom Date: Tue, 3 Nov 2020 21:27:30 -0800 Subject: [PATCH 397/424] Added tests for photometric sequence form --- tom_observations/facilities/lco.py | 8 +- tom_observations/tests/facilities/test_lco.py | 78 +++++++++++++++++-- tom_observations/widgets.py | 2 +- 3 files changed, 79 insertions(+), 9 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 69eb89750..59d9ed8ee 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -112,7 +112,6 @@ def __init__(self, *args, **kwargs): @staticmethod def _get_instruments(): - print('bad') cached_instruments = cache.get('lco_instruments') if not cached_instruments: @@ -551,11 +550,14 @@ def clean(self): return cleaned_data - def instrument_choices(self): + @staticmethod + def instrument_choices(): """ This method returns only the instrument choices available in the current SNEx photometric sequence form. """ - return sorted([(k, v['name']) for k, v in self._get_instruments().items() if k in self.valid_instruments], + return sorted([(k, v['name']) + for k, v in LCOPhotometricSequenceForm._get_instruments().items() + if k in LCOPhotometricSequenceForm.valid_instruments], key=lambda inst: inst[1]) def cadence_layout(self): diff --git a/tom_observations/tests/facilities/test_lco.py b/tom_observations/tests/facilities/test_lco.py index b5141d3e2..e0f0ec353 100644 --- a/tom_observations/tests/facilities/test_lco.py +++ b/tom_observations/tests/facilities/test_lco.py @@ -1,3 +1,4 @@ +from datetime import datetime, timedelta import json from requests import Response from unittest.mock import patch @@ -7,7 +8,7 @@ from tom_common.exceptions import ImproperCredentialsException from tom_observations.facilities.lco import make_request from tom_observations.facilities.lco import LCOBaseForm, LCOBaseObservationForm, LCOImagingObservationForm -from tom_observations.facilities.lco import LCOSpectroscopyObservationForm +from tom_observations.facilities.lco import LCOPhotometricSequenceForm, LCOSpectroscopyObservationForm from tom_observations.tests.factories import SiderealTargetFactory, NonSiderealTargetFactory @@ -122,9 +123,9 @@ def test_proposal_choices(self, mock_make_request): self.assertNotIn(('InactiveProposal', 'Inactive (InactiveProposal)'), proposal_choices) -@patch('tom_observations.facilities.lco.LCOBaseForm.proposal_choices') -@patch('tom_observations.facilities.lco.LCOBaseForm.filter_choices') -@patch('tom_observations.facilities.lco.LCOBaseForm.instrument_choices') +@patch('tom_observations.facilities.lco.LCOBaseObservationForm.proposal_choices') +@patch('tom_observations.facilities.lco.LCOBaseObservationForm.filter_choices') +@patch('tom_observations.facilities.lco.LCOBaseObservationForm.instrument_choices') @patch('tom_observations.facilities.lco.LCOBaseObservationForm.validate_at_facility') class TestLCOBaseObservationForm(TestCase): @@ -411,7 +412,74 @@ def test_build_instrument_config(self, mock_validate, mock_insts, mock_filters, class TestLCOPhotometricSequenceForm(TestCase): - pass + + def setUp(self): + self.st = SiderealTargetFactory.create() + self.valid_form_data = { + 'name': 'test', 'facility': 'LCO', 'target_id': self.st.id, 'ipp_value': 0.5, 'max_airmass': 3, + 'min_lunar_distance': 20, 'observation_mode': 'NORMAL', 'proposal': 'sampleproposal', + 'instrument_type': '0M4-SCICAM-SBIG', 'cadence_frequency': 24, + 'U_0': 30.0, 'U_1': 1, 'U_2': 1, 'B_0': 60.0, 'B_1': 2, 'B_2': 1, + } + self.instrument_choices = [(k, v['name']) for k, v in instrument_response.items() if 'SOAR' not in k] + self.filter_choices = set([ + (f['code'], f['name']) for ins in instrument_response.values() for f in + ins['optical_elements'].get('filters', []) + ins['optical_elements'].get('slits', []) + ]) + self.proposal_choices = [('sampleproposal', 'Sample Proposal')] + + @patch('tom_observations.facilities.lco.LCOPhotometricSequenceForm._get_instruments') + def test_instrument_choices(self, mock_get_instruments): + """Test LCOPhotometricSequenceForm._instrument_choices.""" + mock_get_instruments.return_value = {k: v for k, v in instrument_response.items() if 'SOAR' not in k} + + inst_choices = LCOPhotometricSequenceForm.instrument_choices() + self.assertIn(('0M4-SCICAM-SBIG', '0.4 meter SBIG'), inst_choices) + self.assertNotIn(('2M0-FLOYDS-SCICAM', '2.0 meter FLOYDS'), inst_choices) + self.assertEqual(len(inst_choices), 1) + + @patch('tom_observations.facilities.lco.LCOPhotometricSequenceForm.proposal_choices') + @patch('tom_observations.facilities.lco.LCOPhotometricSequenceForm.filter_choices') + @patch('tom_observations.facilities.lco.LCOPhotometricSequenceForm.instrument_choices') + @patch('tom_observations.facilities.lco.LCOPhotometricSequenceForm.validate_at_facility') + def test_build_instrument_config(self, mock_validate, mock_insts, mock_filters, mock_proposals): + mock_validate.return_value = [] + mock_insts.return_value = self.instrument_choices + mock_filters.return_value = self.filter_choices + mock_proposals.return_value = self.proposal_choices + + form = LCOPhotometricSequenceForm(self.valid_form_data) + self.assertTrue(form.is_valid()) + inst_config = form._build_instrument_config() + self.assertEqual(len(inst_config), 2) + self.assertIn({'exposure_count': 1, 'exposure_time': 30.0, 'optical_elements': {'filter': 'U'}}, inst_config) + self.assertIn({'exposure_count': 2, 'exposure_time': 60.0, 'optical_elements': {'filter': 'B'}}, inst_config) + + @patch('tom_observations.facilities.lco.LCOPhotometricSequenceForm.proposal_choices') + @patch('tom_observations.facilities.lco.LCOPhotometricSequenceForm.filter_choices') + @patch('tom_observations.facilities.lco.LCOPhotometricSequenceForm.instrument_choices') + @patch('tom_observations.facilities.lco.LCOPhotometricSequenceForm.validate_at_facility') + def test_clean(self, mock_validate, mock_insts, mock_filters, mock_proposals): + mock_validate.return_value = [] + mock_insts.return_value = self.instrument_choices + mock_filters.return_value = self.filter_choices + mock_proposals.return_value = self.proposal_choices + + # Test that a valid form returns True, and that start and end are cleaned properly + form = LCOPhotometricSequenceForm(self.valid_form_data) + self.assertTrue(form.is_valid()) + self.assertAlmostEqual(datetime.strftime(datetime.now(), '%Y-%m-%dT%H:%M:%S'), form.cleaned_data['start']) + self.assertAlmostEqual( + datetime.strftime( + datetime.now() + timedelta(hours=form.cleaned_data['cadence_frequency']), '%Y-%m-%dT%H:%M:%S' + ), + form.cleaned_data['end'] + ) + + # Test that an invalid form returns False + self.valid_form_data.pop('target_id') + form = LCOPhotometricSequenceForm(self.valid_form_data) + self.assertFalse(form.is_valid()) class TestLCOSpectroscopicSequenceForm(TestCase): diff --git a/tom_observations/widgets.py b/tom_observations/widgets.py index 8c3a31abb..0c166da1b 100644 --- a/tom_observations/widgets.py +++ b/tom_observations/widgets.py @@ -24,7 +24,7 @@ class FilterField(forms.MultiValueField): widget = FilterConfigurationWidget def __init__(self, *args, **kwargs): - fields = (forms.IntegerField(), forms.IntegerField(), forms.IntegerField()) + fields = (forms.FloatField(), forms.IntegerField(), forms.IntegerField()) super().__init__(fields, *args, **kwargs) def compress(self, data_list): From ae5a3dd3f21aa0ec960fc8c5cf3c9b55169dba3c Mon Sep 17 00:00:00 2001 From: David Collom Date: Wed, 4 Nov 2020 13:34:36 -0800 Subject: [PATCH 398/424] Adding file for soar tests --- tom_observations/tests/facilities/test_soar.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tom_observations/tests/facilities/test_soar.py diff --git a/tom_observations/tests/facilities/test_soar.py b/tom_observations/tests/facilities/test_soar.py new file mode 100644 index 000000000..e69de29bb From 67434fb4c3c05edd9e4515d1384a46f95c34d1e9 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2020 13:54:53 +0000 Subject: [PATCH 399/424] Bump djangorestframework from 3.12.1 to 3.12.2 Bumps [djangorestframework](https://github.com/encode/django-rest-framework) from 3.12.1 to 3.12.2. - [Release notes](https://github.com/encode/django-rest-framework/releases) - [Commits](https://github.com/encode/django-rest-framework/compare/3.12.1...3.12.2) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cdbfe5366..2586c7a3c 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ 'beautifulsoup4==4.9.3', 'dataclasses; python_version < "3.7"', 'django==3.1.3', # TOM Toolkit requires db math functions - 'djangorestframework==3.12.1', + 'djangorestframework==3.12.2', 'django-bootstrap4==2.3.1', 'django-contrib-comments==1.9.2', # Earlier version are incompatible with Django >= 3.0 'django-crispy-forms==1.9.2', From bf8ad17d5715f1911d3669a79f6f234ac9c7096a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2020 13:28:07 +0000 Subject: [PATCH 400/424] Bump requests from 2.24.0 to 2.25.0 Bumps [requests](https://github.com/psf/requests) from 2.24.0 to 2.25.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/master/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.24.0...v2.25.0) Signed-off-by: dependabot-preview[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2586c7a3c..f36713614 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ 'pillow==8.0.1', 'plotly==4.12.0', 'python-dateutil==2.8.1', - 'requests==2.24.0', + 'requests==2.25.0', 'specutils==1.1', ], extras_require={ From 1cf6eddb88d1f4159a3dadd5e4e793fcc5756c07 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 12 Nov 2020 16:17:33 -0800 Subject: [PATCH 401/424] Added form validation to Lasair and added basic tests --- tom_alerts/brokers/lasair.py | 9 ++++ tom_alerts/tests/brokers/test_lasair.py | 62 +++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 tom_alerts/tests/brokers/test_lasair.py diff --git a/tom_alerts/brokers/lasair.py b/tom_alerts/brokers/lasair.py index ee21f8e54..c68d34d79 100644 --- a/tom_alerts/brokers/lasair.py +++ b/tom_alerts/brokers/lasair.py @@ -11,6 +11,15 @@ class LasairBrokerForm(GenericQueryForm): cone = forms.CharField(required=False, label='Object Cone Search', help_text='Object RA and Dec') sqlquery = forms.CharField(required=False, label='Freeform SQL query', help_text='SQL query') + def clean(self): + cleaned_data = super().clean() + + # Ensure that either cone search or sqlquery are populated + if not (cleaned_data['cone'] or cleaned_data['sqlquery']): + raise forms.ValidationError('One of either Object Cone Search or Freeform SQL Query must be populated.') + + return cleaned_data + def get_lasair_object(objectId): url = LASAIR_URL + '/object/' + objectId + '/json/' diff --git a/tom_alerts/tests/brokers/test_lasair.py b/tom_alerts/tests/brokers/test_lasair.py new file mode 100644 index 000000000..fd01e6032 --- /dev/null +++ b/tom_alerts/tests/brokers/test_lasair.py @@ -0,0 +1,62 @@ +from unittest import mock + +from django.test import override_settings, tag, TestCase + +from tom_alerts.alerts import get_service_class +from tom_alerts.brokers.lasair import LasairBroker, LasairBrokerForm + + +class TestLasairBrokerForm(TestCase): + def setUp(self): + pass + + def test_clean(self): + form_parameters = {'query_name': 'Test Lasair', 'broker': 'Lasair', 'name': 'ZTF18abbkloa', + 'cone': '', 'sqlquery': ''} + + with self.subTest(): + form = LasairBrokerForm(form_parameters) + self.assertFalse(form.is_valid()) + self.assertIn('One of either Object Cone Search or Freeform SQL Query must be populated.', + form.non_field_errors()) + + test_parameters_list = [{'cone': '1, 2', 'sqlquery': ''}, {'cone': '', 'sqlquery': 'select * from objects;'}] + for test_params in test_parameters_list: + with self.subTest(): + form_parameters.update(test_params) + form = LasairBrokerForm(form_parameters) + self.assertTrue(form.is_valid()) + + +@override_settings(TOM_ALERT_CLASSES=['tom_alerts.brokers.lasair.LasairBroker']) +class TestLasairBrokerClass(TestCase): + """ Test the functionality of the LasairBroker, we modify the django settings to make sure + it is the only installed broker. + """ + def setUp(self): + pass + + def test_get_broker_class(self): + self.assertEqual(LasairBroker, get_service_class('Lasair')) + + @mock.patch('tom_alerts.brokers.lasair.requests.get') + def test_fetch_alerts(self, mock_requests_get): + pass + + def test_to_target(self): + pass + + def test_to_generic_alert(self): + pass + + +@tag('canary') +class TestLasairModuleCanary(TestCase): + def setUp(self): + self.broker = LasairBroker() + + def test_fetch_alerts(self): + pass + + def test_fetch_alert(self): + pass From b9a8f1105884661dded46dccd9f60d2f0572a5f8 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 12 Nov 2020 17:08:35 -0800 Subject: [PATCH 402/424] Added tests for spectroscopic sequence form --- tom_observations/facilities/lco.py | 26 +-- tom_observations/tests/facilities/test_lco.py | 157 +++++++++++++++++- 2 files changed, 170 insertions(+), 13 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index 59d9ed8ee..bbcaab060 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -607,7 +607,7 @@ def layout(self): class LCOSpectroscopicSequenceForm(LCOBaseObservationForm): site = forms.ChoiceField(choices=(('any', 'Any'), ('ogg', 'Hawaii'), ('coj', 'Australia'))) - acquisition_radius = forms.FloatField(min_value=0) + acquisition_radius = forms.FloatField(min_value=0, required=False) guider_mode = forms.ChoiceField(choices=[('on', 'On'), ('off', 'Off'), ('optional', 'Optional')], required=True) guider_exposure_time = forms.IntegerField(min_value=0) cadence_frequency = forms.IntegerField(required=True, @@ -655,13 +655,13 @@ def _build_instrument_config(self): def _build_acquisition_config(self): acquisition_config = super()._build_acquisition_config() # SNEx uses WCS mode if no acquisition radius is specified, and BRIGHTEST otherwise - acquisition_mode = 'BRIGHTEST' if not self.cleaned_data['acquisition_radius']: - acquisition_mode = 'WCS' - acquisition_config['mode'] = acquisition_mode - acquisition_config['extra_params'] = { - 'acquire_radius': self.cleaned_data['acquisition_radius'] - } + acquisition_config['mode'] = 'WCS' + else: + acquisition_config['mode'] = 'BRIGHTEST' + acquisition_config['extra_params'] = { + 'acquire_radius': self.cleaned_data['acquisition_radius'] + } return acquisition_config @@ -697,15 +697,19 @@ def clean(self): return cleaned_data - def instrument_choices(self): + @staticmethod + def instrument_choices(): # SNEx only uses the Spectroscopic Sequence Form with FLOYDS # This doesn't need to be sorted because it will only return one instrument - return [(k, v['name']) for k, v in self._get_instruments().items() if k == '2M0-FLOYDS-SCICAM'] + return [(k, v['name']) + for k, v in LCOSpectroscopicSequenceForm._get_instruments().items() + if k == '2M0-FLOYDS-SCICAM'] - def filter_choices(self): + @staticmethod + def filter_choices(): # SNEx only uses the Spectroscopic Sequence Form with FLOYDS return sorted(set([ - (f['code'], f['name']) for name, ins in self._get_instruments().items() for f in + (f['code'], f['name']) for name, ins in LCOSpectroscopicSequenceForm._get_instruments().items() for f in ins['optical_elements'].get('slits', []) if name == '2M0-FLOYDS-SCICAM' ]), key=lambda filter_tuple: filter_tuple[1]) diff --git a/tom_observations/tests/facilities/test_lco.py b/tom_observations/tests/facilities/test_lco.py index e0f0ec353..253625422 100644 --- a/tom_observations/tests/facilities/test_lco.py +++ b/tom_observations/tests/facilities/test_lco.py @@ -8,7 +8,8 @@ from tom_common.exceptions import ImproperCredentialsException from tom_observations.facilities.lco import make_request from tom_observations.facilities.lco import LCOBaseForm, LCOBaseObservationForm, LCOImagingObservationForm -from tom_observations.facilities.lco import LCOPhotometricSequenceForm, LCOSpectroscopyObservationForm +from tom_observations.facilities.lco import LCOPhotometricSequenceForm, LCOSpectroscopicSequenceForm +from tom_observations.facilities.lco import LCOSpectroscopyObservationForm from tom_observations.tests.factories import SiderealTargetFactory, NonSiderealTargetFactory @@ -483,7 +484,159 @@ def test_clean(self, mock_validate, mock_insts, mock_filters, mock_proposals): class TestLCOSpectroscopicSequenceForm(TestCase): - pass + def setUp(self): + self.st = SiderealTargetFactory.create() + self.valid_form_data = { + 'name': 'test', 'facility': 'LCO', 'target_id': self.st.id, 'exposure_count': 1, 'exposure_time': 30, + 'max_airmass': 3, 'min_lunar_distance': 20, 'site': 'any', 'ipp_value': 0.5, 'filter': 'slit_1.2as', + 'observation_mode': 'NORMAL', 'proposal': 'sampleproposal', 'acquisition_radius': 1, + 'guider_mode': 'on', 'guider_exposure_time': 30, 'instrument_type': '0M4-SCICAM-SBIG', + 'cadence_frequency': 24 + } + self.instrument_choices = [(k, v['name']) for k, v in instrument_response.items() if 'SOAR' not in k] + self.filter_choices = set([ + (f['code'], f['name']) for ins in instrument_response.values() for f in + ins['optical_elements'].get('filters', []) + ins['optical_elements'].get('slits', []) + ]) + self.proposal_choices = [('sampleproposal', 'Sample Proposal')] + + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm._get_instruments') + def test_instrument_choices(self, mock_get_instruments): + """Test LCOSpectroscopicSequenceForm._instrument_choices.""" + mock_get_instruments.return_value = {k: v for k, v in instrument_response.items() if 'SOAR' not in k} + + inst_choices = LCOSpectroscopicSequenceForm.instrument_choices() + self.assertIn(('2M0-FLOYDS-SCICAM', '2.0 meter FLOYDS'), inst_choices) + self.assertNotIn(('0M4-SCICAM-SBIG', '0.4 meter SBIG'), inst_choices) + self.assertEqual(len(inst_choices), 1) + + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm._get_instruments') + def test_filter_choices(self, mock_get_instruments): + """Test LCOSpectroscopicSequenceForm._instrument_choices.""" + mock_get_instruments.return_value = {k: v for k, v in instrument_response.items() if 'SOAR' not in k} + + filter_choices = LCOSpectroscopicSequenceForm.filter_choices() + for expected in [('slit_6.0as', '6.0 arcsec slit'), ('slit_1.6as', '1.6 arcsec slit'), + ('slit_2.0as', '2.0 arcsec slit'), ('slit_1.2as', '1.2 arcsec slit')]: + self.assertIn(expected, filter_choices) + for not_expected in [('opaque', 'Opaque'), ('100um-Pinhole', '100um Pinhole')]: + self.assertNotIn(not_expected, filter_choices) + self.assertEqual(len(filter_choices), 4) + + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.proposal_choices') + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.filter_choices') + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.instrument_choices') + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.validate_at_facility') + def test_build_instrument_config(self, mock_validate, mock_insts, mock_filters, mock_proposals): + mock_validate.return_value = [] + mock_insts.return_value = self.instrument_choices + mock_filters.return_value = self.filter_choices + mock_proposals.return_value = self.proposal_choices + + form = LCOSpectroscopicSequenceForm(self.valid_form_data) + self.assertTrue(form.is_valid()) + inst_config = form._build_instrument_config() + self.assertEqual(len(inst_config), 1) + self.assertIn({'exposure_count': 1, 'exposure_time': 30.0, 'optical_elements': {'slit': 'slit_1.2as'}}, + inst_config) + self.assertNotIn('filter', inst_config[0]['optical_elements']) + + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.proposal_choices') + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.filter_choices') + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.instrument_choices') + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.validate_at_facility') + def test_build_acquisition_config(self, mock_validate, mock_insts, mock_filters, mock_proposals): + mock_validate.return_value = [] + mock_insts.return_value = self.instrument_choices + mock_filters.return_value = self.filter_choices + mock_proposals.return_value = self.proposal_choices + + with self.subTest(): + form = LCOSpectroscopicSequenceForm(self.valid_form_data) + self.assertTrue(form.is_valid()) + acquisition_config = form._build_acquisition_config() + self.assertDictEqual({'mode': 'BRIGHTEST', 'extra_params': {'acquire_radius': 1}}, + acquisition_config) + + with self.subTest(): + self.valid_form_data.pop('acquisition_radius') + form = LCOSpectroscopicSequenceForm(self.valid_form_data) + self.assertTrue(form.is_valid()) + acquisition_config = form._build_acquisition_config() + self.assertDictEqual({'mode': 'WCS'}, acquisition_config) + + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.proposal_choices') + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.filter_choices') + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.instrument_choices') + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.validate_at_facility') + def test_build_guiding_config(self, mock_validate, mock_insts, mock_filters, mock_proposals): + mock_validate.return_value = [] + mock_insts.return_value = self.instrument_choices + mock_filters.return_value = self.filter_choices + mock_proposals.return_value = self.proposal_choices + + test_params = [ + ({'guider_mode': 'on'}, {'mode': 'ON', 'optional': 'false'}), + ({'guider_mode': 'off'}, {'mode': 'OFF', 'optional': 'false'}), + ({'guider_mode': 'optional'}, {'mode': 'ON', 'optional': 'true'}) + ] + for params in test_params: + with self.subTest(): + self.valid_form_data.update(params[0]) + form = LCOSpectroscopicSequenceForm(self.valid_form_data) + self.assertTrue(form.is_valid()) + guiding_config = form._build_guiding_config() + self.assertDictEqual(params[1], guiding_config) + + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.proposal_choices') + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.filter_choices') + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.instrument_choices') + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.validate_at_facility') + def test_build_location(self, mock_validate, mock_insts, mock_filters, mock_proposals): + mock_validate.return_value = [] + mock_insts.return_value = self.instrument_choices + mock_filters.return_value = self.filter_choices + mock_proposals.return_value = self.proposal_choices + + test_params = [ + ({'site': 'ogg'}, {'site': 'ogg'}), + ({'site': 'coj'}, {'site': 'coj'}), + ({'site': 'any'}, {}) + ] + for params in test_params: + with self.subTest(): + self.valid_form_data.update(params[0]) + form = LCOSpectroscopicSequenceForm(self.valid_form_data) + self.assertTrue(form.is_valid()) + location = form._build_location() + self.assertDictContainsSubset(params[1], location) + + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.proposal_choices') + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.filter_choices') + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.instrument_choices') + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.validate_at_facility') + def test_clean(self, mock_validate, mock_insts, mock_filters, mock_proposals): + mock_validate.return_value = [] + mock_insts.return_value = self.instrument_choices + mock_filters.return_value = self.filter_choices + mock_proposals.return_value = self.proposal_choices + + # Test that a valid form returns True, and that start and end are cleaned properly + form = LCOSpectroscopicSequenceForm(self.valid_form_data) + self.assertTrue(form.is_valid()) + self.assertEqual(form.cleaned_data['instrument_type'], '2M0-FLOYDS-SCICAM') + self.assertAlmostEqual(datetime.strftime(datetime.now(), '%Y-%m-%dT%H:%M:%S'), form.cleaned_data['start']) + self.assertAlmostEqual( + datetime.strftime( + datetime.now() + timedelta(hours=form.cleaned_data['cadence_frequency']), '%Y-%m-%dT%H:%M:%S' + ), + form.cleaned_data['end'] + ) + + # Test that an invalid form returns False + self.valid_form_data.pop('target_id') + form = LCOSpectroscopicSequenceForm(self.valid_form_data) + self.assertFalse(form.is_valid()) class TestLCOObservationTemplateForm(TestCase): From de3e6a3c2e6b0b754c6c64a2e8daa0b292e8cd55 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 12 Nov 2020 17:16:44 -0800 Subject: [PATCH 403/424] Fixing missed mock --- tom_observations/tests/facilities/test_lco.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tom_observations/tests/facilities/test_lco.py b/tom_observations/tests/facilities/test_lco.py index 253625422..095c723c7 100644 --- a/tom_observations/tests/facilities/test_lco.py +++ b/tom_observations/tests/facilities/test_lco.py @@ -592,7 +592,9 @@ def test_build_guiding_config(self, mock_validate, mock_insts, mock_filters, moc @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.filter_choices') @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.instrument_choices') @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm.validate_at_facility') - def test_build_location(self, mock_validate, mock_insts, mock_filters, mock_proposals): + @patch('tom_observations.facilities.lco.LCOSpectroscopicSequenceForm._get_instruments') + def test_build_location(self, mock_get_instruments, mock_validate, mock_insts, mock_filters, mock_proposals): + mock_get_instruments.return_value = {k: v for k, v in instrument_response.items() if 'SOAR' not in k} mock_validate.return_value = [] mock_insts.return_value = self.instrument_choices mock_filters.return_value = self.filter_choices From 299e820d76b7a593033a9e3f1866315ca4a6bfa0 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 12 Nov 2020 18:15:36 -0800 Subject: [PATCH 404/424] Added soar tests --- tom_observations/facilities/soar.py | 36 ++- .../tests/facilities/test_soar.py | 211 ++++++++++++++++++ 2 files changed, 241 insertions(+), 6 deletions(-) diff --git a/tom_observations/facilities/soar.py b/tom_observations/facilities/soar.py index e42a84ff1..022fe4dcd 100644 --- a/tom_observations/facilities/soar.py +++ b/tom_observations/facilities/soar.py @@ -34,7 +34,8 @@ def make_request(*args, **kwargs): class SOARBaseObservationForm(LCOBaseObservationForm): - def _get_instruments(self): + @staticmethod + def _get_instruments(): cached_instruments = cache.get('soar_instruments') if not cached_instruments: @@ -49,7 +50,8 @@ def _get_instruments(self): return cached_instruments - def instrument_to_type(self, instrument_type): + @staticmethod + def instrument_to_type(instrument_type): if 'IMAGER' in instrument_type: return 'EXPOSE' else: @@ -57,14 +59,36 @@ def instrument_to_type(self, instrument_type): class SOARImagingObservationForm(SOARBaseObservationForm, LCOImagingObservationForm): - pass + + @staticmethod + def instrument_choices(): + return sorted( + [(k, v['name']) for k, v in SOARBaseObservationForm._get_instruments().items() if 'IMAGE' in v['type']], + key=lambda inst: inst[1] + ) + + @staticmethod + def filter_choices(): + return sorted(set([ + (f['code'], f['name']) for ins in SOARBaseObservationForm._get_instruments().values() for f in + ins['optical_elements'].get('filters', []) + ]), key=lambda filter_tuple: filter_tuple[1]) class SOARSpectroscopyObservationForm(SOARBaseObservationForm, LCOSpectroscopyObservationForm): - def filter_choices(self): + @staticmethod + def instrument_choices(): + return sorted( + [(k, v['name']) + for k, v in SOARSpectroscopyObservationForm._get_instruments().items() + if 'SPECTRA' in v['type']], + key=lambda inst: inst[1]) + + @staticmethod + def filter_choices(): return set([ - (f['code'], f['name']) for ins in self._get_instruments().values() for f in + (f['code'], f['name']) for ins in SOARSpectroscopyObservationForm._get_instruments().values() for f in ins['optical_elements'].get('slits', []) ]) @@ -82,7 +106,7 @@ def _build_instrument_config(self): } } - return instrument_config + return [instrument_config] class SOARFacility(LCOFacility): diff --git a/tom_observations/tests/facilities/test_soar.py b/tom_observations/tests/facilities/test_soar.py index e69de29bb..2ee26ad75 100644 --- a/tom_observations/tests/facilities/test_soar.py +++ b/tom_observations/tests/facilities/test_soar.py @@ -0,0 +1,211 @@ +import json +from requests import Response +from unittest.mock import patch + +from django.test import TestCase + +from tom_common.exceptions import ImproperCredentialsException +from tom_observations.facilities.soar import make_request, SOARBaseObservationForm, SOARImagingObservationForm +from tom_observations.facilities.soar import SOARSpectroscopyObservationForm +from tom_observations.tests.factories import NonSiderealTargetFactory, SiderealTargetFactory + + +instrument_response = { + '2M0-FLOYDS-SCICAM': { + 'type': 'SPECTRA', 'class': '2m0', 'name': '2.0 meter FLOYDS', 'optical_elements': { + 'slits': [ + {'name': '6.0 arcsec slit', 'code': 'slit_6.0as', 'schedulable': True, 'default': False}, + {'name': '1.6 arcsec slit', 'code': 'slit_1.6as', 'schedulable': True, 'default': False}, + {'name': '2.0 arcsec slit', 'code': 'slit_2.0as', 'schedulable': True, 'default': False}, + {'name': '1.2 arcsec slit', 'code': 'slit_1.2as', 'schedulable': True, 'default': False} + ] + } + }, + '0M4-SCICAM-SBIG': { + 'type': 'IMAGE', 'class': '0m4', 'name': '0.4 meter SBIG', 'optical_elements': { + 'filters': [ + {'name': 'Opaque', 'code': 'opaque', 'schedulable': False, 'default': False}, + {'name': '100um Pinhole', 'code': '100um-Pinhole', 'schedulable': False, 'default': False}, + ] + }, + }, + 'SOAR_GHTS_REDCAM': { + 'type': 'SPECTRA', 'class': '4m0', 'name': 'Goodman Spectrograph RedCam', 'optical_elements': { + 'gratings': [ + {'name': '400 line grating', 'code': 'SYZY_400', 'schedulable': True, 'default': True}, + ], + 'slits': [ + {'name': '1.0 arcsec slit', 'code': 'slit_1.0as', 'schedulable': True, 'default': True} + ] + }, + }, + 'SOAR_GHTS_REDCAM_IMAGER': { + 'type': 'IMAGE', 'class': '4m0', 'name': 'Goodman Spectrograph RedCam Imager', 'optical_elements': { + 'filters': [ + {'name': 'Clear', 'code': 'air', 'schedulable': True, 'default': False}, + {'name': 'GHTS u-SDSS', 'code': 'u-SDSS', 'schedulable': False, 'default': False}, + {'name': 'GHTS g-SDSS', 'code': 'g-SDSS', 'schedulable': True, 'default': False}, + {'name': 'GHTS r-SDSS', 'code': 'r-SDSS', 'schedulable': True, 'default': True}, + {'name': 'GHTS i-SDSS', 'code': 'i-SDSS', 'schedulable': True, 'default': False}, + {'name': 'GHTS z-SDSS', 'code': 'z-SDSS', 'schedulable': False, 'default': False}, + {'name': 'GHTS VR', 'code': 'VR', 'schedulable': True, 'default': False} + ] + }, + } +} + + +class TestMakeRequest(TestCase): + + @patch('tom_observations.facilities.soar.requests.request') + def test_make_request(self, mock_request): + mock_response = Response() + mock_response._content = str.encode(json.dumps({'test': 'test'})) + mock_response.status_code = 200 + mock_request.return_value = mock_response + + self.assertDictEqual({'test': 'test'}, make_request('GET', 'google.com', headers={'test': 'test'}).json()) + + mock_response.status_code = 403 + mock_request.return_value = mock_response + with self.assertRaises(ImproperCredentialsException): + make_request('GET', 'google.com', headers={'test': 'test'}) + + +class TestSOARBaseObservationForm(TestCase): + + def setUp(self): + self.st = SiderealTargetFactory.create() + self.nst = NonSiderealTargetFactory.create(scheme='MPC_MINOR_PLANET') + self.valid_form_data = { + 'name': 'test', 'facility': 'SOAR', 'target_id': self.st.id, 'ipp_value': 0.5, 'start': '2020-11-03', + 'end': '2020-11-04', 'exposure_count': 1, 'exposure_time': 30, 'max_airmass': 3, + 'min_lunar_distance': 20, 'observation_mode': 'NORMAL', + 'proposal': 'sampleproposal', 'filter': 'opaque', 'instrument_type': 'SOAR_GHTS_REDCAM_IMAGER' + } + self.instrument_choices = [(k, v['name']) for k, v in instrument_response.items() if 'SOAR' in k] + self.filter_choices = set([ + (f['code'], f['name']) for ins in instrument_response.values() for f in + ins['optical_elements'].get('filters', []) + ins['optical_elements'].get('slits', []) + ]) + self.proposal_choices = [('sampleproposal', 'Sample Proposal')] + + @patch('tom_observations.facilities.soar.make_request') + @patch('tom_observations.facilities.soar.cache') + def test_get_instruments(self, mock_cache, mock_make_request): + mock_response = Response() + mock_response._content = str.encode(json.dumps(instrument_response)) + mock_response.status_code = 200 + mock_make_request.return_value = mock_response + + # Test that cached value is returned + with self.subTest(): + test_instruments = {'test instrument': {'type': 'IMAGE'}} + mock_cache.get.return_value = test_instruments + + instruments = SOARBaseObservationForm._get_instruments() + self.assertDictContainsSubset({'test instrument': {'type': 'IMAGE'}}, instruments) + self.assertNotIn('0M4-SCICAM-SBIG', instruments) + + # Test that empty cache results in mock_instruments, and cache.set is called + with self.subTest(): + mock_cache.get.return_value = None + + instruments = SOARBaseObservationForm._get_instruments() + self.assertIn('SOAR_GHTS_REDCAM_IMAGER', instruments) + self.assertDictContainsSubset({'type': 'IMAGE'}, instruments['SOAR_GHTS_REDCAM_IMAGER']) + self.assertNotIn('0M4-SCICAM-SBIG', instruments) + mock_cache.set.assert_called() + + @patch('tom_observations.facilities.soar.SOARBaseObservationForm.proposal_choices') + @patch('tom_observations.facilities.soar.SOARBaseObservationForm.filter_choices') + @patch('tom_observations.facilities.soar.SOARBaseObservationForm.instrument_choices') + @patch('tom_observations.facilities.soar.SOARBaseObservationForm.validate_at_facility') + def test_instrument_to_type(self, mock_validate, mock_insts, mock_filters, mock_proposals): + """Test instrument_to_type method.""" + self.assertEqual('EXPOSE', SOARBaseObservationForm.instrument_to_type('SOAR_GHTS_REDCAM_IMAGER')) + self.assertEqual('SPECTRUM', SOARBaseObservationForm.instrument_to_type('SOAR_GHTS_REDCAM')) + + +@patch('tom_observations.facilities.soar.SOARImagingObservationForm._get_instruments') +class TestSOARImagingObservationForm(TestCase): + def test_instrument_choices(self, mock_get_instruments): + """Test SOARImagingObservationForm._instrument_choices.""" + mock_get_instruments.return_value = {k: v for k, v in instrument_response.items() if 'SOAR' in k} + + inst_choices = SOARImagingObservationForm.instrument_choices() + self.assertIn(('SOAR_GHTS_REDCAM_IMAGER', 'Goodman Spectrograph RedCam Imager'), inst_choices) + self.assertNotIn(('SOAR_GHTS_REDCAM', 'Goodman Spectrograph RedCam'), inst_choices) + self.assertEqual(len(inst_choices), 1) + + def test_filter_choices(self, mock_get_instruments): + """Test SOARImagingObservationForm._filter_choices.""" + mock_get_instruments.return_value = {k: v for k, v in instrument_response.items() if 'SOAR' in k} + + filter_choices = SOARImagingObservationForm.filter_choices() + for expected in [('air', 'Clear'), ('u-SDSS', 'GHTS u-SDSS'), ('g-SDSS', 'GHTS g-SDSS'), + ('r-SDSS', 'GHTS r-SDSS'), ('i-SDSS', 'GHTS i-SDSS'), ('z-SDSS', 'GHTS z-SDSS'), + ('VR', 'GHTS VR')]: + self.assertIn(expected, filter_choices) + for not_expected in [('slit_1.0as', '1.0 arcsec slit')]: + self.assertNotIn(not_expected, filter_choices) + self.assertEqual(len(filter_choices), 7) + + +class TestSOARSpectroscopyObservationForm(TestCase): + + @patch('tom_observations.facilities.soar.SOARSpectroscopyObservationForm._get_instruments') + def test_instrument_choices(self, mock_get_instruments): + """Test SOARSpectroscopyObservationForm._instrument_choices.""" + mock_get_instruments.return_value = {k: v for k, v in instrument_response.items() if 'SOAR' in k} + + inst_choices = SOARSpectroscopyObservationForm.instrument_choices() + self.assertIn(('SOAR_GHTS_REDCAM', 'Goodman Spectrograph RedCam'), inst_choices) + self.assertNotIn(('SOAR_GHTS_REDCAM_IMAGER', 'Goodman Spectrograph RedCam Imager'), inst_choices) + self.assertEqual(len(inst_choices), 1) + + @patch('tom_observations.facilities.soar.SOARSpectroscopyObservationForm._get_instruments') + def test_filter_choices(self, mock_get_instruments): + """Test SOARSpectroscopyObservationForm._filter_choices.""" + mock_get_instruments.return_value = {k: v for k, v in instrument_response.items() if 'SOAR' in k} + + filter_choices = SOARSpectroscopyObservationForm.filter_choices() + for expected in [('slit_1.0as', '1.0 arcsec slit')]: + self.assertIn(expected, filter_choices) + for not_expected in [('u-SDSS', 'GHTS u-SDSS'), ('i-SDSS', 'GHTS i-SDSS')]: + self.assertNotIn(not_expected, filter_choices) + self.assertEqual(len(filter_choices), 1) + + @patch('tom_observations.facilities.soar.SOARSpectroscopyObservationForm.proposal_choices') + @patch('tom_observations.facilities.soar.SOARSpectroscopyObservationForm.filter_choices') + @patch('tom_observations.facilities.soar.SOARSpectroscopyObservationForm.instrument_choices') + @patch('tom_observations.facilities.soar.SOARSpectroscopyObservationForm.validate_at_facility') + def test_build_instrument_config(self, mock_validate, mock_insts, mock_filters, mock_proposals): + mock_validate.return_value = [] + mock_insts.return_value = [(k, v['name']) for k, v in instrument_response.items() if 'SPECTRA' in v['type']] + mock_filters.return_value = set([ + (f['code'], f['name']) for ins in instrument_response.values() for f in + ins['optical_elements'].get('slits', []) + ] + [('None', 'None')]) + mock_proposals.return_value = [('sampleproposal', 'Sample Proposal')] + + st = SiderealTargetFactory.create() + valid_form_data = { + 'name': 'test', 'facility': 'SOAR', 'target_id': st.id, 'ipp_value': 0.5, 'start': '2020-11-03', + 'end': '2020-11-04', 'exposure_count': 1, 'exposure_time': 30.0, 'max_airmass': 3, + 'min_lunar_distance': 20, 'observation_mode': 'NORMAL', 'proposal': 'sampleproposal', + 'filter': 'slit_1.0as', 'instrument_type': 'SOAR_GHTS_REDCAM', 'rotator_angle': 1.0 + } + + # Test that optical_elements['slit'] and optical_elements['grating] are populated when filter is included + with self.subTest(): + form = SOARSpectroscopyObservationForm(valid_form_data) + self.assertTrue(form.is_valid()) + self.assertEqual( + [{'exposure_count': valid_form_data['exposure_count'], + 'exposure_time': valid_form_data['exposure_time'], + 'optical_elements': {'slit': valid_form_data['filter'], 'grating': 'SYZY_400'}, + 'rotator_mode': 'SKY', + 'extra_params': {'rotator_angle': valid_form_data['rotator_angle']} + }], form._build_instrument_config() + ) From 33c15749d540a599bf0676b5bda401d817584948 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 12 Nov 2020 18:39:09 -0800 Subject: [PATCH 405/424] Fixing static method call in SOAR --- tom_observations/facilities/soar.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tom_observations/facilities/soar.py b/tom_observations/facilities/soar.py index 022fe4dcd..8179d5f6b 100644 --- a/tom_observations/facilities/soar.py +++ b/tom_observations/facilities/soar.py @@ -37,6 +37,7 @@ class SOARBaseObservationForm(LCOBaseObservationForm): @staticmethod def _get_instruments(): cached_instruments = cache.get('soar_instruments') + cached_instruments = None if not cached_instruments: response = make_request( @@ -63,14 +64,14 @@ class SOARImagingObservationForm(SOARBaseObservationForm, LCOImagingObservationF @staticmethod def instrument_choices(): return sorted( - [(k, v['name']) for k, v in SOARBaseObservationForm._get_instruments().items() if 'IMAGE' in v['type']], + [(k, v['name']) for k, v in SOARImagingObservationForm._get_instruments().items() if 'IMAGE' in v['type']], key=lambda inst: inst[1] ) @staticmethod def filter_choices(): return sorted(set([ - (f['code'], f['name']) for ins in SOARBaseObservationForm._get_instruments().values() for f in + (f['code'], f['name']) for ins in SOARImagingObservationForm._get_instruments().values() for f in ins['optical_elements'].get('filters', []) ]), key=lambda filter_tuple: filter_tuple[1]) From 3bb66801d62faddc6a93bae7881c1eaa2c1a83ac Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 12 Nov 2020 18:44:55 -0800 Subject: [PATCH 406/424] re-enabled cache after mistakenly disabling --- tom_observations/facilities/soar.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tom_observations/facilities/soar.py b/tom_observations/facilities/soar.py index 8179d5f6b..a68b23319 100644 --- a/tom_observations/facilities/soar.py +++ b/tom_observations/facilities/soar.py @@ -37,7 +37,6 @@ class SOARBaseObservationForm(LCOBaseObservationForm): @staticmethod def _get_instruments(): cached_instruments = cache.get('soar_instruments') - cached_instruments = None if not cached_instruments: response = make_request( From 82101a92a9c19f0ff8ab0f59ecb758bc47824252 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 12 Nov 2020 20:54:17 -0800 Subject: [PATCH 407/424] Removed tom_publications --- MANIFEST.in | 4 +- tom_base/settings.py | 6 - tom_common/urls.py | 1 - .../observationgroup_list.html | 4 +- tom_publications/__init__.py | 0 tom_publications/admin.py | 1 - tom_publications/apps.py | 5 - tom_publications/forms.py | 14 --- tom_publications/latex.py | 103 ------------------ tom_publications/migrations/0001_initial.py | 23 ---- tom_publications/migrations/__init__.py | 0 tom_publications/models.py | 7 -- tom_publications/processors/__init__.py | 0 .../observation_group_latex_processor.py | 44 -------- .../processors/target_list_latex_processor.py | 47 -------- .../tom_publications/latex_table.html | 32 ------ .../partials/latex_button.html | 1 - tom_publications/templatetags/__init__.py | 0 .../templatetags/publication_extras.py | 12 -- tom_publications/urls.py | 9 -- tom_publications/views.py | 47 -------- tom_setup/templates/tom_setup/settings.tmpl | 6 - .../templates/tom_targets/target_detail.html | 2 +- .../tom_targets/target_grouping.html | 4 +- 24 files changed, 4 insertions(+), 368 deletions(-) delete mode 100644 tom_publications/__init__.py delete mode 100644 tom_publications/admin.py delete mode 100644 tom_publications/apps.py delete mode 100644 tom_publications/forms.py delete mode 100644 tom_publications/latex.py delete mode 100644 tom_publications/migrations/0001_initial.py delete mode 100644 tom_publications/migrations/__init__.py delete mode 100644 tom_publications/models.py delete mode 100644 tom_publications/processors/__init__.py delete mode 100644 tom_publications/processors/observation_group_latex_processor.py delete mode 100644 tom_publications/processors/target_list_latex_processor.py delete mode 100644 tom_publications/templates/tom_publications/latex_table.html delete mode 100644 tom_publications/templates/tom_publications/partials/latex_button.html delete mode 100644 tom_publications/templatetags/__init__.py delete mode 100644 tom_publications/templatetags/publication_extras.py delete mode 100644 tom_publications/urls.py delete mode 100644 tom_publications/views.py diff --git a/MANIFEST.in b/MANIFEST.in index 2d1f67976..3ebd22483 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -11,6 +11,4 @@ recursive-include tom_setup/templates * recursive-include tom_observations/static * recursive-include tom_observations/templates * recursive-include tom_dataproducts/static * -recursive-include tom_dataproducts/templates * -recursive-include tom_publications/static * -recursive-include tom_publications/templates * \ No newline at end of file +recursive-include tom_dataproducts/templates * \ No newline at end of file diff --git a/tom_base/settings.py b/tom_base/settings.py index 319ecb151..a560c455b 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -53,7 +53,6 @@ 'tom_catalogs', 'tom_observations', 'tom_dataproducts', - 'tom_publications', ] SITE_ID = 1 @@ -206,11 +205,6 @@ 'spectroscopy': 'tom_dataproducts.processors.spectroscopy_processor.SpectroscopyProcessor', } -TOM_LATEX_PROCESSORS = { - 'ObservationGroup': 'tom_publications.processors.observation_group_latex_processor.ObservationGroupLatexProcessor', - 'TargetList': 'tom_publications.processors.target_list_latex_processor.TargetListLatexProcessor' -} - TOM_FACILITY_CLASSES = [ 'tom_observations.facilities.lco.LCOFacility', 'tom_observations.facilities.gemini.GEMFacility', diff --git a/tom_common/urls.py b/tom_common/urls.py index f9c183fc1..ae7faf1ab 100644 --- a/tom_common/urls.py +++ b/tom_common/urls.py @@ -44,7 +44,6 @@ path('catalogs/', include('tom_catalogs.urls')), path('observations/', include('tom_observations.urls', namespace='observations')), path('dataproducts/', include('tom_dataproducts.urls', namespace='dataproducts')), - path('publications/', include('tom_publications.urls', namespace='publications')), path('users/', UserListView.as_view(), name='user-list'), path('users//changepassword/', UserPasswordChangeView.as_view(), name='admin-user-change-password'), path('users/create/', UserCreateView.as_view(), name='user-create'), diff --git a/tom_observations/templates/tom_observations/observationgroup_list.html b/tom_observations/templates/tom_observations/observationgroup_list.html index 57cdf346e..212bb953b 100644 --- a/tom_observations/templates/tom_observations/observationgroup_list.html +++ b/tom_observations/templates/tom_observations/observationgroup_list.html @@ -1,5 +1,5 @@ {% extends 'tom_common/base.html' %} -{% load bootstrap4 publication_extras %} +{% load bootstrap4 %} {% block title %}Target Groups{% endblock %} {% block content %}

    Observation Groups

    @@ -17,7 +17,6 @@

    Observation Groups

    Name Total Observations - Generate Latex Delete @@ -32,7 +31,6 @@

    Observation Groups

    {{ group.observation_records.count }} - {% latex_button group %} Delete diff --git a/tom_publications/__init__.py b/tom_publications/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tom_publications/admin.py b/tom_publications/admin.py deleted file mode 100644 index 846f6b406..000000000 --- a/tom_publications/admin.py +++ /dev/null @@ -1 +0,0 @@ -# Register your models here. diff --git a/tom_publications/apps.py b/tom_publications/apps.py deleted file mode 100644 index 974c5693e..000000000 --- a/tom_publications/apps.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.apps import AppConfig - - -class TomPublicationssConfig(AppConfig): - name = 'tom_publicationss' diff --git a/tom_publications/forms.py b/tom_publications/forms.py deleted file mode 100644 index 1322da272..000000000 --- a/tom_publications/forms.py +++ /dev/null @@ -1,14 +0,0 @@ -from django import forms - - -class LatexTableForm(forms.Form): - - model_pk = forms.IntegerField( - widget=forms.HiddenInput(), - required=True - ) - model_name = forms.CharField( - widget=forms.HiddenInput(), - required=True - ) - template = forms.CharField(widget=forms.HiddenInput(), required=False) diff --git a/tom_publications/latex.py b/tom_publications/latex.py deleted file mode 100644 index 121a3f8f3..000000000 --- a/tom_publications/latex.py +++ /dev/null @@ -1,103 +0,0 @@ -from importlib import import_module -import io - -from abc import ABC -from astropy.io import ascii -from crispy_forms.helper import FormHelper -from crispy_forms.layout import Layout, Submit -from django import forms -from django.conf import settings - - -DEFAULT_LATEX_PROCESSOR_CLASSES = { - 'ObservationGroup': 'tom_publications.processors.observation_group_latex_processor.ObservationGroupLatexProcessor', - 'TargetList': 'tom_publications.processors.target_list_latex_processor.TargetListLatexProcessor' -} - - -def get_latex_processor(model_name): - try: - processor_class = settings.TOM_LATEX_PROCESSORS[model_name] - except AttributeError: - processor_class = DEFAULT_LATEX_PROCESSOR_CLASSES[model_name] - - try: - mod_name, class_name = processor_class.rsplit('.', 1) - mod = import_module(mod_name) - clazz = getattr(mod, class_name) - except (ImportError, AttributeError): - raise ImportError('Could not import {}. Did you provide the correct path?'.format(processor_class)) - - latex_processor = clazz() - return latex_processor - - -class GenericLatexForm(forms.Form): - - model_pk = forms.IntegerField( - widget=forms.HiddenInput(), - required=True - ) - model_name = forms.CharField( - widget=forms.HiddenInput(), - required=True - ) - table_header = forms.CharField( - required=False, - widget=forms.TextInput() - ) - table_footer = forms.CharField( - required=False, - widget=forms.TextInput() - ) - template = forms.CharField(widget=forms.HiddenInput(), required=False) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.helper = FormHelper() - self.helper.add_input(Submit('create-latex', 'Create Table')) - # if self.is_bound: - # self.helper.add_input(Submit('save-latex', 'Save Latex Config')) - self.common_layout = Layout('model_pk', 'model_name', 'table_header', 'table_footer', 'template') - - -class GenericLatexProcessor(ABC): - """ - The latex processor class contains the logic to render a Latex-formatted table using the fields from a TOM model. - All abstract methods need to be implemented by any subclasses of the GenericLatexProcessor. In order to make use of - a latex processor, add the model type and processor path to ``TOM_LATEX_PROCESSORS`` in your ``settings.py``. - """ - - form_class = GenericLatexForm - - def get_form(self, data=None, **kwargs): - """ - This method returns the form class specified for the processor class. - """ - return self.form_class(data, **kwargs) - - def create_latex_table_data(self, cleaned_data): - """ - This method creates the actual table data to be passed to the latex generator. - - :param cleaned_data: Cleaned form data from a Django form - :type cleaned_data: dict - - :returns: dict of tabular data. Keys should be column headers, with values being lists of ordered data. - :rtype: dict - """ - return {} - - def generate_latex(self, cleaned_data): - """ - This method takes in the data from a form.clean() and returns a string of latex. - """ - - table_data = self.create_latex_table_data(cleaned_data) - - latex_dict = ascii.latex.latexdicts['AA'] - latex_dict.update({'caption': cleaned_data.get('table_header'), 'tablefoot': cleaned_data.get('table_footer')}) - - latex = io.StringIO() - ascii.write(table_data, latex, format='latex', latexdict=latex_dict) - return latex.getvalue() diff --git a/tom_publications/migrations/0001_initial.py b/tom_publications/migrations/0001_initial.py deleted file mode 100644 index fa790dec0..000000000 --- a/tom_publications/migrations/0001_initial.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 2.2.9 on 2020-01-27 18:05 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='LatexConfiguration', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('fields', models.TextField(default='')), - ('model_name', models.CharField(default='', max_length=120)), - ('template', models.CharField(default='', max_length=200)), - ], - ), - ] diff --git a/tom_publications/migrations/__init__.py b/tom_publications/migrations/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tom_publications/models.py b/tom_publications/models.py deleted file mode 100644 index 78f25616b..000000000 --- a/tom_publications/models.py +++ /dev/null @@ -1,7 +0,0 @@ -from django.db import models - - -class LatexConfiguration(models.Model): - fields = models.TextField(blank=False, default='') - model_name = models.CharField(blank=False, default='', max_length=120) - template = models.CharField(blank=False, default='', max_length=200) diff --git a/tom_publications/processors/__init__.py b/tom_publications/processors/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tom_publications/processors/observation_group_latex_processor.py b/tom_publications/processors/observation_group_latex_processor.py deleted file mode 100644 index 511385053..000000000 --- a/tom_publications/processors/observation_group_latex_processor.py +++ /dev/null @@ -1,44 +0,0 @@ -from crispy_forms.bootstrap import InlineCheckboxes -from crispy_forms.layout import Layout -from django import forms -from django.core.exceptions import FieldDoesNotExist -from django.db.models import Field - -from tom_publications.latex import GenericLatexProcessor, GenericLatexForm -from tom_observations.models import ObservationRecord, ObservationGroup - - -class ObservationGroupLatexForm(GenericLatexForm): - field_list = forms.MultipleChoiceField( - choices=[(v.name, v.verbose_name) for v in ObservationRecord._meta.get_fields() if issubclass(type(v), Field)], - required=True, - widget=forms.CheckboxSelectMultiple() - ) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.helper.layout = Layout( - self.common_layout, - InlineCheckboxes('field_list') - ) - - -class ObservationGroupLatexProcessor(GenericLatexProcessor): - - form_class = ObservationGroupLatexForm - - def create_latex_table_data(self, cleaned_data): - # TODO: enable user to modify column header - # TODO: add preview PDF - observation_group = ObservationGroup.objects.get(pk=cleaned_data.get('model_pk')) - - table_data = {} - for field in cleaned_data.get('field_list', []): - for obs_record in observation_group.observation_records.all(): - try: - verbose_name = ObservationRecord._meta.get_field(field).verbose_name - table_data.setdefault(verbose_name, []).append(getattr(obs_record, field)) - except FieldDoesNotExist: - pass - - return table_data diff --git a/tom_publications/processors/target_list_latex_processor.py b/tom_publications/processors/target_list_latex_processor.py deleted file mode 100644 index d46c9b635..000000000 --- a/tom_publications/processors/target_list_latex_processor.py +++ /dev/null @@ -1,47 +0,0 @@ -from crispy_forms.bootstrap import InlineCheckboxes -from crispy_forms.layout import Layout -from django import forms -from django.conf import settings -from django.core.exceptions import FieldDoesNotExist -from django.db.models import Field - -from tom_publications.latex import GenericLatexProcessor, GenericLatexForm -from tom_targets.models import Target, TargetExtra, TargetList - - -class TargetListLatexForm(GenericLatexForm): - field_list = forms.MultipleChoiceField( - choices=[(v.name, v.verbose_name) for v in Target._meta.get_fields() - if issubclass(type(v), Field)] + [(e['name'], e['name']) for e in settings.EXTRA_FIELDS], - required=True, - widget=forms.CheckboxSelectMultiple() - ) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.helper.layout = Layout( - self.common_layout, - InlineCheckboxes('field_list') - ) - - -class TargetListLatexProcessor(GenericLatexProcessor): - - form_class = TargetListLatexForm - - def create_latex_table_data(self, cleaned_data): - # TODO: enable user to modify column header - # TODO: add preview PDF - target_list = TargetList.objects.get(pk=cleaned_data.get('model_pk')) - - table_data = {} - for field in cleaned_data.get('field_list', []): - for target in target_list.targets.all(): - try: - verbose_name = Target._meta.get_field(field).verbose_name - table_data.setdefault(verbose_name, []).append(getattr(target, field)) - except FieldDoesNotExist: - table_data.setdefault(field, []).append(TargetExtra.objects.filter(target=target, - key=field).first().value) - - return table_data diff --git a/tom_publications/templates/tom_publications/latex_table.html b/tom_publications/templates/tom_publications/latex_table.html deleted file mode 100644 index faa2ff97a..000000000 --- a/tom_publications/templates/tom_publications/latex_table.html +++ /dev/null @@ -1,32 +0,0 @@ -{% extends 'tom_common/base.html' %} -{% load bootstrap4 crispy_forms_tags %} -{% bootstrap_javascript jquery='True' %} -{% block title %}Target {{ object.name }}{% endblock %} -{% block extra_javascript %} - -{% endblock %} -{% block content %} -
    -
    -

    Generate latex table for {{ object.name }}

    -
    - {% csrf_token %} - {% crispy latex_form %} -
    -
    -
    -
    -
    - {% if latex %} - - {% endif %} -
    - -
    -{% endblock %} \ No newline at end of file diff --git a/tom_publications/templates/tom_publications/partials/latex_button.html b/tom_publications/templates/tom_publications/partials/latex_button.html deleted file mode 100644 index dcaeae127..000000000 --- a/tom_publications/templates/tom_publications/partials/latex_button.html +++ /dev/null @@ -1 +0,0 @@ -Generate Latex \ No newline at end of file diff --git a/tom_publications/templatetags/__init__.py b/tom_publications/templatetags/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tom_publications/templatetags/publication_extras.py b/tom_publications/templatetags/publication_extras.py deleted file mode 100644 index b99785015..000000000 --- a/tom_publications/templatetags/publication_extras.py +++ /dev/null @@ -1,12 +0,0 @@ -from django import template - -register = template.Library() - - -@register.inclusion_tag('tom_publications/partials/latex_button.html') -def latex_button(object): - """ - Renders a button that redirects to the LaTeX table generation page for the specified model instance. Requires an - object, which is generally the object in the context for the page on which the templatetag will be used. - """ - return {'model_name': object._meta.label, 'model_pk': object.id} diff --git a/tom_publications/urls.py b/tom_publications/urls.py deleted file mode 100644 index 1a7e61f6b..000000000 --- a/tom_publications/urls.py +++ /dev/null @@ -1,9 +0,0 @@ -from django.urls import path - -from tom_publications.views import LatexTableView - -app_name = 'tom_publications' - -urlpatterns = [ - path('latex/create/', LatexTableView.as_view(), name='create-latex'), -] diff --git a/tom_publications/views.py b/tom_publications/views.py deleted file mode 100644 index 9d62041ef..000000000 --- a/tom_publications/views.py +++ /dev/null @@ -1,47 +0,0 @@ -from django.apps import apps -from django.contrib.auth.mixins import LoginRequiredMixin -from django.views.generic import TemplateView - -from tom_publications.latex import get_latex_processor -from tom_publications.models import LatexConfiguration - - -class LatexTableView(LoginRequiredMixin, TemplateView): - template_name = 'tom_publications/latex_table.html' - - def get(self, request, *args, **kwargs): - context = super().get_context_data(**kwargs) - - model_name = request.GET.get('model_name') - obj = None - if not model_name: - raise Exception - else: - model = apps.get_model(model_name) - obj = model.objects.get(pk=request.GET.get('model_pk')) - - processor = get_latex_processor(model_name.split('.')[1]) - - latex = [] - if 'create-latex' in request.GET or 'save-latex' in request.GET: - latex_form = processor.get_form(request.GET) - if latex_form.is_valid(): - latex_form.clean() - - latex = processor.generate_latex( - latex_form.cleaned_data - ) - if request.GET.get('save-latex'): - config = LatexConfiguration( - fields=','.join(latex_form.cleaned_data['field_list']), - model_name=latex_form.cleaned_data['model_name'] - ) - config.save() - else: - latex_form = processor.get_form(initial=request.GET) - - context['object'] = obj - context['latex_form'] = latex_form - context['latex'] = latex - - return self.render_to_response(context) diff --git a/tom_setup/templates/tom_setup/settings.tmpl b/tom_setup/templates/tom_setup/settings.tmpl index ef8d8c111..2da1a214f 100644 --- a/tom_setup/templates/tom_setup/settings.tmpl +++ b/tom_setup/templates/tom_setup/settings.tmpl @@ -54,7 +54,6 @@ INSTALLED_APPS = [ 'tom_catalogs', 'tom_observations', 'tom_dataproducts', - 'tom_publications' ] SITE_ID = 1 @@ -230,11 +229,6 @@ DATA_PROCESSORS = { 'spectroscopy': 'tom_dataproducts.processors.spectroscopy_processor.SpectroscopyProcessor', } -TOM_LATEX_PROCESSORS = { - 'ObservationGroup': 'tom_publications.processors.latex_processor.ObservationGroupLatexProcessor', - 'TargetList': 'tom_publications.processors.target_list_latex_processor.TargetListLatexProcessor' -} - TOM_FACILITY_CLASSES = [ 'tom_observations.facilities.lco.LCOFacility', 'tom_observations.facilities.gemini.GEMFacility', diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index 0af781e01..95e6d5cd0 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -1,5 +1,5 @@ {% extends 'tom_common/base.html' %} -{% load comments bootstrap4 tom_common_extras targets_extras observation_extras dataproduct_extras publication_extras static cache %} +{% load comments bootstrap4 tom_common_extras targets_extras observation_extras dataproduct_extras static cache %} {% block title %}Target {{ object.name }}{% endblock %} {% block additional_css %} diff --git a/tom_targets/templates/tom_targets/target_grouping.html b/tom_targets/templates/tom_targets/target_grouping.html index 9e4294a26..d5889db1a 100644 --- a/tom_targets/templates/tom_targets/target_grouping.html +++ b/tom_targets/templates/tom_targets/target_grouping.html @@ -1,5 +1,5 @@ {% extends 'tom_common/base.html' %} -{% load bootstrap4 publication_extras %} +{% load bootstrap4 %} {% block title %}Target Groups{% endblock %} {% block content %}

    Target Groupings

    @@ -17,7 +17,6 @@

    Target Groupings

    Group Total Targets - Generate Latex Delete @@ -26,7 +25,6 @@

    Target Groupings

    {{ group.targets.count }} - {% latex_button group %} Delete {% empty %} From 848a25062fad9378bc05a939ed9cda9b0a0b4311 Mon Sep 17 00:00:00 2001 From: David Collom Date: Thu, 12 Nov 2020 21:07:15 -0800 Subject: [PATCH 408/424] Updating canary test data so it won't break when a new observation happens --- tom_alerts/tests/brokers/test_alerce.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_alerts/tests/brokers/test_alerce.py b/tom_alerts/tests/brokers/test_alerce.py index ef52d498d..fd61eaac1 100644 --- a/tom_alerts/tests/brokers/test_alerce.py +++ b/tom_alerts/tests/brokers/test_alerce.py @@ -313,7 +313,7 @@ def test_fetch_alert(self): self.assertDictContainsSubset({ 'oid': 'ZTF20acnsdjd', - 'last_magpsf_r': 17.8492107391357, + 'first_magpsf_g': 17.3446006774902, 'first_magpsf_r': 17.0198993682861, 'firstmjd': 59149.1119328998, }, alert) From 1c3b74e31b53913ac8b3f3bfffcbd08e565a111c Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 13 Nov 2020 13:45:28 -0800 Subject: [PATCH 409/424] Added release notes to top level and put a note in --- README-dev.md | 1 + docs/api/tom_alerts/brokers.rst | 5 ++++- docs/requirements.txt | 4 +++- docs/common/releasenotes.md => releasenotes.md | 4 ++++ 4 files changed, 12 insertions(+), 2 deletions(-) rename docs/common/releasenotes.md => releasenotes.md (98%) diff --git a/README-dev.md b/README-dev.md index fdff3b2b0..1e52200a8 100644 --- a/README-dev.md +++ b/README-dev.md @@ -27,6 +27,7 @@ Following deployment of a release, a Github Release is created, and this should ### Pre-release deployment 1. Meet pre-deployment criteria. + * Includes appropriate release notes, including breaking changes, in `releasenotes.md`. * Pass [Codacy code quality check](https://app.codacy.com/gh/TOMToolkit/tom_base/pullRequests). * Doesn't decrease [Coveralls test coverage](https://coveralls.io/github/TOMToolkit/tom_base). * Passes [Travis tests and code style check](https://travis-ci.com/github/TOMToolkit/tom_base/branches). diff --git a/docs/api/tom_alerts/brokers.rst b/docs/api/tom_alerts/brokers.rst index 5fa260da4..9514512ba 100644 --- a/docs/api/tom_alerts/brokers.rst +++ b/docs/api/tom_alerts/brokers.rst @@ -22,6 +22,8 @@ ALeRCE ANTARES ******* +.. automodule:: tom_antares.antares + :members: ****** Lasair @@ -43,7 +45,8 @@ MARS SCIMMA ****** - +.. automodule:: tom_scimma.scimma + :members: ***** Scout diff --git a/docs/requirements.txt b/docs/requirements.txt index 664569358..e9077fea9 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -18,4 +18,6 @@ python-dateutil recommonmark requests specutils -sphinx>=2.1.2 \ No newline at end of file +sphinx>=2.1.2 +tom_antares +tom_scimma \ No newline at end of file diff --git a/docs/common/releasenotes.md b/releasenotes.md similarity index 98% rename from docs/common/releasenotes.md rename to releasenotes.md index faa4665ae..d5618b33f 100644 --- a/docs/common/releasenotes.md +++ b/releasenotes.md @@ -1,5 +1,9 @@ # Release Notes +### 1.13.0 + +- + ### 1.6.1 - This release pins the Django version in order to address a security vulnerability. From a7ca4b39030e9451d9e90d71e5a24ef8752ab006 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 13 Nov 2020 14:27:29 -0800 Subject: [PATCH 410/424] Fixing merge conflict --- tom_observations/facilities/soar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_observations/facilities/soar.py b/tom_observations/facilities/soar.py index c574345cd..1d3b03f97 100644 --- a/tom_observations/facilities/soar.py +++ b/tom_observations/facilities/soar.py @@ -101,7 +101,7 @@ def _build_instrument_config(self): } instrument_configs[0]['rotator_mode'] = 'SKY' - return [instrument_config] + return instrument_configs class SOARFacility(LCOFacility): From 8d008df201ff7a63cf72056700d7c4b5986bf543 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 13 Nov 2020 15:24:47 -0800 Subject: [PATCH 411/424] Added 2.0 release notes --- releasenotes.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/releasenotes.md b/releasenotes.md index d5618b33f..5db615724 100644 --- a/releasenotes.md +++ b/releasenotes.md @@ -1,8 +1,29 @@ # Release Notes -### 1.13.0 - -- +### 2.0.0 + +- Renamed `ALERT_CREDENTIALS` and `BROKER_CREDENTIALS` to `BROKERS` as a catchall for any broker-specific values. +- Added support for custom `CadenceStrategy` layouts. +- Moved settings for `TNSHarvester` into `settings.HARVESTERS` to maintain consistency. +- Updated `tom_alerts.GenericBroker` interface to support submission upstream to a broker, if implemented. +- Fixed `TNSBroker` to get the correct object name. +- Added stub `SCIMMABroker`. +- Removed `tom_publications` from `tom_base`, and placed it in a separate `tom_publications` repository. +- Upgraded a number of dependencies, including `astroplan`, `astropy`, and multiple `django`-related libraries. +- Added tests for `lco.py`, `soar.py`, `alerce.py`, and `mars.py`. +- Added canary tests for `mars.py` and `alerce.py`. + +#### Breaking changes + +- Migrations are required for this version. +- Due to the renaming of `BROKER_CREDENTIALS` and `ALERT_CREDENTIALS` to `BROKERS`, TOM Toolkit users will need to consolidate their broker configurations in `settings.py` into the `BROKERS` dict. +- Because the built-in cadence strategies were moved into their own files, users of the cadence strategies will need to update their `settings.TOM_CADENCE_STRATEGIES` to include the values as seen in this commit: https://github.com/TOMToolkit/tom_base/blob/82101a92a9c19f0ff8ab0f59ecb758bc47824252/tom_base/settings.py#L214 +- Users of the `TNSHarvester` will need to introduce a dict in `settings` called `HARVESTERS` with a sub-dict `TNS` to store the relevant `api_key`. +- Due to the removal of `tom_publications`, TOM Toolkit users will need to either add `tom_publications` to their dependencies, or: + - Remove `tom_publications` from `INSTALLED_APPS`. + - Remove `publications_extras` from the following templates, if they've been customized: `observation_groups.html`, `target_grouping.html`. + - Remove references to `latex_button_group` from the templates referenced above, if they've been customized. +- The `LCOBaseForm` methods `instrument_choices`, `instrument_to_type`, and `filter_choices` were re-implemented as static methods, and any subclasses will need to add a `staticmethod` decorator, modify the method signature, and replace calls to `self` within the method to calls to the class name. ### 1.6.1 From 651c5d9a3be8a3e8c4d2a028f48ac8fd7bd1e627 Mon Sep 17 00:00:00 2001 From: David Collom Date: Fri, 13 Nov 2020 15:28:31 -0800 Subject: [PATCH 412/424] Updated travis for renaming of dev branch --- .travis.yml | 12 ++++++------ README-dev.md | 16 ++++++++-------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index bf6501936..b462e0a0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ stages: - "Style Checks" - "test" - "Canary Tests" - - "Deploy Development" + - "Deploy Dev" - "Deploy Master" jobs: @@ -44,7 +44,7 @@ jobs: script: python manage.py test --tag=canary # Deploy alpha releases to PyPi and generate draft release on Github Releases (incomplete) - - stage: "Deploy Development" + - stage: "Deploy Dev" if: - tag IS present - type != cron @@ -54,7 +54,7 @@ jobs: skip_existing: true cleanup: false on: - branch: development + branch: dev tags: true condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+\-(alpha)\.[0-9]+$ username: "__token__" @@ -62,7 +62,7 @@ jobs: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" distributions: "sdist bdist_wheel" - - stage: "Deploy Development" + - stage: "Deploy Dev" if: - tag IS present - type != cron @@ -73,7 +73,7 @@ jobs: token: secure: "V5t/Rk0jK6ggR6Oc2sk6Kr8+mC4vcKD4Vh1/CVhFoB+/QJTJR4wR1ZzdmxSTHVqyYRWMhtbQg4GEenVu3CCE6Aqcpb8FuXagBGDjrydYMvBz0QV0tGpc0QfAGFgihoMxRXYYy8x0qJoImpIgweIcBP6qCIbWlHNn7uSbJgQAcpHI1KP5//SJbiZ5+h1tYa3AImfhuRIjtw5X5XCkbNe04DpJZZrR7et4BnZwjBKmkPBHAoDhfuGzA5PF1S6jHCpWIddhRQ60v6T7yp8jcBEn5yvQSmusfUju61GI0+irFV8LksaWMaLFkxdR2ne8DtHGuXoRs8G/hOLlYZGBMM/JkV2VcuX9F61vspQW2bDrNgAzMEHknYKeTamvt5HV+zYly7X/OnBMAdbDl3ZOjP5h7zYjdeDza+t6dG1+iJKfCjz5dekT20NDVlwJvIMGhqxNOxDRfnlHopoTT/16/Jd7SPDOY2gpDoZxA8uPGZJR52y6T8tZqrR/tamoXRZk5WR4zeIW2rFK85hZwmz59dWQywlhzcIxMHhBcH6r6O3p1NwFA1LPIzgEQHKig/5DxFk52reU70fA9L2nZ1AHSES2RtG9kDQZLx7gbnOaOGx7Jbd6mECFMzliEBKiSnYxN404bNVAoTBEwcuxTho9zJRccQVeIdTD617K4sAqPWWhFwk=" on: - branch: development + branch: dev tags: true file_glob: true file: dist/* @@ -112,7 +112,7 @@ jobs: token: secure: "V5t/Rk0jK6ggR6Oc2sk6Kr8+mC4vcKD4Vh1/CVhFoB+/QJTJR4wR1ZzdmxSTHVqyYRWMhtbQg4GEenVu3CCE6Aqcpb8FuXagBGDjrydYMvBz0QV0tGpc0QfAGFgihoMxRXYYy8x0qJoImpIgweIcBP6qCIbWlHNn7uSbJgQAcpHI1KP5//SJbiZ5+h1tYa3AImfhuRIjtw5X5XCkbNe04DpJZZrR7et4BnZwjBKmkPBHAoDhfuGzA5PF1S6jHCpWIddhRQ60v6T7yp8jcBEn5yvQSmusfUju61GI0+irFV8LksaWMaLFkxdR2ne8DtHGuXoRs8G/hOLlYZGBMM/JkV2VcuX9F61vspQW2bDrNgAzMEHknYKeTamvt5HV+zYly7X/OnBMAdbDl3ZOjP5h7zYjdeDza+t6dG1+iJKfCjz5dekT20NDVlwJvIMGhqxNOxDRfnlHopoTT/16/Jd7SPDOY2gpDoZxA8uPGZJR52y6T8tZqrR/tamoXRZk5WR4zeIW2rFK85hZwmz59dWQywlhzcIxMHhBcH6r6O3p1NwFA1LPIzgEQHKig/5DxFk52reU70fA9L2nZ1AHSES2RtG9kDQZLx7gbnOaOGx7Jbd6mECFMzliEBKiSnYxN404bNVAoTBEwcuxTho9zJRccQVeIdTD617K4sAqPWWhFwk=" on: - branch: development + branch: master tags: true file_glob: true file: dist/* diff --git a/README-dev.md b/README-dev.md index 1e52200a8..ac23b2dc1 100644 --- a/README-dev.md +++ b/README-dev.md @@ -3,12 +3,12 @@ isn't pertinent to the wider community. ## Deployment The [PyPi](https://pypi.org/project/tomtoolkit/) package is kept under the Las Cumbres Observatory PyPi account. The -development and master branches are deployed automatically by TravisCI upon tagging either branch. +dev and master branches are deployed automatically by TravisCI upon tagging either branch. -In order to trigger a PyPi deployment of either development or master, the branch must be given an annotated tag that +In order to trigger a PyPi deployment of either dev or master, the branch must be given an annotated tag that matches the correct version format. The version formats are as follows: -| | Development | Master | All other branches | +| | Dev | Master | All other branches | |-------------|--------------|--------------|--------------------| | Tagged | Push to PyPi | Push to PyPi | No effect | | Not tagged | No effect | No effect | No effect | @@ -16,7 +16,7 @@ matches the correct version format. The version formats are as follows: Tagged branches must follow the [semantic versioning syntax](https://semver.org/). Tagged versions will not be deployed unless they match the validation regex. The version format is as follows: -| | Development | Master | +| | Dev | Master | |---|---------------|--------| | | x.y.z-alpha.w | x.y.z | @@ -34,8 +34,8 @@ Following deployment of a release, a Github Release is created, and this should * Successfully builds [ReadTheDocs documentation](https://readthedocs.org/projects/tom-toolkit/builds/) (not an automated check) (TODO: fix webhook). * One review approval by a repository owner. -2. Merge your feature branch into the `development` branch - * `git checkout development` +2. Merge your feature branch into the `dev` branch + * `git checkout dev` * `git merge feature/your_feature_branch` 3. Tag the release, triggering GitHub and PyPI actions: @@ -74,7 +74,7 @@ Following deployment of a release, a Github Release is created, and this should The public release deployment workflow parallels the pre-release deployment work flow and more details for a particular step may be found above. -1. Create PR: `master <- development` +1. Create PR: `master <- dev` 2. Meet pre-deployment criteria. * Include docstrings for any new or updated methods * Include tutorial documentation for any new major features as needed @@ -98,7 +98,7 @@ and more details for a particular step may be found above. 6. Update Release Notes in GitHub draft release. This should be the accumulation of the all - the development-release release notes: For example, release notes for releases x.y.z-alpha.1, + the dev-release release notes: For example, release notes for releases x.y.z-alpha.1, x.y.z-alpha.2, etc. should be combined into release notes for release x.y.z. 7. Publish Release From 574303bd876301417c52b33caf0f9d28f39cb365 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 16 Nov 2020 12:01:51 -0800 Subject: [PATCH 413/424] Renamed main branch in travis --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index bf6501936..4846f0e85 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ stages: - "test" - "Canary Tests" - "Deploy Development" - - "Deploy Master" + - "Deploy Main" jobs: include: @@ -82,7 +82,7 @@ jobs: prerelease: true # Deploy full releases to PyPi and generate draft release on Github Releases (incomplete) - - stage: "Deploy Master" + - stage: "Deploy Main" if: - tag IS present - type != cron @@ -93,7 +93,7 @@ jobs: skip_existing: true cleanup: false on: - branch: master + branch: main tags: true condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$ username: "__token__" @@ -101,7 +101,7 @@ jobs: secure: "W11OdJOvj9hvdEsJB48SuPOvylT0awsSG2wScYdtM+mnFx6u5pv7/0oxz0nyhr2/1xD+Mz7QBy5wH2ijXcpruhHwm7loopaB9Lc28au3vnVuDgwdU3yUttXjjzmPHh+q3ggsjAJS7aV4X4CKl0/cM9h3MTsr91MiIvta0cjiPYBTn/3YTrVyO+VypQnsnC2bV7bj1H/c6hYi6mzq4xKtpM1QSMs5baWo7vui1LqbHGeZSID+r9Z3Zx8c8UyAbDG9Qhqi2+TyZxhond1R13IrGtIlvjey87aCnlOFwnA+CXPAbTWsUxq+gE+QO7BCAA2oMZGkLgVeCgHmEHf8gPU/N+XEpdh9FEmgaoU7LIrPOjQI+4ijhEoadhpUxHaQG0j2qhHeE4THz+dfV0XQVPUlzAwX5ZQyLwbAHje8E1wdeqS+pPzodIB18Vtw6Lz0c2ppmieXB1IiSSG0nnxcuLgyFJ/tYJyp63+5fCLK/Itafn3SLT1HRNS/PccbZdUo5L4oFKQsMXgIU9v33Z59CyXpPiNruC+AEwujuEcDrZN1/pYc8+Zmgh42fx+qHZ3/2QitKopl+XMRMnz6X7JIS2QTgY74aLBp/djv5DHtTJnN1cdZ8wMo8K515R/p+C5HboMh9bNEM+imL8pk6Whw9QMGapNoNst99aQz4fUz3n6sPr0=" distributions: "sdist bdist_wheel" - - stage: "Deploy Master" + - stage: "Deploy Main" if: - tag IS present - type != cron From 5144ffe16929f5c64991cc87533460b90b944d7f Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 16 Nov 2020 14:07:13 -0800 Subject: [PATCH 414/424] Making codacy happy and removing references to master in the docs --- README-dev.md | 10 +-- README.md | 4 +- docs/customization/adding_pages.rst | 2 +- .../customization/customize_template_tags.rst | 2 +- docs/customization/customize_templates.rst | 2 +- docs/introduction/contributing.rst | 4 +- docs/introduction/tomarchitecture.rst | 28 ++++---- docs/observing/customize_observations.rst | 2 +- docs/observing/observation_module.rst | 10 +-- releasenotes.md | 68 +++++++++---------- tom_alerts/alerts.py | 2 +- tom_base/settings.py | 2 +- tom_observations/facility.py | 8 +-- 13 files changed, 72 insertions(+), 72 deletions(-) diff --git a/README-dev.md b/README-dev.md index 1e52200a8..75f2e2ecc 100644 --- a/README-dev.md +++ b/README-dev.md @@ -3,12 +3,12 @@ isn't pertinent to the wider community. ## Deployment The [PyPi](https://pypi.org/project/tomtoolkit/) package is kept under the Las Cumbres Observatory PyPi account. The -development and master branches are deployed automatically by TravisCI upon tagging either branch. +development and main branches are deployed automatically by TravisCI upon tagging either branch. -In order to trigger a PyPi deployment of either development or master, the branch must be given an annotated tag that +In order to trigger a PyPi deployment of either development or main, the branch must be given an annotated tag that matches the correct version format. The version formats are as follows: -| | Development | Master | All other branches | +| | Development | Main | All other branches | |-------------|--------------|--------------|--------------------| | Tagged | Push to PyPi | Push to PyPi | No effect | | Not tagged | No effect | No effect | No effect | @@ -16,7 +16,7 @@ matches the correct version format. The version formats are as follows: Tagged branches must follow the [semantic versioning syntax](https://semver.org/). Tagged versions will not be deployed unless they match the validation regex. The version format is as follows: -| | Development | Master | +| | Development | Main | |---|---------------|--------| | | x.y.z-alpha.w | x.y.z | @@ -74,7 +74,7 @@ Following deployment of a release, a Github Release is created, and this should The public release deployment workflow parallels the pre-release deployment work flow and more details for a particular step may be found above. -1. Create PR: `master <- development` +1. Create PR: `main <- dev` 2. Meet pre-deployment criteria. * Include docstrings for any new or updated methods * Include tutorial documentation for any new major features as needed diff --git a/README.md b/README.md index 7e1ce00ae..a9e968e74 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # TOM Toolkit -[![Build Status](https://travis-ci.com/TOMToolkit/tom_base.svg?branch=master)](https://travis-ci.com/TOMToolkit/tom_base) +[![Build Status](https://travis-ci.com/TOMToolkit/tom_base.svg?branch=main)](https://travis-ci.com/TOMToolkit/tom_base) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/9846cee7c4904cae8864525101030169)](https://www.codacy.com/gh/observatorycontrolsystem/observation-portal?utm_source=github.com&utm_medium=referral&utm_content=observatorycontrolsystem/observation-portal&utm_campaign=Badge_Grade) -[![Coverage Status](https://coveralls.io/repos/github/TOMToolkit/tom_base/badge.svg?branch=master)](https://coveralls.io/github/TOMToolkit/tom_base?branch=master) +[![Coverage Status](https://coveralls.io/repos/github/TOMToolkit/tom_base/badge.svg?branch=main)](https://coveralls.io/github/TOMToolkit/tom_base?branch=main) [![Documentation Status](https://readthedocs.org/projects/tom-toolkit/badge/?version=stable)](https://tom-toolkit.readthedocs.io/en/stable/?badge=stable) [Documentation](https://tom-toolkit.readthedocs.io/en/latest/) diff --git a/docs/customization/adding_pages.rst b/docs/customization/adding_pages.rst index 63597c897..3d6e730bc 100644 --- a/docs/customization/adding_pages.rst +++ b/docs/customization/adding_pages.rst @@ -65,7 +65,7 @@ That’s progress, but our new page is pretty ugly. The navigation bar is missing and we don’t have any of the nice CSS that makes the rest of the TOM pages look good! But wait, before you start copying in lines of HTML, know that all we need to do is extend -`tom_common/base.html `__ +`tom_common/base.html `__ to get all that back. You can read more about extending templates from the guide on `Customizing TOM Templates `__. Let’s modify diff --git a/docs/customization/customize_template_tags.rst b/docs/customization/customize_template_tags.rst index 4d871e7ee..38d80fe9c 100644 --- a/docs/customization/customize_template_tags.rst +++ b/docs/customization/customize_template_tags.rst @@ -284,7 +284,7 @@ your project. Your project directory should look like this: Then, you’ll need to copy the contents of ``target_detail.html`` in the base TOM Toolkit to your ``target_detail.html``. You can find that file on -`Github `__. +`Github `__. Near the top of the file, there’s a series of template tags that are loaded in. Add ``custom_extras`` to that list: diff --git a/docs/customization/customize_templates.rst b/docs/customization/customize_templates.rst index a5f91441d..49b35346c 100644 --- a/docs/customization/customize_templates.rst +++ b/docs/customization/customize_templates.rst @@ -35,7 +35,7 @@ Since the template we want to override is already part of the TOM Toolkit source code, we can use it as a starting point for our customized template. In fact, we’ll copy and paste the entire thing from the `source code of TOM -Toolkit `__. +Toolkit `__. and place it in our project. The template we are looking for is ``tom_common/index.html`` diff --git a/docs/introduction/contributing.rst b/docs/introduction/contributing.rst index a1a78ece8..a44bcecbf 100644 --- a/docs/introduction/contributing.rst +++ b/docs/introduction/contributing.rst @@ -53,7 +53,7 @@ following: :: git fetch upstream - git merge upstream/master + git merge upstream/main 5. Create and checkout a branch for your changes (see `Branch Naming <#branch-naming>`__). @@ -63,7 +63,7 @@ following: git checkout -b 6. Commit frequently, and push your changes to Github. Be sure to merge - master in before submitting your pull request. + main in before submitting your pull request. :: diff --git a/docs/introduction/tomarchitecture.rst b/docs/introduction/tomarchitecture.rst index 86920d4b9..431797ca4 100644 --- a/docs/introduction/tomarchitecture.rst +++ b/docs/introduction/tomarchitecture.rst @@ -59,7 +59,7 @@ Django, and by extension the toolkit, rely heavily on object oriented programming, especially inheritance. Most customization in the TOM toolkit comes from subclassing classes that provide generic functionality and overriding or extending methods. An experienced Django developer would feel right at home. For example, the -`ObservationRecordDetailView `_ +`ObservationRecordDetailView `_ in the ``tom_observations`` module of the toolkit inherits from Django's `DetailView `_. This means TOM developers are able to take full advantage of the power of Django @@ -81,7 +81,7 @@ fills in its own logic. This structure makes it easy for developers to write their own plugins which can then be shared and installed by others or even contributed to the main codebase. -The `gemini.py module `_ +The `gemini.py module `_ is an observation module plugin contributed by Bryan Miller to enable the triggering of observation requests on the Gemini telescope via the TOM Toolkit. Thanks Bryan! @@ -144,7 +144,7 @@ The following describes each app that ships with the toolkit and its purpose. TOM Targets ----------- -The `tom_targets `_ +The `tom_targets `_ app is central to the entire TOM Toolkit project. It provides the database definitions for the storage and retrieval of targets and target lists. It also provides the views (pages) for viewing, creating, modifying and visualizing @@ -157,23 +157,23 @@ Nearly every app depends on the ``tom_targets`` module in some way. TOM Observations ---------------- -The `tom_observations `_ +The `tom_observations `_ app handles all the logic for submitting and querying observations of targets at observatories. It defines the database models for observation requests and provides some views for working with them. -`facility.py `_ +`facility.py `_ defines an interface that external facilities (observatories) can implement in order to integrate with the toolkit: -`gemini.py `_ +`gemini.py `_ and -`lco.py `_ +`lco.py `_ are two examples, and we expect more in the future. TOM Data Products ----------------- Straddling both the ``tom_targets`` and ``tom_observations`` packages is -`tom_dataproducts `_. +`tom_dataproducts `_. This package contains the logic required for storing data related to targets and observations within the toolkit. Some data products are fetched from on-line archives (handled by an observatory's observation module) but data can also be @@ -187,12 +187,12 @@ specialized data processing, analytics or pipelining is required. TOM Alerts ---------- -The `tom_alerts `_ +The `tom_alerts `_ app contains modules related to the functionality of ingesting targets from various external services. These services, usually called brokers, provide rapidly changing target lists that are of interest to time domain astronomers. The -`alerts.py `_ +`alerts.py `_ module provides a generic interface that other modules can implement, giving them the ability to integrate these brokers with the toolkit. Currently, there are modules available for `Lasair `_, @@ -203,22 +203,22 @@ TOM Catalogs ------------ The -`tom_catalogs `_ +`tom_catalogs `_ app contains functionality related to querying astronomical catalogs. These "harvester" modules enable the querying and translation of targets found in databases such as Simbad and JPL Horizons directly into targets within the toolkit. The -`harvester.py `_ +`harvester.py `_ module provides the basic interface, and there are several modules already written for Simbad, NED, the MPC, JPL Horizons and the Transient Name Server. TOM Setup and TOM Common ------------------------ -The `tom_setup `_ +The `tom_setup `_ package is special in that its sole purpose is to help TOM developers bootstrap new TOMs. See the :doc:`getting started ` guide for an example. -The `tom_common `_ +The `tom_common `_ package contains logic and data that doesn't fit anywhere else. Database Layout diff --git a/docs/observing/customize_observations.rst b/docs/observing/customize_observations.rst index 7d0db1bbf..93d060120 100644 --- a/docs/observing/customize_observations.rst +++ b/docs/observing/customize_observations.rst @@ -189,7 +189,7 @@ like this: form = LCOMultiFilterForm Take a look at the layout and compare it to the `existing lco -layout `__. +layout `__. A second row has been added that includes all the filter choices. Note that the original ``filter`` and ``exposure_time`` have been moved from their original location to the new row. diff --git a/docs/observing/observation_module.rst b/docs/observing/observation_module.rst index 6dca67ae1..996680f7e 100644 --- a/docs/observing/observation_module.rst +++ b/docs/observing/observation_module.rst @@ -22,9 +22,9 @@ A TOM Toolkit observing facility module is a python module which contains the code necessary to provide an interface to an observing facility in a TOM. Some examples of existing modules are the `Las Cumbres -Observatory `__ +Observatory `__ and the -`Gemini `__ +`Gemini `__ modules. Both allow the submission of observation requests to their respective observatories through a TOM. @@ -174,7 +174,7 @@ second value of each tuple is what will be displayed on the webpage, as different tabs of observation types to submit. The first value of each tuple is what should be used to distinguish different observation types in your code. To see a demonstration of this, check out the `Las Cumbres -Observatory `__ +Observatory `__ facility’s ``observation_types`` and ``get_form``. Now let’s populate the form. Let’s assume our observatory only requires @@ -270,7 +270,7 @@ data, but when you adapt it to work with a real observatory you should fill them in with the correct logic so that the whole module works correctly with the TOM. You can view explanations of each method `in the source -code `__ +code `__ ###Airmass plotting for new facilities The last step in adding a new facility is to get it to appear on airmass plots. If you input two dates @@ -281,7 +281,7 @@ at LCO and Gemini sites. In our ``MyObservationFacility`` class, let’s define a new variable called ``SITES``. Modeling our ``SITES`` on the one defined for `Las Cumbres -Observatory `__, +Observatory `__, we can easily put new sites into the airmass plots: .. code:: python diff --git a/releasenotes.md b/releasenotes.md index 5db615724..ac194f07c 100644 --- a/releasenotes.md +++ b/releasenotes.md @@ -1,50 +1,50 @@ # Release Notes -### 2.0.0 +## 2.0.0 -- Renamed `ALERT_CREDENTIALS` and `BROKER_CREDENTIALS` to `BROKERS` as a catchall for any broker-specific values. -- Added support for custom `CadenceStrategy` layouts. -- Moved settings for `TNSHarvester` into `settings.HARVESTERS` to maintain consistency. -- Updated `tom_alerts.GenericBroker` interface to support submission upstream to a broker, if implemented. -- Fixed `TNSBroker` to get the correct object name. -- Added stub `SCIMMABroker`. -- Removed `tom_publications` from `tom_base`, and placed it in a separate `tom_publications` repository. -- Upgraded a number of dependencies, including `astroplan`, `astropy`, and multiple `django`-related libraries. -- Added tests for `lco.py`, `soar.py`, `alerce.py`, and `mars.py`. -- Added canary tests for `mars.py` and `alerce.py`. + - Renamed `ALERT_CREDENTIALS` and `BROKER_CREDENTIALS` to `BROKERS` as a catchall for any broker-specific values. + - Added support for custom `CadenceStrategy` layouts. + - Moved settings for `TNSHarvester` into `settings.HARVESTERS` to maintain consistency. + - Updated `tom_alerts.GenericBroker` interface to support submission upstream to a broker, if implemented. + - Fixed `TNSBroker` to get the correct object name. + - Added stub `SCIMMABroker`. + - Removed `tom_publications` from `tom_base`, and placed it in a separate `tom_publications` repository. + - Upgraded a number of dependencies, including `astroplan`, `astropy`, and multiple `django`-related libraries. + - Added tests for `lco.py`, `soar.py`, `alerce.py`, and `mars.py`. + - Added canary tests for `mars.py` and `alerce.py`. -#### Breaking changes +### Breaking changes -- Migrations are required for this version. -- Due to the renaming of `BROKER_CREDENTIALS` and `ALERT_CREDENTIALS` to `BROKERS`, TOM Toolkit users will need to consolidate their broker configurations in `settings.py` into the `BROKERS` dict. -- Because the built-in cadence strategies were moved into their own files, users of the cadence strategies will need to update their `settings.TOM_CADENCE_STRATEGIES` to include the values as seen in this commit: https://github.com/TOMToolkit/tom_base/blob/82101a92a9c19f0ff8ab0f59ecb758bc47824252/tom_base/settings.py#L214 -- Users of the `TNSHarvester` will need to introduce a dict in `settings` called `HARVESTERS` with a sub-dict `TNS` to store the relevant `api_key`. -- Due to the removal of `tom_publications`, TOM Toolkit users will need to either add `tom_publications` to their dependencies, or: - - Remove `tom_publications` from `INSTALLED_APPS`. - - Remove `publications_extras` from the following templates, if they've been customized: `observation_groups.html`, `target_grouping.html`. - - Remove references to `latex_button_group` from the templates referenced above, if they've been customized. -- The `LCOBaseForm` methods `instrument_choices`, `instrument_to_type`, and `filter_choices` were re-implemented as static methods, and any subclasses will need to add a `staticmethod` decorator, modify the method signature, and replace calls to `self` within the method to calls to the class name. + - Migrations are required for this version. + - Due to the renaming of `BROKER_CREDENTIALS` and `ALERT_CREDENTIALS` to `BROKERS`, TOM Toolkit users will need to consolidate their broker configurations in `settings.py` into the `BROKERS` dict. + - Because the built-in cadence strategies were moved into their own files, users of the cadence strategies will need to update their `settings.TOM_CADENCE_STRATEGIES` to include the values as seen in this commit: https://github.com/TOMToolkit/tom_base/blob/82101a92a9c19f0ff8ab0f59ecb758bc47824252/tom_base/settings.py#L214 + - Users of the `TNSHarvester` will need to introduce a dict in `settings` called `HARVESTERS` with a sub-dict `TNS` to store the relevant `api_key`. + - Due to the removal of `tom_publications`, TOM Toolkit users will need to either add `tom_publications` to their dependencies, or: + - Remove `tom_publications` from `INSTALLED_APPS`. + - Remove `publications_extras` from the following templates, if they've been customized: `observation_groups.html`, `target_grouping.html`. + - Remove references to `latex_button_group` from the templates referenced above, if they've been customized. + - The `LCOBaseForm` methods `instrument_choices`, `instrument_to_type`, and `filter_choices` were re-implemented as static methods, and any subclasses will need to add a `staticmethod` decorator, modify the method signature, and replace calls to `self` within the method to calls to the class name. -### 1.6.1 +## 1.6.1 -- This release pins the Django version in order to address a security vulnerability. + - This release pins the Django version in order to address a security vulnerability. -#### What to watch out for +### What to watch out for -- The Django version is now pinned at 3.0.7, where previously it allowed >=2.2. You'll need to ensure that any custom code is compatible with Django >=3.0.7. + - The Django version is now pinned at 3.0.7, where previously it allowed >=2.2. You'll need to ensure that any custom code is compatible with Django >=3.0.7. -### 1.6.0 +## 1.6.0 -- New methods expand the Facility API to support reporting Facility status and weather: `get_facility_status()` and `get_facility_weather_url()`. When these methods are implemented by a Facility provider, this information can be made available in your TOM. -- A new template tag, `facility_status()`, is available to present this information. + - New methods expand the Facility API to support reporting Facility status and weather: `get_facility_status()` and `get_facility_weather_url()`. When these methods are implemented by a Facility provider, this information can be made available in your TOM. + - A new template tag, `facility_status()`, is available to present this information. -### 1.5.0 +## 1.5.0 -- Introduced a manual facility interface for classical observing. -- Introduced a view and corresponding form to add existing API-based observations to a Target. -- Introduced a view and corresponding form to update an existing manual observation with an API-based observation ID. + - Introduced a manual facility interface for classical observing. + - Introduced a view and corresponding form to add existing API-based observations to a Target. + - Introduced a view and corresponding form to update an existing manual observation with an API-based observation ID. -#### What to watch out for +### What to watch out for -- For facility implementers: in order to support a Manual Facility Interface, the team created a `BaseObservationFacility` and two abstract implementations of it, `BaseRoboticObservationFacility` and `BaseManualObservationFacility`. `BaseRoboticObservationFacility` was aliased as `GenericObservationFacility` to support backwards compatibility, but will be removed in 2.0. \ No newline at end of file + - For facility implementers: in order to support a Manual Facility Interface, the team created a `BaseObservationFacility` and two abstract implementations of it, `BaseRoboticObservationFacility` and `BaseManualObservationFacility`. `BaseRoboticObservationFacility` was aliased as `GenericObservationFacility` to support backwards compatibility, but will be removed in 2.0. \ No newline at end of file diff --git a/tom_alerts/alerts.py b/tom_alerts/alerts.py index ccd2825f0..35f8a1902 100644 --- a/tom_alerts/alerts.py +++ b/tom_alerts/alerts.py @@ -184,7 +184,7 @@ class GenericBroker(ABC): make use of a broker module, add the path to ``TOM_ALERT_CLASSES`` in your ``settings.py``. For an implementation example, please see - https://github.com/TOMToolkit/tom_base/blob/master/tom_alerts/brokers/mars.py + https://github.com/TOMToolkit/tom_base/blob/main/tom_alerts/brokers/mars.py """ alert_submission_form = GenericUpstreamSubmissionForm diff --git a/tom_base/settings.py b/tom_base/settings.py index a560c455b..d5cd1a8d9 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -21,7 +21,7 @@ # See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'dxja^_6p35x46dx0rx+c$(^31(10^n(twe1#ax3o8xl=n^p37q' +SECRET_KEY = '' # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True diff --git a/tom_observations/facility.py b/tom_observations/facility.py index ea5142cc9..39665fc6e 100644 --- a/tom_observations/facility.py +++ b/tom_observations/facility.py @@ -63,7 +63,7 @@ class BaseObservationForm(forms.Form): the other BaseObservationForms. For an implementation example please see - https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/lco.py#L132 + https://github.com/TOMToolkit/tom_base/blob/main/tom_observations/facilities/lco.py#L132 """ facility = forms.CharField(required=True, max_length=50, widget=forms.HiddenInput()) target_id = forms.IntegerField(required=True, widget=forms.HiddenInput()) @@ -131,7 +131,7 @@ class BaseRoboticObservationForm(BaseObservationForm): This specific class is intended for use with robotic facilities, such as LCO, Gemini, and SOAR. For an implementation example please see - https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/lco.py#L132 + https://github.com/TOMToolkit/tom_base/blob/main/tom_observations/facilities/lco.py#L132 """ pass @@ -153,7 +153,7 @@ class BaseManualObservationForm(BaseObservationForm): This specific class is intended for use with classical-style manual facilities. For an implementation example please see - https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/lco.py#L132 + https://github.com/TOMToolkit/tom_base/blob/main/tom_observations/facilities/lco.py#L132 """ name = forms.CharField() start = forms.CharField(widget=forms.TextInput(attrs={'type': 'date'})) @@ -332,7 +332,7 @@ class BaseRoboticObservationFacility(BaseObservationFacility): This specific class is intended for use with robotic facilities, such as LCO, Gemini, and SOAR. For an implementation example, please see - https://github.com/TOMToolkit/tom_base/blob/master/tom_observations/facilities/lco.py + https://github.com/TOMToolkit/tom_base/blob/main/tom_observations/facilities/lco.py """ name = 'BaseRobotic' # rename in concrete subclasses From 12d3c3e134a05a03c18aba5b1c84a307ca98dab9 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 16 Nov 2020 14:09:17 -0800 Subject: [PATCH 415/424] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a9e968e74..8bc9ab3e4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TOM Toolkit [![Build Status](https://travis-ci.com/TOMToolkit/tom_base.svg?branch=main)](https://travis-ci.com/TOMToolkit/tom_base) -[![Codacy Badge](https://api.codacy.com/project/badge/Grade/9846cee7c4904cae8864525101030169)](https://www.codacy.com/gh/observatorycontrolsystem/observation-portal?utm_source=github.com&utm_medium=referral&utm_content=observatorycontrolsystem/observation-portal&utm_campaign=Badge_Grade) +[![Codacy Badge](https://app.codacy.com/project/badge/Grade/578e468dbd01494696d4446288858252)](https://www.codacy.com/gh/TOMToolkit/tom_base/dashboard?utm_source=github.com&utm_medium=referral&utm_content=TOMToolkit/tom_base&utm_campaign=Badge_Grade) [![Coverage Status](https://coveralls.io/repos/github/TOMToolkit/tom_base/badge.svg?branch=main)](https://coveralls.io/github/TOMToolkit/tom_base?branch=main) [![Documentation Status](https://readthedocs.org/projects/tom-toolkit/badge/?version=stable)](https://tom-toolkit.readthedocs.io/en/stable/?badge=stable) [Documentation](https://tom-toolkit.readthedocs.io/en/latest/) From a788323b2ce1ff65d373ed377f8701d5b549c916 Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 16 Nov 2020 14:17:59 -0800 Subject: [PATCH 416/424] More modifications to release notes style --- releasenotes.md | 38 +++++++++++++++++++------------------- tom_base/settings.py | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/releasenotes.md b/releasenotes.md index ac194f07c..44453b7f7 100644 --- a/releasenotes.md +++ b/releasenotes.md @@ -2,28 +2,28 @@ ## 2.0.0 - - Renamed `ALERT_CREDENTIALS` and `BROKER_CREDENTIALS` to `BROKERS` as a catchall for any broker-specific values. - - Added support for custom `CadenceStrategy` layouts. - - Moved settings for `TNSHarvester` into `settings.HARVESTERS` to maintain consistency. - - Updated `tom_alerts.GenericBroker` interface to support submission upstream to a broker, if implemented. - - Fixed `TNSBroker` to get the correct object name. - - Added stub `SCIMMABroker`. - - Removed `tom_publications` from `tom_base`, and placed it in a separate `tom_publications` repository. - - Upgraded a number of dependencies, including `astroplan`, `astropy`, and multiple `django`-related libraries. - - Added tests for `lco.py`, `soar.py`, `alerce.py`, and `mars.py`. - - Added canary tests for `mars.py` and `alerce.py`. +- Renamed `ALERT_CREDENTIALS` and `BROKER_CREDENTIALS` to `BROKERS` as a catchall for any broker-specific values. +- Added support for custom `CadenceStrategy` layouts. +- Moved settings for `TNSHarvester` into `settings.HARVESTERS` to maintain consistency. +- Updated `tom_alerts.GenericBroker` interface to support submission upstream to a broker, if implemented. +- Fixed `TNSBroker` to get the correct object name. +- Added stub `SCIMMABroker`. +- Removed `tom_publications` from `tom_base`, and placed it in a separate `tom_publications` repository. +- Upgraded a number of dependencies, including `astroplan`, `astropy`, and multiple `django`-related libraries. +- Added tests for `lco.py`, `soar.py`, `alerce.py`, and `mars.py`. +- Added canary tests for `mars.py` and `alerce.py`. ### Breaking changes - - Migrations are required for this version. - - Due to the renaming of `BROKER_CREDENTIALS` and `ALERT_CREDENTIALS` to `BROKERS`, TOM Toolkit users will need to consolidate their broker configurations in `settings.py` into the `BROKERS` dict. - - Because the built-in cadence strategies were moved into their own files, users of the cadence strategies will need to update their `settings.TOM_CADENCE_STRATEGIES` to include the values as seen in this commit: https://github.com/TOMToolkit/tom_base/blob/82101a92a9c19f0ff8ab0f59ecb758bc47824252/tom_base/settings.py#L214 - - Users of the `TNSHarvester` will need to introduce a dict in `settings` called `HARVESTERS` with a sub-dict `TNS` to store the relevant `api_key`. - - Due to the removal of `tom_publications`, TOM Toolkit users will need to either add `tom_publications` to their dependencies, or: - - Remove `tom_publications` from `INSTALLED_APPS`. - - Remove `publications_extras` from the following templates, if they've been customized: `observation_groups.html`, `target_grouping.html`. - - Remove references to `latex_button_group` from the templates referenced above, if they've been customized. - - The `LCOBaseForm` methods `instrument_choices`, `instrument_to_type`, and `filter_choices` were re-implemented as static methods, and any subclasses will need to add a `staticmethod` decorator, modify the method signature, and replace calls to `self` within the method to calls to the class name. +- Migrations are required for this version. +- Due to the renaming of `BROKER_CREDENTIALS` and `ALERT_CREDENTIALS` to `BROKERS`, TOM Toolkit users will need to consolidate their broker configurations in `settings.py` into the `BROKERS` dict. +- Because the built-in cadence strategies were moved into their own files, users of the cadence strategies will need to update their `settings.TOM_CADENCE_STRATEGIES` to include the values as seen in this commit: https://github.com/TOMToolkit/tom_base/blob/82101a92a9c19f0ff8ab0f59ecb758bc47824252/tom_base/settings.py#L214 +- Users of the `TNSHarvester` will need to introduce a dict in `settings` called `HARVESTERS` with a sub-dict `TNS` to store the relevant `api_key`. +- Due to the removal of `tom_publications`, TOM Toolkit users will need to either add `tom_publications` to their dependencies, or: + - Remove `tom_publications` from `INSTALLED_APPS`. + - Remove `publications_extras` from the following templates, if they've been customized: `observation_groups.html`, `target_grouping.html`. + - Remove references to `latex_button_group` from the templates referenced above, if they've been customized. +- The `LCOBaseForm` methods `instrument_choices`, `instrument_to_type`, and `filter_choices` were re-implemented as static methods, and any subclasses will need to add a `staticmethod` decorator, modify the method signature, and replace calls to `self` within the method to calls to the class name. ## 1.6.1 diff --git a/tom_base/settings.py b/tom_base/settings.py index 0d13bf1d0..e8cbcc1d6 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -21,7 +21,7 @@ # See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'testkey' +SECRET_KEY = os.getenv('SECRET_KEY', 'testkey') # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True From 275f15939e4af36ee898d7a8deee67bca9053b5c Mon Sep 17 00:00:00 2001 From: David Collom Date: Mon, 16 Nov 2020 14:20:02 -0800 Subject: [PATCH 417/424] Fixing merge conflicts in travisfile --- .travis.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index a4ddaea19..e6683037c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,13 +22,8 @@ stages: - "Style Checks" - "test" - "Canary Tests" -<<<<<<< HEAD - "Deploy Dev" - "Deploy Master" -======= - - "Deploy Development" - - "Deploy Main" ->>>>>>> main jobs: include: From 88fc6db76f27f6407728f52841ede63a086e6d65 Mon Sep 17 00:00:00 2001 From: fraserw Date: Tue, 17 Nov 2020 09:18:06 -0800 Subject: [PATCH 418/424] --changes to reflect updated Aladin view for MOPS and tiler testing. --- tom_observations/forms.py | 23 +- .../tom_observations/observation_form.html | 12 +- .../templatetags/observation_extras.py | 2 +- tom_targets/forms.py | 42 ++++ tom_targets/models.py | 1 + .../tom_targets/partials/aladin.html | 2 +- .../templates/tom_targets/target_detail.html | 4 +- tom_targets/templatetags/targets_extras.py | 81 ++++++- tom_targets/utils.py | 198 +++++++++--------- tom_targets/views.py | 7 +- 10 files changed, 257 insertions(+), 115 deletions(-) diff --git a/tom_observations/forms.py b/tom_observations/forms.py index d45597c55..71a10f57b 100644 --- a/tom_observations/forms.py +++ b/tom_observations/forms.py @@ -76,7 +76,7 @@ class TileForm(forms.Form): ra_uncertainty = forms.DecimalField(required=False, label='R.A. Uncertainty (")') dec_uncertainty = forms.DecimalField(required=False, label='Dec. Uncertainty (")') selected_date = forms.DateTimeField(required=False, label='Date', widget=forms.TextInput(attrs={'type': 'date'})) - selected_time = forms.DateTimeField(required=False, label='Time', widget=forms.TextInput(attrs={'type': 'time'})) + selected_time = forms.TimeField(required=False, label='Time', widget=forms.TextInput(attrs={'type': 'time'})) def clean(self): cleaned_data = super().clean() @@ -94,12 +94,17 @@ def __init__(self, *args, **kwargs): def layout(self): return Div( - Div( - Div('field_overlap', css_class='col'), - Div('ra_uncertainty', css_class='col'), - Div('dec_uncertainty', css_class='col'), - Div('min_fill_fraction', css_class='col'), - Div('shimmy_factor', css_class='col'), - css_class='form-row'), - Div('selected_date', 'selected_time'), + Row( + Column('field_overlap', css_class='col'), + Column('ra_uncertainty', css_class='col'), + Column('dec_uncertainty', css_class='col'), + ), + Row( + Column('min_fill_fraction', css_class='col'), + Column('shimmy_factor', css_class='col'), + ), + Row( + Column('selected_date'), + Column('selected_time'), + ), ) diff --git a/tom_observations/templates/tom_observations/observation_form.html b/tom_observations/templates/tom_observations/observation_form.html index f1c06dc9f..7658c4ee5 100644 --- a/tom_observations/templates/tom_observations/observation_form.html +++ b/tom_observations/templates/tom_observations/observation_form.html @@ -1,5 +1,5 @@ {% extends 'tom_common/base.html' %} -{% load bootstrap4 static crispy_forms_tags observation_extras targets_extras %} +{% load bootstrap4 static crispy_forms_tags observation_extras targets_extras nonsidereal_airmass_extras %} {% block title %}Submit Observation{% endblock %} {% block additional_css %} @@ -11,7 +11,13 @@

    Submit an observation to {{ form.facility.value }}

    {% if target.type == 'SIDEREAL' %}
    - {% observation_plan target form.facility.value %} + {% observation_plan target form.facility.value %} +
    +
    +{% elif target.type == 'NON_SIDEREAL' %} +
    +
    + {% observation_plan_nonsidereal target form.facility.value %}
    {% endif %} @@ -44,4 +50,4 @@

    Submit an observation to {{ form.facility.value }}

    -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/tom_observations/templatetags/observation_extras.py b/tom_observations/templatetags/observation_extras.py index 95b834347..ccb8a7c7c 100644 --- a/tom_observations/templatetags/observation_extras.py +++ b/tom_observations/templatetags/observation_extras.py @@ -293,7 +293,7 @@ def tile_plan(context): 'target': context['object'] }) if tile_form.is_valid(): - field_overlap = float(request.GET['field_overlap']) + field_overlap = float(request.GET.get('field_overlap')) min_fill_fraction = float(request.GET.get('min_fill_fraction')) shimmy_factor = float(request.GET.get('shimmy_factor')) if request.GET.get('ra_uncertainty') and request.GET.get('dec_uncertainty'): diff --git a/tom_targets/forms.py b/tom_targets/forms.py index bd1d2e41b..a3b28a19a 100644 --- a/tom_targets/forms.py +++ b/tom_targets/forms.py @@ -1,10 +1,17 @@ from django import forms from astropy.coordinates import Angle from astropy import units as u +from astropy.time import Time from django.forms import ValidationError, inlineformset_factory from django.conf import settings from django.contrib.auth.models import Group from guardian.shortcuts import assign_perm, get_groups_with_perms, remove_perm +from crispy_forms.helper import FormHelper +from crispy_forms.layout import Column, Layout, Row, Div + +import datetime +import json +import numpy as np from .models import ( Target, TargetExtra, TargetName, SIDEREAL_FIELDS, NON_SIDEREAL_FIELDS, REQUIRED_SIDEREAL_FIELDS, @@ -161,6 +168,41 @@ def clean(self): if target.type == 'NON_SIDEREAL': raise forms.ValidationError('Airmass plotting is only supported for sidereal targets') +class AladinNonSiderealForm(forms.Form): + #selected_date = forms.DateTimeField(required=True, label='Date', widget=forms.TextInput(attrs={'type': 'date', 'value': datetime.datetime.now().strftime("%d-%m-%Y")})) + #selected_time = forms.DateTimeField(required=True, label='Time', widget=forms.TextInput(attrs={'type': 'time', 'value': datetime.now().strftime("%H:%M:%S")})) + selected_date = forms.DateTimeField(required=True, label='Date (UTC)', widget=forms.TextInput(attrs={'type': 'date'}), initial=datetime.date.today) + selected_time = forms.TimeField(required=True, label='Start Time (UTC)', widget=forms.TextInput(attrs={'type': 'time'}), initial=datetime.datetime.now().strftime("%H:%M")) + duration = forms.DecimalField(required=True, label='Duration (hrs)', initial=24.0) + + def clean(self): + cleaned_data = super().clean() + selected_date = cleaned_data.get('selected_date') + selected_time = cleaned_data.get('selected_time') + duration = cleaned_data.get('duration') + target = self.data['target'] + + if self.data['target'].scheme == 'EPHEMERIS': + t = Time(selected_date.strftime("%Y-%m-%dT")+selected_time.strftime("%H:%M:00.0")) + + eph_json = json.loads(self.data['target'].eph_json) + keys = list(eph_json.keys()) + mjd = [] + for i in eph_json[keys[0]]: + mjd.append(i['t']) + mjd = np.array(mjd, dtype='float64') + min_mjd, max_mjd = np.min(mjd), np.max(mjd) + + if t.mjd < min_mjd or t.mjd > max_mjd: + raise ValidationError('The selected date must be between {} and {} utc.'.format(Time(min_mjd, format='mjd').isot, Time(max_mjd, format='mjd').isot)) + + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.helper = FormHelper() + self.helper.layout = Layout( + Row(Column('selected_date'), Column('selected_time')) + ) diff --git a/tom_targets/models.py b/tom_targets/models.py index 0c72609f6..d0be46c21 100644 --- a/tom_targets/models.py +++ b/tom_targets/models.py @@ -132,6 +132,7 @@ class Target(models.Model): SIDEREAL = 'SIDEREAL' NON_SIDEREAL = 'NON_SIDEREAL' + EPHEMERIS = 'EPHEMERIS' TARGET_TYPES = ((SIDEREAL, 'Sidereal'), (NON_SIDEREAL, 'Non-sidereal')) TARGET_SCHEMES = ( diff --git a/tom_targets/templates/tom_targets/partials/aladin.html b/tom_targets/templates/tom_targets/partials/aladin.html index d33607434..e2752f3fe 100644 --- a/tom_targets/templates/tom_targets/partials/aladin.html +++ b/tom_targets/templates/tom_targets/partials/aladin.html @@ -1,7 +1,7 @@ -

    Survey View

    +

    Survey View

    diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index f1b1e6b11..4521dbe9f 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -1,5 +1,5 @@ {% extends 'tom_common/base.html' %} -{% load comments bootstrap4 tom_common_extras targets_extras observation_extras dataproduct_extras publication_extras static cache nonsidereal_airmass_extras%} +{% load comments bootstrap4 tom_common_extras targets_extras observation_extras dataproduct_extras publication_extras static cache nonsidereal_airmass_extras %} {% block title %}Target {{ object.name }}{% endblock %} {% block additional_css %} @@ -24,6 +24,8 @@ {% recent_photometry object limit=3 %} {% if object.type == 'SIDEREAL' %} {% aladin object %} + {% elif object.type == 'NON_SIDEREAL' %} + {% aladin_nonsidereal %} {% endif %}
    diff --git a/tom_targets/templatetags/targets_extras.py b/tom_targets/templatetags/targets_extras.py index e0e10f974..d0346f33d 100644 --- a/tom_targets/templatetags/targets_extras.py +++ b/tom_targets/templatetags/targets_extras.py @@ -14,8 +14,9 @@ from tom_observations.utils import get_sidereal_visibility from tom_targets.models import Target, TargetExtra, TargetList -from tom_targets.forms import TargetVisibilityForm +from tom_targets.forms import TargetVisibilityForm, AladinNonSiderealForm +from scipy import interpolate as interp import json from astroquery.jplhorizons import Horizons @@ -273,6 +274,84 @@ def aladin(target): return {'target': target} +@register.inclusion_tag('tom_targets/partials/aladin_nonsidereal.html', takes_context=True) +def aladin_nonsidereal(context): + """ + Testing in prep of aladin_nonsidereal + """ + request = context['request'] + aladin_form = AladinNonSiderealForm() + + selected_date = datetime.now().strftime("%d-%m-%Y") + selected_time = datetime.now().strftime("%H:%M") + duration = 24.0 + + if all(request.GET.get(x) for x in ['selected_date']): + aladin_form = AladinNonSiderealForm({ + 'selected_date': request.GET.get('selected_date'), + 'selected_time': request.GET.get('selected_time'), + 'duration': request.GET.get('duration'), + 'target': context['object'] + }) + if aladin_form.is_valid(): + selected_date = request.GET.get('selected_date') + selected_time = request.GET.get('selected_time') + duration = float(request.GET.get('duration')) + + if context['object'].type == 'NON_SIDEREAL': + if context['object'].scheme == 'EPHEMERIS': + + # this logic can probably be pulled from tom_observations.utils + # but this is actually lighter weight + eph_json = json.loads(context['object'].eph_json) + keys = list(eph_json.keys()) + mjd, ra, dec = [], [], [] + for i in eph_json[keys[0]]: + mjd.append(i['t']) + ra.append(i['R']) + dec.append(i['D']) + mjd = np.array(mjd, dtype='float64') + ra = np.array(ra, dtype='float64') + dec = np.array(dec, dtype='float64') + + fra = interp.interp1d(mjd, ra) + fdec = interp.interp1d(mjd, dec) + try: + fra = interp.interp1d(mjd, ra) + fdec = interp.interp1d(mjd, dec) + t = Time(selected_date+'T'+selected_time+':00') + context['object'].ra = fra(t.mjd) + context['object'].dec = fdec(t.mjd) + context['object'].ra1 = fra(t.mjd+duration/24.0) + context['object'].dec1 = fdec(t.mjd+duration/24.0) + + except: + context['object'].ra = None + context['object'].dec = None + else: + try: + t = Time(selected_date+'T'+selected_time+':00') + + # if there is a space in the nane, assume the first string is an acceptable name + obj = Horizons(id=context['object'].names[0].split()[0], epochs=[t.jd, (t+duration/24.0).jd]) + context['object'].ra = obj.ephemerides()['RA'][0] + context['object'].dec = obj.ephemerides()['DEC'][0] + context['object'].ra1 = obj.ephemerides()['RA'][1] + context['object'].dec1 = obj.ephemerides()['DEC'][1] + except: + context['object'].ra = None + context['object'].dec = None + context['object'].ra1 = None + context['object'].dec1 = None + pass + + # return the html you need + return { + 'form': aladin_form, + 'target': context['object'], + } + + @register.filter def eph_json_to_value_ra(value): """ diff --git a/tom_targets/utils.py b/tom_targets/utils.py index cdce056ae..6a0f7b113 100644 --- a/tom_targets/utils.py +++ b/tom_targets/utils.py @@ -1,15 +1,17 @@ from django.db.models import Count - +from django.conf import settings import csv from .models import Target, TargetExtra, TargetName from io import StringIO + import json + # this dictionary should contain as key entires text sufficient to uniquely # identify the observatory name from the common English names used by JPL for # that site. For example, Sunderland is probably unique enough to identify SAAO # there may be a better way to handle this. -site_names = {'Mauna Kea': '568', +SITE_NAMES = {'Mauna Kea': 'mko', 'Haleakala': 'ogg', 'McDonald': 'elp', 'Tololo': 'lsc', @@ -17,101 +19,11 @@ 'Sutherland': 'cpt', 'Wise': 'tlv', 'Siding Spring': 'coj', + 'Gemini South': 'cpo', + 'SOAR': 'sor', } -# NOTE: This saves locally. To avoid this, create file buffer. -# referenced https://www.codingforentrepreneurs.com/blog/django-queryset-to-csv-files-datasets/ -def export_targets(qs): - """ - Exports all the specified targets into a csv file in folder csvTargetFiles - NOTE: This saves locally. To avoid this, create file buffer. - - :param qs: List of targets to export - :type qs: QuerySet - - :returns: String buffer of exported targets - :rtype: StringIO - """ - qs_pk = [data['id'] for data in qs] - data_list = list(qs) - target_fields = [field.name for field in Target._meta.get_fields()] - target_extra_fields = list({field.key for field in TargetExtra.objects.filter(target__in=qs_pk)}) - # Gets the count of the target names for the target with the most aliases in the database - # This is to construct enough row headers of format "name2, name3, name4, etc" for exporting aliases - # The alias headers are then added to the set of fields for export - aliases = TargetName.objects.filter(target__in=qs_pk).values('target_id').annotate(count=Count('target_id')) - max_alias_count = 0 - if aliases: - max_alias_count = max([alias['count'] for alias in aliases]) - all_fields = target_fields + target_extra_fields + [f'name{index+1}' for index in range(1, max_alias_count+1)] - for key in ['id', 'targetlist', 'dataproduct', 'observationrecord', 'reduceddatum', 'aliases', 'targetextra']: - all_fields.remove(key) - - file_buffer = StringIO() - writer = csv.DictWriter(file_buffer, fieldnames=all_fields) - writer.writeheader() - for target_data in data_list: - extras = list(TargetExtra.objects.filter(target_id=target_data['id'])) - names = list(TargetName.objects.filter(target_id=target_data['id'])) - for e in extras: - target_data[e.key] = e.value - name_index = 2 - for name in names: - target_data[f'name{str(name_index)}'] = name.name - name_index += 1 - del target_data['id'] # do not export 'id' - writer.writerow(target_data) - return file_buffer - - -def import_targets(targets): - """ - Imports a set of targets into the TOM and saves them to the database. - - :param targets: String buffer of targets - :type targets: StringIO - - :returns: dictionary of successfully imported targets, as well errors - :rtype: dict - """ - # TODO: Replace this with an in memory iterator - targetreader = csv.DictReader(targets, dialect=csv.excel) - targets = [] - errors = [] - base_target_fields = [field.name for field in Target._meta.get_fields()] - for index, row in enumerate(targetreader): - # filter out empty values in base fields, otherwise converting empty string to float will throw error - row = {k: v for (k, v) in row.items() if not (k in base_target_fields and not v)} - target_extra_fields = [] - target_names = [] - target_fields = {} - for k in row: - # All fields starting with 'name' (e.g. name2, name3) that aren't literally 'name' will be added as - # TargetNames - if k != 'name' and k.startswith('name'): - target_names.append(row[k]) - elif k not in base_target_fields: - target_extra_fields.append((k, row[k])) - else: - target_fields[k] = row[k] - for extra in target_extra_fields: - row.pop(extra[0]) - try: - target = Target.objects.create(**target_fields) - for extra in target_extra_fields: - TargetExtra.objects.create(target=target, key=extra[0], value=extra[1]) - for name in target_names: - if name: - TargetName.objects.create(target=target, name=name) - targets.append(target) - except Exception as e: - error = 'Error on line {0}: {1}'.format(index + 2, str(e)) - errors.append(error) - - return {'targets': targets, 'errors': errors} - - def import_ephemeris_target(stream): """ Reads in a custom ephemeris from provided file stream. @@ -137,7 +49,7 @@ def import_ephemeris_target(stream): if 'Center-site name' in eph[i]: num_sites += 1 - if num_sites != 8: + if num_sites < 8: errors.append(Warning('WARNING: Provided file does not have ephemerides for all 8 LCO sites.')) eph_json = {} @@ -155,9 +67,9 @@ def import_ephemeris_target(stream): for i in range(end_ind, len(eph)): if 'Center-site name' in eph[i]: s = eph[i].split(': ')[-1] - for j in site_names.keys(): + for j in SITE_NAMES.keys(): if j in s: - centre_site_name = site_names[j] + centre_site_name = SITE_NAMES[j] site_name_found = True break if not site_name_found: @@ -242,3 +154,95 @@ def import_ephemeris_target(stream): errors.append(str(e)) return {'targets': targets, 'errors': errors} + + +# NOTE: This saves locally. To avoid this, create file buffer. +# referenced https://www.codingforentrepreneurs.com/blog/django-queryset-to-csv-files-datasets/ +def export_targets(qs): + """ + Exports all the specified targets into a csv file in folder csvTargetFiles + NOTE: This saves locally. To avoid this, create file buffer. + + :param qs: List of targets to export + :type qs: QuerySet + + :returns: String buffer of exported targets + :rtype: StringIO + """ + qs_pk = [data['id'] for data in qs] + data_list = list(qs) + target_fields = [field.name for field in Target._meta.get_fields()] + target_extra_fields = list({field.key for field in TargetExtra.objects.filter(target__in=qs_pk)}) + # Gets the count of the target names for the target with the most aliases in the database + # This is to construct enough row headers of format "name2, name3, name4, etc" for exporting aliases + # The alias headers are then added to the set of fields for export + aliases = TargetName.objects.filter(target__in=qs_pk).values('target_id').annotate(count=Count('target_id')) + max_alias_count = 0 + if aliases: + max_alias_count = max([alias['count'] for alias in aliases]) + all_fields = target_fields + target_extra_fields + [f'name{index+1}' for index in range(1, max_alias_count+1)] + for key in ['id', 'targetlist', 'dataproduct', 'observationrecord', 'reduceddatum', 'aliases', 'targetextra']: + all_fields.remove(key) + + file_buffer = StringIO() + writer = csv.DictWriter(file_buffer, fieldnames=all_fields) + writer.writeheader() + for target_data in data_list: + extras = list(TargetExtra.objects.filter(target_id=target_data['id'])) + names = list(TargetName.objects.filter(target_id=target_data['id'])) + for e in extras: + target_data[e.key] = e.value + name_index = 2 + for name in names: + target_data[f'name{str(name_index)}'] = name.name + name_index += 1 + del target_data['id'] # do not export 'id' + writer.writerow(target_data) + return file_buffer + + +def import_targets(targets): + """ + Imports a set of targets into the TOM and saves them to the database. + + :param targets: String buffer of targets + :type targets: StringIO + + :returns: dictionary of successfully imported targets, as well errors + :rtype: dict + """ + # TODO: Replace this with an in memory iterator + targetreader = csv.DictReader(targets, dialect=csv.excel) + targets = [] + errors = [] + base_target_fields = [field.name for field in Target._meta.get_fields()] + for index, row in enumerate(targetreader): + # filter out empty values in base fields, otherwise converting empty string to float will throw error + row = {k: v for (k, v) in row.items() if not (k in base_target_fields and not v)} + target_extra_fields = [] + target_names = [] + target_fields = {} + for k in row: + # All fields starting with 'name' (e.g. name2, name3) that aren't literally 'name' will be added as + # TargetNames + if k != 'name' and k.startswith('name'): + target_names.append(row[k]) + elif k not in base_target_fields: + target_extra_fields.append((k, row[k])) + else: + target_fields[k] = row[k] + for extra in target_extra_fields: + row.pop(extra[0]) + try: + target = Target.objects.create(**target_fields) + for extra in target_extra_fields: + TargetExtra.objects.create(target=target, key=extra[0], value=extra[1]) + for name in target_names: + if name: + TargetName.objects.create(target=target, name=name) + targets.append(target) + except Exception as e: + error = 'Error on line {0}: {1}'.format(index + 2, str(e)) + errors.append(error) + + return {'targets': targets, 'errors': errors} diff --git a/tom_targets/views.py b/tom_targets/views.py index 8c6b43bd9..6d356a078 100644 --- a/tom_targets/views.py +++ b/tom_targets/views.py @@ -40,7 +40,6 @@ ) from tom_targets.models import Target, TargetList from tom_targets.utils import import_targets, export_targets, import_ephemeris_target -from tom_targets.filters import TargetFilter logger = logging.getLogger(__name__) @@ -317,7 +316,10 @@ class TargetSSOISView(RedirectView): model = Target def get_redirect_url(*args, **kwargs): - + """ + Produce a redirect to the Solar System Object Image Search at the + Canadian Astronomy Data Centre, for the target. + """ now = datetime.now() targ_name_guess = kwargs['pk'].split()[0].split('-')[0] url = 'http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/cadcbin/ssos/ssosclf.pl?lang=en&object={}'.format(targ_name_guess.split()[0]) @@ -325,6 +327,7 @@ def get_redirect_url(*args, **kwargs): url += '&eellipse=&eunits=arcseconds&extres=no&xyres=no' return url + class TargetDetailView(Raise403PermissionRequiredMixin, DetailView): """ View that handles the display of the target details. Requires authorization. From b9e3b4a5f35bc0555c04e25b60dba45c79423b58 Mon Sep 17 00:00:00 2001 From: fraserw Date: Tue, 17 Nov 2020 09:24:44 -0800 Subject: [PATCH 419/424] --adding missing files --- .../tom_dataproducts/upload_dataproduct.html | 12 ++ .../observation_plan_nonsidereal.html | 4 + .../partials/aladin_nonsidereal.html | 190 ++++++++++++++++++ 3 files changed, 206 insertions(+) create mode 100644 tom_dataproducts/templates/tom_dataproducts/upload_dataproduct.html create mode 100644 tom_observations/templates/tom_observations/partials/observation_plan_nonsidereal.html create mode 100644 tom_targets/templates/tom_targets/partials/aladin_nonsidereal.html diff --git a/tom_dataproducts/templates/tom_dataproducts/upload_dataproduct.html b/tom_dataproducts/templates/tom_dataproducts/upload_dataproduct.html new file mode 100644 index 000000000..cb0577df9 --- /dev/null +++ b/tom_dataproducts/templates/tom_dataproducts/upload_dataproduct.html @@ -0,0 +1,12 @@ +{% load bootstrap4 static %} +

    Upload a data product

    +

    + For example CSVs, see the photometry and spectroscopy sample files. FITS is supported for spectra. +

    + + {% csrf_token %} + {% bootstrap_form data_product_form %} + {% buttons %} + + {% endbuttons %} + diff --git a/tom_observations/templates/tom_observations/partials/observation_plan_nonsidereal.html b/tom_observations/templates/tom_observations/partials/observation_plan_nonsidereal.html new file mode 100644 index 000000000..21a0b25ca --- /dev/null +++ b/tom_observations/templates/tom_observations/partials/observation_plan_nonsidereal.html @@ -0,0 +1,4 @@ +{% load bootstrap4 %} +
    + {{ visibility_graph|safe }} +
    \ No newline at end of file diff --git a/tom_targets/templates/tom_targets/partials/aladin_nonsidereal.html b/tom_targets/templates/tom_targets/partials/aladin_nonsidereal.html new file mode 100644 index 000000000..874362317 --- /dev/null +++ b/tom_targets/templates/tom_targets/partials/aladin_nonsidereal.html @@ -0,0 +1,190 @@ +{% load bootstrap4 %} + +

    Target Location

    +{% if target.ra is None %} + {% if target.scheme == 'EPHEMERIS' %} +
    Displaying an invalid image!! Selected date beyond range of available ephemeris!
    + {% else %} +
    Unable to query JPL. If refreshing with update doesn't help, probably can't parse the target name.
    + {% endif %} +{% endif %} +
    +
    +
    + {% csrf_token %} + {% bootstrap_form form %} +
    +
    +
    + Field of view +
    + +
    + +
    +
    +
    +
    +
    +
    + +
    + +
    + +
    +
    +
    + {% buttons %} + + {% endbuttons %} +
    + * - The line is the approximate ephemeris assuming linear path, with the start shown in the red circle. +
    + + From f14154d982a371bb1a5186945e47060ec881688d Mon Sep 17 00:00:00 2001 From: fraserw Date: Tue, 17 Nov 2020 10:14:58 -0800 Subject: [PATCH 420/424] --the conflict handling didn't take for some rason. --- tom_observations/facilities/lco.py | 34 ++++--------------- .../templates/tom_targets/target_detail.html | 6 +--- 2 files changed, 8 insertions(+), 32 deletions(-) diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index aae8ed8b4..d188acb82 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -524,7 +524,6 @@ def _expand_cadence_request(self, payload): return response.json() def observation_payload(self): -<<<<<<< HEAD if not self.eph_target: payload = { "name": self.cleaned_data['name'], @@ -539,8 +538,13 @@ def observation_payload(self): { "start": self.cleaned_data['start'], "end": self.cleaned_data['end'] - }]}] + } + ], + 'location': self._build_location() + } + ] } + if self.cleaned_data.get('period') and self.cleaned_data.get('jitter'): payload = self._expand_cadence_request(payload) @@ -555,11 +559,11 @@ def observation_payload(self): # is done in tom_base/utils.py. obs_module = get_service_class(self.cleaned_data['facility']) requests = self._build_ephemeris_requests() + """ locations = [] for j in range(len(requests)): if requests[j]['location'] not in locations: locations.append(requests[j]['location']) - """ # I dont understand why the following is inappropriate when selecting a single # telescope location, but MANY seems to always be required now. if len(locations) > 1: @@ -576,30 +580,6 @@ def observation_payload(self): "operator": operator, "observation_type": self.cleaned_data['observation_mode'], "requests": requests -======= - payload = { - 'name': self.cleaned_data['name'], - 'proposal': self.cleaned_data['proposal'], - 'ipp_value': self.cleaned_data['ipp_value'], - 'operator': 'SINGLE', - 'observation_type': self.cleaned_data['observation_mode'], - 'requests': [ - { - 'configurations': [self._build_configuration()], - 'windows': [ - { - 'start': self.cleaned_data['start'], - 'end': self.cleaned_data['end'] - } - ], - 'location': self._build_location() - } - ] - } - if self.cleaned_data.get('period') and self.cleaned_data.get('jitter'): - payload = self._expand_cadence_request(payload) ->>>>>>> 275f15939e4af36ee898d7a8deee67bca9053b5c - }) if len(errors) > 0: diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index 547c7afc9..2e21167df 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -1,9 +1,5 @@ {% extends 'tom_common/base.html' %} -<<<<<<< HEAD -{% load comments bootstrap4 tom_common_extras targets_extras observation_extras dataproduct_extras publication_extras static cache nonsidereal_airmass_extras %} -======= -{% load comments bootstrap4 tom_common_extras targets_extras observation_extras dataproduct_extras static cache %} ->>>>>>> 275f15939e4af36ee898d7a8deee67bca9053b5c +{% load comments bootstrap4 tom_common_extras targets_extras observation_extras publication_extras static cache nonsidereal_airmass_extras %} {% block title %}Target {{ object.name }}{% endblock %} {% block additional_css %} From 2effc1af10853b695c197f0c5e6866bf9a9a2a92 Mon Sep 17 00:00:00 2001 From: fraserw Date: Tue, 17 Nov 2020 14:23:31 -0800 Subject: [PATCH 421/424] --fixed a dataproducts load bug --- tom_targets/templates/tom_targets/target_detail.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index 2e21167df..fc256b951 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -1,5 +1,5 @@ {% extends 'tom_common/base.html' %} -{% load comments bootstrap4 tom_common_extras targets_extras observation_extras publication_extras static cache nonsidereal_airmass_extras %} +{% load comments bootstrap4 tom_common_extras targets_extras observation_extras dataproduct_extras static cache nonsidereal_airmass_extras %} {% block title %}Target {{ object.name }}{% endblock %} {% block additional_css %} From a4691270fe542054eeb192fb472679e30cf83437 Mon Sep 17 00:00:00 2001 From: fraserw Date: Mon, 7 Dec 2020 11:14:33 -0800 Subject: [PATCH 422/424] -- testing that I can still commit and push --- Wes Sandbox/Aladin+airmass notes.txt | 61 ++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 Wes Sandbox/Aladin+airmass notes.txt diff --git a/Wes Sandbox/Aladin+airmass notes.txt b/Wes Sandbox/Aladin+airmass notes.txt new file mode 100644 index 000000000..adb545b47 --- /dev/null +++ b/Wes Sandbox/Aladin+airmass notes.txt @@ -0,0 +1,61 @@ +Summary of changes +~~~~~~~~~~~~~~~~~~ +The visibility(airmass) plot function, and the target distribution function for non-sidereal targets +from from functions I added to nonsidereal_airmass_extras.py + +The aladin plot view comes from changes I made to tom_extras.py and forms.py in tom_targets. + + +files that get changed with non-sidereal-airmass: + +observation_form.html +~~~~~~~~~~~~~~~~~~~~~~ + +{load nonsidereal_airmass_extras} + +and then + +{% if target.type == 'SIDEREAL' %} +
    +
    + {% observation_plan target form.facility.value %} +
    +
    +{% elif target.type == 'NON_SIDEREAL' %} +
    +
    + {% observation_plan_nonsidereal target form.facility.value %} +
    +
    +{% endif %} + + + + + + +For aladin non-sidereal and SSOIS: + +Files added to tom_targets: +templates/tom_targets/partials/aladin_nonsidereal.html +templates/tom_targets/partials/target_ssois.html + +Files updated: +templatetags/target_extras.py + +target_detail.html +~~~~~~~~~~~~~~~~~~ + +After the line containing {% target_buttons object %} + +{% if object.type == 'NON_SIDEREAL' %} +{% target_ssois object %} +{% endif %} + +and after recent photometry object replace the aladin call with + +{% if object.type == 'SIDEREAL' %} +{% aladin object %} +{% elif object.type == 'NON_SIDEREAL' %} +{% aladin_nonsidereal %} +{% endif %} From 457ed9b45e6b679abd0d5c7b74e5afb30b23bd04 Mon Sep 17 00:00:00 2001 From: fraserw Date: Mon, 7 Dec 2020 11:17:49 -0800 Subject: [PATCH 423/424] -- a lot of stuff so D. can see what's going wrong --- tom_observations/facilities/gemini.py | 5 +- tom_observations/facilities/lco.py | 24 ++-- tom_observations/facilities/utils.py | 2 +- tom_observations/forms.py | 24 +++- .../tom_observations/observation_form.html | 5 + .../tom_observations/partials/tile_plan.html | 9 +- .../partials/tile_plan_observations.html | 8 ++ .../templatetags/observation_extras.py | 39 +++++-- tom_observations/tiler.py | 12 +- tom_targets/forms.py | 4 +- .../tom_targets/partials/aladin.html | 2 +- .../partials/aladin_nonsidereal.html | 15 +-- .../templates/tom_targets/target_detail.html | 1 + tom_targets/templatetags/targets_extras.py | 103 +++++++++--------- 14 files changed, 153 insertions(+), 100 deletions(-) create mode 100644 tom_observations/templates/tom_observations/partials/tile_plan_observations.html diff --git a/tom_observations/facilities/gemini.py b/tom_observations/facilities/gemini.py index 7f955eb7c..00cc425cf 100644 --- a/tom_observations/facilities/gemini.py +++ b/tom_observations/facilities/gemini.py @@ -277,8 +277,8 @@ def __init__(self, *args, **kwargs): if target.scheme == 'EPHEMERIS': self.eph_target = True eph_json = json.loads(target.eph_json) - self.eph_GN = reconstruct_gemini_eph_note(eph_json, site='568') - self.eph_GS = reconstruct_gemini_eph_note(eph_json, site='lsc') + self.eph_GN = reconstruct_gemini_eph_note(eph_json, site='mko') + self.eph_GS = reconstruct_gemini_eph_note(eph_json, site='cpo') super().__init__(*args, **kwargs) self.helper.layout = Layout( @@ -448,7 +448,6 @@ def isodatetime(value): note_text = self.eph_GS[0][0:4] + self.eph_GS[0][mjd_k:mjd_K] + self.eph_GS[0][-2:] payload['note'] += "\n\n" payload['note'] += "\n".join(note_text) - print(payload['note']) if self.cleaned_data['brightness'] is not None: smags = str(self.cleaned_data['brightness']).strip() + '/' + \ self.cleaned_data['brightness_band'] + '/' + \ diff --git a/tom_observations/facilities/lco.py b/tom_observations/facilities/lco.py index d188acb82..44c45f2ca 100644 --- a/tom_observations/facilities/lco.py +++ b/tom_observations/facilities/lco.py @@ -26,6 +26,8 @@ ) from tom_observations.utils import get_radec_ephemeris import json +from tom_observations.forms import TileForm, camera_fovs +from tom_observations.tiler import * # Determine settings for this module. @@ -219,6 +221,9 @@ class LCOBaseObservationForm(BaseRoboticObservationForm, LCOBaseForm): imaging_interval = forms.FloatField( label='Interval (hrs). Will schedule exposure count per interval.' ) + min_fill_fraction = forms.DecimalField(required=True, label='Minimum Fill Fraction', initial=0.5) + field_overlap = forms.DecimalField(required=True, label='Field Overlap', initial=0.3) + def __init__(self, *args, **kwargs): # the ephemeris target stuff must come before super() @@ -274,7 +279,10 @@ def layout(self): def extra_layout(self): # If you just want to add some fields to the end of the form, add them here. if self.eph_target: - return Div('site', 'imaging_interval') + return Div( + Div('site', 'imaging_interval'), + #Div('field_overlap', 'min_fill_fraction'), + ) return Div() def clean_start(self): @@ -559,18 +567,6 @@ def observation_payload(self): # is done in tom_base/utils.py. obs_module = get_service_class(self.cleaned_data['facility']) requests = self._build_ephemeris_requests() - """ - locations = [] - for j in range(len(requests)): - if requests[j]['location'] not in locations: - locations.append(requests[j]['location']) - # I dont understand why the following is inappropriate when selecting a single - # telescope location, but MANY seems to always be required now. - if len(locations) > 1: - operator = "MANY" - else: - operator = "MANY"#"SINGLE" - """ operator = "MANY" errors = obs_module().validate_observation({ @@ -998,7 +994,7 @@ class LCOFacility(BaseRoboticObservationFacility): 'IMAGING': LCOImagingObservationForm, 'SPECTRA': LCOSpectroscopyObservationForm, 'PHOTOMETRIC_SEQUENCE': LCOPhotometricSequenceForm, - 'SPECTROSCOPIC_SEQUENCE': LCOSpectroscopicSequenceForm + 'SPECTROSCOPIC_SEQUENCE': LCOSpectroscopicSequenceForm, } # The SITES dictionary is used to calculate visibility intervals in the # planning tool. All entries should contain latitude, longitude, elevation diff --git a/tom_observations/facilities/utils.py b/tom_observations/facilities/utils.py index ab1b0c12b..378d9a849 100644 --- a/tom_observations/facilities/utils.py +++ b/tom_observations/facilities/utils.py @@ -42,7 +42,7 @@ def add_month(t): T = T.replace('-06-','-Jun-').replace('-07-','-Jul-').replace('-08-','-Aug-').replace('-09-','-Sep-').replace('-10-','-Oct-') return T.replace('-11-', '-Nov-').replace('-12-', '-Dec-') -def reconstruct_gemini_eph_note(eph, site='568'): +def reconstruct_gemini_eph_note(eph, site='mko'): mk = eph[site] ras = [] diff --git a/tom_observations/forms.py b/tom_observations/forms.py index 71a10f57b..d922652eb 100644 --- a/tom_observations/forms.py +++ b/tom_observations/forms.py @@ -9,6 +9,13 @@ def facility_choices(): return [(k, k) for k in get_service_classes().keys()] +# camera fields of view in arcmin +camera_fovs = ((26.0, "SINISTRO - 26'"), + (9.3, "MuSCAT3 - 9.3'"), + (29.0, "SBIG 0.4m - 29'"), + (15.8, "SBIG 1.0m - 15.8'"), + (5.0, "Merope - 5'"), + (5.5, "GMOS - 5.5'")) class AddExistingObservationForm(forms.Form): """ @@ -69,7 +76,9 @@ def __init__(self, *args, **kwargs): ) ) + class TileForm(forms.Form): + instrument = forms.ChoiceField(required=True, label='Instrument', choices=camera_fovs) field_overlap = forms.DecimalField(required=True, label='Field Overlap', initial=0.3) min_fill_fraction = forms.DecimalField(required=True, label='Minimum Fill Fraction', initial=0.5) shimmy_factor = forms.DecimalField(required=True, label='Shimmy Factor', initial=0.0) @@ -82,29 +91,36 @@ def clean(self): cleaned_data = super().clean() field_overlap = cleaned_data.get('field_overlap') min_fill_fraction = cleaned_data.get('min_fill_fraction') - target = self.data['target'] + target = self.data.get('target') + instrument = cleaned_data.get('instrument') def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.helper = FormHelper() self.helper.layout = Layout( - self.layout() + self.layout(), ) def layout(self): return Div( Row( Column('field_overlap', css_class='col'), - Column('ra_uncertainty', css_class='col'), - Column('dec_uncertainty', css_class='col'), + Column('instrument', css_class='col'), ), Row( Column('min_fill_fraction', css_class='col'), Column('shimmy_factor', css_class='col'), ), + Row( + Column('ra_uncertainty', css_class='col'), + Column('dec_uncertainty', css_class='col'), + ), Row( Column('selected_date'), Column('selected_time'), ), + ButtonHolder( + Submit('submit', 'Tile') + ), ) diff --git a/tom_observations/templates/tom_observations/observation_form.html b/tom_observations/templates/tom_observations/observation_form.html index 7658c4ee5..5bd6c4126 100644 --- a/tom_observations/templates/tom_observations/observation_form.html +++ b/tom_observations/templates/tom_observations/observation_form.html @@ -30,6 +30,11 @@

    Submit an observation to {{ form.facility.value }}

    Lunar Distance {% moon_distance target %}
    + {% if target.type == 'SIDEREAL' %} + {% aladin target %} + {% elif target.type == 'NON_SIDEREAL' %} + {% aladin_nonsidereal %} + {% endif %}