From 806bba1e17ed94185efb514b9df8301debd96a63 Mon Sep 17 00:00:00 2001 From: Brandon Drumheller Date: Sun, 18 Feb 2018 19:15:13 -0600 Subject: [PATCH 01/51] update djoser version --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 8cc734b..3d3bd57 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "metpetdb_api/vendor/djoser"] path = metpetdb_api/vendor/djoser - url = https://github.com/kristallizer/djoser.git + url = https://github.com/bdrumheller/djoser.git From b8779a3f829213945e2fd2b57cd80c55a773d0ac Mon Sep 17 00:00:00 2001 From: Brandon Drumheller Date: Sun, 18 Feb 2018 19:36:47 -0600 Subject: [PATCH 02/51] update submodule on parent --- metpetdb_api/vendor/djoser | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metpetdb_api/vendor/djoser b/metpetdb_api/vendor/djoser index 0e3b5d6..713d07b 160000 --- a/metpetdb_api/vendor/djoser +++ b/metpetdb_api/vendor/djoser @@ -1 +1 @@ -Subproject commit 0e3b5d62d88cec8c6d949785dce487753d8edf59 +Subproject commit 713d07bb736771b300a0be23270093a83abbab25 From 9941b6c23b378f8d1c7fd85e171554b06a343435 Mon Sep 17 00:00:00 2001 From: Brandon Drumheller Date: Sun, 18 Feb 2018 20:21:44 -0600 Subject: [PATCH 03/51] upgrade to drf 1.11 LTS --- metpetdb_api/api/chemical_analyses/v1/serializers.py | 3 +++ metpetdb_api/api/samples/v1/serializers.py | 12 ++++++++++++ metpetdb_api/apps/chemical_analyses/models.py | 3 +++ metpetdb_api/apps/samples/models.py | 12 +++++++++++- metpetdb_api/manage.py | 2 +- metpetdb_api/requirements/dev.txt | 6 +++--- metpetdb_api/requirements/staging.txt | 6 +++--- 7 files changed, 36 insertions(+), 8 deletions(-) diff --git a/metpetdb_api/api/chemical_analyses/v1/serializers.py b/metpetdb_api/api/chemical_analyses/v1/serializers.py index 6343b77..466763e 100644 --- a/metpetdb_api/api/chemical_analyses/v1/serializers.py +++ b/metpetdb_api/api/chemical_analyses/v1/serializers.py @@ -73,6 +73,7 @@ class ChemicalAnalysisSerializer(DynamicFieldsModelSerializer): class Meta: model = ChemicalAnalysis depth = 1 + fields = '__all__' def is_valid(self, raise_exception=False): super().is_valid(raise_exception) @@ -119,8 +120,10 @@ def update(self, instance, validated_data): class ElementSerializer(DynamicFieldsModelSerializer): class Meta: model = Element + fields = '__all__' class OxideSerializer(DynamicFieldsModelSerializer): class Meta: model = Oxide + fields = '__all__' diff --git a/metpetdb_api/api/samples/v1/serializers.py b/metpetdb_api/api/samples/v1/serializers.py index 4e911a0..102cca5 100644 --- a/metpetdb_api/api/samples/v1/serializers.py +++ b/metpetdb_api/api/samples/v1/serializers.py @@ -30,11 +30,13 @@ class RockTypeSerializer(DynamicFieldsModelSerializer): class Meta: model = RockType + fields = '__all__' class MineralSerializer(DynamicFieldsModelSerializer): class Meta: model = Mineral + fields = '__all__' class SampleMineralSerializer(DynamicFieldsModelSerializer): @@ -60,6 +62,7 @@ class SampleSerializer(DynamicFieldsModelSerializer): class Meta: model = Sample depth = 1 + fields = '__all__' def is_valid(self, raise_exception=False): super().is_valid(raise_exception) @@ -111,6 +114,7 @@ class SubsampleSerializer(DynamicFieldsModelSerializer): class Meta: model = Subsample depth = 1 + fields = '__all__' def is_valid(self, raise_exception=False): super().is_valid(raise_exception) @@ -154,32 +158,40 @@ class SubsampleTypeSerializer(DynamicFieldsModelSerializer): class Meta: model = SubsampleType depth = 1 + fields = '__all__' class MetamorphicGradeSerializer(DynamicFieldsModelSerializer): class Meta: model = MetamorphicGrade + fields = '__all__' class MetamorphicRegionSerializer(DynamicFieldsModelSerializer): class Meta: model = MetamorphicRegion + fields = '__all__' class GeoReferenceSerializer(DynamicFieldsModelSerializer): class Meta: model = GeoReference + fields = '__all__' + class RegionSerializer(DynamicFieldsModelSerializer): class Meta: model = Region + fields = '__all__' class ReferenceSerializer(DynamicFieldsModelSerializer): class Meta: model = Reference + fields = '__all__' class CollectorSerializer(DynamicFieldsModelSerializer): class Meta: model = Collector + fields = '__all__' diff --git a/metpetdb_api/apps/chemical_analyses/models.py b/metpetdb_api/apps/chemical_analyses/models.py index 5848045..758aef5 100644 --- a/metpetdb_api/apps/chemical_analyses/models.py +++ b/metpetdb_api/apps/chemical_analyses/models.py @@ -37,6 +37,7 @@ class ChemicalAnalysis(models.Model): class Meta: db_table = 'chemical_analyses' + ordering = ['id'] class Element(models.Model): @@ -50,6 +51,7 @@ class Element(models.Model): class Meta: db_table = 'elements' + ordering = ['id'] class Oxide(models.Model): @@ -64,6 +66,7 @@ class Oxide(models.Model): class Meta: db_table = 'oxides' + ordering = ['id'] class ChemicalAnalysisElement(models.Model): diff --git a/metpetdb_api/apps/samples/models.py b/metpetdb_api/apps/samples/models.py index ebc8c24..fa82060 100644 --- a/metpetdb_api/apps/samples/models.py +++ b/metpetdb_api/apps/samples/models.py @@ -15,6 +15,7 @@ class RockType(models.Model): class Meta: db_table = 'rock_types' + ordering = ['id'] class Sample(models.Model): @@ -64,6 +65,7 @@ class Sample(models.Model): class Meta: db_table = 'samples' + ordering = ['id'] class SubsampleType(models.Model): @@ -72,6 +74,7 @@ class SubsampleType(models.Model): class Meta: db_table = 'subsample_types' + ordering = ['id'] class Subsample(models.Model): @@ -85,6 +88,7 @@ class Subsample(models.Model): class Meta: db_table = 'subsamples' + ordering = ['id'] class Grid(models.Model): @@ -105,6 +109,7 @@ class MetamorphicGrade(models.Model): class Meta: db_table = 'metamorphic_grades' + ordering = ['id'] class MetamorphicRegion(models.Model): @@ -116,6 +121,7 @@ class MetamorphicRegion(models.Model): class Meta: db_table = 'metamorphic_regions' + ordering = ['id'] class Mineral(models.Model): @@ -130,6 +136,7 @@ class Mineral(models.Model): class Meta: db_table = 'minerals' + ordering = ['id'] class SampleMineral(models.Model): @@ -179,6 +186,7 @@ class GeoReference(models.Model): class Meta: db_table = 'georeferences' + ordering = ['id'] # Following are models for easy retrieval of sample-related free-text fields @@ -208,6 +216,7 @@ class Region(models.Model): class Meta: db_table = 'regions' + ordering = ['id'] class Reference(models.Model): @@ -216,6 +225,7 @@ class Reference(models.Model): class Meta: db_table = 'references' + ordering = ['id'] class Collector(models.Model): @@ -224,7 +234,7 @@ class Collector(models.Model): class Meta: db_table = 'collectors' - + ordering = ['id'] # A mapping table to help the migration of old samples to new samples; can # be gotten rid of once thi app goes into production. diff --git a/metpetdb_api/manage.py b/metpetdb_api/manage.py index 939380d..c4b654e 100755 --- a/metpetdb_api/manage.py +++ b/metpetdb_api/manage.py @@ -8,7 +8,7 @@ if __name__ == "__main__": PROJECT_ROOT = os.path.dirname(__file__) sys.path.insert(-1, os.path.join(PROJECT_ROOT, "vendor/djoser")) - + dotenv.read_dotenv('api.env') os.environ.setdefault("DJANGO_SETTINGS_MODULE", env('API_SETTINGS')) diff --git a/metpetdb_api/requirements/dev.txt b/metpetdb_api/requirements/dev.txt index 236460b..190c72b 100644 --- a/metpetdb_api/requirements/dev.txt +++ b/metpetdb_api/requirements/dev.txt @@ -1,11 +1,11 @@ -Django>=1.8.4,<=1.9 +Django>=1.11.0,<=1.11.10 django-concurrency>=0.9 django-debug-toolbar>=1.3.2,<=1.4 django-dotenv>=1.3.0 django-extensions>=1.5.5,<=2.0 django-getenv>=1.3.1 -djangorestframework>=3.2,<=3.3 -psycopg2>=2.6.1 +djangorestframework>=3.7,<=3.8 +psycopg2-binary>=2.6.1 six>=1.9.0 sqlparse>=0.1.15 wheel>=0.24.0 diff --git a/metpetdb_api/requirements/staging.txt b/metpetdb_api/requirements/staging.txt index 838cc2f..15d5906 100644 --- a/metpetdb_api/requirements/staging.txt +++ b/metpetdb_api/requirements/staging.txt @@ -1,11 +1,11 @@ -Django>=1.8.4,<=1.9 +Django>=1.11.0,<=1.11.10 django-concurrency>=0.9 django-dotenv>=1.3.0 django-extensions>=1.5.5,<=2.0 django-getenv>=1.3.1 -djangorestframework>=3.2,<=3.3 +djangorestframework>=3.7,<=3.8 gunicorn>=19.3.0 -psycopg2>=2.6.1 +psycopg2-binary>=2.6.1 six>=1.9.0 sqlparse>=0.1.15 wheel>=0.24.0 From e442758c459f4d67dee72f0b984e10c978b8c609 Mon Sep 17 00:00:00 2001 From: Brandon Drumheller Date: Sun, 18 Feb 2018 20:39:58 -0600 Subject: [PATCH 04/51] remove valid password check for password reset --- metpetdb_api/vendor/djoser | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metpetdb_api/vendor/djoser b/metpetdb_api/vendor/djoser index 713d07b..8239ecf 160000 --- a/metpetdb_api/vendor/djoser +++ b/metpetdb_api/vendor/djoser @@ -1 +1 @@ -Subproject commit 713d07bb736771b300a0be23270093a83abbab25 +Subproject commit 8239ecf52b164f8e16997f6970ebd30969fffedb From 811c862d2f78c9b9cf678c9858b77c26c44aadad Mon Sep 17 00:00:00 2001 From: Brandon Drumheller Date: Tue, 20 Feb 2018 00:11:26 -0600 Subject: [PATCH 05/51] add images to samples, subsamples, chemical analysis --- .../api/chemical_analyses/v1/serializers.py | 2 + metpetdb_api/api/images/__init__.py | 0 metpetdb_api/api/images/v1/__init__.py | 0 metpetdb_api/api/images/v1/serializers.py | 45 ++++++++++++++ metpetdb_api/api/images/v1/tests.py | 0 metpetdb_api/api/images/v1/views.py | 18 ++++++ metpetdb_api/api/samples/lib/query.py | 7 +-- metpetdb_api/api/samples/v1/serializers.py | 7 ++- metpetdb_api/api/samples/v1/views.py | 8 --- .../migrations/0004_auto_20180220_0409.py | 27 +++++++++ metpetdb_api/apps/chemical_analyses/models.py | 2 +- metpetdb_api/apps/images/__init__.py | 0 .../apps/images/migrations/0001_initial.py | 48 +++++++++++++++ .../images/migrations/0002_image_samples.py | 22 +++++++ .../migrations/0003_auto_20180220_0539.py | 26 ++++++++ .../images/migrations/0004_image_subsample.py | 22 +++++++ .../0005_image_chemical_analysis.py | 22 +++++++ .../apps/images/migrations/__init__.py | 0 metpetdb_api/apps/images/models.py | 52 ++++++++++++++++ .../migrations/0003_auto_20180220_0409.py | 59 +++++++++++++++++++ metpetdb_api/apps/users/models.py | 4 +- metpetdb_api/metpetdb_api/urls.py | 9 ++- metpetdb_api/requirements/dev.txt | 3 +- metpetdb_api/requirements/staging.txt | 3 +- metpetdb_api/settings/staging.py | 56 ++++++++++++++++++ 25 files changed, 422 insertions(+), 20 deletions(-) create mode 100644 metpetdb_api/api/images/__init__.py create mode 100644 metpetdb_api/api/images/v1/__init__.py create mode 100644 metpetdb_api/api/images/v1/serializers.py create mode 100644 metpetdb_api/api/images/v1/tests.py create mode 100644 metpetdb_api/api/images/v1/views.py create mode 100644 metpetdb_api/apps/chemical_analyses/migrations/0004_auto_20180220_0409.py create mode 100644 metpetdb_api/apps/images/__init__.py create mode 100644 metpetdb_api/apps/images/migrations/0001_initial.py create mode 100644 metpetdb_api/apps/images/migrations/0002_image_samples.py create mode 100644 metpetdb_api/apps/images/migrations/0003_auto_20180220_0539.py create mode 100644 metpetdb_api/apps/images/migrations/0004_image_subsample.py create mode 100644 metpetdb_api/apps/images/migrations/0005_image_chemical_analysis.py create mode 100644 metpetdb_api/apps/images/migrations/__init__.py create mode 100644 metpetdb_api/apps/images/models.py create mode 100644 metpetdb_api/apps/samples/migrations/0003_auto_20180220_0409.py diff --git a/metpetdb_api/api/chemical_analyses/v1/serializers.py b/metpetdb_api/api/chemical_analyses/v1/serializers.py index 466763e..760fb92 100644 --- a/metpetdb_api/api/chemical_analyses/v1/serializers.py +++ b/metpetdb_api/api/chemical_analyses/v1/serializers.py @@ -12,6 +12,7 @@ ) from apps.samples.models import Subsample, Mineral from apps.users.models import User +from api.images.v1.serializers import ImageSerializer CHEMICAL_ANALYSIS_FIELDS = ('reference_x', 'reference_y', 'stage_x', 'stage_y', 'analysis_method', 'where_done', 'analyst', @@ -59,6 +60,7 @@ class Meta: class ChemicalAnalysisSerializer(DynamicFieldsModelSerializer): mineral = MineralSerializer(read_only=True) owner = UserSerializer(read_only=True) + image = ImageSerializer(many=False, read_only=True) elements = ChemicalAnalysisElementSerializer( many=True, source='chemicalanalysiselement_set', diff --git a/metpetdb_api/api/images/__init__.py b/metpetdb_api/api/images/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/metpetdb_api/api/images/v1/__init__.py b/metpetdb_api/api/images/v1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/metpetdb_api/api/images/v1/serializers.py b/metpetdb_api/api/images/v1/serializers.py new file mode 100644 index 0000000..8db916f --- /dev/null +++ b/metpetdb_api/api/images/v1/serializers.py @@ -0,0 +1,45 @@ +import tempfile +from os import walk, sep +from rest_framework import serializers +from apps.images.models import Image, ImageContainer +from django.core.files import File +import urllib.request +from versatileimagefield.serializers import VersatileImageFieldSerializer +import zipfile + + +class ImageSerializer(serializers.ModelSerializer): + class Meta: + model = Image + fields = ('id', 'image') + image = VersatileImageFieldSerializer(sizes='image_sizes', required=False) + + +class ImageContainerSerializer(serializers.ModelSerializer): + class Meta: + model = ImageContainer + fields = ('id', 'description', 'url', 'images') + + images = ImageSerializer(many=True, read_only=True) + + def create(self, validated_data): + new_image_container = ImageContainer.objects.create(**validated_data) + url = validated_data['url'] # zip file url + + with tempfile.TemporaryDirectory() as tmp_dir: + with tempfile.TemporaryFile() as zip_tmp: + with urllib.request.urlopen(url) as image_zip: + zip_tmp.write(image_zip.read()) + + with zipfile.ZipFile(zip_tmp, 'r') as zip_ref: + zip_ref.extractall(tmp_dir) + + for (dir_path, dir_names, file_names) in walk(tmp_dir): + for file_name in file_names: + print(sep.join((dir_path, file_name))) + with open(sep.join((dir_path, file_name)), 'rb') as file_contents: + new_image = Image.objects.create(image_container=new_image_container) + new_image.image.save(file_name, File(file_contents)) + new_image.save() + + return new_image_container diff --git a/metpetdb_api/api/images/v1/tests.py b/metpetdb_api/api/images/v1/tests.py new file mode 100644 index 0000000..e69de29 diff --git a/metpetdb_api/api/images/v1/views.py b/metpetdb_api/api/images/v1/views.py new file mode 100644 index 0000000..1085cae --- /dev/null +++ b/metpetdb_api/api/images/v1/views.py @@ -0,0 +1,18 @@ +from rest_framework import viewsets +from api.images.v1.serializers import ImageContainerSerializer, ImageSerializer +from apps.images.models import ImageContainer, Image +from rest_framework import permissions, status, viewsets +from api.lib.permissions import IsOwnerOrReadOnly, IsSuperuserOrReadOnly + + +class ImageContainerViewSet(viewsets.ModelViewSet): + queryset = ImageContainer.objects.all() + serializer_class = ImageContainerSerializer + # FIXME + # permission_classes = (permissions.IsAuthenticatedOrReadOnly, + # IsOwnerOrReadOnly,) + +class ImageViewSet(viewsets.ModelViewSet): + queryset = Image.objects.all() + serializer_class = ImageSerializer + # TODO add permissions diff --git a/metpetdb_api/api/samples/lib/query.py b/metpetdb_api/api/samples/lib/query.py index 7aae9ad..8f186a2 100644 --- a/metpetdb_api/api/samples/lib/query.py +++ b/metpetdb_api/api/samples/lib/query.py @@ -13,12 +13,11 @@ def sample_query(user, params, qs): qs = qs.filter(Q(owner=user) | Q(public_data=True)) if params.get('provenance'): - if params['provenance']=="Public": + if params['provenance'] == "Public": qs = qs.filter(Q(public_data=True)) elif params['provenance'] == "Private": qs = qs.filter(Q(public_data=False)) - if params.get('ids'): qs = qs.filter(pk__in=params['ids'].split(',')) @@ -32,7 +31,7 @@ def sample_query(user, params, qs): qs = qs.filter(country__in=params['countries'].split(',')) if params.get('location_bbox'): - bbox = Polygon.from_bbox(params['location_bbox'].split(',')) + bbox = Polygon.from_bbox(params['location_bbox'].split(',')) qs = qs.filter(location_coords__contained=bbox) if params.get('polygon_coords'): @@ -49,7 +48,7 @@ def sample_query(user, params, qs): if params.get('metamorphic_regions'): metamorphic_regions = params['metamorphic_regions'].split(',') - qs =qs.filter(metamorphic_regions__name__in=metamorphic_regions) + qs = qs.filter(metamorphic_regions__name__in=metamorphic_regions) if params.get('minerals'): minerals = params['minerals'].split(',') diff --git a/metpetdb_api/api/samples/v1/serializers.py b/metpetdb_api/api/samples/v1/serializers.py index 102cca5..8410176 100644 --- a/metpetdb_api/api/samples/v1/serializers.py +++ b/metpetdb_api/api/samples/v1/serializers.py @@ -2,6 +2,7 @@ from api.lib.serializers import DynamicFieldsModelSerializer from api.users.v1.serializers import UserSerializer +from api.images.v1.serializers import ImageSerializer from apps.chemical_analyses.models import ChemicalAnalysis from apps.samples.models import ( @@ -23,7 +24,7 @@ SAMPLE_FIELDS = ('number', 'aliases', 'collection_date', 'description', 'location_name', 'location_coords', 'location_error', 'date_precision', 'country', 'regions', 'collector_name', - 'collector_id', 'sesar_number',) + 'collector_id', 'sesar_number', 'image') SUBSAMPLE_FIELDS = ('name') @@ -55,6 +56,8 @@ class SampleSerializer(DynamicFieldsModelSerializer): many=True) owner = UserSerializer(read_only=True) + images = ImageSerializer(many=True, read_only=True) + # TODO: figure out if there is a better, more efficient way to do this subsample_ids = serializers.SerializerMethodField() chemical_analyses_ids = serializers.SerializerMethodField() @@ -88,7 +91,6 @@ def create(self, validated_data): instance = super().create(validated_data) return instance - def update(self, instance, validated_data): for attr, value in validated_data.items(): if attr in SAMPLE_FIELDS: @@ -110,6 +112,7 @@ def get_chemical_analyses_ids(self, obj): class SubsampleSerializer(DynamicFieldsModelSerializer): sample = SampleSerializer(read_only=True) owner = UserSerializer(read_only=True) + images = ImageSerializer(many=True, read_only=True) class Meta: model = Subsample diff --git a/metpetdb_api/api/samples/v1/views.py b/metpetdb_api/api/samples/v1/views.py index ef1aa9c..7e2c5d9 100644 --- a/metpetdb_api/api/samples/v1/views.py +++ b/metpetdb_api/api/samples/v1/views.py @@ -49,7 +49,6 @@ def get_serializer(self, *args, **kwargs): kwargs['partial'] = True return super().get_serializer(*args, **kwargs) - def list(self, request, *args, **kwargs): params = request.query_params @@ -81,7 +80,6 @@ def list(self, request, *args, **kwargs): serializer = self.get_serializer(qs, many=True) return Response(serializer.data) - def _handle_metamorphic_regions(self, instance, ids): metamorphic_regions = [] for id in ids: @@ -94,7 +92,6 @@ def _handle_metamorphic_regions(self, instance, ids): metamorphic_regions.append(metamorphic_region) instance.metamorphic_regions = metamorphic_regions - def _handle_metamorphic_grades(self, instance, ids): metamorphic_grades = [] for id in ids: @@ -106,7 +103,6 @@ def _handle_metamorphic_grades(self, instance, ids): metamorphic_grades.append(metamorphic_grade) instance.metamorphic_grades = metamorphic_grades - def _handle_minerals(self, instance, minerals): to_add = [] for record in minerals: @@ -123,7 +119,6 @@ def _handle_minerals(self, instance, minerals): mineral=record['mineral'], amount=record['amount']) - def _handle_references(self, instance, references): to_add = [] @@ -198,7 +193,6 @@ def create(self, request, *args, **kwargs): status=status.HTTP_201_CREATED, headers=headers) - def update(self, request, *args, **kwargs): params = request.data partial = kwargs.pop('partial', False) @@ -286,7 +280,6 @@ def list(self, request, *args, **kwargs): serializer = self.get_serializer(qs, many=True) return Response(serializer.data) - def perform_create(self, serializer): return serializer.save() @@ -323,7 +316,6 @@ def update(self, request, *args, **kwargs): serializer = self.get_serializer(instance) return Response(serializer.data) - class SubsampleTypeViewSet(viewsets.ModelViewSet): queryset = SubsampleType.objects.all() diff --git a/metpetdb_api/apps/chemical_analyses/migrations/0004_auto_20180220_0409.py b/metpetdb_api/apps/chemical_analyses/migrations/0004_auto_20180220_0409.py new file mode 100644 index 0000000..8e7cfe5 --- /dev/null +++ b/metpetdb_api/apps/chemical_analyses/migrations/0004_auto_20180220_0409.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-02-20 04:09 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('chemical_analyses', '0003_chemicalanalysis_stage_y'), + ] + + operations = [ + migrations.AlterModelOptions( + name='chemicalanalysis', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='element', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='oxide', + options={'ordering': ['id']}, + ), + ] diff --git a/metpetdb_api/apps/chemical_analyses/models.py b/metpetdb_api/apps/chemical_analyses/models.py index 758aef5..a11cf7a 100644 --- a/metpetdb_api/apps/chemical_analyses/models.py +++ b/metpetdb_api/apps/chemical_analyses/models.py @@ -1,6 +1,6 @@ import uuid -from concurrency.fields import AutoIncVersionField +from concurrency.fields import AutoIncVersionField from django.conf import settings from django.contrib.gis.db import models diff --git a/metpetdb_api/apps/images/__init__.py b/metpetdb_api/apps/images/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/metpetdb_api/apps/images/migrations/0001_initial.py b/metpetdb_api/apps/images/migrations/0001_initial.py new file mode 100644 index 0000000..1ea9ddf --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0001_initial.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-02-20 04:20 +from __future__ import unicode_literals + +import apps.images.models +from django.db import migrations, models +import django.db.models.deletion +import uuid +import versatileimagefield.fields + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Image', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('image', versatileimagefield.fields.VersatileImageField(blank=True, null=True, upload_to=apps.images.models.Image.generate_filename, verbose_name='Image')), + ], + options={ + 'db_table': 'images', + 'ordering': ('id',), + }, + ), + migrations.CreateModel( + name='ImageContainer', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('description', models.CharField(blank=True, max_length=100, null=True)), + ('url', models.CharField(blank=True, max_length=500, null=True)), + ], + options={ + 'db_table': 'image_container', + 'ordering': ('id',), + }, + ), + migrations.AddField( + model_name='image', + name='image_container', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='images', to='images.ImageContainer'), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0002_image_samples.py b/metpetdb_api/apps/images/migrations/0002_image_samples.py new file mode 100644 index 0000000..9fb6444 --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0002_image_samples.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-02-20 05:21 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('samples', '0003_auto_20180220_0409'), + ('images', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='image', + name='samples', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='image', to='samples.Sample'), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0003_auto_20180220_0539.py b/metpetdb_api/apps/images/migrations/0003_auto_20180220_0539.py new file mode 100644 index 0000000..abde7aa --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0003_auto_20180220_0539.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-02-20 05:39 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('samples', '0003_auto_20180220_0409'), + ('images', '0002_image_samples'), + ] + + operations = [ + migrations.RemoveField( + model_name='image', + name='samples', + ), + migrations.AddField( + model_name='image', + name='sample', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='images', to='samples.Sample'), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0004_image_subsample.py b/metpetdb_api/apps/images/migrations/0004_image_subsample.py new file mode 100644 index 0000000..ee57aa8 --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0004_image_subsample.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-02-20 05:46 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('samples', '0003_auto_20180220_0409'), + ('images', '0003_auto_20180220_0539'), + ] + + operations = [ + migrations.AddField( + model_name='image', + name='subsample', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='images', to='samples.Subsample'), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0005_image_chemical_analysis.py b/metpetdb_api/apps/images/migrations/0005_image_chemical_analysis.py new file mode 100644 index 0000000..cf62c97 --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0005_image_chemical_analysis.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-02-20 06:09 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('chemical_analyses', '0004_auto_20180220_0409'), + ('images', '0004_image_subsample'), + ] + + operations = [ + migrations.AddField( + model_name='image', + name='chemical_analysis', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='image', to='chemical_analyses.ChemicalAnalysis'), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/__init__.py b/metpetdb_api/apps/images/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/metpetdb_api/apps/images/models.py b/metpetdb_api/apps/images/models.py new file mode 100644 index 0000000..a8c6a00 --- /dev/null +++ b/metpetdb_api/apps/images/models.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import os +import uuid + +from django.contrib.gis.db import models +from django.dispatch import receiver +from versatileimagefield.fields import VersatileImageField +from versatileimagefield.image_warmer import VersatileImageFieldWarmer +from apps.samples.models import Sample, Subsample +from apps.chemical_analyses.models import ChemicalAnalysis + + +class ImageContainer(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + description = models.CharField(max_length=100, blank=True, null=True) + url = models.CharField(max_length=500, blank=True, null=True) + + class Meta: + db_table = 'image_container' + ordering = ('id',) + + +class Image(models.Model): + def generate_filename(instance, filename): + f_hash = str(uuid.uuid4()).replace('-', '') + assert (len(f_hash) % 2 == 0) + return "{}{}{}".format(os.sep.join(x + y for x, y in zip(f_hash[::2], f_hash[1::2])), + os.sep, filename) + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + image = VersatileImageField('Image', upload_to=generate_filename, blank=True, null=True) + image_container = models.ForeignKey(ImageContainer, on_delete=models.CASCADE, blank=True, null=True, related_name='images') + sample = models.ForeignKey(Sample, on_delete=models.CASCADE, blank=True, null=True, related_name='images') + subsample = models.ForeignKey(Subsample, on_delete=models.CASCADE, blank=True, null=True, related_name='images') + chemical_analysis = models.ForeignKey(ChemicalAnalysis, on_delete=models.CASCADE, blank=True, null=True, related_name='image') + + class Meta: + db_table = 'images' + ordering = ('id',) + + +@receiver(models.signals.post_save, sender=Image) +def warm_images(sender, instance, **kwargs): + """Create all image size on POST""" + image_warmer = VersatileImageFieldWarmer( + instance_or_queryset=instance, + rendition_key_set='image_sizes', + image_attr='image' + ) + image_warmer.warm() diff --git a/metpetdb_api/apps/samples/migrations/0003_auto_20180220_0409.py b/metpetdb_api/apps/samples/migrations/0003_auto_20180220_0409.py new file mode 100644 index 0000000..761b62e --- /dev/null +++ b/metpetdb_api/apps/samples/migrations/0003_auto_20180220_0409.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-02-20 04:09 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('samples', '0002_auto_20170425_1902'), + ] + + operations = [ + migrations.AlterModelOptions( + name='collector', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='georeference', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='metamorphicgrade', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='metamorphicregion', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='mineral', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='reference', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='region', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='rocktype', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='sample', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='subsample', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='subsampletype', + options={'ordering': ['id']}, + ), + ] diff --git a/metpetdb_api/apps/users/models.py b/metpetdb_api/apps/users/models.py index b477b4a..7cc7527 100644 --- a/metpetdb_api/apps/users/models.py +++ b/metpetdb_api/apps/users/models.py @@ -1,12 +1,12 @@ -from datetime import datetime import uuid +from datetime import datetime -from django.db import models from django.contrib.auth.models import ( AbstractBaseUser, PermissionsMixin, BaseUserManager, ) +from django.db import models from rest_framework.authtoken.models import Token diff --git a/metpetdb_api/metpetdb_api/urls.py b/metpetdb_api/metpetdb_api/urls.py index e054705..7cfc513 100644 --- a/metpetdb_api/metpetdb_api/urls.py +++ b/metpetdb_api/metpetdb_api/urls.py @@ -45,8 +45,13 @@ ) from api.users.v1.views import UserViewSet +from api.images.v1.views import ImageContainerViewSet, ImageViewSet + from api.bulk_upload.v1.views import BulkUploadViewSet +from django.conf import settings +from django.conf.urls.static import static + router = routers.DefaultRouter() router.register(r'users', UserViewSet) @@ -65,6 +70,8 @@ router.register(r'references', ReferenceViewSet) router.register(r'collectors', CollectorViewSet) router.register(r'bulk_upload', BulkUploadViewSet) +router.register(r'image_sets', ImageContainerViewSet) +router.register(r'images', ImageViewSet) urlpatterns = [ url(r'^api/', include(router.urls)), @@ -74,4 +81,4 @@ url(r'^api/sample_numbers/$', SampleNumbersView.as_view()), url(r'^api/country_names/$', CountryNamesView.as_view()), url(r'^api/sample_owner_names/$', SampleOwnerNamesView.as_view()), -] +] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/metpetdb_api/requirements/dev.txt b/metpetdb_api/requirements/dev.txt index 190c72b..70647d9 100644 --- a/metpetdb_api/requirements/dev.txt +++ b/metpetdb_api/requirements/dev.txt @@ -9,4 +9,5 @@ psycopg2-binary>=2.6.1 six>=1.9.0 sqlparse>=0.1.15 wheel>=0.24.0 -pathlib \ No newline at end of file +pathlib +django-versatileimagefield diff --git a/metpetdb_api/requirements/staging.txt b/metpetdb_api/requirements/staging.txt index 15d5906..51e696a 100644 --- a/metpetdb_api/requirements/staging.txt +++ b/metpetdb_api/requirements/staging.txt @@ -9,4 +9,5 @@ psycopg2-binary>=2.6.1 six>=1.9.0 sqlparse>=0.1.15 wheel>=0.24.0 -pathlib==1.0.1 \ No newline at end of file +pathlib==1.0.1 +django-versatileimagefield diff --git a/metpetdb_api/settings/staging.py b/metpetdb_api/settings/staging.py index 35b09d6..a84825e 100644 --- a/metpetdb_api/settings/staging.py +++ b/metpetdb_api/settings/staging.py @@ -62,6 +62,8 @@ 'apps.samples', 'apps.users', 'apps.core', + 'versatileimagefield', + 'apps.images' ) MIDDLEWARE_CLASSES = ( @@ -176,3 +178,57 @@ 'ACTIVATION_URL': 'login#/activate/{uid}/{token}', 'SEND_ACTIVATION_EMAIL' : True } + +# A dictionary that allows you to fine-tune how django-versatileimagefield works: +VERSATILEIMAGEFIELD_SETTINGS = { + # The amount of time, in seconds, that references to created images + # should be stored in the cache. Defaults to `2592000` (30 days) + 'cache_length': 2592000, + # The name of the cache you'd like `django-versatileimagefield` to use. + # Defaults to 'versatileimagefield_cache'. If no cache exists with the name + # provided, the 'default' cache will be used instead. + 'cache_name': 'versatileimagefield_cache', + # The save quality of modified JPEG images. More info here: + # https://pillow.readthedocs.io/en/latest/handbook/image-file-formats.html#jpeg + # Defaults to 70 + 'jpeg_resize_quality': 70, + # The name of the top-level folder within storage classes to save all + # sized images. Defaults to '__sized__' + 'sized_directory_name': '__sized__', + # The name of the directory to save all filtered images within. + # Defaults to '__filtered__': + 'filtered_directory_name': '__filtered__', + # The name of the directory to save placeholder images within. + # Defaults to '__placeholder__': + 'placeholder_directory_name': '__placeholder__', + # Whether or not to create new images on-the-fly. Set this to `False` for + # speedy performance but don't forget to 'pre-warm' to ensure they're + # created and available at the appropriate URL. + 'create_images_on_demand': False, + # A dot-notated python path string to a function that processes sized + # image keys. Typically used to md5-ify the 'image key' portion of the + # filename, giving each a uniform length. + # `django-versatileimagefield` ships with two post processors: + # 1. 'versatileimagefield.processors.md5' Returns a full length (32 char) + # md5 hash of `image_key`. + # 2. 'versatileimagefield.processors.md5_16' Returns the first 16 chars + # of the 32 character md5 hash of `image_key`. + # By default, image_keys are unprocessed. To write your own processor, + # just define a function (that can be imported from your project's + # python path) that takes a single argument, `image_key` and returns + # a string. + 'image_key_post_processor': None, + # Whether to create progressive JPEGs. Read more about progressive JPEGs + # here: https://optimus.io/support/progressive-jpeg/ + 'progressive_jpeg': False +} + +VERSATILEIMAGEFIELD_RENDITION_KEY_SETS = { + 'image_sizes': [ + ('full_size', 'url'), + ('thumbnail', 'thumbnail__100x100') + ] +} + +MEDIA_ROOT = os.path.join(BASE_DIR, 'media_root/') +MEDIA_URL = '/images/' From 6bda95f2d481ef9f121756ec3480a511ab9340b5 Mon Sep 17 00:00:00 2001 From: metpetdb Date: Tue, 13 Mar 2018 22:13:56 +0000 Subject: [PATCH 06/51] added var for sitename for new email template --- metpetdb_api/settings/staging.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metpetdb_api/settings/staging.py b/metpetdb_api/settings/staging.py index 35b09d6..a95deac 100644 --- a/metpetdb_api/settings/staging.py +++ b/metpetdb_api/settings/staging.py @@ -172,6 +172,7 @@ DJOSER = { 'DOMAIN': env('FRONT_END_URL'), + 'SITE_NAME': env('FRONT_END_SITE_NAME'), 'PASSWORD_RESET_CONFIRM_URL': 'reset-password#/{uid}/{token}', 'ACTIVATION_URL': 'login#/activate/{uid}/{token}', 'SEND_ACTIVATION_EMAIL' : True From 447608403354369af6b018b7f31d39c22f7988cc Mon Sep 17 00:00:00 2001 From: metpetdb Date: Tue, 13 Mar 2018 22:36:46 +0000 Subject: [PATCH 07/51] updated djoser submodule locaiton --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 8cc734b..6ca1359 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "metpetdb_api/vendor/djoser"] path = metpetdb_api/vendor/djoser - url = https://github.com/kristallizer/djoser.git + url = https://github.com/IreLynn/djoser.git From f2917b4b658e73f64ef6ebf97c91f304cf5b7e7b Mon Sep 17 00:00:00 2001 From: metpetDB Date: Sat, 14 Jul 2018 15:28:34 -0400 Subject: [PATCH 08/51] make chemical analysis query orderable --- metpetdb_api/api/chemical_analyses/lib/query.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metpetdb_api/api/chemical_analyses/lib/query.py b/metpetdb_api/api/chemical_analyses/lib/query.py index 5727ed0..c8c00af 100644 --- a/metpetdb_api/api/chemical_analyses/lib/query.py +++ b/metpetdb_api/api/chemical_analyses/lib/query.py @@ -54,4 +54,7 @@ def chemical_analysis_query(user, params, qs): if params.get('subsample_ids'): qs = qs.filter(subsample_id__in=params.get('subsample_ids').split(',')) + if params.get('ordering'): + qs = qs.order_by(params['ordering']) + return qs From b18425eb06908d69abc051cb78ae8e29af86a271 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Thu, 9 Aug 2018 20:10:23 -0400 Subject: [PATCH 09/51] added csv renderer for chemical analyses & updated serializer and view --- .../api/chemical_analyses/v1/renderers.py | 108 ++++++++++++++++++ .../api/chemical_analyses/v1/serializers.py | 101 ++++++++++++---- .../api/chemical_analyses/v1/views.py | 19 ++- 3 files changed, 197 insertions(+), 31 deletions(-) create mode 100644 metpetdb_api/api/chemical_analyses/v1/renderers.py diff --git a/metpetdb_api/api/chemical_analyses/v1/renderers.py b/metpetdb_api/api/chemical_analyses/v1/renderers.py new file mode 100644 index 0000000..d51ff2e --- /dev/null +++ b/metpetdb_api/api/chemical_analyses/v1/renderers.py @@ -0,0 +1,108 @@ +from rest_framework_csv import renderers as r + +class ChemicalAnalysisCSVRenderer (r.CSVRenderer): + + def __init__(self): + self.header = [ + 'sample', + 'subsample', + 'spot_id', + 'mineral', + 'analysis_method', + 'subsample_type', + 'where_done', + 'analysis_date', + 'analyst', + 'reference', + # 'reference_image' + 'reference_x', + 'reference_y', + 'elements', + # 'oxides', + 'total', + 'stage_x', + 'stage_y', + 'description', + ] + self.labels = { + 'sample':'Sample', + 'subsample':'Subsample', + 'mineral':'Mineral', + 'analysis_method':'Method', + 'subsample_type':'Subsample Type', + 'reference':'Reference', + 'spot_id':'Point', + 'where_done':'Analytical Facility', + 'analysis_date':'Analysis Date', + 'analyst':'Analyst', + 'reference_x':'Reference X', + 'reference_y':'Reference Y', + 'stage_x':'Stage X', + 'stage_y':'Stage Y', + 'description':'Comment', + # 'elements', + # 'oxides', + 'total':'Total', + } + + self.num_elements = 0 + self.num_oxides = 0 + self.elements = set() + self.oxides = set() + + def tablize(self, data, header=None, labels=None): + if data: + data = self.flatten_data(data) + + if 'elements' in header: + data = tuple(data) + header.remove('elements') + els = list(self.elements) + els.sort() + oxs = list(self.oxides) + oxs.sort() + header[12:12] = oxs + header[12:12] = els + + if labels: + yield [labels.get(x,x) for x in header] + else: + yield header + + for item in data: + # print(item) + row = [item.get(key,None) for key in header] + yield row + + elif header: + if labels: + yield [labels.get(x,x) for x in header] + else: + yield header + else: + pass + + def flatten_data(self,data): + for item in data: + self.handle_elements(item) + self.handle_oxides(item) + flat_item = self.flatten_item(item) + yield flat_item + + def handle_elements(self,item): + for e in item['elements']: + col = '{} ({})'.format(e['symbol'],e['measurement_unit']) + prec = '{} Precision ({})'.format(e['symbol'],e['precision_type']) + item[col] = e['amount'] + item[prec] = e['precision'] + self.elements.add(col) + self.elements.add(prec) + + def handle_oxides(self,item): + for o in item['oxides']: + col = '{} ({})'.format(o['species'],o['measurement_unit']) + prec = '{} Precision ({})'.format(o['species'],o['precision_type']) + item[col] = o['amount'] + item[prec] = o['precision'] + self.oxides.add(col) + self.oxides.add(prec) diff --git a/metpetdb_api/api/chemical_analyses/v1/serializers.py b/metpetdb_api/api/chemical_analyses/v1/serializers.py index 6343b77..ab8cc0c 100644 --- a/metpetdb_api/api/chemical_analyses/v1/serializers.py +++ b/metpetdb_api/api/chemical_analyses/v1/serializers.py @@ -20,45 +20,72 @@ class ChemicalAnalysisElementSerializer(DynamicFieldsModelSerializer): - id = serializers.ReadOnlyField(source='element.id') - name = serializers.ReadOnlyField(source='element.name') - alternate_name = serializers.ReadOnlyField(source='element.alternate_name') + # id = serializers.ReadOnlyField(source='element.id') + # name = serializers.ReadOnlyField(source='element.name') + # alternate_name = serializers.ReadOnlyField(source='element.alternate_name') symbol = serializers.ReadOnlyField(source='element.symbol') - atomic_number = serializers.ReadOnlyField(source='element.atomic_number') - weight = serializers.ReadOnlyField(source='element.weight') - order_id = serializers.ReadOnlyField(source='element.order_id') + # atomic_number = serializers.ReadOnlyField(source='element.atomic_number') + # weight = serializers.ReadOnlyField(source='element.weight') + # order_id = serializers.ReadOnlyField(source='element.order_id') class Meta: model = ChemicalAnalysisElement - fields = ('id', 'name', 'alternate_name', 'symbol', 'atomic_number', - 'weight', 'order_id', 'amount', 'precision', - 'precision_type', 'measurement_unit', 'min_amount', - 'max_amount') + fields = ( + # 'id', + # 'name', + # 'alternate_name', + 'symbol', + # 'atomic_number', + # 'weight', + # 'order_id', + 'amount', + 'precision', + 'precision_type', + 'measurement_unit', + # 'min_amount', + # 'max_amount' + ) class ChemicalAnalysisOxideSerializer(DynamicFieldsModelSerializer): - id = serializers.ReadOnlyField(source='oxide.id') - element_id = serializers.ReadOnlyField(source='oxide.element_id') - oxidation_state = serializers.ReadOnlyField(source='oxide.oxidation_state') + # id = serializers.ReadOnlyField(source='oxide.id') + # element_id = serializers.ReadOnlyField(source='oxide.element_id') + # oxidation_state = serializers.ReadOnlyField(source='oxide.oxidation_state') species = serializers.ReadOnlyField(source='oxide.species') - weight = serializers.ReadOnlyField(source='oxide.weight') - cations_per_oxide = serializers.ReadOnlyField( - source='oxide.cations_per_oxide') - conversion_factor = serializers.ReadOnlyField( - source='oxide.conversion_factor') - order_id = serializers.ReadOnlyField(source='oxide.order_id') + # weight = serializers.ReadOnlyField(source='oxide.weight') + # cations_per_oxide = serializers.ReadOnlyField( + # source='oxide.cations_per_oxide') + # conversion_factor = serializers.ReadOnlyField( + # source='oxide.conversion_factor') + # order_id = serializers.ReadOnlyField(source='oxide.order_id') class Meta: model = ChemicalAnalysisOxide - fields = ('id', 'element_id', 'oxidation_state', 'species', 'weight', - 'cations_per_oxide', 'conversion_factor', 'order_id', - 'amount', 'precision', 'precision_type', 'measurement_unit', - 'min_amount', 'max_amount' ) + fields = ( + # 'id', + # 'element_id', + # 'oxidation_state', + 'species', + # 'weight', + # 'cations_per_oxide', + # 'conversion_factor', + # 'order_id', + 'amount', + 'precision', + 'precision_type', + 'measurement_unit', + # 'min_amount', + # 'max_amount' + ) class ChemicalAnalysisSerializer(DynamicFieldsModelSerializer): - mineral = MineralSerializer(read_only=True) - owner = UserSerializer(read_only=True) + owner = serializers.ReadOnlyField(source='owner.name') + sample = serializers.ReadOnlyField(source='subsample.sample.number') + subsample = serializers.ReadOnlyField(source='subsample.name') + subsample_type = serializers.ReadOnlyField(source='subsample.subsample_type.name') + mineral = serializers.ReadOnlyField(source='mineral.name') + elements = ChemicalAnalysisElementSerializer( many=True, source='chemicalanalysiselement_set', @@ -73,6 +100,28 @@ class ChemicalAnalysisSerializer(DynamicFieldsModelSerializer): class Meta: model = ChemicalAnalysis depth = 1 + fields = ( + 'owner', + 'sample', + 'subsample', + 'subsample_type', + 'mineral', + 'analysis_method', + 'reference', + 'spot_id', + 'where_done', + 'analysis_date', + # 'date_precision', + 'analyst', + 'reference_x', + 'reference_y', + 'stage_x', + 'stage_y', + 'description', + 'elements', + 'oxides', + 'total', + ) def is_valid(self, raise_exception=False): super().is_valid(raise_exception) @@ -116,6 +165,8 @@ def update(self, instance, validated_data): return instance + + class ElementSerializer(DynamicFieldsModelSerializer): class Meta: model = Element diff --git a/metpetdb_api/api/chemical_analyses/v1/views.py b/metpetdb_api/api/chemical_analyses/v1/views.py index 9086093..c82c458 100644 --- a/metpetdb_api/api/chemical_analyses/v1/views.py +++ b/metpetdb_api/api/chemical_analyses/v1/views.py @@ -1,5 +1,7 @@ from rest_framework import permissions, status, viewsets from rest_framework.response import Response +from rest_framework.renderers import JSONRenderer, BrowsableAPIRenderer +from api.chemical_analyses.v1.renderers import ChemicalAnalysisCSVRenderer from api.chemical_analyses.lib.query import chemical_analysis_query from api.chemical_analyses.v1.serializers import ( @@ -25,6 +27,7 @@ class ChemicalAnalysisViewSet(viewsets.ModelViewSet): queryset = ChemicalAnalysis.objects.all() serializer_class = ChemicalAnalysisSerializer + renderer_classes = (JSONRenderer, BrowsableAPIRenderer, ChemicalAnalysisCSVRenderer) permission_classes = (permissions.IsAuthenticatedOrReadOnly, IsOwnerOrReadOnly,) @@ -50,13 +53,17 @@ def list(self, request, *args, **kwargs): qs = chemical_analyses_qs_optimizer(params, qs) - page = self.paginate_queryset(qs) - if page is not None: - serializer = self.get_serializer(page, many=True) - return self.get_paginated_response(serializer.data) + if params.get('format') == 'csv': + serializer = self.get_serializer(qs,many=True) + return Response(serializer.data) + else: + page = self.paginate_queryset(qs) + if page is not None: + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) - serializer = self.get_serializer(qs, many=True) - return Response(serializer.data) + serializer = self.get_serializer(qs, many=True) + return Response(serializer.data) def _handle_elements(self, instance, params): From e6cf15e1c0155c98cff2785a8477ca93c3656cf0 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Thu, 9 Aug 2018 20:12:11 -0400 Subject: [PATCH 10/51] cleaned up some methods in sample serializer --- metpetdb_api/api/samples/v1/renderers.py | 49 ++++++++++++---------- metpetdb_api/api/samples/v1/serializers.py | 11 +---- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/metpetdb_api/api/samples/v1/renderers.py b/metpetdb_api/api/samples/v1/renderers.py index ca97fb1..86b4768 100644 --- a/metpetdb_api/api/samples/v1/renderers.py +++ b/metpetdb_api/api/samples/v1/renderers.py @@ -4,23 +4,19 @@ class SampleCSVRenderer (r.CSVRenderer): def __init__(self): - self.header = ['number', - 'rock_type', - 'description', - 'latitude', - 'longitude', - 'location_error', - 'country', - 'collector_name', - 'collection_date', - 'location_name', - # 'references.0', - # 'metamorphic_grades.0', - 'minerals', - # 'igsn', - # 'subsample_ids', - # 'chemical_analyses_ids', - ] + self.header = [ + 'number', + 'rock_type', + 'description', + 'latitude', + 'longitude', + 'location_error', + 'country', + 'collector_name', + 'collection_date', + 'location_name', + 'minerals', + ] self.labels = { 'number': 'Sample', 'rock_type': 'Rock Type', @@ -32,14 +28,11 @@ def __init__(self): 'collector_name': 'Collector', 'collection_date': 'Date of Collection', 'location_name': 'Present Sample Location', - # 'references.0': 'Reference', - # 'metamorphic_grades.0': 'Metamorphic Grade', 'minerals': 'Mineral', - # 'Subsamples': 'Number of Subsamples', - # 'Chemical_Analyses': 'Number of Chemical Analyses' } self.num_regions = 0 + self.num_meta_regions = 0 self.num_refs = 0 self.num_grades = 0 self.minerals = set() @@ -60,11 +53,17 @@ def tablize(self, data, header=None, labels=None): region_headers.append('regions.' + str(i)) labels[region_headers[-1]] = 'Region' header[6:6] = region_headers + meta_region_headers = [] + for i in range(self.num_meta_regions): + meta_region_headers.append('metamorphic_regions.' + str(i)) + labels[meta_region_headers[-1]] = 'Metamorphic Region' + offset = 6 + self.num_regions + header[offset:offset] = meta_region_headers ref_headers = [] for i in range(self.num_refs): ref_headers.append('references.' + str(i)) labels[ref_headers[-1]] = 'Reference' - offset = 10 + self.num_regions + offset += 4 + self.num_meta_regions header[offset:offset] = ref_headers grade_headers = [] for i in range(self.num_grades): @@ -111,9 +110,13 @@ def handle_minerals(self,item): self.minerals.add(m) def handle_regions(self,item): - regions = {r.title() for r in item['regions']} | {r.title() for r in item['metamorphic_regions']} + regions = {r.title() for r in item['regions']} self.num_regions = max(self.num_regions,len(regions)) + def handle_meta_regions(self,item): + meta_regions = {r.title() for r in item['metamorphic_regions']} + self.num_meta_regions = max(self.num_meta_regions,len(meta_regions)) + def handle_references(self,item): refs = {r for r in item['references']} self.num_refs = max(self.num_refs, len(refs)) diff --git a/metpetdb_api/api/samples/v1/serializers.py b/metpetdb_api/api/samples/v1/serializers.py index 07b4bd6..82efc7b 100644 --- a/metpetdb_api/api/samples/v1/serializers.py +++ b/metpetdb_api/api/samples/v1/serializers.py @@ -51,9 +51,8 @@ class Meta: class SampleSerializer(DynamicFieldsModelSerializer): minerals = SampleMineralSerializer(source='samplemineral_set', many=True) - # owner = UserSerializer(read_only=True) - owner = serializers.SerializerMethodField(read_only=True) - rock_type = serializers.SerializerMethodField(read_only=True) + owner = serializers.ReadOnlyField(source='owner.name') + rock_type = serializers.ReadOnlyField(source='rock_type.name') metamorphic_grades = serializers.SerializerMethodField(read_only=True) metamorphic_regions = serializers.SerializerMethodField(read_only=True) minerals = serializers.SerializerMethodField(read_only=True) @@ -124,12 +123,6 @@ def update(self, instance, validated_data): return instance - def get_owner(self,obj): - return obj.owner.name - - def get_rock_type(self,obj): - return obj.rock_type.name - def get_metamorphic_grades(self,obj): return [g.name for g in obj.metamorphic_grades.all()] From e4f54041952da70f9f1fc918e129be8868bf816a Mon Sep 17 00:00:00 2001 From: metpetDB Date: Wed, 22 Aug 2018 18:47:43 -0400 Subject: [PATCH 11/51] chemical analysis CSV renderer works, and adapts to fields argument --- .../api/chemical_analyses/v1/renderers.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/metpetdb_api/api/chemical_analyses/v1/renderers.py b/metpetdb_api/api/chemical_analyses/v1/renderers.py index d51ff2e..03aa297 100644 --- a/metpetdb_api/api/chemical_analyses/v1/renderers.py +++ b/metpetdb_api/api/chemical_analyses/v1/renderers.py @@ -49,6 +49,7 @@ def __init__(self): self.num_oxides = 0 self.elements = set() self.oxides = set() + self.fields = set() def tablize(self, data, header=None, labels=None): if data: @@ -63,6 +64,7 @@ def tablize(self, data, header=None, labels=None): oxs.sort() header[12:12] = oxs header[12:12] = els + header = [x for x in header if (x in self.fields or x in self.elements or x in self.oxides)] if labels: yield [labels.get(x,x) for x in header] @@ -84,13 +86,19 @@ def tablize(self, data, header=None, labels=None): def flatten_data(self,data): for item in data: - self.handle_elements(item) - self.handle_oxides(item) + self.fields |= item.keys() + if (item.get('elements')): + self.handle_elements(item) + if (item.get('oxides')): + self.handle_oxides(item) flat_item = self.flatten_item(item) yield flat_item def handle_elements(self,item): for e in item['elements']: + if e['precision_type'] == 'REL': + e['precision'] *= e['amount'] + e['precision_type'] = 'ABS' col = '{} ({})'.format(e['symbol'],e['measurement_unit']) prec = '{} Precision ({})'.format(e['symbol'],e['precision_type']) item[col] = e['amount'] @@ -100,6 +108,9 @@ def handle_elements(self,item): def handle_oxides(self,item): for o in item['oxides']: + if o['precision_type'] == 'REL': + o['precision'] *= o['amount'] + o['precision_type'] = 'ABS' col = '{} ({})'.format(o['species'],o['measurement_unit']) prec = '{} Precision ({})'.format(o['species'],o['precision_type']) item[col] = o['amount'] From 2ec37ade8c41517dc5e9c4482af90187d45a509b Mon Sep 17 00:00:00 2001 From: metpetDB Date: Wed, 22 Aug 2018 18:59:31 -0400 Subject: [PATCH 12/51] added missing call to handle_meta_regions & added functionality to adapt to selected fields --- metpetdb_api/api/samples/v1/renderers.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/metpetdb_api/api/samples/v1/renderers.py b/metpetdb_api/api/samples/v1/renderers.py index 86b4768..2397f50 100644 --- a/metpetdb_api/api/samples/v1/renderers.py +++ b/metpetdb_api/api/samples/v1/renderers.py @@ -36,6 +36,7 @@ def __init__(self): self.num_refs = 0 self.num_grades = 0 self.minerals = set() + self.fields = set() def tablize(self, data, header=None, labels=None): if data: @@ -74,6 +75,7 @@ def tablize(self, data, header=None, labels=None): mins = list(self.minerals) mins.sort() header.extend(mins) + header = [x for x in header if (x in self.fields or x in self.minerals or x in region_headers or x in meta_region_headers or x in ref_headers or x in grade_headers)] if labels: yield [labels.get(x,x) for x in header] @@ -95,13 +97,18 @@ def tablize(self, data, header=None, labels=None): def flatten_data(self,data): for item in data: - self.handle_regions(item) - self.handle_minerals(item) - self.handle_references(item) - self.handle_meta_grades(item) - # print(item) + self.fields |= item.keys() + if (item.get('regions')): + self.handle_regions(item) + if (item.get('metamorphic_regions')): + self.handle_meta_regions(item) + if (item.get('minerals')): + self.handle_minerals(item) + if (item.get('references')): + self.handle_references(item) + if (item.get('metamorphic_grades')): + self.handle_meta_grades(item) flat_item = self.flatten_item(item) - # print(flat_item) yield flat_item def handle_minerals(self,item): From 0e740aac76fd5337f7b18e882f2ac166e2c1aac5 Mon Sep 17 00:00:00 2001 From: Brandon Drumheller Date: Mon, 27 Aug 2018 01:24:21 -0500 Subject: [PATCH 13/51] add support for bulk image upload via image container list; update image import to match wiki documentation; update legacy import to include chemical analysis reference image, image comments --- metpetdb_api/__init__.py | 0 metpetdb_api/api/bulk_upload/v1/tests.py | 6 +- metpetdb_api/api/bulk_upload/v1/views.py | 4 +- .../api/chemical_analyses/v1/serializers.py | 7 +- .../api/chemical_analyses/v1/views.py | 3 +- metpetdb_api/api/images/v1/serializers.py | 225 ++++++++++++++-- metpetdb_api/api/images/v1/views.py | 13 +- metpetdb_api/api/samples/v1/views.py | 35 ++- .../migrate_legacy_chemical_analyses.py | 11 +- .../0005_chemicalanalysis_reference_image.py} | 10 +- metpetdb_api/apps/chemical_analyses/models.py | 33 +-- .../apps/chemical_analyses/shared_models.py | 32 +++ .../apps/images/migrations/0001_initial.py | 43 +++- ...220_0539.py => 0002_auto_20180618_0640.py} | 11 +- .../images/migrations/0002_image_samples.py | 22 -- .../migrations/0003_auto_20180618_0655.py | 22 ++ .../migrations/0004_auto_20180618_0754.py | 31 +++ .../images/migrations/0004_image_subsample.py | 22 -- .../migrations/0005_auto_20180618_0821.py | 20 ++ .../migrations/0006_auto_20180619_0335.py | 20 ++ .../apps/images/migrations/0007_xrayimage.py | 30 +++ .../migrations/0008_auto_20180619_0454.py | 21 ++ .../migrations/0009_auto_20180619_0454.py | 21 ++ .../migrations/0010_auto_20180619_0455.py | 21 ++ .../migrations/0011_auto_20180826_1618.py | 25 ++ .../0012_remove_image_chemical_analysis.py | 19 ++ .../images/migrations/0013_imagecomments.py | 29 +++ .../migrations/0014_auto_20180827_0336.py | 21 ++ .../migrations/0015_auto_20180827_0344.py | 19 ++ metpetdb_api/apps/images/models.py | 59 ++++- .../commands/migrate_legacy_samples.py | 236 ++++++++++------- metpetdb_api/apps/samples/models.py | 3 +- metpetdb_api/legacy/models.py | 241 ++++++++++-------- metpetdb_api/requirements/dev.txt | 1 + metpetdb_api/requirements/staging.txt | 1 + 35 files changed, 963 insertions(+), 354 deletions(-) create mode 100644 metpetdb_api/__init__.py rename metpetdb_api/apps/{images/migrations/0005_image_chemical_analysis.py => chemical_analyses/migrations/0005_chemicalanalysis_reference_image.py} (58%) create mode 100644 metpetdb_api/apps/chemical_analyses/shared_models.py rename metpetdb_api/apps/images/migrations/{0003_auto_20180220_0539.py => 0002_auto_20180618_0640.py} (52%) delete mode 100644 metpetdb_api/apps/images/migrations/0002_image_samples.py create mode 100644 metpetdb_api/apps/images/migrations/0003_auto_20180618_0655.py create mode 100644 metpetdb_api/apps/images/migrations/0004_auto_20180618_0754.py delete mode 100644 metpetdb_api/apps/images/migrations/0004_image_subsample.py create mode 100644 metpetdb_api/apps/images/migrations/0005_auto_20180618_0821.py create mode 100644 metpetdb_api/apps/images/migrations/0006_auto_20180619_0335.py create mode 100644 metpetdb_api/apps/images/migrations/0007_xrayimage.py create mode 100644 metpetdb_api/apps/images/migrations/0008_auto_20180619_0454.py create mode 100644 metpetdb_api/apps/images/migrations/0009_auto_20180619_0454.py create mode 100644 metpetdb_api/apps/images/migrations/0010_auto_20180619_0455.py create mode 100644 metpetdb_api/apps/images/migrations/0011_auto_20180826_1618.py create mode 100644 metpetdb_api/apps/images/migrations/0012_remove_image_chemical_analysis.py create mode 100644 metpetdb_api/apps/images/migrations/0013_imagecomments.py create mode 100644 metpetdb_api/apps/images/migrations/0014_auto_20180827_0336.py create mode 100644 metpetdb_api/apps/images/migrations/0015_auto_20180827_0344.py diff --git a/metpetdb_api/__init__.py b/metpetdb_api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/metpetdb_api/api/bulk_upload/v1/tests.py b/metpetdb_api/api/bulk_upload/v1/tests.py index bd4288b..2a227c7 100644 --- a/metpetdb_api/api/bulk_upload/v1/tests.py +++ b/metpetdb_api/api/bulk_upload/v1/tests.py @@ -12,10 +12,8 @@ Subsample ) -from apps.chemical_analyses.models import( - Element, - Oxide, -) +from apps.chemical_analyses.shared_models import Element, Oxide + class BulkUploadTests(APITransactionTestCase): """ diff --git a/metpetdb_api/api/bulk_upload/v1/views.py b/metpetdb_api/api/bulk_upload/v1/views.py index dfe2acb..aa30b3a 100644 --- a/metpetdb_api/api/bulk_upload/v1/views.py +++ b/metpetdb_api/api/bulk_upload/v1/views.py @@ -56,10 +56,8 @@ ChemicalAnalysis, ChemicalAnalysisElement, ChemicalAnalysisOxide, - Element, - Oxide, ) - +from apps.chemical_analyses.shared_models import Element, Oxide from api.bulk_upload.v1 import upload_templates import json diff --git a/metpetdb_api/api/chemical_analyses/v1/serializers.py b/metpetdb_api/api/chemical_analyses/v1/serializers.py index 760fb92..c1d0c8b 100644 --- a/metpetdb_api/api/chemical_analyses/v1/serializers.py +++ b/metpetdb_api/api/chemical_analyses/v1/serializers.py @@ -7,9 +7,8 @@ ChemicalAnalysis, ChemicalAnalysisElement, ChemicalAnalysisOxide, - Element, - Oxide, ) +from apps.chemical_analyses.shared_models import Element, Oxide from apps.samples.models import Subsample, Mineral from apps.users.models import User from api.images.v1.serializers import ImageSerializer @@ -54,13 +53,13 @@ class Meta: fields = ('id', 'element_id', 'oxidation_state', 'species', 'weight', 'cations_per_oxide', 'conversion_factor', 'order_id', 'amount', 'precision', 'precision_type', 'measurement_unit', - 'min_amount', 'max_amount' ) + 'min_amount', 'max_amount') class ChemicalAnalysisSerializer(DynamicFieldsModelSerializer): mineral = MineralSerializer(read_only=True) owner = UserSerializer(read_only=True) - image = ImageSerializer(many=False, read_only=True) + reference_image = ImageSerializer(many=False, read_only=True) elements = ChemicalAnalysisElementSerializer( many=True, source='chemicalanalysiselement_set', diff --git a/metpetdb_api/api/chemical_analyses/v1/views.py b/metpetdb_api/api/chemical_analyses/v1/views.py index 9086093..f17cf8e 100644 --- a/metpetdb_api/api/chemical_analyses/v1/views.py +++ b/metpetdb_api/api/chemical_analyses/v1/views.py @@ -17,9 +17,8 @@ ChemicalAnalysis, ChemicalAnalysisElement, ChemicalAnalysisOxide, - Element, - Oxide, ) +from apps.chemical_analyses.shared_models import Element, Oxide class ChemicalAnalysisViewSet(viewsets.ModelViewSet): diff --git a/metpetdb_api/api/images/v1/serializers.py b/metpetdb_api/api/images/v1/serializers.py index 8db916f..72aca89 100644 --- a/metpetdb_api/api/images/v1/serializers.py +++ b/metpetdb_api/api/images/v1/serializers.py @@ -1,18 +1,59 @@ import tempfile -from os import walk, sep +from os import sep, listdir from rest_framework import serializers -from apps.images.models import Image, ImageContainer +from apps.images.models import Image, ImageContainer, ImageType, XrayImage, ImageComments +from apps.samples.models import Sample, Subsample, SubsampleType from django.core.files import File import urllib.request +from urllib.parse import urlparse, urlencode, urlunparse from versatileimagefield.serializers import VersatileImageFieldSerializer import zipfile +import xlrd +import re +from django.db import transaction +from api.users.v1.serializers import UserSerializer +from apps.users.models import User +from rest_framework.response import Response + +SAMPLE_NUMBER = 'samplenumber' +SUBSAMPLE = 'subsample' +SUBSAMPLE_TYPE = 'subsampletype' +PATH = 'path' +FILE = 'file' +IMAGE_TYPE = 'imagetype' +SCALE = 'scale' +COLLECTOR = 'collector' +COMMENT = 'comment' +ELEMENT = 'element' +DWELL_TIME = 'dwelltime' +CURRENT = 'current' +VOLTAGE = 'voltage' +XRAY_IMAGE = 'xrayimage' + +required_headers = {SAMPLE_NUMBER, IMAGE_TYPE} + + +class ImageTypeSerializer(serializers.ModelSerializer): + class Meta: + model = ImageType + fields = ('id', 'image_type', 'abbreviation', 'comments') + + +class ImageCommentsSerializer(serializers.ModelSerializer): + class Meta: + model = ImageComments + fields = ('comment_id', 'comment_text', 'version') class ImageSerializer(serializers.ModelSerializer): class Meta: model = Image - fields = ('id', 'image') + fields = ('id', 'image', 'version', 'image_type', 'collector', 'owner', 'public_data', 'scale', + 'description', 'comments') image = VersatileImageFieldSerializer(sizes='image_sizes', required=False) + image_type = ImageTypeSerializer(read_only=True) + comments = ImageCommentsSerializer(many=True) + owner = UserSerializer(read_only=True) class ImageContainerSerializer(serializers.ModelSerializer): @@ -22,24 +63,172 @@ class Meta: images = ImageSerializer(many=True, read_only=True) + @staticmethod + def get_worksheet_row_values(worksheet, row_number): + return [x.value for x in worksheet.row(row_number)] + + @staticmethod + def extract_zip_file_from_url(url, destination): + with tempfile.TemporaryFile() as zip_tmp: + with urllib.request.urlopen(url) as image_zip: + zip_tmp.write(image_zip.read()) + + with zipfile.ZipFile(zip_tmp, 'r') as zip_ref: + zip_ref.extractall(destination) + + @staticmethod + def get_xls_file_name(directory): + xls_file_names = [file_name for file_name in listdir(directory) if file_name.endswith('.xls')] + if len(xls_file_names) != 1: + return Response( + data={'error': 'Expected exactly 1 .xls file, but {} files were provided'.format(len(xls_file_names))}, + status=400 + ) + + return xls_file_names[0] + + @staticmethod + def get_worksheet_header_mappings(directory, xls_file): + workbook = xlrd.open_workbook('{}{}{}'.format(directory, sep, xls_file)) + worksheet = workbook.sheet_by_index(0) + header_row = [x.value.replace(' ', '').lower() for x in worksheet.row(0)] + + if not set(header_row) >= required_headers: + raise serializers.ValidationError('Missing required headers {}'.format(required_headers-set(header_row))) + if PATH not in header_row and FILE not in header_row: + raise serializers.ValidationError('Missing required header path/file') + + header_to_index = {COMMENT: set()} + for index, header in enumerate(header_row): + if header == COMMENT: + header_to_index[COMMENT].add(index) + else: + header_to_index[header] = index + print(header_to_index) + return worksheet, header_to_index + + @staticmethod + def parse_dropbox_url(url): + # first add the ?dl=1 to the url if not present or if 0 + parsed_url = list(urlparse(url)) + drop_box_params = {'dl': '1'} + parsed_url[4] = urlencode(drop_box_params) + return urlunparse(parsed_url) + + def create_image(self, base_directory, path, image_container, image_type, collector, scale, + sample, subsample, public_data): + with open(sep.join((base_directory, path)), 'rb') as file_contents: + try: + image_type_obj = ImageType.objects.get(image_type__iexact=image_type) + except ImageType.DoesNotExist: + image_type_obj = None + if not image_type_obj: + image_type_obj = ImageType.objects.get(abbreviation__iexact=image_type) + + owner = None + if self.initial_data.get('owner'): + owner = User.objects.get(pk=self.initial_data['owner']) + + new_image = Image.objects.create(image_container=image_container, + image_type=image_type_obj, + collector=collector, + scale=scale, + sample=sample, + subsample=subsample, + public_data=public_data, + owner=owner) + + file_name = path[path.rfind(sep) + 1:] + new_image.image.save(file_name, File(file_contents)) + new_image.save() + return new_image + + def get_sample_subsample_public_data(self, subsample_name, sample_number, subsample_type): + sample, subsample, public_data = None, None, False + if len(subsample_name) == 0: + sample = Sample.objects.get(number=sample_number) + public_data = sample.public_data + else: + subsample_sample = Sample.objects.get(number=sample_number) + try: + subsample = Subsample.objects.get(sample=subsample_sample, name=subsample_name) + except Subsample.DoesNotExist: + owner = None + if self.initial_data.get('owner'): + owner = User.objects.get(pk=self.initial_data['owner']) + Subsample.objects.create( + name=subsample_name, + sample=subsample_sample, + subsample_type=SubsampleType.objects.get(name=subsample_type), + owner=owner + ) + public_data = subsample.public_data + return sample, subsample, public_data + + @staticmethod + def create_xray_image(image_type, values, header_to_index, created_image, dwell_time, current, voltage): + image_type_value = re.sub('[^a-z]+', '', image_type.lower()) + if image_type_value == XRAY_IMAGE: + element = values[header_to_index[ELEMENT]] + if not element: + raise serializers.ValidationError('Expected element for xray image, but none provided') + xray_image = XrayImage.objects.create(image=created_image, + dwell_time=dwell_time, + current=current, + voltage=voltage, + element=element) + xray_image.save() + + @staticmethod + def create_image_comments(image, comments): + ImageComments.objects.bulk_create([ImageComments( + image=image, + comment_text=comment + ) for comment in comments]) + + def process_worksheet(self, worksheet, header_to_index, base_directory, image_container): + num_rows = worksheet.nrows + + for i in range(1, num_rows): + values = ImageContainerSerializer.get_worksheet_row_values(worksheet, i) + sample_number = values[header_to_index[SAMPLE_NUMBER]] # required + subsample_name = values[header_to_index[SUBSAMPLE]] if SUBSAMPLE in header_to_index else None + subsample_type = values[header_to_index[SUBSAMPLE_TYPE]] if SUBSAMPLE_TYPE in header_to_index else None + path_to_replace = values[header_to_index[PATH]] if PATH in header_to_index else values[header_to_index[FILE]] + path = re.sub(r'[\\/]', sep, path_to_replace) + image_type = values[header_to_index[IMAGE_TYPE]] # required + scale = values[header_to_index[SCALE]] if SCALE in header_to_index else None + collector = values[header_to_index[COLLECTOR]] if COLLECTOR in header_to_index else None + comments = {values[comment_index] for comment_index in header_to_index[COMMENT] if len(values[comment_index].strip()) > 0} if COMMENT in header_to_index else None + dwell_time = values[header_to_index[DWELL_TIME]] if DWELL_TIME in header_to_index else None + current = values[header_to_index[CURRENT]] if CURRENT in header_to_index else None + voltage = values[header_to_index[VOLTAGE]] if VOLTAGE in header_to_index else None + + if not scale: + scale = None + + sample, subsample, public_data = self.get_sample_subsample_public_data(subsample_name, + sample_number, subsample_type) + + created_image = self.create_image(base_directory, path, image_container, image_type, + collector, scale, sample, subsample, public_data) + + if image_type: + ImageContainerSerializer.create_xray_image(image_type, values, header_to_index, created_image, + dwell_time, current, voltage) + + if comments: + ImageContainerSerializer.create_image_comments(created_image, comments) + + @transaction.atomic def create(self, validated_data): new_image_container = ImageContainer.objects.create(**validated_data) - url = validated_data['url'] # zip file url + url = self.parse_dropbox_url(validated_data['url']) with tempfile.TemporaryDirectory() as tmp_dir: - with tempfile.TemporaryFile() as zip_tmp: - with urllib.request.urlopen(url) as image_zip: - zip_tmp.write(image_zip.read()) - - with zipfile.ZipFile(zip_tmp, 'r') as zip_ref: - zip_ref.extractall(tmp_dir) - - for (dir_path, dir_names, file_names) in walk(tmp_dir): - for file_name in file_names: - print(sep.join((dir_path, file_name))) - with open(sep.join((dir_path, file_name)), 'rb') as file_contents: - new_image = Image.objects.create(image_container=new_image_container) - new_image.image.save(file_name, File(file_contents)) - new_image.save() + self.extract_zip_file_from_url(url, tmp_dir) + xls_file = self.get_xls_file_name(tmp_dir) + worksheet, header_to_index = self.get_worksheet_header_mappings(tmp_dir, xls_file) + self.process_worksheet(worksheet, header_to_index, tmp_dir, new_image_container) return new_image_container diff --git a/metpetdb_api/api/images/v1/views.py b/metpetdb_api/api/images/v1/views.py index 1085cae..db9952f 100644 --- a/metpetdb_api/api/images/v1/views.py +++ b/metpetdb_api/api/images/v1/views.py @@ -1,18 +1,19 @@ -from rest_framework import viewsets +from rest_framework import viewsets, permissions + from api.images.v1.serializers import ImageContainerSerializer, ImageSerializer from apps.images.models import ImageContainer, Image -from rest_framework import permissions, status, viewsets from api.lib.permissions import IsOwnerOrReadOnly, IsSuperuserOrReadOnly class ImageContainerViewSet(viewsets.ModelViewSet): queryset = ImageContainer.objects.all() serializer_class = ImageContainerSerializer - # FIXME - # permission_classes = (permissions.IsAuthenticatedOrReadOnly, - # IsOwnerOrReadOnly,) + permission_classes = (permissions.IsAuthenticatedOrReadOnly, + IsOwnerOrReadOnly,) + class ImageViewSet(viewsets.ModelViewSet): queryset = Image.objects.all() serializer_class = ImageSerializer - # TODO add permissions + permission_classes = (permissions.IsAuthenticatedOrReadOnly, + IsOwnerOrReadOnly) diff --git a/metpetdb_api/api/samples/v1/views.py b/metpetdb_api/api/samples/v1/views.py index 7e2c5d9..46728ad 100644 --- a/metpetdb_api/api/samples/v1/views.py +++ b/metpetdb_api/api/samples/v1/views.py @@ -141,11 +141,9 @@ def _handle_references(self, instance, references): instance.references.clear() instance.references.add(*to_add) - def perform_create(self, serializer): return serializer.save() - def create(self, request, *args, **kwargs): serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) @@ -265,18 +263,17 @@ def get_serializer(self, *args, **kwargs): kwargs['partial'] = True return super().get_serializer(*args, **kwargs) - def list(self, request, *args, **kwargs): - + params = request.query_params qs = self.get_queryset().distinct() - + page = self.paginate_queryset(qs) if page is not None: serializer = self.get_serializer(page, many=True) return self.get_paginated_response(serializer.data) - + serializer = self.get_serializer(qs, many=True) return Response(serializer.data) @@ -309,7 +306,7 @@ def update(self, request, *args, **kwargs): return Response(data={'error': 'Invalid subsample type'}, status=400) else: - instance.subsample_type = subsample_type + instance.subsample_type = subsample_type instance.save() # refresh the data before returning a response @@ -384,10 +381,10 @@ class SampleNumbersView(APIView): def get(self, request, format=None): sample_numbers = ( Sample - .objects - .all() - .values_list('number', flat=True) - .distinct() + .objects + .all() + .values_list('number', flat=True) + .distinct() ) return Response({'sample_numbers': sample_numbers}) @@ -396,10 +393,10 @@ class CountryNamesView(APIView): def get(self, request, format=None): country_names = ( Country - .objects - .all() - .values_list('name', flat=True) - .distinct() + .objects + .all() + .values_list('name', flat=True) + .distinct() ) return Response({'country_names': country_names}) @@ -408,9 +405,9 @@ class SampleOwnerNamesView(APIView): def get(self, request, format=None): sample_owner_names = ( Sample - .objects - .all() - .values_list('owner__name', flat=True) - .distinct() + .objects + .all() + .values_list('owner__name', flat=True) + .distinct() ) return Response({'sample_owner_names': sample_owner_names}) diff --git a/metpetdb_api/apps/chemical_analyses/management/commands/migrate_legacy_chemical_analyses.py b/metpetdb_api/apps/chemical_analyses/management/commands/migrate_legacy_chemical_analyses.py index a5a2be0..643eda9 100644 --- a/metpetdb_api/apps/chemical_analyses/management/commands/migrate_legacy_chemical_analyses.py +++ b/metpetdb_api/apps/chemical_analyses/management/commands/migrate_legacy_chemical_analyses.py @@ -6,9 +6,8 @@ ChemicalAnalysis, ChemicalAnalysisElement, ChemicalAnalysisOxide, - Element, - Oxide, ) +from apps.chemical_analyses.shared_models import Element, Oxide from apps.common.utils import queryset_iterator from apps.samples.models import ( Mineral, @@ -19,6 +18,8 @@ Reference, ) +from apps.images.models import Image + from legacy.models import ( ChemicalAnalyses as LegacyChemicalAnalyses, ChemicalAnalysisElements as LegacyChemicalAnalysisElements, @@ -81,6 +82,11 @@ def _migrate_chemical_analyses(self): except AttributeError: reference = None + try: + reference_image = Image.get(subsample=subsample) + except Image.DoesNotExist: + reference_image = None + chem_analysis = ChemicalAnalysis.objects.create( subsample=subsample, public_data=True if record.public_data == 'Y' else False, @@ -99,6 +105,7 @@ def _migrate_chemical_analyses(self): mineral=mineral, owner=User.objects.get(email=record.user.email), reference=reference, + reference_image=reference_image ) legacy_cae = LegacyChemicalAnalysisElements.objects.filter( diff --git a/metpetdb_api/apps/images/migrations/0005_image_chemical_analysis.py b/metpetdb_api/apps/chemical_analyses/migrations/0005_chemicalanalysis_reference_image.py similarity index 58% rename from metpetdb_api/apps/images/migrations/0005_image_chemical_analysis.py rename to metpetdb_api/apps/chemical_analyses/migrations/0005_chemicalanalysis_reference_image.py index cf62c97..f2ac4e8 100644 --- a/metpetdb_api/apps/images/migrations/0005_image_chemical_analysis.py +++ b/metpetdb_api/apps/chemical_analyses/migrations/0005_chemicalanalysis_reference_image.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.10 on 2018-02-20 06:09 +# Generated by Django 1.11.10 on 2018-08-27 02:29 from __future__ import unicode_literals from django.db import migrations, models @@ -9,14 +9,14 @@ class Migration(migrations.Migration): dependencies = [ + ('images', '0012_remove_image_chemical_analysis'), ('chemical_analyses', '0004_auto_20180220_0409'), - ('images', '0004_image_subsample'), ] operations = [ migrations.AddField( - model_name='image', - name='chemical_analysis', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='image', to='chemical_analyses.ChemicalAnalysis'), + model_name='chemicalanalysis', + name='reference_image', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='chemical_analyses', to='images.Image'), ), ] diff --git a/metpetdb_api/apps/chemical_analyses/models.py b/metpetdb_api/apps/chemical_analyses/models.py index a11cf7a..204d527 100644 --- a/metpetdb_api/apps/chemical_analyses/models.py +++ b/metpetdb_api/apps/chemical_analyses/models.py @@ -3,6 +3,7 @@ from concurrency.fields import AutoIncVersionField from django.conf import settings from django.contrib.gis.db import models +from apps.images.models import Image class ChemicalAnalysis(models.Model): @@ -31,6 +32,9 @@ class ChemicalAnalysis(models.Model): through='ChemicalAnalysisElement') oxides = models.ManyToManyField('Oxide', through='ChemicalAnalysisOxide') + reference_image = models.ForeignKey(Image, on_delete=models.CASCADE, blank=True, null=True, + related_name='chemical_analyses') + # Free-text field; stored as an CharField to avoid joining to the # references table every time we retrieve chemical analyses reference = models.CharField(max_length=100, blank=True, null=True) @@ -40,35 +44,6 @@ class Meta: ordering = ['id'] -class Element(models.Model): - id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - name = models.CharField(unique=True, max_length=100) - alternate_name = models.CharField(max_length=100, blank=True, null=True) - symbol = models.CharField(unique=True, max_length=4) - atomic_number = models.IntegerField() - weight = models.FloatField(blank=True, null=True) - order_id = models.IntegerField(blank=True, null=True) - - class Meta: - db_table = 'elements' - ordering = ['id'] - - -class Oxide(models.Model): - id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - element = models.ForeignKey('Element') - oxidation_state = models.SmallIntegerField(blank=True, null=True) - species = models.CharField(unique=True, max_length=20, blank=True, null=True) - weight = models.FloatField(blank=True, null=True) - cations_per_oxide = models.SmallIntegerField(blank=True, null=True) - conversion_factor = models.FloatField() - order_id = models.IntegerField(blank=True, null=True) - - class Meta: - db_table = 'oxides' - ordering = ['id'] - - class ChemicalAnalysisElement(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) chemical_analysis = models.ForeignKey('ChemicalAnalysis') diff --git a/metpetdb_api/apps/chemical_analyses/shared_models.py b/metpetdb_api/apps/chemical_analyses/shared_models.py new file mode 100644 index 0000000..39b8cc6 --- /dev/null +++ b/metpetdb_api/apps/chemical_analyses/shared_models.py @@ -0,0 +1,32 @@ +import uuid + +from django.contrib.gis.db import models + + +class Element(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(unique=True, max_length=100) + alternate_name = models.CharField(max_length=100, blank=True, null=True) + symbol = models.CharField(unique=True, max_length=4) + atomic_number = models.IntegerField() + weight = models.FloatField(blank=True, null=True) + order_id = models.IntegerField(blank=True, null=True) + + class Meta: + db_table = 'elements' + ordering = ['id'] + + +class Oxide(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + element = models.ForeignKey('Element') + oxidation_state = models.SmallIntegerField(blank=True, null=True) + species = models.CharField(unique=True, max_length=20, blank=True, null=True) + weight = models.FloatField(blank=True, null=True) + cations_per_oxide = models.SmallIntegerField(blank=True, null=True) + conversion_factor = models.FloatField() + order_id = models.IntegerField(blank=True, null=True) + + class Meta: + db_table = 'oxides' + ordering = ['id'] \ No newline at end of file diff --git a/metpetdb_api/apps/images/migrations/0001_initial.py b/metpetdb_api/apps/images/migrations/0001_initial.py index 1ea9ddf..ab8eb1b 100644 --- a/metpetdb_api/apps/images/migrations/0001_initial.py +++ b/metpetdb_api/apps/images/migrations/0001_initial.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.10 on 2018-02-20 04:20 +# Generated by Django 1.11.10 on 2018-06-18 06:15 from __future__ import unicode_literals import apps.images.models +from django.conf import settings from django.db import migrations, models import django.db.models.deletion import uuid @@ -14,6 +15,9 @@ class Migration(migrations.Migration): initial = True dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('samples', '0003_auto_20180220_0409'), + ('chemical_analyses', '0004_auto_20180220_0409'), ] operations = [ @@ -22,10 +26,14 @@ class Migration(migrations.Migration): fields=[ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), ('image', versatileimagefield.fields.VersatileImageField(blank=True, null=True, upload_to=apps.images.models.Image.generate_filename, verbose_name='Image')), + ('version', models.IntegerField(default=0)), + ('collector', models.CharField(blank=True, max_length=50)), + ('public_data', models.BooleanField(default=False)), + ('chemical_analysis', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='image', to='chemical_analyses.ChemicalAnalysis')), ], options={ - 'db_table': 'images', 'ordering': ('id',), + 'db_table': 'images', }, ), migrations.CreateModel( @@ -36,13 +44,42 @@ class Migration(migrations.Migration): ('url', models.CharField(blank=True, max_length=500, null=True)), ], options={ - 'db_table': 'image_container', 'ordering': ('id',), + 'db_table': 'image_container', }, ), + migrations.CreateModel( + name='ImageType', + fields=[ + ('id', models.SmallIntegerField(primary_key=True, serialize=False)), + ('image_type', models.CharField(max_length=100)), + ('abbreviation', models.CharField(max_length=10)), + ('comments', models.CharField(max_length=250)), + ], + ), migrations.AddField( model_name='image', name='image_container', field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='images', to='images.ImageContainer'), ), + migrations.AddField( + model_name='image', + name='image_type_id', + field=models.ForeignKey(blank=True, default='', on_delete=django.db.models.deletion.CASCADE, to='images.ImageType'), + ), + migrations.AddField( + model_name='image', + name='owner', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='images', to=settings.AUTH_USER_MODEL), + ), + migrations.AddField( + model_name='image', + name='sample', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='image', to='samples.Sample'), + ), + migrations.AddField( + model_name='image', + name='subsample', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='image', to='samples.Subsample'), + ), ] diff --git a/metpetdb_api/apps/images/migrations/0003_auto_20180220_0539.py b/metpetdb_api/apps/images/migrations/0002_auto_20180618_0640.py similarity index 52% rename from metpetdb_api/apps/images/migrations/0003_auto_20180220_0539.py rename to metpetdb_api/apps/images/migrations/0002_auto_20180618_0640.py index abde7aa..e1dd390 100644 --- a/metpetdb_api/apps/images/migrations/0003_auto_20180220_0539.py +++ b/metpetdb_api/apps/images/migrations/0002_auto_20180618_0640.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.10 on 2018-02-20 05:39 +# Generated by Django 1.11.10 on 2018-06-18 06:40 from __future__ import unicode_literals from django.db import migrations, models @@ -9,18 +9,17 @@ class Migration(migrations.Migration): dependencies = [ - ('samples', '0003_auto_20180220_0409'), - ('images', '0002_image_samples'), + ('images', '0001_initial'), ] operations = [ migrations.RemoveField( model_name='image', - name='samples', + name='image_type_id', ), migrations.AddField( model_name='image', - name='sample', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='images', to='samples.Sample'), + name='image_type', + field=models.ForeignKey(blank=True, default=0, on_delete=django.db.models.deletion.CASCADE, to='images.ImageType'), ), ] diff --git a/metpetdb_api/apps/images/migrations/0002_image_samples.py b/metpetdb_api/apps/images/migrations/0002_image_samples.py deleted file mode 100644 index 9fb6444..0000000 --- a/metpetdb_api/apps/images/migrations/0002_image_samples.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.10 on 2018-02-20 05:21 -from __future__ import unicode_literals - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('samples', '0003_auto_20180220_0409'), - ('images', '0001_initial'), - ] - - operations = [ - migrations.AddField( - model_name='image', - name='samples', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='image', to='samples.Sample'), - ), - ] diff --git a/metpetdb_api/apps/images/migrations/0003_auto_20180618_0655.py b/metpetdb_api/apps/images/migrations/0003_auto_20180618_0655.py new file mode 100644 index 0000000..1c5fe46 --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0003_auto_20180618_0655.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-06-18 06:55 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0002_auto_20180618_0640'), + ] + + operations = [ + migrations.AlterField( + model_name='image', + name='owner', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='images', to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0004_auto_20180618_0754.py b/metpetdb_api/apps/images/migrations/0004_auto_20180618_0754.py new file mode 100644 index 0000000..838d761 --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0004_auto_20180618_0754.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-06-18 07:54 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0003_auto_20180618_0655'), + ] + + operations = [ + migrations.AlterField( + model_name='image', + name='chemical_analysis', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='images', to='chemical_analyses.ChemicalAnalysis'), + ), + migrations.AlterField( + model_name='image', + name='sample', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='images', to='samples.Sample'), + ), + migrations.AlterField( + model_name='image', + name='subsample', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='images', to='samples.Subsample'), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0004_image_subsample.py b/metpetdb_api/apps/images/migrations/0004_image_subsample.py deleted file mode 100644 index ee57aa8..0000000 --- a/metpetdb_api/apps/images/migrations/0004_image_subsample.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.10 on 2018-02-20 05:46 -from __future__ import unicode_literals - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('samples', '0003_auto_20180220_0409'), - ('images', '0003_auto_20180220_0539'), - ] - - operations = [ - migrations.AddField( - model_name='image', - name='subsample', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='images', to='samples.Subsample'), - ), - ] diff --git a/metpetdb_api/apps/images/migrations/0005_auto_20180618_0821.py b/metpetdb_api/apps/images/migrations/0005_auto_20180618_0821.py new file mode 100644 index 0000000..c91681b --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0005_auto_20180618_0821.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-06-18 08:21 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0004_auto_20180618_0754'), + ] + + operations = [ + migrations.AlterField( + model_name='imagetype', + name='comments', + field=models.CharField(blank=True, max_length=250, null=True), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0006_auto_20180619_0335.py b/metpetdb_api/apps/images/migrations/0006_auto_20180619_0335.py new file mode 100644 index 0000000..464fb11 --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0006_auto_20180619_0335.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-06-19 03:35 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0005_auto_20180618_0821'), + ] + + operations = [ + migrations.AlterField( + model_name='image', + name='collector', + field=models.CharField(blank=True, max_length=50, null=True), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0007_xrayimage.py b/metpetdb_api/apps/images/migrations/0007_xrayimage.py new file mode 100644 index 0000000..c121c2c --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0007_xrayimage.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-06-19 04:53 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0006_auto_20180619_0335'), + ] + + operations = [ + migrations.CreateModel( + name='XrayImage', + fields=[ + ('image', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='images.Image')), + ('element', models.CharField(blank=True, max_length=256, null=True)), + ('dwelltime', models.SmallIntegerField(blank=True, null=True)), + ('current', models.SmallIntegerField(blank=True, null=True)), + ('voltage', models.SmallIntegerField(blank=True, null=True)), + ], + options={ + 'db_table': 'xray_image', + 'ordering': ('image_id',), + }, + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0008_auto_20180619_0454.py b/metpetdb_api/apps/images/migrations/0008_auto_20180619_0454.py new file mode 100644 index 0000000..6680c43 --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0008_auto_20180619_0454.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-06-19 04:54 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0007_xrayimage'), + ] + + operations = [ + migrations.AlterField( + model_name='xrayimage', + name='image', + field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='images.Image'), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0009_auto_20180619_0454.py b/metpetdb_api/apps/images/migrations/0009_auto_20180619_0454.py new file mode 100644 index 0000000..5211b7d --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0009_auto_20180619_0454.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-06-19 04:54 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0008_auto_20180619_0454'), + ] + + operations = [ + migrations.AlterField( + model_name='xrayimage', + name='image', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='images.Image', unique=True), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0010_auto_20180619_0455.py b/metpetdb_api/apps/images/migrations/0010_auto_20180619_0455.py new file mode 100644 index 0000000..c7478da --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0010_auto_20180619_0455.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-06-19 04:55 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0009_auto_20180619_0454'), + ] + + operations = [ + migrations.AlterField( + model_name='xrayimage', + name='image', + field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='images.Image'), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0011_auto_20180826_1618.py b/metpetdb_api/apps/images/migrations/0011_auto_20180826_1618.py new file mode 100644 index 0000000..41409c1 --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0011_auto_20180826_1618.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-08-26 16:18 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0010_auto_20180619_0455'), + ] + + operations = [ + migrations.AddField( + model_name='image', + name='description', + field=models.CharField(blank=True, max_length=1024, null=True), + ), + migrations.AddField( + model_name='image', + name='scale', + field=models.SmallIntegerField(blank=True, null=True), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0012_remove_image_chemical_analysis.py b/metpetdb_api/apps/images/migrations/0012_remove_image_chemical_analysis.py new file mode 100644 index 0000000..03e3bee --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0012_remove_image_chemical_analysis.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-08-27 02:29 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0011_auto_20180826_1618'), + ] + + operations = [ + migrations.RemoveField( + model_name='image', + name='chemical_analysis', + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0013_imagecomments.py b/metpetdb_api/apps/images/migrations/0013_imagecomments.py new file mode 100644 index 0000000..f423fbe --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0013_imagecomments.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-08-27 03:34 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0012_remove_image_chemical_analysis'), + ] + + operations = [ + migrations.CreateModel( + name='ImageComments', + fields=[ + ('comment_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('comment_text', models.TextField()), + ('version', models.IntegerField(default=0)), + ('image', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='images.Image')), + ], + options={ + 'db_table': 'image_comments', + }, + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0014_auto_20180827_0336.py b/metpetdb_api/apps/images/migrations/0014_auto_20180827_0336.py new file mode 100644 index 0000000..3d86265 --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0014_auto_20180827_0336.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-08-27 03:36 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0013_imagecomments'), + ] + + operations = [ + migrations.AlterField( + model_name='imagecomments', + name='image', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='comments', to='images.Image'), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0015_auto_20180827_0344.py b/metpetdb_api/apps/images/migrations/0015_auto_20180827_0344.py new file mode 100644 index 0000000..89615b0 --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0015_auto_20180827_0344.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-08-27 03:44 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0014_auto_20180827_0336'), + ] + + operations = [ + migrations.AlterModelOptions( + name='imagecomments', + options={'ordering': ('comment_id',)}, + ), + ] diff --git a/metpetdb_api/apps/images/models.py b/metpetdb_api/apps/images/models.py index a8c6a00..8dc2b59 100644 --- a/metpetdb_api/apps/images/models.py +++ b/metpetdb_api/apps/images/models.py @@ -4,12 +4,12 @@ import os import uuid +from apps.samples.models import Sample, Subsample +from django.conf import settings from django.contrib.gis.db import models from django.dispatch import receiver from versatileimagefield.fields import VersatileImageField from versatileimagefield.image_warmer import VersatileImageFieldWarmer -from apps.samples.models import Sample, Subsample -from apps.chemical_analyses.models import ChemicalAnalysis class ImageContainer(models.Model): @@ -22,6 +22,24 @@ class Meta: ordering = ('id',) +class ImageType(models.Model): + id = models.SmallIntegerField(primary_key=True) + image_type = models.CharField(max_length=100, null=False) + abbreviation = models.CharField(max_length=10) + comments = models.CharField(max_length=250, null=True, blank=True) + + +class ImageComments(models.Model): + comment_id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + comment_text = models.TextField(blank=False, null=False) + version = models.IntegerField(null=False, default=0) + image = models.ForeignKey('Image', related_name='comments') + + class Meta: + db_table = 'image_comments' + ordering = ('comment_id',) + + class Image(models.Model): def generate_filename(instance, filename): f_hash = str(uuid.uuid4()).replace('-', '') @@ -31,22 +49,43 @@ def generate_filename(instance, filename): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) image = VersatileImageField('Image', upload_to=generate_filename, blank=True, null=True) - image_container = models.ForeignKey(ImageContainer, on_delete=models.CASCADE, blank=True, null=True, related_name='images') + version = models.IntegerField(null=False, default=0) + scale = models.SmallIntegerField(null=True, blank=True) + description = models.CharField(max_length=1024, null=True, blank=True) + image_type = models.ForeignKey(ImageType, null=False, blank=True, default=0) + collector = models.CharField(max_length=50, blank=True, null=True) + owner = models.ForeignKey(settings.AUTH_USER_MODEL, related_name='images', null=True) + public_data = models.BooleanField(null=False, default=False) + image_container = models.ForeignKey(ImageContainer, on_delete=models.CASCADE, blank=True, null=True, + related_name='images') + # move to respective classes sample = models.ForeignKey(Sample, on_delete=models.CASCADE, blank=True, null=True, related_name='images') subsample = models.ForeignKey(Subsample, on_delete=models.CASCADE, blank=True, null=True, related_name='images') - chemical_analysis = models.ForeignKey(ChemicalAnalysis, on_delete=models.CASCADE, blank=True, null=True, related_name='image') class Meta: db_table = 'images' ordering = ('id',) +class XrayImage(models.Model): + image = models.OneToOneField(Image, primary_key=True) + element = models.CharField(max_length=256, blank=True, null=True) + dwelltime = models.SmallIntegerField(blank=True, null=True) + current = models.SmallIntegerField(blank=True, null=True) + voltage = models.SmallIntegerField(blank=True, null=True) + + class Meta: + db_table = 'xray_image' + ordering = ('image_id',) + + @receiver(models.signals.post_save, sender=Image) def warm_images(sender, instance, **kwargs): """Create all image size on POST""" - image_warmer = VersatileImageFieldWarmer( - instance_or_queryset=instance, - rendition_key_set='image_sizes', - image_attr='image' - ) - image_warmer.warm() + if instance.image: + image_warmer = VersatileImageFieldWarmer( + instance_or_queryset=instance, + rendition_key_set='image_sizes', + image_attr='image' + ) + image_warmer.warm() diff --git a/metpetdb_api/apps/samples/management/commands/migrate_legacy_samples.py b/metpetdb_api/apps/samples/management/commands/migrate_legacy_samples.py index f2bfb96..fc463b5 100644 --- a/metpetdb_api/apps/samples/management/commands/migrate_legacy_samples.py +++ b/metpetdb_api/apps/samples/management/commands/migrate_legacy_samples.py @@ -1,6 +1,7 @@ from django.contrib.auth import get_user_model from django.core.management import BaseCommand from django.db import transaction +import uuid from apps.common.utils import queryset_iterator from apps.samples.models import ( @@ -21,6 +22,12 @@ Subsample, SubsampleType, ) +from apps.images.models import ( + ImageType, + Image, + XrayImage, + ImageComments +) from legacy.models import ( Georeference as LegacyGeoreference, Grids as LegacyGrid, @@ -40,7 +47,19 @@ SampleRegions as LegacySampleRegion, SampleReference as LegacySampleReference, Users as LegacyUser, + ImageType as LegacyImageTypes, + Images as LegacyImages, + XrayImage as LegacyXrayImage, + ImageComments as LegacyImageComments +) +from apps.users.models import ( + User ) +from django.core.files import File + +BASE_DIR = '/mnt/volume-nyc1-01/images' +FAILED_IMAGES_FILE = 'failed_images.txt' + class Command(BaseCommand): help = 'Migrates legacy samples to the new data model' @@ -53,75 +72,43 @@ def handle(self, *args, **options): self._migrate_mineral_relationships() self._migrate_subsample_types() self._migrate_references() + self._migrate_image_types() self._migrate_samples() self._migrate_countries() - @transaction.atomic def _migrate_samples(self): print("Migrating samples...") User = get_user_model() - all_collectors = [] - all_regions = [] + all_collectors = set() + all_regions = set() - old_samples = queryset_iterator(LegacySample.objects.all(), - chunksize=1000) + old_samples = queryset_iterator(LegacySample.objects.all(), chunksize=1000) Sample.objects.all().delete() for old_sample in old_samples: - print("Migrating old sample #{0}: {1}" - .format(old_sample.pk, old_sample.number)) - rock_type = (RockType - .objects - .filter(name=old_sample.rock_type.rock_type)[0]) + print("Migrating old sample #{0}: {1}".format(old_sample.pk, old_sample.number)) + rock_type = (RockType.objects.filter(name=old_sample.rock_type.rock_type)[0]) old_user = LegacyUser.objects.get(pk=old_sample.user_id) new_user = User.objects.get(email=old_user.email) if old_sample.collector: - all_collectors.extend([old_sample.collector]) + all_collectors.add(old_sample.collector) - regions = [lsr.region.name - for lsr in LegacySampleRegion - .objects - .filter(sample=old_sample)] + regions = [lsr.region.name for lsr in LegacySampleRegion.objects.filter(sample=old_sample)] if regions: - all_regions.extend(regions) - - references = [lsr.reference.name - for lsr in LegacySampleReference - .objects - .filter(sample=old_sample)] - new_georeferences = GeoReference.objects.filter(name__in=references) - - old_metamorphic_regions = (lsa.metamorphic_region.name - for lsa in - LegacySampleMetamorphicRegion - .objects - .filter(sample=old_sample)) - new_metamorphic_regions = ( - MetamorphicRegion - .objects - .filter(name__in=old_metamorphic_regions) - ) + all_regions.update(set(regions)) - old_metamorphic_grades = (lsa.metamorphic_grade.name - for lsa in - LegacySampleMetamorphicGrades - .objects - .filter(sample=old_sample)) - new_metamorphic_grades = (MetamorphicGrade - .objects - .filter(name__in=old_metamorphic_grades)) + new_georeferences = self._get_georeferences(old_sample) + new_metamorphic_regions = self._get_metamorphic_regions(old_sample) + new_metamorphic_grades = self._get_metamorphic_grades(old_sample) - aliases = [lsa.alias - for lsa in LegacySampleAlias - .objects - .filter(sample=old_sample)] + aliases = [lsa.alias for lsa in LegacySampleAlias.objects.filter(sample=old_sample)] new_sample = Sample.objects.create( - public_data=True if old_sample.public_data == 'Y' else False, + public_data=old_sample.public_data == 'Y', number=old_sample.number, owner=new_user, aliases=aliases, @@ -137,43 +124,56 @@ def _migrate_samples(self): collector_name=old_sample.collector ) - SampleMapping.objects.create(old_sample_id=old_sample.pk, - new_sample_id=new_sample.pk) + SampleMapping.objects.create(old_sample_id=old_sample.pk, new_sample_id=new_sample.pk) + + self._migrate_subsamples(old_sample, new_sample=new_sample) - self._migrate_subsamples(old_sample, new_sample) + old_images = LegacyImages.objects.filter(sample=old_sample) + self._migrate_images(old_images, new_sample) new_sample.metamorphic_regions.add(*new_metamorphic_regions) new_sample.metamorphic_grades.add(*new_metamorphic_grades) new_sample.references.add(*new_georeferences) - - old_minerals = (LegacySampleMineral - .objects - .filter(sample=old_sample)) - old_mineral_names = [om.mineral.name for om in old_minerals] - new_minerals = Mineral.objects.filter(name__in=old_mineral_names) - - for mineral in new_minerals: - old_sample_mineral = (LegacySampleMineral - .objects - .get(mineral__name=mineral.name, - sample=old_sample)) - SampleMineral.objects.create(sample=new_sample, - mineral=mineral, - amount=old_sample_mineral.amount) - + self._migrate_sample_minerals(old_sample, new_sample) Region.objects.all().delete() - Region.objects.bulk_create([Region(name=region) - for region in set(all_regions)]) + Region.objects.bulk_create([Region(name=region) for region in all_regions]) Collector.objects.all().delete() - Collector.objects.bulk_create([Collector(name=collector) - for collector in set(all_collectors)]) - + Collector.objects.bulk_create([Collector(name=collector) for collector in all_collectors]) + + @staticmethod + def _migrate_sample_minerals(old_sample, new_sample): + old_minerals = (LegacySampleMineral.objects.filter(sample=old_sample)) + old_mineral_names = [om.mineral.name for om in old_minerals] + new_minerals = Mineral.objects.filter(name__in=old_mineral_names) + + for mineral in new_minerals: + old_sample_mineral = (LegacySampleMineral.objects.get(mineral__name=mineral.name, sample=old_sample)) + SampleMineral.objects.create(sample=new_sample, mineral=mineral, amount=old_sample_mineral.amount) + + @staticmethod + def _get_metamorphic_regions(old_sample): + old_metamorphic_regions = (lsa.metamorphic_region.name for lsa in + LegacySampleMetamorphicRegion.objects.filter(sample=old_sample)) + return MetamorphicRegion.objects.filter(name__in=old_metamorphic_regions) + + @staticmethod + def _get_metamorphic_grades(old_sample): + old_metamorphic_grades = (lsa.metamorphic_grade.name for lsa in + LegacySampleMetamorphicGrades.objects.filter(sample=old_sample)) + return MetamorphicGrade.objects.filter(name__in=old_metamorphic_grades) + + @staticmethod + def _get_georeferences(old_sample): + references = [lsr.reference.name for lsr in LegacySampleReference.objects.filter(sample=old_sample)] + return GeoReference.objects.filter(name__in=references) + @transaction.atomic def _migrate_references(self): print("Migrating references...") old_georeferences = LegacyGeoreference.objects.all() + GeoReference.objects.all().delete() for record in old_georeferences: Reference.objects.get_or_create(name=record.reference_number) @@ -196,6 +196,51 @@ def _migrate_references(self): Reference.objects.get_or_create(name=record.name) GeoReference.objects.get_or_create(name=record.name) + @staticmethod + def _migrate_images(old_images, new_sample=None, new_subsample=None): + errors = [] + + for old_image in old_images: + old_user = LegacyUser.objects.get(pk=old_image.user_id) + new_user = User.objects.get(email=old_user.email) + new_image = Image.objects.create( + id=uuid.uuid4(), + version=old_image.version, + image=None, + image_type=ImageType.objects.get(id=old_image.image_type_id), + collector=old_image.collector, + owner=new_user, + public_data=old_image.public_data == 'Y', + sample=new_sample, + subsample=new_subsample + ) + + checksum = old_image.checksum + try: + with open('{}/{}/{}/{}'.format(BASE_DIR, checksum[0:2], checksum[2:4], checksum[4:]), 'rb') as image_file: + new_image.image.save(old_image.filename, File(image_file)) + new_image.save() + except Exception as ex: + print('ERROR saving image with checksum {}. Errors logged to {}'.format(checksum, FAILED_IMAGES_FILE)) + errors.append(checksum) + + old_xray = LegacyXrayImage.objects.filter(image=old_image).first() + if old_xray: + XrayImage.objects.create( + image=new_image, + element=old_xray.element, + dwelltime=old_xray.dwelltime, + current=old_xray.current, + voltage=old_xray.voltage + ) + old_comments = LegacyImageComments.objects.filter(image=old_image) + + ImageComments.objects.bulk_create([ImageComments(image=new_image, comment_text=old_comment.comment_text, + version=old_comment.version) for old_comment in old_comments]) + + if errors: + with open(FAILED_IMAGES_FILE, 'a+', encoding='utf-8') as failures: + failures.write('{}{}'.format('\n', '\n'.join(errors))) def _migrate_subsamples(self, old_sample, new_sample): old_records = LegacySubsample.objects.filter(sample=old_sample) @@ -208,29 +253,31 @@ def _migrate_subsamples(self, old_sample, new_sample): subsample = Subsample.objects.create( name=record.name, sample=new_sample, - public_data=record.public_data, + public_data=record.public_data == 'Y', owner=new_user, subsample_type=subsample_type ) + old_images = LegacyImages.objects.filter(subsample=record.subsample_id) + self._migrate_images(old_images, new_subsample=subsample) + old_grids = LegacyGrid.objects.filter(subsample=record) if old_grids: for grid in old_grids: Grid.objects.create(subsample=subsample, width=grid.width, height=grid.height, - public_data=grid.public_data) - + public_data=grid.public_data == 'Y') @transaction.atomic def _migrate_subsample_types(self): print("Migrating legacy subsample types...") old_records = LegacySubsampleType.objects.all() + SubsampleType.objects.all().delete() for record in old_records: SubsampleType.objects.create(name=record.subsample_type) - @transaction.atomic def _migrate_rock_types(self): print("Migrating rock types...") @@ -240,7 +287,6 @@ def _migrate_rock_types(self): for record in old_records: RockType.objects.create(name=record.rock_type) - @transaction.atomic def _migrate_metamorphic_grades(self): print("Migrating metamorphic grades...") @@ -250,7 +296,6 @@ def _migrate_metamorphic_grades(self): for record in old_records: MetamorphicGrade.objects.create(name=record.name) - @transaction.atomic def _migrate_metamorphic_regions(self): print("Migrating metamorphic regions...") @@ -265,7 +310,6 @@ def _migrate_metamorphic_regions(self): label_location=record.label_location ) - @transaction.atomic def _migrate_minerals(self): print("Migrating minerals...") @@ -281,12 +325,9 @@ def _migrate_minerals(self): for record in old_records: mineral = Mineral.objects.get(name=record.name) - mineral.real_mineral = (Mineral - .objects - .get(name=record.real_mineral.name)) + mineral.real_mineral = Mineral.objects.get(name=record.real_mineral.name) mineral.save() - @transaction.atomic def _migrate_mineral_relationships(self): print("Migrating mineral relationships...") @@ -295,22 +336,27 @@ def _migrate_mineral_relationships(self): for record in old_records: MineralRelationship.objects.create( - parent_mineral=Mineral.objects.get( - name=record.parent_mineral.name - ), - child_mineral=Mineral.objects.get( - name=record.child_mineral.name - ) + parent_mineral=Mineral.objects.get(name=record.parent_mineral.name), + child_mineral=Mineral.objects.get(name=record.child_mineral.name) + ) + + @transaction.atomic + def _migrate_image_types(self): + print('Migrating image types...') + old_records = LegacyImageTypes.objects.all() + ImageType.objects.all().delete() + + for record in old_records: + ImageType.objects.create( + id=record.image_type_id, + image_type=record.image_type, + abbreviation=record.abbreviation, + comments=record.comments ) @transaction.atomic def _migrate_countries(self): print("Migrating country names...") - country_names = (Sample - .objects - .all() - .values_list('country', flat=True) - .distinct()) - country_names = [name for name in country_names if name is not None] - Country.objects.bulk_create([Country(name=name) - for name in country_names]) + Country.objects.all().delete() + country_names = set(filter(lambda x: x, Sample.objects.all().values_list('country', flat=True).distinct())) + Country.objects.bulk_create([Country(name=name) for name in country_names]) diff --git a/metpetdb_api/apps/samples/models.py b/metpetdb_api/apps/samples/models.py index fa82060..a515b57 100644 --- a/metpetdb_api/apps/samples/models.py +++ b/metpetdb_api/apps/samples/models.py @@ -4,7 +4,8 @@ from django.conf import settings from django.contrib.gis.db import models from django.contrib.postgres.fields import ArrayField -from apps.chemical_analyses.models import Element, Oxide +from apps.chemical_analyses.shared_models import Element, Oxide + class BulkUpload(models.Model): pass diff --git a/metpetdb_api/legacy/models.py b/metpetdb_api/legacy/models.py index b67678f..3894107 100644 --- a/metpetdb_api/legacy/models.py +++ b/metpetdb_api/legacy/models.py @@ -136,6 +136,7 @@ class Meta: db_table = 'sample_minerals' unique_together = (('mineral', 'sample'),) + class Reference(models.Model): reference_id = models.BigIntegerField(primary_key=True) name = models.CharField(unique=True, max_length=100) @@ -144,6 +145,7 @@ class Meta: managed = False db_table = 'reference' + class SampleReference(models.Model): sample = models.ForeignKey('Samples') reference = models.ForeignKey(Reference) @@ -162,6 +164,42 @@ class Meta: managed = False db_table = 'regions' + +# class ImageTypes(models.Model): +# image_type_id = models.SmallIntegerField(primary_key=True) +# image_type = models.CharField(null=False, max_length=100) +# abbreviation = models.CharField(max_length=10) +# comments = models.CharField(max_length=250) +# +# class Meta: +# managed = False +# db_table = 'image_type' +# +# +# class Images(models.Model): +# image_id = models.BigIntegerField(primary_key=True) +# checksum = models.CharField(max_length=50, null=False) +# version = models.IntegerField(null=False) +# sample_id = models.BigIntegerField +# subsample_id = models.BigIntegerField +# image_format_id = models.SmallIntegerField +# image_type_id = models.SmallIntegerField(null=False) +# width = models.SmallIntegerField(null=False) +# height = models.SmallIntegerField(null=False) +# collector = models.CharField(max_length=50) +# description = models.CharField(max_length=1024) +# scale = models.SmallIntegerField +# user_id = models.IntegerField(null=False) +# public_data = models.CharField(max_length=1, null=False) +# checksum_64x64 = models.CharField(max_length=50, null=False) +# checksum_half = models.CharField(max_length=50, null=False) +# filename = models.CharField(max_length=256, null=False) +# checksum_mobile = models.CharField(max_length=50) +# +# class Meta: +# managed = False +# db_table = 'images' + class SampleRegions(models.Model): sample = models.ForeignKey('Samples') region = models.ForeignKey(Regions) @@ -240,7 +278,7 @@ class ChemicalAnalyses(models.Model): reference_y = models.FloatField(blank=True, null=True) stage_x = models.FloatField(blank=True, null=True) stage_y = models.FloatField(blank=True, null=True) - # image = models.ForeignKey('Images', blank=True, null=True) + image = models.ForeignKey('Images', blank=True, null=True) analysis_method = models.CharField(max_length=50, blank=True, null=True) where_done = models.CharField(max_length=50, blank=True, null=True) analyst = models.CharField(max_length=50, blank=True, null=True) @@ -331,98 +369,94 @@ class Georeference(models.Model): class Meta: managed = False db_table = 'georeference' -# -# -# -# -# class ImageComments(models.Model): -# comment_id = models.BigIntegerField(primary_key=True) -# image = models.ForeignKey('Images') -# comment_text = models.TextField() -# version = models.IntegerField() -# -# class Meta: -# managed = False -# db_table = 'image_comments' -# -# -# class ImageFormat(models.Model): -# image_format_id = models.SmallIntegerField(primary_key=True) -# name = models.CharField(unique=True, max_length=100) -# -# class Meta: -# managed = False -# db_table = 'image_format' -# -# -# class ImageOnGrid(models.Model): -# image_on_grid_id = models.BigIntegerField(primary_key=True) -# grid = models.ForeignKey(Grids) -# image = models.ForeignKey('Images') -# top_left_x = models.FloatField() -# top_left_y = models.FloatField() -# z_order = models.SmallIntegerField() -# opacity = models.SmallIntegerField() -# resize_ratio = models.FloatField() -# width = models.SmallIntegerField() -# height = models.SmallIntegerField() -# checksum = models.CharField(max_length=50) -# checksum_64x64 = models.CharField(max_length=50) -# checksum_half = models.CharField(max_length=50) -# locked = models.CharField(max_length=1) -# angle = models.FloatField(blank=True, null=True) -# -# class Meta: -# managed = False -# db_table = 'image_on_grid' -# -# -# class ImageReference(models.Model): -# image = models.ForeignKey('Images') -# reference = models.ForeignKey('Reference') -# -# class Meta: -# managed = False -# db_table = 'image_reference' -# unique_together = (('image_id', 'reference_id'),) -# -# -# class ImageType(models.Model): -# image_type_id = models.SmallIntegerField(primary_key=True) -# image_type = models.CharField(unique=True, max_length=100) -# abbreviation = models.CharField(unique=True, max_length=10, blank=True, null=True) -# comments = models.CharField(max_length=250, blank=True, null=True) -# -# class Meta: -# managed = False -# db_table = 'image_type' -# -# -# class Images(models.Model): -# image_id = models.BigIntegerField(primary_key=True) -# checksum = models.CharField(max_length=50) -# version = models.IntegerField() -# sample = models.ForeignKey('Samples', blank=True, null=True) -# subsample = models.ForeignKey('Subsamples', blank=True, null=True) -# image_format = models.ForeignKey(ImageFormat, blank=True, null=True) -# image_type = models.ForeignKey(ImageType) -# width = models.SmallIntegerField() -# height = models.SmallIntegerField() -# collector = models.CharField(max_length=50, blank=True, null=True) -# description = models.CharField(max_length=1024, blank=True, null=True) -# scale = models.SmallIntegerField(blank=True, null=True) -# user = models.ForeignKey('Users') -# public_data = models.CharField(max_length=1) -# checksum_64x64 = models.CharField(max_length=50) -# checksum_half = models.CharField(max_length=50) -# filename = models.CharField(max_length=256) -# checksum_mobile = models.CharField(max_length=50, blank=True, null=True) -# -# class Meta: -# managed = False -# db_table = 'images' -# unique_together = (('sample_id', 'None'), ('subsample_id', 'None'),) -# + + +class ImageComments(models.Model): + comment_id = models.BigIntegerField(primary_key=True) + image = models.ForeignKey('Images') + comment_text = models.TextField() + version = models.IntegerField() + + class Meta: + managed = False + db_table = 'image_comments' + + +class ImageFormat(models.Model): + image_format_id = models.SmallIntegerField(primary_key=True) + name = models.CharField(unique=True, max_length=100) + + class Meta: + managed = False + db_table = 'image_format' + + +class ImageOnGrid(models.Model): + image_on_grid_id = models.BigIntegerField(primary_key=True) + grid = models.ForeignKey(Grids) + image = models.ForeignKey('Images') + top_left_x = models.FloatField() + top_left_y = models.FloatField() + z_order = models.SmallIntegerField() + opacity = models.SmallIntegerField() + resize_ratio = models.FloatField() + width = models.SmallIntegerField() + height = models.SmallIntegerField() + checksum = models.CharField(max_length=50) + checksum_64x64 = models.CharField(max_length=50) + checksum_half = models.CharField(max_length=50) + locked = models.CharField(max_length=1) + angle = models.FloatField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'image_on_grid' + + +class ImageReference(models.Model): + image = models.ForeignKey('Images') + reference = models.ForeignKey('Reference') + + class Meta: + managed = False + db_table = 'image_reference' + + +class ImageType(models.Model): + image_type_id = models.SmallIntegerField(primary_key=True) + image_type = models.CharField(unique=True, max_length=100) + abbreviation = models.CharField(unique=True, max_length=10, blank=True, null=True) + comments = models.CharField(max_length=250, blank=True, null=True) + + class Meta: + managed = False + db_table = 'image_type' + + +class Images(models.Model): + image_id = models.BigIntegerField(primary_key=True) + checksum = models.CharField(max_length=50) + version = models.IntegerField() + sample = models.ForeignKey('Samples', blank=True, null=True) + subsample = models.ForeignKey('Subsamples', blank=True, null=True) + image_format = models.ForeignKey(ImageFormat, blank=True, null=True) + image_type = models.ForeignKey(ImageType) + width = models.SmallIntegerField() + height = models.SmallIntegerField() + collector = models.CharField(max_length=50, blank=True, null=True) + description = models.CharField(max_length=1024, blank=True, null=True) + scale = models.SmallIntegerField(blank=True, null=True) + user = models.ForeignKey('Users') + public_data = models.CharField(max_length=1) + checksum_64x64 = models.CharField(max_length=50) + checksum_half = models.CharField(max_length=50) + filename = models.CharField(max_length=256) + checksum_mobile = models.CharField(max_length=50, blank=True, null=True) + + class Meta: + managed = False + db_table = 'images' + # # # @@ -595,13 +629,14 @@ class Meta: # db_table = 'users_roles' # # -# class XrayImage(models.Model): -# image = models.ForeignKey(Images, primary_key=True) -# element = models.CharField(max_length=256, blank=True, null=True) -# dwelltime = models.SmallIntegerField(blank=True, null=True) -# current = models.SmallIntegerField(blank=True, null=True) -# voltage = models.SmallIntegerField(blank=True, null=True) -# -# class Meta: -# managed = False -# db_table = 'xray_image' + +class XrayImage(models.Model): + image = models.OneToOneField(Images, primary_key=True) + element = models.CharField(max_length=256, blank=True, null=True) + dwelltime = models.SmallIntegerField(blank=True, null=True) + current = models.SmallIntegerField(blank=True, null=True) + voltage = models.SmallIntegerField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'xray_image' diff --git a/metpetdb_api/requirements/dev.txt b/metpetdb_api/requirements/dev.txt index 70647d9..3887003 100644 --- a/metpetdb_api/requirements/dev.txt +++ b/metpetdb_api/requirements/dev.txt @@ -11,3 +11,4 @@ sqlparse>=0.1.15 wheel>=0.24.0 pathlib django-versatileimagefield +xlrd diff --git a/metpetdb_api/requirements/staging.txt b/metpetdb_api/requirements/staging.txt index 51e696a..3262b60 100644 --- a/metpetdb_api/requirements/staging.txt +++ b/metpetdb_api/requirements/staging.txt @@ -11,3 +11,4 @@ sqlparse>=0.1.15 wheel>=0.24.0 pathlib==1.0.1 django-versatileimagefield +xlrd From a778face5c0ee6d44738fd9749b09200b747b40d Mon Sep 17 00:00:00 2001 From: Brandon Drumheller Date: Mon, 27 Aug 2018 01:36:37 -0500 Subject: [PATCH 14/51] remove print --- metpetdb_api/api/images/v1/serializers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metpetdb_api/api/images/v1/serializers.py b/metpetdb_api/api/images/v1/serializers.py index 72aca89..96f6508 100644 --- a/metpetdb_api/api/images/v1/serializers.py +++ b/metpetdb_api/api/images/v1/serializers.py @@ -104,7 +104,6 @@ def get_worksheet_header_mappings(directory, xls_file): header_to_index[COMMENT].add(index) else: header_to_index[header] = index - print(header_to_index) return worksheet, header_to_index @staticmethod From bc9ff1e5c2bf0c3a9c09dfdc184711272291738d Mon Sep 17 00:00:00 2001 From: metpetDB Date: Mon, 3 Sep 2018 14:10:08 -0400 Subject: [PATCH 15/51] added missing comma --- metpetdb_api/settings/staging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metpetdb_api/settings/staging.py b/metpetdb_api/settings/staging.py index ed33434..2094f41 100644 --- a/metpetdb_api/settings/staging.py +++ b/metpetdb_api/settings/staging.py @@ -63,7 +63,7 @@ 'apps.users', 'apps.core', 'versatileimagefield', - 'apps.images' + 'apps.images', 'rest_framework_csv', ) From 39f9b6c060b4aec07285fc45ec40f5c2286ab069 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Mon, 3 Sep 2018 16:16:49 -0400 Subject: [PATCH 16/51] updated url routing to match newest version of djoser --- metpetdb_api/metpetdb_api/urls.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metpetdb_api/metpetdb_api/urls.py b/metpetdb_api/metpetdb_api/urls.py index d791ec3..1958655 100644 --- a/metpetdb_api/metpetdb_api/urls.py +++ b/metpetdb_api/metpetdb_api/urls.py @@ -71,6 +71,7 @@ urlpatterns = [ url(r'^api/', include(router.urls)), url(r'^api/admin/', include(admin.site.urls)), + url(r'^api/auth/', include('djoser.urls')), url(r'^api/auth/', include('djoser.urls.authtoken')), url(r'^api/sample_numbers/$', SampleNumbersView.as_view()), From 72e4e059f431b14acb9052f049aaff6a1773f9c5 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Mon, 3 Sep 2018 22:45:04 -0400 Subject: [PATCH 17/51] added images to fields so it stops breaking (hopefully....) --- metpetdb_api/api/samples/v1/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metpetdb_api/api/samples/v1/serializers.py b/metpetdb_api/api/samples/v1/serializers.py index 1a9f1c0..9f510b7 100644 --- a/metpetdb_api/api/samples/v1/serializers.py +++ b/metpetdb_api/api/samples/v1/serializers.py @@ -91,6 +91,7 @@ class Meta: 'collection_date', 'location_name', 'description', + 'images', 'subsample_ids', 'chemical_analyses_ids', ) From 67a686183b17a3ceceb77bfa199487c2c986b4d9 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Tue, 4 Sep 2018 14:23:49 -0400 Subject: [PATCH 18/51] updated formatting of provided 'DOMAIN' & 'SITE_NAME' vars to work with django-templated-mail (used by new djoser version) --- metpetdb_api/settings/staging.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metpetdb_api/settings/staging.py b/metpetdb_api/settings/staging.py index 2094f41..2c998e2 100644 --- a/metpetdb_api/settings/staging.py +++ b/metpetdb_api/settings/staging.py @@ -179,13 +179,14 @@ STATIC_ROOT = os.path.join(BASE_DIR, "static/") DJOSER = { - 'DOMAIN': env('FRONT_END_URL'), - 'SITE_NAME': env('FRONT_END_SITE_NAME'), 'PASSWORD_RESET_CONFIRM_URL': 'reset-password#/{uid}/{token}', 'ACTIVATION_URL': 'login#/activate/{uid}/{token}', 'SEND_ACTIVATION_EMAIL' : True } +DOMAIN = env('FRONT_END_URL') +SITE_NAME = env('FRONT_END_SITE_NAME') + # A dictionary that allows you to fine-tune how django-versatileimagefield works: VERSATILEIMAGEFIELD_SETTINGS = { # The amount of time, in seconds, that references to created images From cec2fdf1792b2522e5f192578cfbf9ef2555bdc8 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Thu, 18 Oct 2018 02:15:01 -0400 Subject: [PATCH 19/51] debugged image migration & chemical analysis migration w/ images --- .../migrate_legacy_chemical_analyses.py | 9 +++++-- .../images/migrations/0016_imagemapping.py | 27 +++++++++++++++++++ metpetdb_api/apps/images/models.py | 10 +++++++ .../commands/migrate_legacy_samples.py | 5 +++- 4 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 metpetdb_api/apps/images/migrations/0016_imagemapping.py diff --git a/metpetdb_api/apps/chemical_analyses/management/commands/migrate_legacy_chemical_analyses.py b/metpetdb_api/apps/chemical_analyses/management/commands/migrate_legacy_chemical_analyses.py index 643eda9..ed73de6 100644 --- a/metpetdb_api/apps/chemical_analyses/management/commands/migrate_legacy_chemical_analyses.py +++ b/metpetdb_api/apps/chemical_analyses/management/commands/migrate_legacy_chemical_analyses.py @@ -18,7 +18,7 @@ Reference, ) -from apps.images.models import Image +from apps.images.models import Image, ImageMapping from legacy.models import ( ChemicalAnalyses as LegacyChemicalAnalyses, @@ -83,9 +83,14 @@ def _migrate_chemical_analyses(self): reference = None try: - reference_image = Image.get(subsample=subsample) + reference_image = Image.objects.get( + subsample=subsample, + pk=ImageMapping.objects.get( + old_image_id=record.image.pk).new_image_id) except Image.DoesNotExist: reference_image = None + except AttributeError: + reference_image = None chem_analysis = ChemicalAnalysis.objects.create( subsample=subsample, diff --git a/metpetdb_api/apps/images/migrations/0016_imagemapping.py b/metpetdb_api/apps/images/migrations/0016_imagemapping.py new file mode 100644 index 0000000..27151f4 --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0016_imagemapping.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-10-18 06:13 +from __future__ import unicode_literals + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0015_auto_20180827_0344'), + ] + + operations = [ + migrations.CreateModel( + name='ImageMapping', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('old_image_id', models.IntegerField()), + ('new_image_id', models.UUIDField()), + ], + options={ + 'db_table': 'image_mapping', + }, + ), + ] diff --git a/metpetdb_api/apps/images/models.py b/metpetdb_api/apps/images/models.py index 8dc2b59..f4c589d 100644 --- a/metpetdb_api/apps/images/models.py +++ b/metpetdb_api/apps/images/models.py @@ -78,6 +78,16 @@ class Meta: db_table = 'xray_image' ordering = ('image_id',) +# A mapping table to help migration of old images to new images. +# needed (I think?) for chemical analysis migration. +class ImageMapping(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + old_image_id = models.IntegerField() + new_image_id = models.UUIDField() + + class Meta: + db_table = 'image_mapping' + @receiver(models.signals.post_save, sender=Image) def warm_images(sender, instance, **kwargs): diff --git a/metpetdb_api/apps/samples/management/commands/migrate_legacy_samples.py b/metpetdb_api/apps/samples/management/commands/migrate_legacy_samples.py index fc463b5..f535ade 100644 --- a/metpetdb_api/apps/samples/management/commands/migrate_legacy_samples.py +++ b/metpetdb_api/apps/samples/management/commands/migrate_legacy_samples.py @@ -26,7 +26,8 @@ ImageType, Image, XrayImage, - ImageComments + ImageComments, + ImageMapping ) from legacy.models import ( Georeference as LegacyGeoreference, @@ -215,6 +216,8 @@ def _migrate_images(old_images, new_sample=None, new_subsample=None): subsample=new_subsample ) + ImageMapping.objects.create(old_image_id=old_image.pk,new_image_id=new_image.pk) + checksum = old_image.checksum try: with open('{}/{}/{}/{}'.format(BASE_DIR, checksum[0:2], checksum[2:4], checksum[4:]), 'rb') as image_file: From 5db7105f14e3222b19d94042812a75b27624330a Mon Sep 17 00:00:00 2001 From: metpetDB Date: Thu, 18 Oct 2018 02:49:38 -0400 Subject: [PATCH 20/51] attempt to fix png save issue --- .../apps/samples/management/commands/migrate_legacy_samples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metpetdb_api/apps/samples/management/commands/migrate_legacy_samples.py b/metpetdb_api/apps/samples/management/commands/migrate_legacy_samples.py index f535ade..31676db 100644 --- a/metpetdb_api/apps/samples/management/commands/migrate_legacy_samples.py +++ b/metpetdb_api/apps/samples/management/commands/migrate_legacy_samples.py @@ -221,7 +221,7 @@ def _migrate_images(old_images, new_sample=None, new_subsample=None): checksum = old_image.checksum try: with open('{}/{}/{}/{}'.format(BASE_DIR, checksum[0:2], checksum[2:4], checksum[4:]), 'rb') as image_file: - new_image.image.save(old_image.filename, File(image_file)) + new_image.image.convert('RGB').save(old_image.filename, File(image_file)) new_image.save() except Exception as ex: print('ERROR saving image with checksum {}. Errors logged to {}'.format(checksum, FAILED_IMAGES_FILE)) From a88fdedccb101d15ee990039419d5433b74cd8c5 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Wed, 12 Dec 2018 11:08:53 -0500 Subject: [PATCH 21/51] added sample_id field --- metpetdb_api/api/chemical_analyses/v1/serializers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metpetdb_api/api/chemical_analyses/v1/serializers.py b/metpetdb_api/api/chemical_analyses/v1/serializers.py index 4d558e2..74170a5 100644 --- a/metpetdb_api/api/chemical_analyses/v1/serializers.py +++ b/metpetdb_api/api/chemical_analyses/v1/serializers.py @@ -82,6 +82,7 @@ class Meta: class ChemicalAnalysisSerializer(DynamicFieldsModelSerializer): owner = serializers.ReadOnlyField(source='owner.name') sample = serializers.ReadOnlyField(source='subsample.sample.number') + sample_id = serializers.ReadOnlyField(source='subsample.sample.id') subsample = serializers.ReadOnlyField(source='subsample.name') subsample_type = serializers.ReadOnlyField(source='subsample.subsample_type.name') mineral = serializers.ReadOnlyField(source='mineral.name') @@ -103,11 +104,13 @@ class Meta: fields = ( 'owner', 'sample', + 'sample_id', 'subsample', 'subsample_type', 'mineral', 'analysis_method', 'reference', + 'reference_image', 'spot_id', 'where_done', 'analysis_date', From bad0df946fc5b0cd0bc4307f65e137cfdf36ffcd Mon Sep 17 00:00:00 2001 From: metpetDB Date: Sun, 23 Dec 2018 01:50:22 -0500 Subject: [PATCH 22/51] added ordering to users and countries; added location_coords back to sample serializer for data upload... --- metpetdb_api/api/samples/v1/serializers.py | 2 ++ metpetdb_api/apps/samples/models.py | 1 + metpetdb_api/apps/users/models.py | 1 + 3 files changed, 4 insertions(+) diff --git a/metpetdb_api/api/samples/v1/serializers.py b/metpetdb_api/api/samples/v1/serializers.py index 9f510b7..fca4d27 100644 --- a/metpetdb_api/api/samples/v1/serializers.py +++ b/metpetdb_api/api/samples/v1/serializers.py @@ -74,6 +74,7 @@ class Meta: model = Sample depth = 1 fields = ( + 'id', 'number', 'owner', 'regions', @@ -85,6 +86,7 @@ class Meta: 'references', 'longitude', 'latitude', + 'location_coords', 'location_error', # 'igsn', 'collector_name', diff --git a/metpetdb_api/apps/samples/models.py b/metpetdb_api/apps/samples/models.py index a515b57..0ef6973 100644 --- a/metpetdb_api/apps/samples/models.py +++ b/metpetdb_api/apps/samples/models.py @@ -209,6 +209,7 @@ class Country(models.Model): class Meta: db_table = 'countries' + ordering = ['name'] class Region(models.Model): diff --git a/metpetdb_api/apps/users/models.py b/metpetdb_api/apps/users/models.py index 7cc7527..9114759 100644 --- a/metpetdb_api/apps/users/models.py +++ b/metpetdb_api/apps/users/models.py @@ -76,6 +76,7 @@ class User(AbstractBaseUser, PermissionsMixin): class Meta: db_table = 'users' + ordering = ['name'] def get_full_name(self): """ From b95d803252a88842ad13e128c8b9e21f1279adce Mon Sep 17 00:00:00 2001 From: metpetDB Date: Thu, 27 Dec 2018 16:36:37 -0500 Subject: [PATCH 23/51] modified label: 'Sample'-->'Sample Number' --- metpetdb_api/api/samples/v1/renderers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metpetdb_api/api/samples/v1/renderers.py b/metpetdb_api/api/samples/v1/renderers.py index 2397f50..54a63d8 100644 --- a/metpetdb_api/api/samples/v1/renderers.py +++ b/metpetdb_api/api/samples/v1/renderers.py @@ -18,7 +18,7 @@ def __init__(self): 'minerals', ] self.labels = { - 'number': 'Sample', + 'number': 'Sample Number', 'rock_type': 'Rock Type', 'description': 'Comment', 'latitude': 'Latitude', From f51907f90514a7845511613d54dc87656e08ce61 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Thu, 27 Dec 2018 16:38:13 -0500 Subject: [PATCH 24/51] reworking expected fields for sample template --- .../api/bulk_upload/v1/upload_templates.py | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/metpetdb_api/api/bulk_upload/v1/upload_templates.py b/metpetdb_api/api/bulk_upload/v1/upload_templates.py index 9bff6db..0f8b600 100644 --- a/metpetdb_api/api/bulk_upload/v1/upload_templates.py +++ b/metpetdb_api/api/bulk_upload/v1/upload_templates.py @@ -26,6 +26,25 @@ """ import copy +# labels are case insensitive! +sample_label_mappings = { + 'sample number':'number', + 'rock type':'rock_type_name', + 'latitude':'latitude', + 'longitude':'longitude', + 'location error':'location_error', + 'collector':'collector_name', + 'date of collection':'collection_date', + 'present sample location':'location_name', + 'comment':'comment', + 'country':'country', + # multi-fields + 'reference':'references', + 'region':'regions', + 'metamorphic region':'metamorphic_regions', + 'metamorphic grade':'metamorphic_grades' +} + class Template: def __init__(self, c_types = [], required = [], db_types = [], types = {}): self.complex_types = c_types @@ -189,18 +208,21 @@ def get_amount(self,data=[],i=0,j=0): return 0 def get_meta_header(self,header): - mappings = {'mineral' : 'minerals'} - added = set() + mappings = sample_label_mappings + added = set() meta_header = [] itr = iter(header) for heading in itr: - if heading == 'latitude': + if heading.lower() == 'latitude': for i in range (0,2): heading = next(itr) meta_header.append((('latitude','longitude'),'location_coords')) elif heading not in added: - if heading in mappings.keys(): - meta_header.append((heading, mappings[heading])) + if heading.lower() in mappings.keys(): + meta_header.append((heading, mappings[heading.lower()])) added.add(heading) else: meta_header.append((heading, heading)) + print("\nMETA-HEADER:") + print(meta_header) + print("\n\n") return meta_header From d82e4897c99cb6b6f7865dc862e01a75b17a6078 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Fri, 28 Dec 2018 01:05:10 -0500 Subject: [PATCH 25/51] bulk upload for samples mostly works (but frontend table rendering is broken... oops) --- .../api/bulk_upload/v1/upload_templates.py | 46 ++++---- metpetdb_api/api/bulk_upload/v1/views.py | 100 +++++++++++++----- 2 files changed, 95 insertions(+), 51 deletions(-) diff --git a/metpetdb_api/api/bulk_upload/v1/upload_templates.py b/metpetdb_api/api/bulk_upload/v1/upload_templates.py index 0f8b600..1696ad3 100644 --- a/metpetdb_api/api/bulk_upload/v1/upload_templates.py +++ b/metpetdb_api/api/bulk_upload/v1/upload_templates.py @@ -36,15 +36,16 @@ 'collector':'collector_name', 'date of collection':'collection_date', 'present sample location':'location_name', - 'comment':'comment', 'country':'country', # multi-fields + 'comment':'description', 'reference':'references', 'region':'regions', 'metamorphic region':'metamorphic_regions', 'metamorphic grade':'metamorphic_grades' } + class Template: def __init__(self, c_types = [], required = [], db_types = [], types = {}): self.complex_types = c_types @@ -64,12 +65,13 @@ def check_line_len(self): raise Exception("inconsistent line length. Expected {0}, but was {1}".format(len(data[i-1]), len(data[i]))) def check_required(self, row): - header = row[0] + header = [x for x in row[0]] + for i in range(0,len(header)): + header[i] = header[i].lower() missing ={} - for i in range(0, len(header)): - if self.is_required(header[i]): - if row[1][i] == '': - missing[header[i]] = 'missing' + for i in range(0, len(self.required)): + if self.required[i] not in header: + missing[self.required[i]] = 'missing' return missing def check_type(self, curr_row): @@ -123,7 +125,7 @@ def parse(self, data): for heading in header: if self.is_complex(heading): result_template[heading] = [] else: result_template[heading] = '' - result_template['errors'] = ''; + result_template['errors'] = '' for i in range(1, len(data)): tmp_result = self.TemplateResult(copy.deepcopy(result_template)) @@ -133,17 +135,8 @@ def parse(self, data): for j in range(0,len(data[i])): heading = header[j] - if heading in self.amounts: - name = data[i][j] - amount = self.get_amount(data,i,j) - tmp_result.set_field_complex(heading, {"name": name, "amount": amount}) - continue - - if heading == 'mineral' and heading not in self.amounts: - tmp_result.set_field_complex(heading, {"name": data[i][j]}) - continue - field = data[i][j] + print("{}: {}".format(heading,data[i][j])) if self.is_complex(heading): tmp_result.set_field_complex(heading,field) else: tmp_result.set_field_simple(heading, field) @@ -152,8 +145,8 @@ def parse(self, data): result.append(tmp_result.get_rep()) return result, meta_header - def is_complex(self, name): return name in self.complex_types - def is_required(self, name): return name in self.required + def is_complex(self, name): return name.lower() in self.complex_types + def is_required(self, name): return name.lower() in self.required def is_db_type(self, name): return name in self.db_types class ChemicalAnalysesTemplate(Template): @@ -187,19 +180,19 @@ def get_meta_header(self,header): meta_header.append((heading, mappings[heading])) added.add(heading) else: - meta_header.append((heading, heading)) + meta_header.append((heading, heading)) + added.add(heading) return meta_header class SampleTemplate(Template): def __init__(self): - complex_types = ["comment", "references", "mineral", "metamorphic_region_id", "metamorphic_grade"] - required = ["number", "latitude", "longitude", "rock_type_name"] + complex_types = ["comment", "reference", "region", "metamorphic region", "metamorphic grade"] + required = ["sample number", "latitude", "longitude", "rock type"] types = {"comment": str, "latitude": float, 'longitude': float} db_types = ["minerals"] #selected_types = {'minerals': ['el1', 'el2', 'el3']} Template.__init__(self, complex_types, required, db_types, types) - self.amounts.add('mineral') def check_amounts(self,header): pass @@ -215,13 +208,14 @@ def get_meta_header(self,header): for heading in itr: if heading.lower() == 'latitude': for i in range (0,2): heading = next(itr) - meta_header.append((('latitude','longitude'),'location_coords')) + meta_header.append((('latitude','longitude'),'Location')) elif heading not in added: if heading.lower() in mappings.keys(): - meta_header.append((heading, mappings[heading.lower()])) + meta_header.append((heading.lower(),heading)) added.add(heading) else: - meta_header.append((heading, heading)) + meta_header.append((heading, heading)) + added.add(heading) print("\nMETA-HEADER:") print(meta_header) print("\n\n") diff --git a/metpetdb_api/api/bulk_upload/v1/views.py b/metpetdb_api/api/bulk_upload/v1/views.py index aa30b3a..d1bf491 100644 --- a/metpetdb_api/api/bulk_upload/v1/views.py +++ b/metpetdb_api/api/bulk_upload/v1/views.py @@ -65,6 +65,20 @@ import urllib.request from csv import reader + +sample_labels_dict = { + 'Sample Number':'number', + 'Rock Type':'rock_type_name', + 'Latitude':'latitude', + 'Longitude':'longitude', + 'Location Error':'location_error', + 'Collector':'collector_name', + 'Date of Collection':'collection_date', + 'Present Sample Location':'location_name', + 'Country':'country' +} + + class Parser: def __init__(self, template): self.template = template @@ -85,6 +99,7 @@ def parse(self, url): lined = self.line_split(content) return self.template.parse(lined) # return the JSON ready file except Exception as err: + print(err) raise ValueError(str(err)) class BulkUploadViewSet(viewsets.ModelViewSet): @@ -366,6 +381,61 @@ def parse_samples(self, request, JSON, meta_header): transaction.set_autocommit(False) for i,sample_obj in enumerate(JSON): + + print(sample_obj) + + # REQUIRED FIELDS: + # Sample Number + # Rock Type + # Latitude + # Longitude + + # OPTIONAL FIELDS: + # Location Error + # Metamorphic Region + # Reference + # Present Sample Location + # Region + # Date of Collection + # Comment + # Metamorphic Grade + # Country + # Collector + # [minerals] + + + # PROCEDURE: + ## ensure all required fields are present + ## verify all other fields are valid optional fields or minerals + ## manipulate data for serializer + ## create serializer + + minerals = [] + fields = [x for x in sample_obj.keys()] + for field in fields: + if field.lower() in upload_templates.sample_label_mappings.keys(): + # add proper formatting to date + if field.lower() == 'date of collection': + sample_obj[field] = sample_obj[field] + 'T00:00:00.000Z' + # join comments with newline + elif field.lower() == 'comment': + sample_obj[field] = '\n'.join(sample_obj[field]) + # replace field with corresponding serializer fieldname + sample_obj[upload_templates.sample_label_mappings[field.lower()]] = sample_obj[field] + del(sample_obj[field]) + elif field != 'errors' and field not in upload_templates.sample_label_mappings.values(): # it had better be a mineral + try: + amount = sample_obj[field] + minerals.append( + {'id':Mineral.objects.get(name=field).id, + 'name':field, + 'amount':amount}) + except: + print(field) + return self.set_err(before_parse_json, i, 'minerals', 'Invalid mineral {}'.format(field), meta_header) + + sample_obj['minerals'] = minerals + try: sample_obj['owner'] = request.data.get('owner') except: @@ -375,38 +445,18 @@ def parse_samples(self, request, JSON, meta_header): status = 400 ) - minerals = sample_obj['mineral'] + if 'latitude' in sample_obj.keys() and 'longitude' in sample_obj.keys(): + sample_obj['location_coords'] = u'SRID=4326;POINT ({0} {1})'.format(sample_obj['latitude'], sample_obj['longitude']) + del(sample_obj['latitude']) + del(sample_obj['longitude']) rock_type = sample_obj['rock_type_name'] - to_add = [] - - for mineral in minerals: - try: - to_add.append( - {'id': Mineral.objects.get(name=mineral['name']).id, - 'name': mineral['name'], - 'amount': '0'}) - except: - return self.set_err(before_parse_json, i, 'minerals', 'Invalid mineral {0}'.format(mineral), meta_header) - - sample_obj['minerals'] = to_add - - del(sample_obj['mineral']) - - # Need this for a proper collection date - if sample_obj['collection_date']: - sample_obj['collection_date'] += 'T00:00:00.000Z' - try: sample_obj['rock_type_id'] = RockType.objects.get(name=rock_type).id except: return self.set_err(before_parse_json, i, 'rock_type_id', 'Invalid rock {0}'.format(rock_type), meta_header) - - if 'latitude' in sample_obj.keys() and 'longitude' in sample_obj.keys(): - sample_obj['location_coords'] = u'SRID=4326;POINT ({0} {1})'.format(sample_obj['latitude'], sample_obj['longitude']) - del(sample_obj['latitude']) - del(sample_obj['longitude']) + serializer = self.get_serializer(data=sample_obj) try: From c681cddb7732941ed5c03e792b40eaf98d94cbf5 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Thu, 3 Jan 2019 16:13:01 -0500 Subject: [PATCH 26/51] removed unnecessary time from sample collection date field --- metpetdb_api/apps/samples/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metpetdb_api/apps/samples/models.py b/metpetdb_api/apps/samples/models.py index 0ef6973..c44e7a2 100644 --- a/metpetdb_api/apps/samples/models.py +++ b/metpetdb_api/apps/samples/models.py @@ -28,7 +28,7 @@ class Sample(models.Model): aliases = ArrayField(models.CharField(max_length=35, blank=True), blank=True, null=True) - collection_date = models.DateTimeField(blank=True, null=True) + collection_date = models.DateField(blank=True, null=True) description = models.TextField(blank=True, null=True) location_name = models.CharField(max_length=50, blank=True, null=True) location_coords = models.PointField() From 9fa333202fc0a0dd6bf7a5870bed737aeb7199b9 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Thu, 3 Jan 2019 20:22:41 -0500 Subject: [PATCH 27/51] sample bulk upload appears to be fully functional (in backend) --- metpetdb_api/api/bulk_upload/v1/views.py | 26 ++++++++++++++---------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/metpetdb_api/api/bulk_upload/v1/views.py b/metpetdb_api/api/bulk_upload/v1/views.py index d1bf491..8d89218 100644 --- a/metpetdb_api/api/bulk_upload/v1/views.py +++ b/metpetdb_api/api/bulk_upload/v1/views.py @@ -108,13 +108,15 @@ class BulkUploadViewSet(viewsets.ModelViewSet): IsOwnerOrReadOnly,) http_method_names=['post'] - def _handle_metamorphic_regions(self, instance, ids): + def _handle_metamorphic_regions(self, instance, regions): metamorphic_regions = [] - for id in ids: + for region in regions: + if region == '': + continue try: - metamorphic_region = MetamorphicRegion.objects.get(pk=id) + metamorphic_region = MetamorphicRegion.objects.get(name=region) except: - raise ValueError('Invalid metamorphic_region id: {}' + raise ValueError('Invalid metamorphic region: {}' .format(id)) else: metamorphic_regions.append(metamorphic_region) @@ -124,10 +126,12 @@ def _handle_metamorphic_regions(self, instance, ids): def _handle_metamorphic_grades(self, instance, grades): metamorphic_grades = [] for grade in grades: + if grade == '': + continue try: metamorphic_grade = MetamorphicGrade.objects.get(name=grade) except: - raise ValueError('Invalid metamorphic_grade : {}'.format(grade)) + raise ValueError('Invalid metamorphic grade: {}'.format(grade)) else: metamorphic_grades.append(metamorphic_grade) instance.metamorphic_grades = metamorphic_grades @@ -416,7 +420,7 @@ def parse_samples(self, request, JSON, meta_header): if field.lower() in upload_templates.sample_label_mappings.keys(): # add proper formatting to date if field.lower() == 'date of collection': - sample_obj[field] = sample_obj[field] + 'T00:00:00.000Z' + sample_obj[field] = sample_obj[field] #+ 'T00:00:00.000Z' # join comments with newline elif field.lower() == 'comment': sample_obj[field] = '\n'.join(sample_obj[field]) @@ -466,17 +470,17 @@ def parse_samples(self, request, JSON, meta_header): return self.set_err(before_parse_json, i, 'serialization', str(e), meta_header) - metamorphic_region_ids = sample_obj.get('metamorphic_region_id') - metamorphic_grades = sample_obj.get('metamorphic_grade') + metamorphic_regions = sample_obj.get('metamorphic_regions') + metamorphic_grades = sample_obj.get('metamorphic_grades') references = sample_obj.get('references') minerals = sample_obj.get('minerals') - if metamorphic_region_ids: + if metamorphic_regions: try: self._handle_metamorphic_regions(instance, - metamorphic_region_ids) + metamorphic_regions) except ValueError as err: - return self.set_err(before_parse_json, i, 'metamorphic_region_ids', err.args, meta_header) + return self.set_err(before_parse_json, i, 'metamorphic_regions', err.args, meta_header) if metamorphic_grades: try: From 62789d96788b821a3ee8a8c925249ca5cc5762ab Mon Sep 17 00:00:00 2001 From: metpetDB Date: Wed, 6 Feb 2019 16:13:44 -0500 Subject: [PATCH 28/51] updated sample serializer to automatically handle lat/long geometry --- metpetdb_api/api/samples/v1/serializers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metpetdb_api/api/samples/v1/serializers.py b/metpetdb_api/api/samples/v1/serializers.py index fca4d27..2af429a 100644 --- a/metpetdb_api/api/samples/v1/serializers.py +++ b/metpetdb_api/api/samples/v1/serializers.py @@ -62,7 +62,7 @@ class SampleSerializer(DynamicFieldsModelSerializer): references = serializers.SerializerMethodField(read_only=True) latitude = serializers.SerializerMethodField(read_only=True) longitude = serializers.SerializerMethodField(read_only=True) - collection_date = serializers.SerializerMethodField(read_only=True) + # collection_date = serializers.SerializerMethodField(read_only=True) images = ImageSerializer(many=True, read_only=True) @@ -99,6 +99,9 @@ class Meta: ) def is_valid(self, raise_exception=False): + if self.initial_data.get('latitude') and self.initial_data.get('longitude'): + self.initial_data['location_coords'] = "SRID=4326;POINT ("+str(self.initial_data["latitude"])+" "+str(self.initial_data["longitude"])+")" + super().is_valid(raise_exception) if self.initial_data.get('owner'): From de2f28c68b64f22c49d184b493fa96c3de2f2479 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Thu, 7 Feb 2019 00:48:52 -0500 Subject: [PATCH 29/51] slight modifications to make chemical analyses play nice with frontend --- .../api/chemical_analyses/v1/serializers.py | 21 +++-- .../api/chemical_analyses/v1/views.py | 87 +++++++++++-------- 2 files changed, 62 insertions(+), 46 deletions(-) diff --git a/metpetdb_api/api/chemical_analyses/v1/serializers.py b/metpetdb_api/api/chemical_analyses/v1/serializers.py index 74170a5..d47448b 100644 --- a/metpetdb_api/api/chemical_analyses/v1/serializers.py +++ b/metpetdb_api/api/chemical_analyses/v1/serializers.py @@ -48,8 +48,8 @@ class Meta: class ChemicalAnalysisOxideSerializer(DynamicFieldsModelSerializer): - # id = serializers.ReadOnlyField(source='oxide.id') - # element_id = serializers.ReadOnlyField(source='oxide.element_id') + id = serializers.ReadOnlyField(source='oxide.id') + element_id = serializers.ReadOnlyField(source='oxide.element_id') # oxidation_state = serializers.ReadOnlyField(source='oxide.oxidation_state') species = serializers.ReadOnlyField(source='oxide.species') # weight = serializers.ReadOnlyField(source='oxide.weight') @@ -57,19 +57,19 @@ class ChemicalAnalysisOxideSerializer(DynamicFieldsModelSerializer): # source='oxide.cations_per_oxide') # conversion_factor = serializers.ReadOnlyField( # source='oxide.conversion_factor') - # order_id = serializers.ReadOnlyField(source='oxide.order_id') + order_id = serializers.ReadOnlyField(source='oxide.order_id') class Meta: model = ChemicalAnalysisOxide fields = ( - # 'id', - # 'element_id', + 'id', + 'element_id', # 'oxidation_state', 'species', # 'weight', # 'cations_per_oxide', # 'conversion_factor', - # 'order_id', + 'order_id', 'amount', 'precision', 'precision_type', @@ -84,17 +84,20 @@ class ChemicalAnalysisSerializer(DynamicFieldsModelSerializer): sample = serializers.ReadOnlyField(source='subsample.sample.number') sample_id = serializers.ReadOnlyField(source='subsample.sample.id') subsample = serializers.ReadOnlyField(source='subsample.name') + subsample_id = serializers.ReadOnlyField(source='subsample.id') subsample_type = serializers.ReadOnlyField(source='subsample.subsample_type.name') mineral = serializers.ReadOnlyField(source='mineral.name') elements = ChemicalAnalysisElementSerializer( many=True, source='chemicalanalysiselement_set', required=False, + read_only=True ) oxides = ChemicalAnalysisOxideSerializer( many=True, source='chemicalanalysisoxide_set', - required=False + required=False, + read_only=True ) class Meta: @@ -102,10 +105,12 @@ class Meta: depth = 1 fields = ( + 'id', 'owner', 'sample', 'sample_id', 'subsample', + 'subsample_id', 'subsample_type', 'mineral', 'analysis_method', @@ -124,7 +129,7 @@ class Meta: 'elements', 'oxides', 'total', - ) + ) def is_valid(self, raise_exception=False): super().is_valid(raise_exception) diff --git a/metpetdb_api/api/chemical_analyses/v1/views.py b/metpetdb_api/api/chemical_analyses/v1/views.py index 6160eeb..b070a41 100644 --- a/metpetdb_api/api/chemical_analyses/v1/views.py +++ b/metpetdb_api/api/chemical_analyses/v1/views.py @@ -67,7 +67,8 @@ def list(self, request, *args, **kwargs): def _handle_elements(self, instance, params): to_add = [] - for record in params['elements']: + for record in params: + print(record) try: to_add.append({ 'element': Element.objects.get(pk=record['id']), @@ -79,9 +80,7 @@ def _handle_elements(self, instance, params): 'max_amount': record['max_amount'] }) except Element.DoesNotExist: - return Response( - data={'error': 'Invalid element id'}, - status=400) + raise ValueError('Invalid element id: {}'.format(record['id'])) (ChemicalAnalysisElement .objects @@ -103,7 +102,7 @@ def _handle_elements(self, instance, params): def _handle_oxides(self, instance, params): to_add = [] - for record in params['oxides']: + for record in params: try: to_add.append({ 'oxide': Oxide.objects.get(pk=record['id']), @@ -115,9 +114,7 @@ def _handle_oxides(self, instance, params): 'max_amount': record['max_amount'] }) except Oxide.DoesNotExist: - return Response( - data={'error': 'Invalid oxide id'}, - status=400) + raise ValueError('Invalid oxide id: {}'.format(record['id'])) (ChemicalAnalysisOxide .objects @@ -147,38 +144,52 @@ def create(self, request, *args, **kwargs): instance = self.perform_create(serializer) if request.data.get('elements'): - for record in request.data.get('elements'): - try: - ChemicalAnalysisElement.objects.create( - chemical_analysis=instance, - element=Element.objects.get(pk=record['id']), - amount=record['amount'], - precision=record['precision'], - precision_type=record['precision_type'], - measurement_unit=record['measurement_unit'], - min_amount=record['min_amount'], - max_amount=record['max_amount'], - ) - except Element.DoesNotExist: - return Response(data={'error': 'Invalid element id'}, - status=400) + try: + self._handle_elements(instance,request.data.get('elements')) + except ValueError as err: + return Response( + data={'error':err.args}, + status=400 + ) + # for record in request.data.get('elements'): + # try: + # ChemicalAnalysisElement.objects.create( + # chemical_analysis=instance, + # element=Element.objects.get(pk=record['id']), + # amount=record['amount'], + # precision=record['precision'], + # precision_type=record['precision_type'], + # measurement_unit=record['measurement_unit'], + # min_amount=record['min_amount'], + # max_amount=record['max_amount'], + # ) + # except Element.DoesNotExist: + # return Response(data={'error': 'Invalid element id'}, + # status=400) if request.data.get('oxides'): - for record in request.data.get('oxides'): - try: - ChemicalAnalysisOxide.objects.create( - chemical_analysis=instance, - oxide=Oxide.objects.get(pk=record['id']), - amount=record['amount'], - precision=record['precision'], - precision_type=record['precision_type'], - measurement_unit=record['measurement_unit'], - min_amount=record['min_amount'], - max_amount=record['max_amount'], - ) - except Oxide.DoesNotExist: - return Response(data={'error': 'Invalid oxide id'}, - status=400) + try: + self._handle_oxides(instance,request.data.get('oxides')) + except ValueError as err: + return Response( + data={'error':err.args}, + status=400 + ) + # for record in request.data.get('oxides'): + # try: + # ChemicalAnalysisOxide.objects.create( + # chemical_analysis=instance, + # oxide=Oxide.objects.get(pk=record['id']), + # amount=record['amount'], + # precision=record['precision'], + # precision_type=record['precision_type'], + # measurement_unit=record['measurement_unit'], + # min_amount=record['min_amount'], + # max_amount=record['max_amount'], + # ) + # except Oxide.DoesNotExist: + # return Response(data={'error': 'Invalid oxide id'}, + # status=400) headers = self.get_success_headers(serializer.data) return Response(serializer.data, From 951ffe25df1d6a506d3b68b97398c5861a792505 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Wed, 6 Mar 2019 17:08:54 -0500 Subject: [PATCH 30/51] modified views to look for & assign filenames when csv requested --- metpetdb_api/api/chemical_analyses/v1/views.py | 7 ++++++- metpetdb_api/api/samples/v1/views.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/metpetdb_api/api/chemical_analyses/v1/views.py b/metpetdb_api/api/chemical_analyses/v1/views.py index b070a41..2607d47 100644 --- a/metpetdb_api/api/chemical_analyses/v1/views.py +++ b/metpetdb_api/api/chemical_analyses/v1/views.py @@ -54,7 +54,12 @@ def list(self, request, *args, **kwargs): if params.get('format') == 'csv': serializer = self.get_serializer(qs,many=True) - return Response(serializer.data) + response = Response(serializer.data) + filename = params.get('filename') + if filename is None: + filename = 'search_results.csv' + response['content-disposition'] = "attachment; filename=%s" % filename + return response else: page = self.paginate_queryset(qs) if page is not None: diff --git a/metpetdb_api/api/samples/v1/views.py b/metpetdb_api/api/samples/v1/views.py index 74fc123..40846d8 100644 --- a/metpetdb_api/api/samples/v1/views.py +++ b/metpetdb_api/api/samples/v1/views.py @@ -76,7 +76,12 @@ def list(self, request, *args, **kwargs): if params.get('format') == 'csv': serializer = self.get_serializer(qs, many=True) - return Response(serializer.data) + response = Response(serializer.data) + filename = params.get('filename') + if filename is None: + filename = 'search_results.csv' + response['content-disposition'] = "attachment; filename=%s" % filename + return response else: page = self.paginate_queryset(qs) if page: From 4aa4c56e52021e37a8ce6e6362d8b15953f0586f Mon Sep 17 00:00:00 2001 From: metpetDB Date: Wed, 6 Mar 2019 19:55:02 -0500 Subject: [PATCH 31/51] added ordering on number of associated images & chemical analyses --- metpetdb_api/api/samples/lib/query.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/metpetdb_api/api/samples/lib/query.py b/metpetdb_api/api/samples/lib/query.py index 168103a..15f879c 100644 --- a/metpetdb_api/api/samples/lib/query.py +++ b/metpetdb_api/api/samples/lib/query.py @@ -2,7 +2,7 @@ from django.contrib.auth.models import AnonymousUser from django.contrib.gis.geos import Polygon, GEOSException -from django.db.models import Q, F +from django.db.models import Q, F, Count def sample_query(user, params, qs): @@ -92,6 +92,17 @@ def sample_query(user, params, qs): qs = qs.filter(sesar_number__in=params['sesar_number'].split(',')) if params.get('ordering'): - qs = qs.order_by(params['ordering']) + if params['ordering'] == 'images': + # CHANGE ME WHEN FRONTEND ALLOWS TOGGLE + qs = qs.annotate(image_count=Count('images')).order_by('-image_count') + elif params['ordering'] == '-images': + qs = qs.annotate(image_count=Count('images')).order_by('-image_count') + elif params['ordering'] == 'chemical_analyses': + # CHANGE ME WHEN FRONTEND ALLOWS TOGGLE + qs = qs.annotate(chem_count=Count('subsamples__chemical_analyses')).order_by('-chem_count') + elif params['ordering'] == '-chemical_analyses': + qs = qs.annotate(chem_count=Count('subsamples__chemical_analyses')).order_by('-chem_count') + else: + qs = qs.order_by(params['ordering']) return qs From 962b65ec155f853d2b0549182cf18d3e04133649 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Wed, 6 Mar 2019 20:04:27 -0500 Subject: [PATCH 32/51] added ordering on number of subsamples --- metpetdb_api/api/samples/lib/query.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/metpetdb_api/api/samples/lib/query.py b/metpetdb_api/api/samples/lib/query.py index 15f879c..eaca12c 100644 --- a/metpetdb_api/api/samples/lib/query.py +++ b/metpetdb_api/api/samples/lib/query.py @@ -93,15 +93,20 @@ def sample_query(user, params, qs): if params.get('ordering'): if params['ordering'] == 'images': - # CHANGE ME WHEN FRONTEND ALLOWS TOGGLE + # FIX ME WHEN FRONTEND ALLOWS TOGGLE qs = qs.annotate(image_count=Count('images')).order_by('-image_count') elif params['ordering'] == '-images': qs = qs.annotate(image_count=Count('images')).order_by('-image_count') elif params['ordering'] == 'chemical_analyses': - # CHANGE ME WHEN FRONTEND ALLOWS TOGGLE + # FIX ME qs = qs.annotate(chem_count=Count('subsamples__chemical_analyses')).order_by('-chem_count') elif params['ordering'] == '-chemical_analyses': qs = qs.annotate(chem_count=Count('subsamples__chemical_analyses')).order_by('-chem_count') + elif params['ordering'] == 'subsamples': + # FIX ME + qs = qs.annotate(subsample_count=Count('subsamples')).order_by('-subsample_count') + elif params['ordering'] == '-subsamples': + qs = qs.annotate(subsample_count=Count('subsamples')).order_by('-subsample_count') else: qs = qs.order_by(params['ordering']) From ed24e886f6337fed7255affb52a885a81ee2b7e8 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Wed, 6 Mar 2019 21:20:01 -0500 Subject: [PATCH 33/51] increased max page length for region retrieval in frontend (for search/sample upload/etc) and fixed default ordering for a bunch of things --- metpetdb_api/apps/core/pagination.py | 2 +- metpetdb_api/apps/samples/models.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/metpetdb_api/apps/core/pagination.py b/metpetdb_api/apps/core/pagination.py index 583f6fb..974fa23 100644 --- a/metpetdb_api/apps/core/pagination.py +++ b/metpetdb_api/apps/core/pagination.py @@ -3,4 +3,4 @@ class StandardResultsSetPagination(pagination.PageNumberPagination): page_size = 20 page_size_query_param = 'page_size' - max_page_size = 1000 + max_page_size = 3000 diff --git a/metpetdb_api/apps/samples/models.py b/metpetdb_api/apps/samples/models.py index c44e7a2..560a9d5 100644 --- a/metpetdb_api/apps/samples/models.py +++ b/metpetdb_api/apps/samples/models.py @@ -16,7 +16,7 @@ class RockType(models.Model): class Meta: db_table = 'rock_types' - ordering = ['id'] + ordering = ['name'] class Sample(models.Model): @@ -110,7 +110,7 @@ class MetamorphicGrade(models.Model): class Meta: db_table = 'metamorphic_grades' - ordering = ['id'] + ordering = ['name'] class MetamorphicRegion(models.Model): @@ -122,7 +122,7 @@ class MetamorphicRegion(models.Model): class Meta: db_table = 'metamorphic_regions' - ordering = ['id'] + ordering = ['name'] class Mineral(models.Model): @@ -137,7 +137,7 @@ class Mineral(models.Model): class Meta: db_table = 'minerals' - ordering = ['id'] + ordering = ['name'] class SampleMineral(models.Model): @@ -218,7 +218,7 @@ class Region(models.Model): class Meta: db_table = 'regions' - ordering = ['id'] + ordering = ['name'] class Reference(models.Model): @@ -227,7 +227,7 @@ class Reference(models.Model): class Meta: db_table = 'references' - ordering = ['id'] + ordering = ['name'] class Collector(models.Model): @@ -236,7 +236,7 @@ class Collector(models.Model): class Meta: db_table = 'collectors' - ordering = ['id'] + ordering = ['name'] # A mapping table to help the migration of old samples to new samples; can # be gotten rid of once thi app goes into production. From d98553a624811dd7caa58e7531b82a815cb8fe95 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Mon, 11 Mar 2019 21:46:18 -0400 Subject: [PATCH 34/51] updated default sorting for elements and oxides --- metpetdb_api/apps/chemical_analyses/shared_models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metpetdb_api/apps/chemical_analyses/shared_models.py b/metpetdb_api/apps/chemical_analyses/shared_models.py index 39b8cc6..ad50414 100644 --- a/metpetdb_api/apps/chemical_analyses/shared_models.py +++ b/metpetdb_api/apps/chemical_analyses/shared_models.py @@ -14,7 +14,7 @@ class Element(models.Model): class Meta: db_table = 'elements' - ordering = ['id'] + ordering = ['symbol'] class Oxide(models.Model): @@ -29,4 +29,4 @@ class Oxide(models.Model): class Meta: db_table = 'oxides' - ordering = ['id'] \ No newline at end of file + ordering = ['species'] \ No newline at end of file From 86af47dfc4ee6f3adab50d3e98ee43dda20c8006 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Wed, 13 Mar 2019 00:18:29 -0400 Subject: [PATCH 35/51] fixed some default orderings --- metpetdb_api/apps/samples/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metpetdb_api/apps/samples/models.py b/metpetdb_api/apps/samples/models.py index 560a9d5..7d9cbb6 100644 --- a/metpetdb_api/apps/samples/models.py +++ b/metpetdb_api/apps/samples/models.py @@ -66,7 +66,7 @@ class Sample(models.Model): class Meta: db_table = 'samples' - ordering = ['id'] + ordering = ['number'] class SubsampleType(models.Model): @@ -75,7 +75,7 @@ class SubsampleType(models.Model): class Meta: db_table = 'subsample_types' - ordering = ['id'] + ordering = ['name'] class Subsample(models.Model): @@ -187,7 +187,7 @@ class GeoReference(models.Model): class Meta: db_table = 'georeferences' - ordering = ['id'] + ordering = ['name'] # Following are models for easy retrieval of sample-related free-text fields From f34a553b6ec28acc196ddab43e716aa1407fe076 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Wed, 20 Mar 2019 12:54:03 -0400 Subject: [PATCH 36/51] chem bulk upload parsing correctly --- .../api/bulk_upload/v1/upload_templates.py | 18 ++ metpetdb_api/api/bulk_upload/v1/views.py | 189 +++++++++++++----- 2 files changed, 162 insertions(+), 45 deletions(-) diff --git a/metpetdb_api/api/bulk_upload/v1/upload_templates.py b/metpetdb_api/api/bulk_upload/v1/upload_templates.py index 1696ad3..82801fd 100644 --- a/metpetdb_api/api/bulk_upload/v1/upload_templates.py +++ b/metpetdb_api/api/bulk_upload/v1/upload_templates.py @@ -45,6 +45,24 @@ 'metamorphic grade':'metamorphic_grades' } +chem_analysis_label_mappings = { + 'sample':'sample', + 'subsample':'subsample', + 'mineral':'mineral', + 'method':'analysis_method', + 'subsample type':'subsample_type', + 'reference':'reference', + 'point':'spot_id', + 'analytical facility':'where_done', + 'analysis date':'analysis_date', + 'analyst':'analyst', + 'reference x':'reference_x', + 'reference y':'reference_y', + 'stage x':'stage_x', + 'stage y':'stage_y', + 'total':'total' +} + class Template: def __init__(self, c_types = [], required = [], db_types = [], types = {}): diff --git a/metpetdb_api/api/bulk_upload/v1/views.py b/metpetdb_api/api/bulk_upload/v1/views.py index 8d89218..b83bbf8 100644 --- a/metpetdb_api/api/bulk_upload/v1/views.py +++ b/metpetdb_api/api/bulk_upload/v1/views.py @@ -64,6 +64,7 @@ import sys import urllib.request from csv import reader +import re sample_labels_dict = { @@ -78,6 +79,24 @@ 'Country':'country' } +chem_analysis_labels_dict = { + 'Sample':'sample', + 'Subsample':'subsample', + 'Mineral':'mineral', + 'Method':'analysis_method', + 'Subsample Type':'subsample_type', + 'Reference':'reference', + 'Point':'spot_id', + 'Analytical Facility':'where_done', + 'Analysis Date':'analysis_date', + 'Analyst':'analyst', + 'Reference X':'reference_x', + 'Reference Y':'reference_y', + 'Stage X':'stage_x', + 'Stage Y':'stage_y', + 'Total':'total' +} + class Parser: def __init__(self, template): @@ -268,9 +287,42 @@ def parse_chemical_analyses(self, request, JSON, meta_header): # Manual transaction for ease of exception handling transaction.set_autocommit(False) - for i,chemical_analyses_obj in enumerate(JSON): + for i,analysis_obj in enumerate(JSON): + + print(analysis_obj) + + # REQUIRED FIELDS: + # Sample [Number] + # Subsample [Number] + # Point + # Mineral + # Method + # Subsample Type + + # OPTIONAL FIELDS: + # Analytical Facility + # Analysis Date + # Analyst + # Reference Image (???) + # X Reference + # Y Reference + # X Stage + # Y Stage + # Total + # Comment + # [elements] + # [oxides] + # [precisions] + + + # PROCEDURE: + ## ensure all required fields are present + #### this is NOT enforced by the serializer like with samples! + ## verify all other fields are valid optional fields or minerals + ## manipulate data for serializer + ## create serializer try: - chemical_analyses_obj['owner'] = request.data.get('owner') + analysis_obj['owner'] = request.data.get('owner') except: self.rollback_transaction() return Response( @@ -278,57 +330,104 @@ def parse_chemical_analyses(self, request, JSON, meta_header): status = 400 ) - #fix date formatting - if chemical_analyses_obj['analysis_date']: - chemical_analyses_obj['analysis_date'] += 'T00:00:00.000Z' + element_oxide_pattern = re.compile(r"(.+)\((ppm|wt\%)\)",re.IGNORECASE) + precision_pattern = re.compile(r"(.+) precision \((abs|rel)\)",re.IGNORECASE) + elements_n_oxides = [] + precisions = {} + fields = [x for x in analysis_obj.keys()] + for field in fields: + if field.lower() in upload_templates.chem_analysis_label_mappings.keys(): + # add proper formatting to date (?) + if field.lower() == 'analysis date': + analysis_obj[field] = chemical_analyses_obj[field] #+'T00:00:00.000Z' + # join comments with newline + elif field.lower() == 'comment': + analysis_obj[field] = '\n'.join(analysis_obj[field]) + # replace field with corresponding serializer fieldname + analysis_obj[upload_templates.chem_analysis_label_mappings[field.lower()]] = analysis_obj[field] + del(analysis_obj[field]) + elif field != 'errors' and field not in upload_templates.chem_analysis_label_mappings.values(): # must be element, oxide, or precision + m = element_oxide_pattern.match(field) + if m is not None: + g = m.groups() + print('element or oxide: {}({})'.format(g[0],g[1])) + name = g[0].strip() + elements_n_oxides.append({ + 'name':name, + 'measurement_unit':g[1], + 'amount':analysis_obj[field] + }) + else: + m = precision_pattern.match(field) + if m is not None: + g = m.groups() + print('precision: {} Precision ({})'.format(g[0],g[1])) + precisions[g[0]] = {'type':g[1],'value':analysis_obj[field]} + else: + print('no match: {}'.format(field)) + + # make sure the mineral exists try: - chemical_analyses_obj['mineral_id'] = Mineral.objects.get(name=chemical_analyses_obj['mineral'][0]['name']).id + analysis_obj['mineral_id'] = Mineral.objects.get(name=analysis_obj['mineral']).id except Exception as err: - return self.set_err(before_parse_json, i, 'mineral_id', 'invalid mineral', meta_header) + return self.set_err(before_parse_json, i, 'mineral_id', 'invalid mineral', meta_header) + + # check for required fields that aren't required in model (aaaa...) + if (analysis_obj.get('point')) is None: + return self.set_err(before_parse_json, i, 'point', 'analysis point identifier is required', meta_header) + if (analysis_obj.get('analysis_method')) is None: + return self.set_err(before_parse_json, i, 'analysis_method', 'analysis method is required', meta_header) + - elements_to_add = [] - for element in chemical_analyses_obj['element']: + analysis_obj['oxides'] = [] + analysis_obj['elements'] = [] + + for entry in elements_n_oxides: try: - elements_to_add.append( - {'id': Element.objects.get(name=element['name']).id, - 'amount': element['amount'], + p = None + ptype = None + if (precisions.get(entry['name'])) is not None: + p = precisions[entry['name']]['value'] + ptype = precisions[entry['name']]['type'] + + analysis_obj['elements'].append( + {'id': Element.objects.get(symbol=entry['name']).id, + 'amount': entry['amount'], + 'precision': p, + 'precision_type': ptype, + 'measurement_unit': entry['measurement_unit'], #TODO consider adding these fields #They are not specified in the template - 'precision': None, - 'precision_type': None, - 'measurement_unit': None, 'min_amount': None, 'max_amount': None }) except: - return self.set_err(before_parse_json, i, 'element', 'invalid element {0}'.format(element), meta_header) + try: + analysis_obj['oxides'].append( + {'id' : Oxide.objects.get(species=entry['name']).id, + 'amount': entry['amount'], + 'precision': p, + 'precision_type': ptype, + 'measurement_unit': entry['measurement_unit'], + #TODO consider adding + #They are currently not specified in the template + 'min_amount': None, + 'max_amount': None + }) + except: + return self.set_err(before_parse_json, i, 'element/oxide', 'invalid element or oxide {0}'.format(oxide), meta_header) - chemical_analyses_obj['elements'] = elements_to_add - oxides_to_add = [] + if len(analysis_obj['elements']) == 0: + del(analysis_obj['elements']) + if len(analysis_obj['oxides']) == 0: + del(analysis_obj['oxides']) - for oxide in chemical_analyses_obj['oxide']: - try: - oxides_to_add.append( - { 'id' : Oxide.objects.get(species=oxide['name']).id, - 'amount': oxide['amount'], - #TODO consider adding - #They are currently not specified in the template - 'precision': None, - 'precision_type': None, - 'measurement_unit': None, - 'min_amount': None, - 'max_amount': None - - }) - except: - return self.set_err(before_parse_json, i, 'oxide', 'invalid oxide {0}'.format(oxide), meta_header) - - chemical_analyses_obj['oxides'] = oxides_to_add + - serializer = self.get_serializer(data=chemical_analyses_obj) + serializer = self.get_serializer(data=analysis_obj) try: serializer.is_valid(raise_exception=True) instance = self.perform_create(serializer) @@ -336,8 +435,8 @@ def parse_chemical_analyses(self, request, JSON, meta_header): return self.set_err(before_parse_json, i, 'serialization', str(e), meta_header) - if chemical_analyses_obj.get('elements'): - for record in chemical_analyses_obj.get('elements'): + if analysis_obj.get('elements'): + for record in analysis_obj.get('elements'): try: ChemicalAnalysisElement.objects.create( chemical_analysis=instance, @@ -352,8 +451,8 @@ def parse_chemical_analyses(self, request, JSON, meta_header): except Element.DoesNotExist: return self.set_err(before_parse_json, i, 'elements', 'invalid element id', meta_header) - if chemical_analyses_obj.get('oxides'): - for record in chemical_analyses_obj.get('oxides'): + if analysis_obj.get('oxides'): + for record in analysis_obj.get('oxides'): try: ChemicalAnalysisOxide.objects.create( chemical_analysis=instance, @@ -449,10 +548,10 @@ def parse_samples(self, request, JSON, meta_header): status = 400 ) - if 'latitude' in sample_obj.keys() and 'longitude' in sample_obj.keys(): - sample_obj['location_coords'] = u'SRID=4326;POINT ({0} {1})'.format(sample_obj['latitude'], sample_obj['longitude']) - del(sample_obj['latitude']) - del(sample_obj['longitude']) + # if 'latitude' in sample_obj.keys() and 'longitude' in sample_obj.keys(): + # sample_obj['location_coords'] = u'SRID=4326;POINT ({0} {1})'.format(sample_obj['latitude'], sample_obj['longitude']) + # del(sample_obj['latitude']) + # del(sample_obj['longitude']) rock_type = sample_obj['rock_type_name'] try: From 89ee9eaba860d85ad6685441a5918b69347b723a Mon Sep 17 00:00:00 2001 From: metpetDB Date: Wed, 20 Mar 2019 16:24:00 -0400 Subject: [PATCH 37/51] chem analysis bulk upload (mostly?) working --- .../api/bulk_upload/v1/upload_templates.py | 7 +- metpetdb_api/api/bulk_upload/v1/views.py | 72 ++++++++++++++----- 2 files changed, 59 insertions(+), 20 deletions(-) diff --git a/metpetdb_api/api/bulk_upload/v1/upload_templates.py b/metpetdb_api/api/bulk_upload/v1/upload_templates.py index 82801fd..92a7234 100644 --- a/metpetdb_api/api/bulk_upload/v1/upload_templates.py +++ b/metpetdb_api/api/bulk_upload/v1/upload_templates.py @@ -60,7 +60,8 @@ 'reference y':'reference_y', 'stage x':'stage_x', 'stage y':'stage_y', - 'total':'total' + 'total':'total', + 'comment':'description' } @@ -169,8 +170,8 @@ def is_db_type(self, name): return name in self.db_types class ChemicalAnalysesTemplate(Template): def __init__(self): - complex_types = ["comment", "element", "oxide","mineral"] - required = ["subsample_id", "spot_id", "mineral", "analysis_method"] + complex_types = ["comment","element","oxide"] + required = ["sample","subsample","point","mineral","method","subsample type"] db_types = ["element", "oxide"] types = {"comment": str, "stage_x" : float, "stage_y" : float, "reference_x": float, "reference_y": float} Template.__init__(self, complex_types, required, db_types, types) diff --git a/metpetdb_api/api/bulk_upload/v1/views.py b/metpetdb_api/api/bulk_upload/v1/views.py index b83bbf8..1f4b6ab 100644 --- a/metpetdb_api/api/bulk_upload/v1/views.py +++ b/metpetdb_api/api/bulk_upload/v1/views.py @@ -283,14 +283,12 @@ def set_err(self, JSON, i, field, val, meta_header): def parse_chemical_analyses(self, request, JSON, meta_header): before_parse_json = list(JSON) - + # Manual transaction for ease of exception handling transaction.set_autocommit(False) for i,analysis_obj in enumerate(JSON): - print(analysis_obj) - # REQUIRED FIELDS: # Sample [Number] # Subsample [Number] @@ -340,18 +338,23 @@ def parse_chemical_analyses(self, request, JSON, meta_header): if field.lower() in upload_templates.chem_analysis_label_mappings.keys(): # add proper formatting to date (?) if field.lower() == 'analysis date': - analysis_obj[field] = chemical_analyses_obj[field] #+'T00:00:00.000Z' + if analysis_obj[field] != '': + analysis_obj[field] = analysis_obj[field] +'T00:00:00.000Z' # join comments with newline elif field.lower() == 'comment': analysis_obj[field] = '\n'.join(analysis_obj[field]) # replace field with corresponding serializer fieldname - analysis_obj[upload_templates.chem_analysis_label_mappings[field.lower()]] = analysis_obj[field] - del(analysis_obj[field]) + if (analysis_obj[field] == ''): + analysis_obj[upload_templates.chem_analysis_label_mappings[field.lower()]] = None + else: + analysis_obj[upload_templates.chem_analysis_label_mappings[field.lower()]] = analysis_obj[field] + # print('{} : {}'.format(field,analysis_obj[upload_templates.chem_analysis_label_mappings[field.lower()]])) + # del(analysis_obj[field]) elif field != 'errors' and field not in upload_templates.chem_analysis_label_mappings.values(): # must be element, oxide, or precision m = element_oxide_pattern.match(field) if m is not None: g = m.groups() - print('element or oxide: {}({})'.format(g[0],g[1])) + # print('element or oxide: {}({})'.format(g[0],g[1])) name = g[0].strip() elements_n_oxides.append({ 'name':name, @@ -362,7 +365,7 @@ def parse_chemical_analyses(self, request, JSON, meta_header): m = precision_pattern.match(field) if m is not None: g = m.groups() - print('precision: {} Precision ({})'.format(g[0],g[1])) + # print('precision: {} Precision ({})'.format(g[0],g[1])) precisions[g[0]] = {'type':g[1],'value':analysis_obj[field]} else: print('no match: {}'.format(field)) @@ -375,16 +378,45 @@ def parse_chemical_analyses(self, request, JSON, meta_header): return self.set_err(before_parse_json, i, 'mineral_id', 'invalid mineral', meta_header) # check for required fields that aren't required in model (aaaa...) - if (analysis_obj.get('point')) is None: - return self.set_err(before_parse_json, i, 'point', 'analysis point identifier is required', meta_header) + if (analysis_obj.get('spot_id')) is None: + return self.set_err(before_parse_json, i, 'spot_id', 'analysis point identifier is required', meta_header) if (analysis_obj.get('analysis_method')) is None: return self.set_err(before_parse_json, i, 'analysis_method', 'analysis method is required', meta_header) + # make sure sample exists; if subsample doesn't exist, create it + try: + sample = Sample.objects.get(owner_id=analysis_obj['owner'],number=analysis_obj['sample']) + analysis_obj['sample_id'] = sample.id + try: + analysis_obj['subsample_id'] = Subsample.objects.get(sample_id=analysis_obj['sample_id'],name=analysis_obj['subsample']).id + except Exception as err: + print(err) + try: + sub_type = SubsampleType.objects.get(name=analysis_obj['subsample_type']) + Subsample.objects.create( + name=analysis_obj['subsample'], + sample=sample, + owner_id=analysis_obj['owner'], + subsample_type=sub_type, + ) + analysis_obj['subsample_id'] = Subsample.objects.get( + name=analysis_obj['subsample'], + sample_id=analysis_obj['sample_id'], + owner_id=analysis_obj['owner'], + subsample_type=sub_type).id + except Exception as err: + print(err) + return self.set_err(before_parse_json, i, 'subsample', err, meta_header) + except Exception as err: + print(err) + return self.set_err(before_parse_json, i, 'sample', err, meta_header) analysis_obj['oxides'] = [] analysis_obj['elements'] = [] for entry in elements_n_oxides: + if entry['amount'] == '': + continue try: p = None ptype = None @@ -392,8 +424,9 @@ def parse_chemical_analyses(self, request, JSON, meta_header): p = precisions[entry['name']]['value'] ptype = precisions[entry['name']]['type'] + obj = Element.objects.get(symbol=entry['name']) analysis_obj['elements'].append( - {'id': Element.objects.get(symbol=entry['name']).id, + {'id': obj.id, 'amount': entry['amount'], 'precision': p, 'precision_type': ptype, @@ -405,8 +438,9 @@ def parse_chemical_analyses(self, request, JSON, meta_header): }) except: try: + obj = Oxide.objects.get(species=entry['name']) analysis_obj['oxides'].append( - {'id' : Oxide.objects.get(species=entry['name']).id, + {'id' : obj.id, 'amount': entry['amount'], 'precision': p, 'precision_type': ptype, @@ -432,11 +466,14 @@ def parse_chemical_analyses(self, request, JSON, meta_header): serializer.is_valid(raise_exception=True) instance = self.perform_create(serializer) except Exception as e: + print(e) + print(serializer.initial_data) return self.set_err(before_parse_json, i, 'serialization', str(e), meta_header) if analysis_obj.get('elements'): for record in analysis_obj.get('elements'): + # print(record) try: ChemicalAnalysisElement.objects.create( chemical_analysis=instance, @@ -445,14 +482,15 @@ def parse_chemical_analyses(self, request, JSON, meta_header): precision=record['precision'], precision_type=record['precision_type'], measurement_unit=record['measurement_unit'], - min_amount=record['min_amount'], - max_amount=record['max_amount'], + # min_amount=record['min_amount'], + # max_amount=record['max_amount'], ) except Element.DoesNotExist: return self.set_err(before_parse_json, i, 'elements', 'invalid element id', meta_header) if analysis_obj.get('oxides'): for record in analysis_obj.get('oxides'): + # print(record) try: ChemicalAnalysisOxide.objects.create( chemical_analysis=instance, @@ -461,8 +499,8 @@ def parse_chemical_analyses(self, request, JSON, meta_header): precision=record['precision'], precision_type=record['precision_type'], measurement_unit=record['measurement_unit'], - min_amount=record['min_amount'], - max_amount=record['max_amount'], + # min_amount=record['min_amount'], + # max_amount=record['max_amount'], ) except Oxide.DoesNotExist: return self.set_err(before_parse_json, i, 'oxides', 'invalid oxide id', meta_header) @@ -485,7 +523,7 @@ def parse_samples(self, request, JSON, meta_header): for i,sample_obj in enumerate(JSON): - print(sample_obj) + # print(sample_obj) # REQUIRED FIELDS: # Sample Number From cf92fde40d995a676ff45d8c3c801e46319caa66 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Wed, 10 Apr 2019 14:09:58 -0400 Subject: [PATCH 38/51] commented out error prints (oops) --- .../api/bulk_upload/v1/upload_templates.py | 18 ++++++++--------- metpetdb_api/api/bulk_upload/v1/views.py | 20 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/metpetdb_api/api/bulk_upload/v1/upload_templates.py b/metpetdb_api/api/bulk_upload/v1/upload_templates.py index 92a7234..65da7ce 100644 --- a/metpetdb_api/api/bulk_upload/v1/upload_templates.py +++ b/metpetdb_api/api/bulk_upload/v1/upload_templates.py @@ -155,7 +155,7 @@ def parse(self, data): heading = header[j] field = data[i][j] - print("{}: {}".format(heading,data[i][j])) + # print("{}: {}".format(heading,data[i][j])) if self.is_complex(heading): tmp_result.set_field_complex(heading,field) else: tmp_result.set_field_simple(heading, field) @@ -220,22 +220,22 @@ def get_amount(self,data=[],i=0,j=0): return 0 def get_meta_header(self,header): - mappings = sample_label_mappings + mappings = {} added = set() meta_header = [] itr = iter(header) for heading in itr: - if heading.lower() == 'latitude': - for i in range (0,2): heading = next(itr) - meta_header.append((('latitude','longitude'),'Location')) - elif heading not in added: + # if heading.lower() == 'latitude': + # for i in range (0,2): heading = next(itr) + # meta_header.append((('latitude','longitude'),'Location')) + if heading not in added: if heading.lower() in mappings.keys(): meta_header.append((heading.lower(),heading)) added.add(heading) else: meta_header.append((heading, heading)) added.add(heading) - print("\nMETA-HEADER:") - print(meta_header) - print("\n\n") + # print("\nMETA-HEADER:") + # print(meta_header) + # print("\n\n") return meta_header diff --git a/metpetdb_api/api/bulk_upload/v1/views.py b/metpetdb_api/api/bulk_upload/v1/views.py index 1f4b6ab..1ce98be 100644 --- a/metpetdb_api/api/bulk_upload/v1/views.py +++ b/metpetdb_api/api/bulk_upload/v1/views.py @@ -118,7 +118,7 @@ def parse(self, url): lined = self.line_split(content) return self.template.parse(lined) # return the JSON ready file except Exception as err: - print(err) + # print(err) raise ValueError(str(err)) class BulkUploadViewSet(viewsets.ModelViewSet): @@ -367,8 +367,8 @@ def parse_chemical_analyses(self, request, JSON, meta_header): g = m.groups() # print('precision: {} Precision ({})'.format(g[0],g[1])) precisions[g[0]] = {'type':g[1],'value':analysis_obj[field]} - else: - print('no match: {}'.format(field)) + # else: + # print('no match: {}'.format(field)) # make sure the mineral exists try: @@ -390,7 +390,7 @@ def parse_chemical_analyses(self, request, JSON, meta_header): try: analysis_obj['subsample_id'] = Subsample.objects.get(sample_id=analysis_obj['sample_id'],name=analysis_obj['subsample']).id except Exception as err: - print(err) + # print(err) try: sub_type = SubsampleType.objects.get(name=analysis_obj['subsample_type']) Subsample.objects.create( @@ -405,10 +405,10 @@ def parse_chemical_analyses(self, request, JSON, meta_header): owner_id=analysis_obj['owner'], subsample_type=sub_type).id except Exception as err: - print(err) + # print(err) return self.set_err(before_parse_json, i, 'subsample', err, meta_header) except Exception as err: - print(err) + # print(err) return self.set_err(before_parse_json, i, 'sample', err, meta_header) analysis_obj['oxides'] = [] @@ -466,8 +466,8 @@ def parse_chemical_analyses(self, request, JSON, meta_header): serializer.is_valid(raise_exception=True) instance = self.perform_create(serializer) except Exception as e: - print(e) - print(serializer.initial_data) + # print(e) + # print(serializer.initial_data) return self.set_err(before_parse_json, i, 'serialization', str(e), meta_header) @@ -563,7 +563,7 @@ def parse_samples(self, request, JSON, meta_header): sample_obj[field] = '\n'.join(sample_obj[field]) # replace field with corresponding serializer fieldname sample_obj[upload_templates.sample_label_mappings[field.lower()]] = sample_obj[field] - del(sample_obj[field]) + # del(sample_obj[field]) elif field != 'errors' and field not in upload_templates.sample_label_mappings.values(): # it had better be a mineral try: amount = sample_obj[field] @@ -572,7 +572,7 @@ def parse_samples(self, request, JSON, meta_header): 'name':field, 'amount':amount}) except: - print(field) + # print(field) return self.set_err(before_parse_json, i, 'minerals', 'Invalid mineral {}'.format(field), meta_header) sample_obj['minerals'] = minerals From a63455efbda2c58409dbd3806d2120d21952f1c2 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Thu, 2 May 2019 14:52:51 -0400 Subject: [PATCH 39/51] made sample number & owner id a unique pair --- metpetdb_api/apps/samples/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metpetdb_api/apps/samples/models.py b/metpetdb_api/apps/samples/models.py index 7d9cbb6..413ec70 100644 --- a/metpetdb_api/apps/samples/models.py +++ b/metpetdb_api/apps/samples/models.py @@ -66,6 +66,7 @@ class Sample(models.Model): class Meta: db_table = 'samples' + unique_together = (('number','owner')) ordering = ['number'] @@ -89,6 +90,7 @@ class Subsample(models.Model): class Meta: db_table = 'subsamples' + unique_together = (('name','sample','owner')) ordering = ['id'] From 79aaa0965dd1bbb2e2af56d3e385d414b2cd9e75 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Thu, 2 May 2019 15:53:42 -0400 Subject: [PATCH 40/51] cleaned up views file & added public_data to serializer so field can be set --- metpetdb_api/api/samples/v1/serializers.py | 1 + metpetdb_api/api/samples/v1/views.py | 69 +--------------------- 2 files changed, 2 insertions(+), 68 deletions(-) diff --git a/metpetdb_api/api/samples/v1/serializers.py b/metpetdb_api/api/samples/v1/serializers.py index 2af429a..3968313 100644 --- a/metpetdb_api/api/samples/v1/serializers.py +++ b/metpetdb_api/api/samples/v1/serializers.py @@ -96,6 +96,7 @@ class Meta: 'images', 'subsample_ids', 'chemical_analyses_ids', + 'public_data', ) def is_valid(self, raise_exception=False): diff --git a/metpetdb_api/api/samples/v1/views.py b/metpetdb_api/api/samples/v1/views.py index 40846d8..29289e9 100644 --- a/metpetdb_api/api/samples/v1/views.py +++ b/metpetdb_api/api/samples/v1/views.py @@ -421,71 +421,4 @@ def get(self, request, format=None): .values_list('owner__name', flat=True) .distinct() ) - return Response({'sample_owner_names': sample_owner_names}) - -''' -class SampleCSVRenderer (r.CSVRenderer): - header = ['Sample', 'Rock_Type', 'Comment', 'Latitude', 'Longitude', 'Location_Error', 'Region', 'Country', 'Collector', 'Date_of_Collection', 'Present_Sample_Location', 'Reference', 'Metamorphic_Grade', 'Minerals', 'Subsamples', 'Chemical_Analyses'] - labels = { - 'Sample': 'Sample', - 'Rock_Type': 'Rock Type', - 'Comment': 'Comment', - 'Latitude': 'Latitude', - 'Longitude': 'Longitude', - 'Location_Error': 'Location Error', - 'Region': 'Region', - 'Country': 'Country', - 'Collector': 'Collector', - 'Date_of_Collection': 'Date of Collection', - 'Present_Sample_Location': 'Present Sample Location', - 'Reference': 'Reference', - 'Metamorphic_Grade': 'Metamorphic Grade', - 'Minerals': 'Mineral', - 'Subsamples': 'Number of Subsamples', - 'Chemical_Analyses': 'Number of Chemical Analyses' - } - - - -class SampleSearchView(SampleViewSet): - serializer_class = SampleSearchSerializer - renderer_classes = (JSONRenderer, BrowsableAPIRenderer, SampleCSVRenderer) - - def get_serializer(self, *args, **kwargs): - return super().get_serializer(*args,**kwargs) - - def list(self, request, *args, **kwargs): - params = request.query_params - - if params.get('chemical_analyses_filters') == 'True': - chem_qs = ChemicalAnalysis.objects.all() - chem_qs = chemical_analyses_qs_optimizer(params, chem_qs) - chem_ids = (chemical_analysis_query(request.user, params, chem_qs) - .values_list('id')) - qs = (Sample - .objects - .filter(subsamples__chemical_analyses__id__in=chem_ids)) - else: - qs = self.get_queryset().distinct() - try: - qs = sample_query(request.user, params, qs) - except ValueError as err: - return Response( - data={'error': err.args}, - status=400 - ) - - qs = sample_qs_optimizer(params, qs) - - if params.get('format') == 'csv': - serializer = self.get_serializer(qs, many=True) - return Response(serializer.data) - else: - page = self.paginate_queryset(qs) - if page is not None: - serializer = self.get_serializer(page, many=True) - return self.get_paginated_response(serializer.data) - - serializer = self.get_serializer(qs, many=True) - return Response(serializer.data) -''' + return Response({'sample_owner_names': sample_owner_names}) \ No newline at end of file From 64e58ed6b05f24934a36a3db49feef2de7b2bf6d Mon Sep 17 00:00:00 2001 From: metpetDB Date: Thu, 13 Jun 2019 13:19:15 -0400 Subject: [PATCH 41/51] fixed lat/long bug --- metpetdb_api/api/samples/v1/serializers.py | 63 +++++++++++----------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/metpetdb_api/api/samples/v1/serializers.py b/metpetdb_api/api/samples/v1/serializers.py index 3968313..39f971a 100644 --- a/metpetdb_api/api/samples/v1/serializers.py +++ b/metpetdb_api/api/samples/v1/serializers.py @@ -21,10 +21,33 @@ ) from apps.users.models import User -SAMPLE_FIELDS = ('number', 'aliases', 'collection_date', 'description', - 'location_name', 'location_coords', 'location_error', - 'date_precision', 'country', 'regions', 'collector_name', - 'collector_id', 'sesar_number', 'image') +SAMPLE_FIELDS = ('id', # + 'number', + # 'aliases', + 'owner', # + 'regions', + 'country', + 'rock_type', # + 'metamorphic_grades', # + 'metamorphic_regions', # + 'minerals', # + 'references', # + 'longitude', + 'latitude', + #'location_coords', + 'location_error', + 'sesar_number', + 'collector_name', + # 'collector_id', + 'collection_date', + # 'date_precision', + 'location_name', + 'description', + # 'image', + 'images', + 'subsample_ids', # + 'chemical_analyses_ids', # + 'public_data',) SUBSAMPLE_FIELDS = ('name') @@ -73,38 +96,16 @@ class SampleSerializer(DynamicFieldsModelSerializer): class Meta: model = Sample depth = 1 - fields = ( - 'id', - 'number', - 'owner', - 'regions', - 'country', - 'rock_type', - 'metamorphic_grades', - 'metamorphic_regions', - 'minerals', - 'references', - 'longitude', - 'latitude', - 'location_coords', - 'location_error', - # 'igsn', - 'collector_name', - 'collection_date', - 'location_name', - 'description', - 'images', - 'subsample_ids', - 'chemical_analyses_ids', - 'public_data', - ) + fields = SAMPLE_FIELDS def is_valid(self, raise_exception=False): if self.initial_data.get('latitude') and self.initial_data.get('longitude'): - self.initial_data['location_coords'] = "SRID=4326;POINT ("+str(self.initial_data["latitude"])+" "+str(self.initial_data["longitude"])+")" + self.initial_data['location_coords'] = "SRID=4326;POINT ("+str(self.initial_data["longitude"])+" "+str(self.initial_data["latitude"])+")" super().is_valid(raise_exception) + self._validated_data.update({'location_coords':self.initial_data['location_coords']}) + if self.initial_data.get('owner'): self._validated_data.update( {'owner': User.objects.get(pk=self.initial_data['owner'])}) @@ -128,7 +129,7 @@ def create(self, validated_data): def update(self, instance, validated_data): for attr, value in validated_data.items(): - if attr in SAMPLE_FIELDS: + # if attr in SAMPLE_FIELDS: setattr(instance, attr, value) instance.save() From 718f3d555a7cc13d00424ee7ac69ddaf6b0276fb Mon Sep 17 00:00:00 2001 From: metpetDB Date: Thu, 13 Jun 2019 14:26:22 -0400 Subject: [PATCH 42/51] quick n dirty handling of single image file upload --- metpetdb_api/api/images/v1/views.py | 47 ++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/metpetdb_api/api/images/v1/views.py b/metpetdb_api/api/images/v1/views.py index db9952f..97dda45 100644 --- a/metpetdb_api/api/images/v1/views.py +++ b/metpetdb_api/api/images/v1/views.py @@ -1,4 +1,6 @@ -from rest_framework import viewsets, permissions +from rest_framework import viewsets, permissions, status +from rest_framework.response import Response +from rest_framework.views import APIView from api.images.v1.serializers import ImageContainerSerializer, ImageSerializer from apps.images.models import ImageContainer, Image @@ -6,14 +8,43 @@ class ImageContainerViewSet(viewsets.ModelViewSet): - queryset = ImageContainer.objects.all() - serializer_class = ImageContainerSerializer - permission_classes = (permissions.IsAuthenticatedOrReadOnly, - IsOwnerOrReadOnly,) + queryset = ImageContainer.objects.all() + serializer_class = ImageContainerSerializer + permission_classes = (permissions.IsAuthenticatedOrReadOnly, + IsOwnerOrReadOnly,) class ImageViewSet(viewsets.ModelViewSet): - queryset = Image.objects.all() - serializer_class = ImageSerializer - permission_classes = (permissions.IsAuthenticatedOrReadOnly, + queryset = Image.objects.all() + serializer_class = ImageSerializer + permission_classes = (permissions.IsAuthenticatedOrReadOnly, IsOwnerOrReadOnly) + + def get_serializer(self, *args, **kwargs): + # not sure what this is doing yet + # if self.request.method == 'PUT': + # kwargs['partial'] = True + return super().get_serializer(*args, **kwargs) + + def perform_create(self, serializer): + return serializer.save() + + def create(self, request, *args, **kwargs): + img_file = request.FILES.get('image') + if not img_file: + print("no image file") + return Response(status=404) + + img_data = request.data + img_data['image'] = img_file + + + serializer = self.get_serializer(data=img_data) + serializer.is_valid(raise_exception=True) + + instance = self.perform_create(serializer) + + headers = self.get_success_headers(serializer.data) + return Response(serializer.data, + status=status.HTTP_201_CREATED, + headers=headers) From bfd0ebfbd6fd12144abc8964e2c90041ab3a8896 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Thu, 13 Jun 2019 21:48:26 -0400 Subject: [PATCH 43/51] playing w single image upload --- metpetdb_api/api/images/v1/serializers.py | 38 ++++++++++++++++++----- metpetdb_api/api/images/v1/views.py | 2 +- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/metpetdb_api/api/images/v1/serializers.py b/metpetdb_api/api/images/v1/serializers.py index 96f6508..2315b1c 100644 --- a/metpetdb_api/api/images/v1/serializers.py +++ b/metpetdb_api/api/images/v1/serializers.py @@ -28,7 +28,7 @@ DWELL_TIME = 'dwelltime' CURRENT = 'current' VOLTAGE = 'voltage' -XRAY_IMAGE = 'xrayimage' +XRAY_IMAGE = 'xraymap' required_headers = {SAMPLE_NUMBER, IMAGE_TYPE} @@ -52,9 +52,30 @@ class Meta: 'description', 'comments') image = VersatileImageFieldSerializer(sizes='image_sizes', required=False) image_type = ImageTypeSerializer(read_only=True) - comments = ImageCommentsSerializer(many=True) + comments = ImageCommentsSerializer(many=True,required=False) owner = UserSerializer(read_only=True) + def is_valid(self, raise_exception=False): + super().is_valid(raise_exception) + + if self.initial_data.get('image_type'): + self._validated_data.update({'image_type':ImageType.objects.get(pk=self.initial_data['image_type'])}) + + if self.initial_data.get('owner'): + self._validated_data.update( + {'owner': User.objects.get(pk=self.initial_data['owner'])}) + + def update(self, instance, validated_data): + for attr, value in validated_data.items(): + if attr in self.fields: + setattr(instance, attr, value) + instance.save() + return instance + + def create(self, validated_data): + instance = super().create(validated_data) + return instance + class ImageContainerSerializer(serializers.ModelSerializer): class Meta: @@ -144,18 +165,20 @@ def create_image(self, base_directory, path, image_container, image_type, collec def get_sample_subsample_public_data(self, subsample_name, sample_number, subsample_type): sample, subsample, public_data = None, None, False - if len(subsample_name) == 0: - sample = Sample.objects.get(number=sample_number) + if self.initial_data.get('owner'): + owner = User.objects.get(pk=self.initial_data['owner']) + if not subsample_name or len(subsample_name) == 0: + sample = Sample.objects.get(number=sample_number,owner=owner) public_data = sample.public_data else: - subsample_sample = Sample.objects.get(number=sample_number) + subsample_sample = Sample.objects.get(number=sample_number,owner=owner) try: subsample = Subsample.objects.get(sample=subsample_sample, name=subsample_name) except Subsample.DoesNotExist: owner = None if self.initial_data.get('owner'): owner = User.objects.get(pk=self.initial_data['owner']) - Subsample.objects.create( + subsample = Subsample.objects.create( name=subsample_name, sample=subsample_sample, subsample_type=SubsampleType.objects.get(name=subsample_type), @@ -167,12 +190,13 @@ def get_sample_subsample_public_data(self, subsample_name, sample_number, subsam @staticmethod def create_xray_image(image_type, values, header_to_index, created_image, dwell_time, current, voltage): image_type_value = re.sub('[^a-z]+', '', image_type.lower()) + print(image_type_value) if image_type_value == XRAY_IMAGE: element = values[header_to_index[ELEMENT]] if not element: raise serializers.ValidationError('Expected element for xray image, but none provided') xray_image = XrayImage.objects.create(image=created_image, - dwell_time=dwell_time, + dwelltime=dwell_time, current=current, voltage=voltage, element=element) diff --git a/metpetdb_api/api/images/v1/views.py b/metpetdb_api/api/images/v1/views.py index 97dda45..da20d21 100644 --- a/metpetdb_api/api/images/v1/views.py +++ b/metpetdb_api/api/images/v1/views.py @@ -21,7 +21,7 @@ class ImageViewSet(viewsets.ModelViewSet): IsOwnerOrReadOnly) def get_serializer(self, *args, **kwargs): - # not sure what this is doing yet + # FIXME not sure what this is doing yet # if self.request.method == 'PUT': # kwargs['partial'] = True return super().get_serializer(*args, **kwargs) From 3c3d393fdd58da6b61c540e004155c0e73aa1764 Mon Sep 17 00:00:00 2001 From: metpetdb Date: Fri, 14 Jun 2019 01:50:26 +0000 Subject: [PATCH 44/51] merged stuff --- metpetdb_api/api/images/v1/serializers.py | 6 ++++-- metpetdb_api/api/samples/v1/serializers.py | 7 +++---- metpetdb_api/settings/staging.py | 6 +++--- metpetdb_api/vendor/djoser | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/metpetdb_api/api/images/v1/serializers.py b/metpetdb_api/api/images/v1/serializers.py index 96f6508..60ebedc 100644 --- a/metpetdb_api/api/images/v1/serializers.py +++ b/metpetdb_api/api/images/v1/serializers.py @@ -144,11 +144,13 @@ def create_image(self, base_directory, path, image_container, image_type, collec def get_sample_subsample_public_data(self, subsample_name, sample_number, subsample_type): sample, subsample, public_data = None, None, False + if self.initial_data.get('owner'): + owner = User.objects.get(pk=self.initial_data['owner']); if len(subsample_name) == 0: - sample = Sample.objects.get(number=sample_number) + sample = Sample.objects.get(number=sample_number,owner=owner) public_data = sample.public_data else: - subsample_sample = Sample.objects.get(number=sample_number) + subsample_sample = Sample.objects.get(number=sample_number,owner=owner) try: subsample = Subsample.objects.get(sample=subsample_sample, name=subsample_name) except Subsample.DoesNotExist: diff --git a/metpetdb_api/api/samples/v1/serializers.py b/metpetdb_api/api/samples/v1/serializers.py index 39f971a..452db16 100644 --- a/metpetdb_api/api/samples/v1/serializers.py +++ b/metpetdb_api/api/samples/v1/serializers.py @@ -85,8 +85,7 @@ class SampleSerializer(DynamicFieldsModelSerializer): references = serializers.SerializerMethodField(read_only=True) latitude = serializers.SerializerMethodField(read_only=True) longitude = serializers.SerializerMethodField(read_only=True) - # collection_date = serializers.SerializerMethodField(read_only=True) - + images = ImageSerializer(many=True, read_only=True) # TODO: figure out if there is a better, more efficient way to do this @@ -208,7 +207,7 @@ def create(self, validated_data): def update(self, instance, validated_data): for attr, value in validated_data.items(): - if attr in SUBSAMPLE_FIELDS: + if attr in self.fields: setattr(instance, attr, value) instance.save() @@ -255,4 +254,4 @@ class Meta: class CollectorSerializer(DynamicFieldsModelSerializer): class Meta: model = Collector - fields = '__all__' \ No newline at end of file + fields = '__all__' diff --git a/metpetdb_api/settings/staging.py b/metpetdb_api/settings/staging.py index 2c998e2..42d30f0 100644 --- a/metpetdb_api/settings/staging.py +++ b/metpetdb_api/settings/staging.py @@ -36,7 +36,7 @@ EMAIL_PORT = env('EMAIL_PORT') -ALLOWED_HOSTS = [] +ALLOWED_HOSTS = ['165.227.94.236','api.metpetdb.com','api.metpetdb.org'] AUTH_USER_MODEL = 'users.User' @@ -238,5 +238,5 @@ ] } -MEDIA_ROOT = os.path.join(BASE_DIR, 'media_root/') -MEDIA_URL = '/images/' +MEDIA_ROOT = os.path.join(BASE_DIR, 'images/') +MEDIA_URL = '/api/images/' diff --git a/metpetdb_api/vendor/djoser b/metpetdb_api/vendor/djoser index 8239ecf..eb26c90 160000 --- a/metpetdb_api/vendor/djoser +++ b/metpetdb_api/vendor/djoser @@ -1 +1 @@ -Subproject commit 8239ecf52b164f8e16997f6970ebd30969fffedb +Subproject commit eb26c909125d5b1c32773a62d1830c40684430c3 From c5615965f4b872c64b018b386eb4a85769f3ec23 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Thu, 20 Jun 2019 12:53:02 -0400 Subject: [PATCH 45/51] added association with sample/subsample on image direct upload --- metpetdb_api/api/images/v1/serializers.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/metpetdb_api/api/images/v1/serializers.py b/metpetdb_api/api/images/v1/serializers.py index 2315b1c..6fa44dd 100644 --- a/metpetdb_api/api/images/v1/serializers.py +++ b/metpetdb_api/api/images/v1/serializers.py @@ -3,6 +3,7 @@ from rest_framework import serializers from apps.images.models import Image, ImageContainer, ImageType, XrayImage, ImageComments from apps.samples.models import Sample, Subsample, SubsampleType +from apps.chemical_analyses.models import ChemicalAnalysis from django.core.files import File import urllib.request from urllib.parse import urlparse, urlencode, urlunparse @@ -65,6 +66,22 @@ def is_valid(self, raise_exception=False): self._validated_data.update( {'owner': User.objects.get(pk=self.initial_data['owner'])}) + if self.initial_data.get('sample'): + self._validated_data.update( + {'sample':Sample.objects.get(pk=self.initial_data['sample'])}) + elif self.initial_data.get('subsample'): + self._validated_data.update( + {'subsample':Subsample.objects.get(pk=self.initial_data['subsample'])}) + elif self.initial_data.get('chemical_analysis'): + ca = ChemicalAnalysis.objects.get(pk=self.initial_data['chemical_analysis']) + print(ca) + else: + return Response( + status=404, + data={'error':'Cannot upload image without associated \ + sample, subsample, or chem analysis!'}) + + def update(self, instance, validated_data): for attr, value in validated_data.items(): if attr in self.fields: From 3b3fe6502f96113bdfbd0c817b688582aca9a136 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Thu, 20 Jun 2019 14:25:43 -0400 Subject: [PATCH 46/51] fixed permissions issue with users being allowed to add images to samples/subsamples they didn't own --- metpetdb_api/api/images/v1/serializers.py | 21 +++++++++++++-------- metpetdb_api/api/images/v1/views.py | 17 ++++++++++++++--- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/metpetdb_api/api/images/v1/serializers.py b/metpetdb_api/api/images/v1/serializers.py index 6fa44dd..ddbe101 100644 --- a/metpetdb_api/api/images/v1/serializers.py +++ b/metpetdb_api/api/images/v1/serializers.py @@ -66,20 +66,25 @@ def is_valid(self, raise_exception=False): self._validated_data.update( {'owner': User.objects.get(pk=self.initial_data['owner'])}) + # add data association: sample, subsample, chemical analysis if self.initial_data.get('sample'): - self._validated_data.update( - {'sample':Sample.objects.get(pk=self.initial_data['sample'])}) + s = Sample.objects.get(pk=self.initial_data['sample']) + if s.owner_id == self.initial_data['owner']: + self._validated_data.update({'sample':s}) + else: + raise ValueError("You are not the owner of sample {}".format(self.initial_data['sample'])) elif self.initial_data.get('subsample'): - self._validated_data.update( - {'subsample':Subsample.objects.get(pk=self.initial_data['subsample'])}) + ss = Subsample.objects.get(pk=self.initial_data['subsample']) + if ss['owner_id'] == self.initial_data['owner']: + self._validated_data.update({'subsample':ss}) + else: + raise ValueError("You are not the owner of subsample {}".format(self.initial_data['subsample'])) elif self.initial_data.get('chemical_analysis'): ca = ChemicalAnalysis.objects.get(pk=self.initial_data['chemical_analysis']) print(ca) else: - return Response( - status=404, - data={'error':'Cannot upload image without associated \ - sample, subsample, or chem analysis!'}) + raise ValueError('Cannot upload image without associated \ + sample, subsample, or chem analysis!') def update(self, instance, validated_data): diff --git a/metpetdb_api/api/images/v1/views.py b/metpetdb_api/api/images/v1/views.py index da20d21..51707d0 100644 --- a/metpetdb_api/api/images/v1/views.py +++ b/metpetdb_api/api/images/v1/views.py @@ -2,8 +2,8 @@ from rest_framework.response import Response from rest_framework.views import APIView -from api.images.v1.serializers import ImageContainerSerializer, ImageSerializer -from apps.images.models import ImageContainer, Image +from api.images.v1.serializers import ImageContainerSerializer, ImageSerializer, ImageTypeSerializer +from apps.images.models import ImageContainer, Image, ImageType from api.lib.permissions import IsOwnerOrReadOnly, IsSuperuserOrReadOnly @@ -13,6 +13,12 @@ class ImageContainerViewSet(viewsets.ModelViewSet): permission_classes = (permissions.IsAuthenticatedOrReadOnly, IsOwnerOrReadOnly,) +class ImageTypeViewSet(viewsets.ModelViewSet): + queryset = ImageType.objects.all() + serializer_class = ImageTypeSerializer + permission_classes = (permissions.IsAuthenticatedOrReadOnly, + IsSuperuserOrReadOnly,) + class ImageViewSet(viewsets.ModelViewSet): queryset = Image.objects.all() @@ -40,7 +46,12 @@ def create(self, request, *args, **kwargs): serializer = self.get_serializer(data=img_data) - serializer.is_valid(raise_exception=True) + try: + serializer.is_valid(raise_exception=True) + except Exception as err: + return Response( + data={'error':str(err)}, + status=404) instance = self.perform_create(serializer) From 38be5f7dac07a4b4b4a690c3084d13534e394ddf Mon Sep 17 00:00:00 2001 From: metpetDB Date: Thu, 20 Jun 2019 14:25:59 -0400 Subject: [PATCH 47/51] added image_types endpoint --- metpetdb_api/metpetdb_api/urls.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metpetdb_api/metpetdb_api/urls.py b/metpetdb_api/metpetdb_api/urls.py index 4c064e9..da0b354 100644 --- a/metpetdb_api/metpetdb_api/urls.py +++ b/metpetdb_api/metpetdb_api/urls.py @@ -45,7 +45,7 @@ ) from api.users.v1.views import UserViewSet -from api.images.v1.views import ImageContainerViewSet, ImageViewSet +from api.images.v1.views import ImageContainerViewSet, ImageViewSet, ImageTypeViewSet from api.bulk_upload.v1.views import BulkUploadViewSet @@ -72,6 +72,7 @@ router.register(r'bulk_upload', BulkUploadViewSet) router.register(r'image_sets', ImageContainerViewSet) router.register(r'images', ImageViewSet) +router.register(r'image_types', ImageTypeViewSet) From c24997ea4186edeb9935a635adf2472ba5bfe871 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Thu, 20 Jun 2019 16:40:15 -0400 Subject: [PATCH 48/51] fixed checking of owner ID --- metpetdb_api/api/images/v1/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metpetdb_api/api/images/v1/serializers.py b/metpetdb_api/api/images/v1/serializers.py index ddbe101..0269202 100644 --- a/metpetdb_api/api/images/v1/serializers.py +++ b/metpetdb_api/api/images/v1/serializers.py @@ -69,13 +69,13 @@ def is_valid(self, raise_exception=False): # add data association: sample, subsample, chemical analysis if self.initial_data.get('sample'): s = Sample.objects.get(pk=self.initial_data['sample']) - if s.owner_id == self.initial_data['owner']: + if str(s.owner_id) == self.initial_data['owner']: self._validated_data.update({'sample':s}) else: raise ValueError("You are not the owner of sample {}".format(self.initial_data['sample'])) elif self.initial_data.get('subsample'): ss = Subsample.objects.get(pk=self.initial_data['subsample']) - if ss['owner_id'] == self.initial_data['owner']: + if str(ss.owner_id) == self.initial_data['owner']: self._validated_data.update({'subsample':ss}) else: raise ValueError("You are not the owner of subsample {}".format(self.initial_data['subsample'])) From fcfeb95ebdd2f91603520d69a2661808c27143f0 Mon Sep 17 00:00:00 2001 From: metpetDB Date: Sun, 30 Jun 2019 23:52:33 -0400 Subject: [PATCH 49/51] added element attribute to image model' --- metpetdb_api/api/images/v1/serializers.py | 23 +++++++++++++++++++---- metpetdb_api/apps/images/models.py | 3 +++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/metpetdb_api/api/images/v1/serializers.py b/metpetdb_api/api/images/v1/serializers.py index 0269202..9ce4fe7 100644 --- a/metpetdb_api/api/images/v1/serializers.py +++ b/metpetdb_api/api/images/v1/serializers.py @@ -4,6 +4,7 @@ from apps.images.models import Image, ImageContainer, ImageType, XrayImage, ImageComments from apps.samples.models import Sample, Subsample, SubsampleType from apps.chemical_analyses.models import ChemicalAnalysis +from apps.chemical_analyses.shared_models import Element from django.core.files import File import urllib.request from urllib.parse import urlparse, urlencode, urlunparse @@ -50,22 +51,36 @@ class ImageSerializer(serializers.ModelSerializer): class Meta: model = Image fields = ('id', 'image', 'version', 'image_type', 'collector', 'owner', 'public_data', 'scale', - 'description', 'comments') + 'description', 'element', 'comments') image = VersatileImageFieldSerializer(sizes='image_sizes', required=False) image_type = ImageTypeSerializer(read_only=True) comments = ImageCommentsSerializer(many=True,required=False) owner = UserSerializer(read_only=True) def is_valid(self, raise_exception=False): + + # add an element (for x-ray images) + if str(self.initial_data.get('image_type')) == '13': + if not self.initial_data.get('element'): + raise ValueError('Cannot upload X-Ray Map without associated element!') + else: + try: + e = Element.objects.get(name=self.initial_data['element']) + self.initial_data['element'] = e.pk + # self._validated_data.update({'element':e}) + except: + raise ValueError('Could not find element matching {}'.format(self.initial_data['element'])) super().is_valid(raise_exception) - if self.initial_data.get('image_type'): - self._validated_data.update({'image_type':ImageType.objects.get(pk=self.initial_data['image_type'])}) - if self.initial_data.get('owner'): self._validated_data.update( {'owner': User.objects.get(pk=self.initial_data['owner'])}) + if self.initial_data.get('image_type'): + self._validated_data.update({'image_type':ImageType.objects.get(pk=self.initial_data['image_type'])}) + + + # add data association: sample, subsample, chemical analysis if self.initial_data.get('sample'): s = Sample.objects.get(pk=self.initial_data['sample']) diff --git a/metpetdb_api/apps/images/models.py b/metpetdb_api/apps/images/models.py index f4c589d..98a3fd8 100644 --- a/metpetdb_api/apps/images/models.py +++ b/metpetdb_api/apps/images/models.py @@ -5,6 +5,7 @@ import uuid from apps.samples.models import Sample, Subsample +from apps.chemical_analyses.shared_models import Element from django.conf import settings from django.contrib.gis.db import models from django.dispatch import receiver @@ -62,6 +63,8 @@ def generate_filename(instance, filename): sample = models.ForeignKey(Sample, on_delete=models.CASCADE, blank=True, null=True, related_name='images') subsample = models.ForeignKey(Subsample, on_delete=models.CASCADE, blank=True, null=True, related_name='images') + element = models.ForeignKey(Element, blank=True, null=True) + class Meta: db_table = 'images' ordering = ('id',) From 9c4e4f94b393889dbb53314138c6ab46c963a48e Mon Sep 17 00:00:00 2001 From: metpetdb Date: Mon, 8 Jul 2019 00:47:22 +0000 Subject: [PATCH 50/51] added user ID back to sample serializer so that frontend can properly show/hide edit buttons --- metpetdb_api/api/samples/v1/serializers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metpetdb_api/api/samples/v1/serializers.py b/metpetdb_api/api/samples/v1/serializers.py index 452db16..497e0ac 100644 --- a/metpetdb_api/api/samples/v1/serializers.py +++ b/metpetdb_api/api/samples/v1/serializers.py @@ -77,7 +77,7 @@ class Meta: class SampleSerializer(DynamicFieldsModelSerializer): minerals = SampleMineralSerializer(source='samplemineral_set', many=True) - owner = serializers.ReadOnlyField(source='owner.name') + owner = serializers.SerializerMethodField(read_only=True) rock_type = serializers.ReadOnlyField(source='rock_type.name') metamorphic_grades = serializers.SerializerMethodField(read_only=True) metamorphic_regions = serializers.SerializerMethodField(read_only=True) @@ -134,6 +134,9 @@ def update(self, instance, validated_data): return instance + def get_owner(self,obj): + return {'name':obj.owner.name,'id':obj.owner.id} + def get_metamorphic_grades(self,obj): return [g.name for g in obj.metamorphic_grades.all()] From a86e880795ae2e390ab62c3faa5babebf517dc9b Mon Sep 17 00:00:00 2001 From: metpetdb Date: Thu, 5 Sep 2019 15:01:31 +0000 Subject: [PATCH 51/51] partial query search and modified model for xray images --- metpetdb_api/api/images/v1/serializers.py | 4 ++-- metpetdb_api/api/images/v1/views.py | 4 ++-- metpetdb_api/api/samples/lib/query.py | 24 +++++++++++++++++++---- metpetdb_api/apps/images/models.py | 6 ++++++ 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/metpetdb_api/api/images/v1/serializers.py b/metpetdb_api/api/images/v1/serializers.py index 6759a08..71a8970 100644 --- a/metpetdb_api/api/images/v1/serializers.py +++ b/metpetdb_api/api/images/v1/serializers.py @@ -96,7 +96,7 @@ def is_valid(self, raise_exception=False): raise ValueError("You are not the owner of subsample {}".format(self.initial_data['subsample'])) elif self.initial_data.get('chemical_analysis'): ca = ChemicalAnalysis.objects.get(pk=self.initial_data['chemical_analysis']) - print(ca) + #print(ca) else: raise ValueError('Cannot upload image without associated \ sample, subsample, or chem analysis!') @@ -227,7 +227,7 @@ def get_sample_subsample_public_data(self, subsample_name, sample_number, subsam @staticmethod def create_xray_image(image_type, values, header_to_index, created_image, dwell_time, current, voltage): image_type_value = re.sub('[^a-z]+', '', image_type.lower()) - print(image_type_value) + #print(image_type_value) if image_type_value == XRAY_IMAGE: element = values[header_to_index[ELEMENT]] if not element: diff --git a/metpetdb_api/api/images/v1/views.py b/metpetdb_api/api/images/v1/views.py index 51707d0..a9af749 100644 --- a/metpetdb_api/api/images/v1/views.py +++ b/metpetdb_api/api/images/v1/views.py @@ -38,8 +38,8 @@ def perform_create(self, serializer): def create(self, request, *args, **kwargs): img_file = request.FILES.get('image') if not img_file: - print("no image file") - return Response(status=404) + err = "no image file" + return Response(data={'error':err},status=404) img_data = request.data img_data['image'] = img_file diff --git a/metpetdb_api/api/samples/lib/query.py b/metpetdb_api/api/samples/lib/query.py index eaca12c..38805a8 100644 --- a/metpetdb_api/api/samples/lib/query.py +++ b/metpetdb_api/api/samples/lib/query.py @@ -22,10 +22,18 @@ def sample_query(user, params, qs): qs = qs.filter(pk__in=params['ids'].split(',')) if params.get('collectors'): - qs = qs.filter(collector_name__in=params['collectors'].split(',')) + if params.get('collectors_exact') == 'True': + qs = qs.filter(collector_name__in=params['collectors'].split(',')) + else: + rx = r'(' + params['collectors'].replace(',','|') + ')' + qs = qs.filter(Q(collector_name__regex=rx)) if params.get('numbers'): - qs = qs.filter(number__in=params['numbers'].split(',')) + if params.get('numbers_exact') == 'True': + qs = qs.filter(number__in=params['numbers'].split(',')) + else: + rx = r'(' + params['numbers'].replace(',','|') + ')' + qs = qs.filter(Q(number__regex=rx)) if params.get('countries'): qs = qs.filter(country__in=params['countries'].split(',')) @@ -68,13 +76,21 @@ def sample_query(user, params, qs): qs = qs.filter(minerals__name__in=minerals) if params.get('owners'): - qs = qs.filter(owner__name__in=params['owners'].split(',')) + if params.get('owners_exact') == 'True': + qs = qs.filter(owner__name__in=params['owners'].split(',')) + else: + rx = r'(' + params['owners'].replace(',','|') + ')' + qs = qs.filter(Q(owner__name__regex=rx)) if params.get('emails'): qs = qs.filter(owner__email__in=params['emails'].split(',')) if params.get('references'): - qs = qs.filter(references__name__in=params['references'].split(',')) + if params.get('references_exact') == 'True': + qs = qs.filter(references__name__in=params['references'].split(',')) + else: + rx = r'(' + params['references'].replace(',','|') + ')' + qs = qs.filter(Q(references__name__regex=rx)) if params.get('regions'): qs = qs.filter(regions__overlap=params['regions'].split(',')) diff --git a/metpetdb_api/apps/images/models.py b/metpetdb_api/apps/images/models.py index 98a3fd8..40b7d4d 100644 --- a/metpetdb_api/apps/images/models.py +++ b/metpetdb_api/apps/images/models.py @@ -5,6 +5,7 @@ import uuid from apps.samples.models import Sample, Subsample +import apps.chemical_analyses.models from apps.chemical_analyses.shared_models import Element from django.conf import settings from django.contrib.gis.db import models @@ -62,6 +63,11 @@ def generate_filename(instance, filename): # move to respective classes sample = models.ForeignKey(Sample, on_delete=models.CASCADE, blank=True, null=True, related_name='images') subsample = models.ForeignKey(Subsample, on_delete=models.CASCADE, blank=True, null=True, related_name='images') + chemical_analysis = models.ForeignKey('chemical_analyses.ChemicalAnalysis', + on_delete=models.CASCADE, blank=True, + null=True, related_name='images') + + element = models.ForeignKey(Element,blank=True,null=True) element = models.ForeignKey(Element, blank=True, null=True)