diff --git a/.gitmodules b/.gitmodules index 6ca1359..3d3bd57 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "metpetdb_api/vendor/djoser"] path = metpetdb_api/vendor/djoser - url = https://github.com/IreLynn/djoser.git + url = https://github.com/bdrumheller/djoser.git diff --git a/metpetdb_api/__init__.py b/metpetdb_api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/metpetdb_api/api/bulk_upload/v1/tests.py b/metpetdb_api/api/bulk_upload/v1/tests.py index bd4288b..2a227c7 100644 --- a/metpetdb_api/api/bulk_upload/v1/tests.py +++ b/metpetdb_api/api/bulk_upload/v1/tests.py @@ -12,10 +12,8 @@ Subsample ) -from apps.chemical_analyses.models import( - Element, - Oxide, -) +from apps.chemical_analyses.shared_models import Element, Oxide + class BulkUploadTests(APITransactionTestCase): """ diff --git a/metpetdb_api/api/bulk_upload/v1/upload_templates.py b/metpetdb_api/api/bulk_upload/v1/upload_templates.py index 9bff6db..65da7ce 100644 --- a/metpetdb_api/api/bulk_upload/v1/upload_templates.py +++ b/metpetdb_api/api/bulk_upload/v1/upload_templates.py @@ -26,6 +26,45 @@ """ import copy +# labels are case insensitive! +sample_label_mappings = { + 'sample number':'number', + 'rock type':'rock_type_name', + 'latitude':'latitude', + 'longitude':'longitude', + 'location error':'location_error', + 'collector':'collector_name', + 'date of collection':'collection_date', + 'present sample location':'location_name', + 'country':'country', + # multi-fields + 'comment':'description', + 'reference':'references', + 'region':'regions', + 'metamorphic region':'metamorphic_regions', + 'metamorphic grade':'metamorphic_grades' +} + +chem_analysis_label_mappings = { + 'sample':'sample', + 'subsample':'subsample', + 'mineral':'mineral', + 'method':'analysis_method', + 'subsample type':'subsample_type', + 'reference':'reference', + 'point':'spot_id', + 'analytical facility':'where_done', + 'analysis date':'analysis_date', + 'analyst':'analyst', + 'reference x':'reference_x', + 'reference y':'reference_y', + 'stage x':'stage_x', + 'stage y':'stage_y', + 'total':'total', + 'comment':'description' +} + + class Template: def __init__(self, c_types = [], required = [], db_types = [], types = {}): self.complex_types = c_types @@ -45,12 +84,13 @@ def check_line_len(self): raise Exception("inconsistent line length. Expected {0}, but was {1}".format(len(data[i-1]), len(data[i]))) def check_required(self, row): - header = row[0] + header = [x for x in row[0]] + for i in range(0,len(header)): + header[i] = header[i].lower() missing ={} - for i in range(0, len(header)): - if self.is_required(header[i]): - if row[1][i] == '': - missing[header[i]] = 'missing' + for i in range(0, len(self.required)): + if self.required[i] not in header: + missing[self.required[i]] = 'missing' return missing def check_type(self, curr_row): @@ -104,7 +144,7 @@ def parse(self, data): for heading in header: if self.is_complex(heading): result_template[heading] = [] else: result_template[heading] = '' - result_template['errors'] = ''; + result_template['errors'] = '' for i in range(1, len(data)): tmp_result = self.TemplateResult(copy.deepcopy(result_template)) @@ -114,17 +154,8 @@ def parse(self, data): for j in range(0,len(data[i])): heading = header[j] - if heading in self.amounts: - name = data[i][j] - amount = self.get_amount(data,i,j) - tmp_result.set_field_complex(heading, {"name": name, "amount": amount}) - continue - - if heading == 'mineral' and heading not in self.amounts: - tmp_result.set_field_complex(heading, {"name": data[i][j]}) - continue - field = data[i][j] + # print("{}: {}".format(heading,data[i][j])) if self.is_complex(heading): tmp_result.set_field_complex(heading,field) else: tmp_result.set_field_simple(heading, field) @@ -133,14 +164,14 @@ def parse(self, data): result.append(tmp_result.get_rep()) return result, meta_header - def is_complex(self, name): return name in self.complex_types - def is_required(self, name): return name in self.required + def is_complex(self, name): return name.lower() in self.complex_types + def is_required(self, name): return name.lower() in self.required def is_db_type(self, name): return name in self.db_types class ChemicalAnalysesTemplate(Template): def __init__(self): - complex_types = ["comment", "element", "oxide","mineral"] - required = ["subsample_id", "spot_id", "mineral", "analysis_method"] + complex_types = ["comment","element","oxide"] + required = ["sample","subsample","point","mineral","method","subsample type"] db_types = ["element", "oxide"] types = {"comment": str, "stage_x" : float, "stage_y" : float, "reference_x": float, "reference_y": float} Template.__init__(self, complex_types, required, db_types, types) @@ -168,19 +199,19 @@ def get_meta_header(self,header): meta_header.append((heading, mappings[heading])) added.add(heading) else: - meta_header.append((heading, heading)) + meta_header.append((heading, heading)) + added.add(heading) return meta_header class SampleTemplate(Template): def __init__(self): - complex_types = ["comment", "references", "mineral", "metamorphic_region_id", "metamorphic_grade"] - required = ["number", "latitude", "longitude", "rock_type_name"] + complex_types = ["comment", "reference", "region", "metamorphic region", "metamorphic grade"] + required = ["sample number", "latitude", "longitude", "rock type"] types = {"comment": str, "latitude": float, 'longitude': float} db_types = ["minerals"] #selected_types = {'minerals': ['el1', 'el2', 'el3']} Template.__init__(self, complex_types, required, db_types, types) - self.amounts.add('mineral') def check_amounts(self,header): pass @@ -189,18 +220,22 @@ def get_amount(self,data=[],i=0,j=0): return 0 def get_meta_header(self,header): - mappings = {'mineral' : 'minerals'} - added = set() + mappings = {} + added = set() meta_header = [] itr = iter(header) for heading in itr: - if heading == 'latitude': - for i in range (0,2): heading = next(itr) - meta_header.append((('latitude','longitude'),'location_coords')) - elif heading not in added: - if heading in mappings.keys(): - meta_header.append((heading, mappings[heading])) + # if heading.lower() == 'latitude': + # for i in range (0,2): heading = next(itr) + # meta_header.append((('latitude','longitude'),'Location')) + if heading not in added: + if heading.lower() in mappings.keys(): + meta_header.append((heading.lower(),heading)) added.add(heading) else: - meta_header.append((heading, heading)) + meta_header.append((heading, heading)) + added.add(heading) + # print("\nMETA-HEADER:") + # print(meta_header) + # print("\n\n") return meta_header diff --git a/metpetdb_api/api/bulk_upload/v1/views.py b/metpetdb_api/api/bulk_upload/v1/views.py index dfe2acb..1ce98be 100644 --- a/metpetdb_api/api/bulk_upload/v1/views.py +++ b/metpetdb_api/api/bulk_upload/v1/views.py @@ -56,16 +56,47 @@ ChemicalAnalysis, ChemicalAnalysisElement, ChemicalAnalysisOxide, - Element, - Oxide, ) - +from apps.chemical_analyses.shared_models import Element, Oxide from api.bulk_upload.v1 import upload_templates import json import sys import urllib.request from csv import reader +import re + + +sample_labels_dict = { + 'Sample Number':'number', + 'Rock Type':'rock_type_name', + 'Latitude':'latitude', + 'Longitude':'longitude', + 'Location Error':'location_error', + 'Collector':'collector_name', + 'Date of Collection':'collection_date', + 'Present Sample Location':'location_name', + 'Country':'country' +} + +chem_analysis_labels_dict = { + 'Sample':'sample', + 'Subsample':'subsample', + 'Mineral':'mineral', + 'Method':'analysis_method', + 'Subsample Type':'subsample_type', + 'Reference':'reference', + 'Point':'spot_id', + 'Analytical Facility':'where_done', + 'Analysis Date':'analysis_date', + 'Analyst':'analyst', + 'Reference X':'reference_x', + 'Reference Y':'reference_y', + 'Stage X':'stage_x', + 'Stage Y':'stage_y', + 'Total':'total' +} + class Parser: def __init__(self, template): @@ -87,6 +118,7 @@ def parse(self, url): lined = self.line_split(content) return self.template.parse(lined) # return the JSON ready file except Exception as err: + # print(err) raise ValueError(str(err)) class BulkUploadViewSet(viewsets.ModelViewSet): @@ -95,13 +127,15 @@ class BulkUploadViewSet(viewsets.ModelViewSet): IsOwnerOrReadOnly,) http_method_names=['post'] - def _handle_metamorphic_regions(self, instance, ids): + def _handle_metamorphic_regions(self, instance, regions): metamorphic_regions = [] - for id in ids: + for region in regions: + if region == '': + continue try: - metamorphic_region = MetamorphicRegion.objects.get(pk=id) + metamorphic_region = MetamorphicRegion.objects.get(name=region) except: - raise ValueError('Invalid metamorphic_region id: {}' + raise ValueError('Invalid metamorphic region: {}' .format(id)) else: metamorphic_regions.append(metamorphic_region) @@ -111,10 +145,12 @@ def _handle_metamorphic_regions(self, instance, ids): def _handle_metamorphic_grades(self, instance, grades): metamorphic_grades = [] for grade in grades: + if grade == '': + continue try: metamorphic_grade = MetamorphicGrade.objects.get(name=grade) except: - raise ValueError('Invalid metamorphic_grade : {}'.format(grade)) + raise ValueError('Invalid metamorphic grade: {}'.format(grade)) else: metamorphic_grades.append(metamorphic_grade) instance.metamorphic_grades = metamorphic_grades @@ -247,13 +283,44 @@ def set_err(self, JSON, i, field, val, meta_header): def parse_chemical_analyses(self, request, JSON, meta_header): before_parse_json = list(JSON) - + # Manual transaction for ease of exception handling transaction.set_autocommit(False) - for i,chemical_analyses_obj in enumerate(JSON): + for i,analysis_obj in enumerate(JSON): + + # REQUIRED FIELDS: + # Sample [Number] + # Subsample [Number] + # Point + # Mineral + # Method + # Subsample Type + + # OPTIONAL FIELDS: + # Analytical Facility + # Analysis Date + # Analyst + # Reference Image (???) + # X Reference + # Y Reference + # X Stage + # Y Stage + # Total + # Comment + # [elements] + # [oxides] + # [precisions] + + + # PROCEDURE: + ## ensure all required fields are present + #### this is NOT enforced by the serializer like with samples! + ## verify all other fields are valid optional fields or minerals + ## manipulate data for serializer + ## create serializer try: - chemical_analyses_obj['owner'] = request.data.get('owner') + analysis_obj['owner'] = request.data.get('owner') except: self.rollback_transaction() return Response( @@ -261,66 +328,152 @@ def parse_chemical_analyses(self, request, JSON, meta_header): status = 400 ) - #fix date formatting - if chemical_analyses_obj['analysis_date']: - chemical_analyses_obj['analysis_date'] += 'T00:00:00.000Z' - + element_oxide_pattern = re.compile(r"(.+)\((ppm|wt\%)\)",re.IGNORECASE) + precision_pattern = re.compile(r"(.+) precision \((abs|rel)\)",re.IGNORECASE) + elements_n_oxides = [] + precisions = {} + + fields = [x for x in analysis_obj.keys()] + for field in fields: + if field.lower() in upload_templates.chem_analysis_label_mappings.keys(): + # add proper formatting to date (?) + if field.lower() == 'analysis date': + if analysis_obj[field] != '': + analysis_obj[field] = analysis_obj[field] +'T00:00:00.000Z' + # join comments with newline + elif field.lower() == 'comment': + analysis_obj[field] = '\n'.join(analysis_obj[field]) + # replace field with corresponding serializer fieldname + if (analysis_obj[field] == ''): + analysis_obj[upload_templates.chem_analysis_label_mappings[field.lower()]] = None + else: + analysis_obj[upload_templates.chem_analysis_label_mappings[field.lower()]] = analysis_obj[field] + # print('{} : {}'.format(field,analysis_obj[upload_templates.chem_analysis_label_mappings[field.lower()]])) + # del(analysis_obj[field]) + elif field != 'errors' and field not in upload_templates.chem_analysis_label_mappings.values(): # must be element, oxide, or precision + m = element_oxide_pattern.match(field) + if m is not None: + g = m.groups() + # print('element or oxide: {}({})'.format(g[0],g[1])) + name = g[0].strip() + elements_n_oxides.append({ + 'name':name, + 'measurement_unit':g[1], + 'amount':analysis_obj[field] + }) + else: + m = precision_pattern.match(field) + if m is not None: + g = m.groups() + # print('precision: {} Precision ({})'.format(g[0],g[1])) + precisions[g[0]] = {'type':g[1],'value':analysis_obj[field]} + # else: + # print('no match: {}'.format(field)) + + # make sure the mineral exists try: - chemical_analyses_obj['mineral_id'] = Mineral.objects.get(name=chemical_analyses_obj['mineral'][0]['name']).id + analysis_obj['mineral_id'] = Mineral.objects.get(name=analysis_obj['mineral']).id except Exception as err: - return self.set_err(before_parse_json, i, 'mineral_id', 'invalid mineral', meta_header) + return self.set_err(before_parse_json, i, 'mineral_id', 'invalid mineral', meta_header) + + # check for required fields that aren't required in model (aaaa...) + if (analysis_obj.get('spot_id')) is None: + return self.set_err(before_parse_json, i, 'spot_id', 'analysis point identifier is required', meta_header) + if (analysis_obj.get('analysis_method')) is None: + return self.set_err(before_parse_json, i, 'analysis_method', 'analysis method is required', meta_header) - elements_to_add = [] - for element in chemical_analyses_obj['element']: + # make sure sample exists; if subsample doesn't exist, create it + try: + sample = Sample.objects.get(owner_id=analysis_obj['owner'],number=analysis_obj['sample']) + analysis_obj['sample_id'] = sample.id + try: + analysis_obj['subsample_id'] = Subsample.objects.get(sample_id=analysis_obj['sample_id'],name=analysis_obj['subsample']).id + except Exception as err: + # print(err) + try: + sub_type = SubsampleType.objects.get(name=analysis_obj['subsample_type']) + Subsample.objects.create( + name=analysis_obj['subsample'], + sample=sample, + owner_id=analysis_obj['owner'], + subsample_type=sub_type, + ) + analysis_obj['subsample_id'] = Subsample.objects.get( + name=analysis_obj['subsample'], + sample_id=analysis_obj['sample_id'], + owner_id=analysis_obj['owner'], + subsample_type=sub_type).id + except Exception as err: + # print(err) + return self.set_err(before_parse_json, i, 'subsample', err, meta_header) + except Exception as err: + # print(err) + return self.set_err(before_parse_json, i, 'sample', err, meta_header) + + analysis_obj['oxides'] = [] + analysis_obj['elements'] = [] + + for entry in elements_n_oxides: + if entry['amount'] == '': + continue try: - elements_to_add.append( - {'id': Element.objects.get(name=element['name']).id, - 'amount': element['amount'], + p = None + ptype = None + if (precisions.get(entry['name'])) is not None: + p = precisions[entry['name']]['value'] + ptype = precisions[entry['name']]['type'] + + obj = Element.objects.get(symbol=entry['name']) + analysis_obj['elements'].append( + {'id': obj.id, + 'amount': entry['amount'], + 'precision': p, + 'precision_type': ptype, + 'measurement_unit': entry['measurement_unit'], #TODO consider adding these fields #They are not specified in the template - 'precision': None, - 'precision_type': None, - 'measurement_unit': None, 'min_amount': None, 'max_amount': None }) except: - return self.set_err(before_parse_json, i, 'element', 'invalid element {0}'.format(element), meta_header) - - chemical_analyses_obj['elements'] = elements_to_add - - oxides_to_add = [] - - for oxide in chemical_analyses_obj['oxide']: - try: - oxides_to_add.append( - { 'id' : Oxide.objects.get(species=oxide['name']).id, - 'amount': oxide['amount'], - #TODO consider adding - #They are currently not specified in the template - 'precision': None, - 'precision_type': None, - 'measurement_unit': None, - 'min_amount': None, - 'max_amount': None - - }) - except: - return self.set_err(before_parse_json, i, 'oxide', 'invalid oxide {0}'.format(oxide), meta_header) - - chemical_analyses_obj['oxides'] = oxides_to_add - - serializer = self.get_serializer(data=chemical_analyses_obj) + try: + obj = Oxide.objects.get(species=entry['name']) + analysis_obj['oxides'].append( + {'id' : obj.id, + 'amount': entry['amount'], + 'precision': p, + 'precision_type': ptype, + 'measurement_unit': entry['measurement_unit'], + #TODO consider adding + #They are currently not specified in the template + 'min_amount': None, + 'max_amount': None + }) + except: + return self.set_err(before_parse_json, i, 'element/oxide', 'invalid element or oxide {0}'.format(oxide), meta_header) + + + if len(analysis_obj['elements']) == 0: + del(analysis_obj['elements']) + if len(analysis_obj['oxides']) == 0: + del(analysis_obj['oxides']) + + + + serializer = self.get_serializer(data=analysis_obj) try: serializer.is_valid(raise_exception=True) instance = self.perform_create(serializer) except Exception as e: + # print(e) + # print(serializer.initial_data) return self.set_err(before_parse_json, i, 'serialization', str(e), meta_header) - if chemical_analyses_obj.get('elements'): - for record in chemical_analyses_obj.get('elements'): + if analysis_obj.get('elements'): + for record in analysis_obj.get('elements'): + # print(record) try: ChemicalAnalysisElement.objects.create( chemical_analysis=instance, @@ -329,14 +482,15 @@ def parse_chemical_analyses(self, request, JSON, meta_header): precision=record['precision'], precision_type=record['precision_type'], measurement_unit=record['measurement_unit'], - min_amount=record['min_amount'], - max_amount=record['max_amount'], + # min_amount=record['min_amount'], + # max_amount=record['max_amount'], ) except Element.DoesNotExist: return self.set_err(before_parse_json, i, 'elements', 'invalid element id', meta_header) - if chemical_analyses_obj.get('oxides'): - for record in chemical_analyses_obj.get('oxides'): + if analysis_obj.get('oxides'): + for record in analysis_obj.get('oxides'): + # print(record) try: ChemicalAnalysisOxide.objects.create( chemical_analysis=instance, @@ -345,8 +499,8 @@ def parse_chemical_analyses(self, request, JSON, meta_header): precision=record['precision'], precision_type=record['precision_type'], measurement_unit=record['measurement_unit'], - min_amount=record['min_amount'], - max_amount=record['max_amount'], + # min_amount=record['min_amount'], + # max_amount=record['max_amount'], ) except Oxide.DoesNotExist: return self.set_err(before_parse_json, i, 'oxides', 'invalid oxide id', meta_header) @@ -368,6 +522,61 @@ def parse_samples(self, request, JSON, meta_header): transaction.set_autocommit(False) for i,sample_obj in enumerate(JSON): + + # print(sample_obj) + + # REQUIRED FIELDS: + # Sample Number + # Rock Type + # Latitude + # Longitude + + # OPTIONAL FIELDS: + # Location Error + # Metamorphic Region + # Reference + # Present Sample Location + # Region + # Date of Collection + # Comment + # Metamorphic Grade + # Country + # Collector + # [minerals] + + + # PROCEDURE: + ## ensure all required fields are present + ## verify all other fields are valid optional fields or minerals + ## manipulate data for serializer + ## create serializer + + minerals = [] + fields = [x for x in sample_obj.keys()] + for field in fields: + if field.lower() in upload_templates.sample_label_mappings.keys(): + # add proper formatting to date + if field.lower() == 'date of collection': + sample_obj[field] = sample_obj[field] #+ 'T00:00:00.000Z' + # join comments with newline + elif field.lower() == 'comment': + sample_obj[field] = '\n'.join(sample_obj[field]) + # replace field with corresponding serializer fieldname + sample_obj[upload_templates.sample_label_mappings[field.lower()]] = sample_obj[field] + # del(sample_obj[field]) + elif field != 'errors' and field not in upload_templates.sample_label_mappings.values(): # it had better be a mineral + try: + amount = sample_obj[field] + minerals.append( + {'id':Mineral.objects.get(name=field).id, + 'name':field, + 'amount':amount}) + except: + # print(field) + return self.set_err(before_parse_json, i, 'minerals', 'Invalid mineral {}'.format(field), meta_header) + + sample_obj['minerals'] = minerals + try: sample_obj['owner'] = request.data.get('owner') except: @@ -377,38 +586,18 @@ def parse_samples(self, request, JSON, meta_header): status = 400 ) - minerals = sample_obj['mineral'] + # if 'latitude' in sample_obj.keys() and 'longitude' in sample_obj.keys(): + # sample_obj['location_coords'] = u'SRID=4326;POINT ({0} {1})'.format(sample_obj['latitude'], sample_obj['longitude']) + # del(sample_obj['latitude']) + # del(sample_obj['longitude']) rock_type = sample_obj['rock_type_name'] - to_add = [] - - for mineral in minerals: - try: - to_add.append( - {'id': Mineral.objects.get(name=mineral['name']).id, - 'name': mineral['name'], - 'amount': '0'}) - except: - return self.set_err(before_parse_json, i, 'minerals', 'Invalid mineral {0}'.format(mineral), meta_header) - - sample_obj['minerals'] = to_add - - del(sample_obj['mineral']) - - # Need this for a proper collection date - if sample_obj['collection_date']: - sample_obj['collection_date'] += 'T00:00:00.000Z' - try: sample_obj['rock_type_id'] = RockType.objects.get(name=rock_type).id except: return self.set_err(before_parse_json, i, 'rock_type_id', 'Invalid rock {0}'.format(rock_type), meta_header) - - if 'latitude' in sample_obj.keys() and 'longitude' in sample_obj.keys(): - sample_obj['location_coords'] = u'SRID=4326;POINT ({0} {1})'.format(sample_obj['latitude'], sample_obj['longitude']) - del(sample_obj['latitude']) - del(sample_obj['longitude']) + serializer = self.get_serializer(data=sample_obj) try: @@ -418,17 +607,17 @@ def parse_samples(self, request, JSON, meta_header): return self.set_err(before_parse_json, i, 'serialization', str(e), meta_header) - metamorphic_region_ids = sample_obj.get('metamorphic_region_id') - metamorphic_grades = sample_obj.get('metamorphic_grade') + metamorphic_regions = sample_obj.get('metamorphic_regions') + metamorphic_grades = sample_obj.get('metamorphic_grades') references = sample_obj.get('references') minerals = sample_obj.get('minerals') - if metamorphic_region_ids: + if metamorphic_regions: try: self._handle_metamorphic_regions(instance, - metamorphic_region_ids) + metamorphic_regions) except ValueError as err: - return self.set_err(before_parse_json, i, 'metamorphic_region_ids', err.args, meta_header) + return self.set_err(before_parse_json, i, 'metamorphic_regions', err.args, meta_header) if metamorphic_grades: try: diff --git a/metpetdb_api/api/chemical_analyses/lib/query.py b/metpetdb_api/api/chemical_analyses/lib/query.py index 5727ed0..c8c00af 100644 --- a/metpetdb_api/api/chemical_analyses/lib/query.py +++ b/metpetdb_api/api/chemical_analyses/lib/query.py @@ -54,4 +54,7 @@ def chemical_analysis_query(user, params, qs): if params.get('subsample_ids'): qs = qs.filter(subsample_id__in=params.get('subsample_ids').split(',')) + if params.get('ordering'): + qs = qs.order_by(params['ordering']) + return qs diff --git a/metpetdb_api/api/chemical_analyses/v1/renderers.py b/metpetdb_api/api/chemical_analyses/v1/renderers.py new file mode 100644 index 0000000..03aa297 --- /dev/null +++ b/metpetdb_api/api/chemical_analyses/v1/renderers.py @@ -0,0 +1,119 @@ +from rest_framework_csv import renderers as r + +class ChemicalAnalysisCSVRenderer (r.CSVRenderer): + + def __init__(self): + self.header = [ + 'sample', + 'subsample', + 'spot_id', + 'mineral', + 'analysis_method', + 'subsample_type', + 'where_done', + 'analysis_date', + 'analyst', + 'reference', + # 'reference_image' + 'reference_x', + 'reference_y', + 'elements', + # 'oxides', + 'total', + 'stage_x', + 'stage_y', + 'description', + ] + self.labels = { + 'sample':'Sample', + 'subsample':'Subsample', + 'mineral':'Mineral', + 'analysis_method':'Method', + 'subsample_type':'Subsample Type', + 'reference':'Reference', + 'spot_id':'Point', + 'where_done':'Analytical Facility', + 'analysis_date':'Analysis Date', + 'analyst':'Analyst', + 'reference_x':'Reference X', + 'reference_y':'Reference Y', + 'stage_x':'Stage X', + 'stage_y':'Stage Y', + 'description':'Comment', + # 'elements', + # 'oxides', + 'total':'Total', + } + + self.num_elements = 0 + self.num_oxides = 0 + self.elements = set() + self.oxides = set() + self.fields = set() + + def tablize(self, data, header=None, labels=None): + if data: + data = self.flatten_data(data) + + if 'elements' in header: + data = tuple(data) + header.remove('elements') + els = list(self.elements) + els.sort() + oxs = list(self.oxides) + oxs.sort() + header[12:12] = oxs + header[12:12] = els + header = [x for x in header if (x in self.fields or x in self.elements or x in self.oxides)] + + if labels: + yield [labels.get(x,x) for x in header] + else: + yield header + + for item in data: + # print(item) + row = [item.get(key,None) for key in header] + yield row + + elif header: + if labels: + yield [labels.get(x,x) for x in header] + else: + yield header + else: + pass + + def flatten_data(self,data): + for item in data: + self.fields |= item.keys() + if (item.get('elements')): + self.handle_elements(item) + if (item.get('oxides')): + self.handle_oxides(item) + flat_item = self.flatten_item(item) + yield flat_item + + def handle_elements(self,item): + for e in item['elements']: + if e['precision_type'] == 'REL': + e['precision'] *= e['amount'] + e['precision_type'] = 'ABS' + col = '{} ({})'.format(e['symbol'],e['measurement_unit']) + prec = '{} Precision ({})'.format(e['symbol'],e['precision_type']) + item[col] = e['amount'] + item[prec] = e['precision'] + self.elements.add(col) + self.elements.add(prec) + + def handle_oxides(self,item): + for o in item['oxides']: + if o['precision_type'] == 'REL': + o['precision'] *= o['amount'] + o['precision_type'] = 'ABS' + col = '{} ({})'.format(o['species'],o['measurement_unit']) + prec = '{} Precision ({})'.format(o['species'],o['precision_type']) + item[col] = o['amount'] + item[prec] = o['precision'] + self.oxides.add(col) + self.oxides.add(prec) diff --git a/metpetdb_api/api/chemical_analyses/v1/serializers.py b/metpetdb_api/api/chemical_analyses/v1/serializers.py index 6343b77..d47448b 100644 --- a/metpetdb_api/api/chemical_analyses/v1/serializers.py +++ b/metpetdb_api/api/chemical_analyses/v1/serializers.py @@ -7,11 +7,11 @@ ChemicalAnalysis, ChemicalAnalysisElement, ChemicalAnalysisOxide, - Element, - Oxide, ) +from apps.chemical_analyses.shared_models import Element, Oxide from apps.samples.models import Subsample, Mineral from apps.users.models import User +from api.images.v1.serializers import ImageSerializer CHEMICAL_ANALYSIS_FIELDS = ('reference_x', 'reference_y', 'stage_x', 'stage_y', 'analysis_method', 'where_done', 'analyst', @@ -20,60 +20,117 @@ class ChemicalAnalysisElementSerializer(DynamicFieldsModelSerializer): - id = serializers.ReadOnlyField(source='element.id') - name = serializers.ReadOnlyField(source='element.name') - alternate_name = serializers.ReadOnlyField(source='element.alternate_name') + # id = serializers.ReadOnlyField(source='element.id') + # name = serializers.ReadOnlyField(source='element.name') + # alternate_name = serializers.ReadOnlyField(source='element.alternate_name') symbol = serializers.ReadOnlyField(source='element.symbol') - atomic_number = serializers.ReadOnlyField(source='element.atomic_number') - weight = serializers.ReadOnlyField(source='element.weight') - order_id = serializers.ReadOnlyField(source='element.order_id') + # atomic_number = serializers.ReadOnlyField(source='element.atomic_number') + # weight = serializers.ReadOnlyField(source='element.weight') + # order_id = serializers.ReadOnlyField(source='element.order_id') class Meta: model = ChemicalAnalysisElement - fields = ('id', 'name', 'alternate_name', 'symbol', 'atomic_number', - 'weight', 'order_id', 'amount', 'precision', - 'precision_type', 'measurement_unit', 'min_amount', - 'max_amount') + fields = ( + # 'id', + # 'name', + # 'alternate_name', + 'symbol', + # 'atomic_number', + # 'weight', + # 'order_id', + 'amount', + 'precision', + 'precision_type', + 'measurement_unit', + # 'min_amount', + # 'max_amount' + ) class ChemicalAnalysisOxideSerializer(DynamicFieldsModelSerializer): id = serializers.ReadOnlyField(source='oxide.id') element_id = serializers.ReadOnlyField(source='oxide.element_id') - oxidation_state = serializers.ReadOnlyField(source='oxide.oxidation_state') + # oxidation_state = serializers.ReadOnlyField(source='oxide.oxidation_state') species = serializers.ReadOnlyField(source='oxide.species') - weight = serializers.ReadOnlyField(source='oxide.weight') - cations_per_oxide = serializers.ReadOnlyField( - source='oxide.cations_per_oxide') - conversion_factor = serializers.ReadOnlyField( - source='oxide.conversion_factor') + # weight = serializers.ReadOnlyField(source='oxide.weight') + # cations_per_oxide = serializers.ReadOnlyField( + # source='oxide.cations_per_oxide') + # conversion_factor = serializers.ReadOnlyField( + # source='oxide.conversion_factor') order_id = serializers.ReadOnlyField(source='oxide.order_id') class Meta: model = ChemicalAnalysisOxide - fields = ('id', 'element_id', 'oxidation_state', 'species', 'weight', - 'cations_per_oxide', 'conversion_factor', 'order_id', - 'amount', 'precision', 'precision_type', 'measurement_unit', - 'min_amount', 'max_amount' ) + fields = ( + 'id', + 'element_id', + # 'oxidation_state', + 'species', + # 'weight', + # 'cations_per_oxide', + # 'conversion_factor', + 'order_id', + 'amount', + 'precision', + 'precision_type', + 'measurement_unit', + # 'min_amount', + # 'max_amount' + ) class ChemicalAnalysisSerializer(DynamicFieldsModelSerializer): - mineral = MineralSerializer(read_only=True) - owner = UserSerializer(read_only=True) + owner = serializers.ReadOnlyField(source='owner.name') + sample = serializers.ReadOnlyField(source='subsample.sample.number') + sample_id = serializers.ReadOnlyField(source='subsample.sample.id') + subsample = serializers.ReadOnlyField(source='subsample.name') + subsample_id = serializers.ReadOnlyField(source='subsample.id') + subsample_type = serializers.ReadOnlyField(source='subsample.subsample_type.name') + mineral = serializers.ReadOnlyField(source='mineral.name') elements = ChemicalAnalysisElementSerializer( many=True, source='chemicalanalysiselement_set', required=False, + read_only=True ) oxides = ChemicalAnalysisOxideSerializer( many=True, source='chemicalanalysisoxide_set', - required=False + required=False, + read_only=True ) class Meta: model = ChemicalAnalysis depth = 1 + fields = ( + 'id', + 'owner', + 'sample', + 'sample_id', + 'subsample', + 'subsample_id', + 'subsample_type', + 'mineral', + 'analysis_method', + 'reference', + 'reference_image', + 'spot_id', + 'where_done', + 'analysis_date', + # 'date_precision', + 'analyst', + 'reference_x', + 'reference_y', + 'stage_x', + 'stage_y', + 'description', + 'elements', + 'oxides', + 'total', + ) + def is_valid(self, raise_exception=False): super().is_valid(raise_exception) @@ -116,11 +173,15 @@ def update(self, instance, validated_data): return instance + + class ElementSerializer(DynamicFieldsModelSerializer): class Meta: model = Element + fields = '__all__' class OxideSerializer(DynamicFieldsModelSerializer): class Meta: model = Oxide + fields = '__all__' diff --git a/metpetdb_api/api/chemical_analyses/v1/views.py b/metpetdb_api/api/chemical_analyses/v1/views.py index 9086093..2607d47 100644 --- a/metpetdb_api/api/chemical_analyses/v1/views.py +++ b/metpetdb_api/api/chemical_analyses/v1/views.py @@ -1,5 +1,7 @@ from rest_framework import permissions, status, viewsets from rest_framework.response import Response +from rest_framework.renderers import JSONRenderer, BrowsableAPIRenderer +from api.chemical_analyses.v1.renderers import ChemicalAnalysisCSVRenderer from api.chemical_analyses.lib.query import chemical_analysis_query from api.chemical_analyses.v1.serializers import ( @@ -17,14 +19,14 @@ ChemicalAnalysis, ChemicalAnalysisElement, ChemicalAnalysisOxide, - Element, - Oxide, ) +from apps.chemical_analyses.shared_models import Element, Oxide class ChemicalAnalysisViewSet(viewsets.ModelViewSet): queryset = ChemicalAnalysis.objects.all() serializer_class = ChemicalAnalysisSerializer + renderer_classes = (JSONRenderer, BrowsableAPIRenderer, ChemicalAnalysisCSVRenderer) permission_classes = (permissions.IsAuthenticatedOrReadOnly, IsOwnerOrReadOnly,) @@ -50,18 +52,28 @@ def list(self, request, *args, **kwargs): qs = chemical_analyses_qs_optimizer(params, qs) - page = self.paginate_queryset(qs) - if page is not None: - serializer = self.get_serializer(page, many=True) - return self.get_paginated_response(serializer.data) + if params.get('format') == 'csv': + serializer = self.get_serializer(qs,many=True) + response = Response(serializer.data) + filename = params.get('filename') + if filename is None: + filename = 'search_results.csv' + response['content-disposition'] = "attachment; filename=%s" % filename + return response + else: + page = self.paginate_queryset(qs) + if page is not None: + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) - serializer = self.get_serializer(qs, many=True) - return Response(serializer.data) + serializer = self.get_serializer(qs, many=True) + return Response(serializer.data) def _handle_elements(self, instance, params): to_add = [] - for record in params['elements']: + for record in params: + print(record) try: to_add.append({ 'element': Element.objects.get(pk=record['id']), @@ -73,9 +85,7 @@ def _handle_elements(self, instance, params): 'max_amount': record['max_amount'] }) except Element.DoesNotExist: - return Response( - data={'error': 'Invalid element id'}, - status=400) + raise ValueError('Invalid element id: {}'.format(record['id'])) (ChemicalAnalysisElement .objects @@ -97,7 +107,7 @@ def _handle_elements(self, instance, params): def _handle_oxides(self, instance, params): to_add = [] - for record in params['oxides']: + for record in params: try: to_add.append({ 'oxide': Oxide.objects.get(pk=record['id']), @@ -109,9 +119,7 @@ def _handle_oxides(self, instance, params): 'max_amount': record['max_amount'] }) except Oxide.DoesNotExist: - return Response( - data={'error': 'Invalid oxide id'}, - status=400) + raise ValueError('Invalid oxide id: {}'.format(record['id'])) (ChemicalAnalysisOxide .objects @@ -141,38 +149,52 @@ def create(self, request, *args, **kwargs): instance = self.perform_create(serializer) if request.data.get('elements'): - for record in request.data.get('elements'): - try: - ChemicalAnalysisElement.objects.create( - chemical_analysis=instance, - element=Element.objects.get(pk=record['id']), - amount=record['amount'], - precision=record['precision'], - precision_type=record['precision_type'], - measurement_unit=record['measurement_unit'], - min_amount=record['min_amount'], - max_amount=record['max_amount'], - ) - except Element.DoesNotExist: - return Response(data={'error': 'Invalid element id'}, - status=400) + try: + self._handle_elements(instance,request.data.get('elements')) + except ValueError as err: + return Response( + data={'error':err.args}, + status=400 + ) + # for record in request.data.get('elements'): + # try: + # ChemicalAnalysisElement.objects.create( + # chemical_analysis=instance, + # element=Element.objects.get(pk=record['id']), + # amount=record['amount'], + # precision=record['precision'], + # precision_type=record['precision_type'], + # measurement_unit=record['measurement_unit'], + # min_amount=record['min_amount'], + # max_amount=record['max_amount'], + # ) + # except Element.DoesNotExist: + # return Response(data={'error': 'Invalid element id'}, + # status=400) if request.data.get('oxides'): - for record in request.data.get('oxides'): - try: - ChemicalAnalysisOxide.objects.create( - chemical_analysis=instance, - oxide=Oxide.objects.get(pk=record['id']), - amount=record['amount'], - precision=record['precision'], - precision_type=record['precision_type'], - measurement_unit=record['measurement_unit'], - min_amount=record['min_amount'], - max_amount=record['max_amount'], - ) - except Oxide.DoesNotExist: - return Response(data={'error': 'Invalid oxide id'}, - status=400) + try: + self._handle_oxides(instance,request.data.get('oxides')) + except ValueError as err: + return Response( + data={'error':err.args}, + status=400 + ) + # for record in request.data.get('oxides'): + # try: + # ChemicalAnalysisOxide.objects.create( + # chemical_analysis=instance, + # oxide=Oxide.objects.get(pk=record['id']), + # amount=record['amount'], + # precision=record['precision'], + # precision_type=record['precision_type'], + # measurement_unit=record['measurement_unit'], + # min_amount=record['min_amount'], + # max_amount=record['max_amount'], + # ) + # except Oxide.DoesNotExist: + # return Response(data={'error': 'Invalid oxide id'}, + # status=400) headers = self.get_success_headers(serializer.data) return Response(serializer.data, diff --git a/metpetdb_api/api/images/__init__.py b/metpetdb_api/api/images/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/metpetdb_api/api/images/v1/__init__.py b/metpetdb_api/api/images/v1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/metpetdb_api/api/images/v1/serializers.py b/metpetdb_api/api/images/v1/serializers.py new file mode 100644 index 0000000..71a8970 --- /dev/null +++ b/metpetdb_api/api/images/v1/serializers.py @@ -0,0 +1,294 @@ +import tempfile +from os import sep, listdir +from rest_framework import serializers +from apps.images.models import Image, ImageContainer, ImageType, XrayImage, ImageComments +from apps.samples.models import Sample, Subsample, SubsampleType +from apps.chemical_analyses.models import ChemicalAnalysis +from apps.chemical_analyses.shared_models import Element +from django.core.files import File +import urllib.request +from urllib.parse import urlparse, urlencode, urlunparse +from versatileimagefield.serializers import VersatileImageFieldSerializer +import zipfile +import xlrd +import re +from django.db import transaction +from api.users.v1.serializers import UserSerializer +from apps.users.models import User +from rest_framework.response import Response + +SAMPLE_NUMBER = 'samplenumber' +SUBSAMPLE = 'subsample' +SUBSAMPLE_TYPE = 'subsampletype' +PATH = 'path' +FILE = 'file' +IMAGE_TYPE = 'imagetype' +SCALE = 'scale' +COLLECTOR = 'collector' +COMMENT = 'comment' +ELEMENT = 'element' +DWELL_TIME = 'dwelltime' +CURRENT = 'current' +VOLTAGE = 'voltage' +XRAY_IMAGE = 'xraymap' + +required_headers = {SAMPLE_NUMBER, IMAGE_TYPE} + + +class ImageTypeSerializer(serializers.ModelSerializer): + class Meta: + model = ImageType + fields = ('id', 'image_type', 'abbreviation', 'comments') + + +class ImageCommentsSerializer(serializers.ModelSerializer): + class Meta: + model = ImageComments + fields = ('comment_id', 'comment_text', 'version') + + +class ImageSerializer(serializers.ModelSerializer): + class Meta: + model = Image + fields = ('id', 'image', 'version', 'image_type', 'collector', 'owner', 'public_data', 'scale', + 'description', 'element', 'comments') + image = VersatileImageFieldSerializer(sizes='image_sizes', required=False) + image_type = ImageTypeSerializer(read_only=True) + comments = ImageCommentsSerializer(many=True,required=False) + owner = UserSerializer(read_only=True) + + def is_valid(self, raise_exception=False): + + # add an element (for x-ray images) + if str(self.initial_data.get('image_type')) == '13': + if not self.initial_data.get('element'): + raise ValueError('Cannot upload X-Ray Map without associated element!') + else: + try: + e = Element.objects.get(name=self.initial_data['element']) + self.initial_data['element'] = e.pk + # self._validated_data.update({'element':e}) + except: + raise ValueError('Could not find element matching {}'.format(self.initial_data['element'])) + super().is_valid(raise_exception) + + if self.initial_data.get('owner'): + self._validated_data.update( + {'owner': User.objects.get(pk=self.initial_data['owner'])}) + + if self.initial_data.get('image_type'): + self._validated_data.update({'image_type':ImageType.objects.get(pk=self.initial_data['image_type'])}) + + + + # add data association: sample, subsample, chemical analysis + if self.initial_data.get('sample'): + s = Sample.objects.get(pk=self.initial_data['sample']) + if str(s.owner_id) == self.initial_data['owner']: + self._validated_data.update({'sample':s}) + else: + raise ValueError("You are not the owner of sample {}".format(self.initial_data['sample'])) + elif self.initial_data.get('subsample'): + ss = Subsample.objects.get(pk=self.initial_data['subsample']) + if str(ss.owner_id) == self.initial_data['owner']: + self._validated_data.update({'subsample':ss}) + else: + raise ValueError("You are not the owner of subsample {}".format(self.initial_data['subsample'])) + elif self.initial_data.get('chemical_analysis'): + ca = ChemicalAnalysis.objects.get(pk=self.initial_data['chemical_analysis']) + #print(ca) + else: + raise ValueError('Cannot upload image without associated \ + sample, subsample, or chem analysis!') + + + def update(self, instance, validated_data): + for attr, value in validated_data.items(): + if attr in self.fields: + setattr(instance, attr, value) + instance.save() + return instance + + def create(self, validated_data): + instance = super().create(validated_data) + return instance + + +class ImageContainerSerializer(serializers.ModelSerializer): + class Meta: + model = ImageContainer + fields = ('id', 'description', 'url', 'images') + + images = ImageSerializer(many=True, read_only=True) + + @staticmethod + def get_worksheet_row_values(worksheet, row_number): + return [x.value for x in worksheet.row(row_number)] + + @staticmethod + def extract_zip_file_from_url(url, destination): + with tempfile.TemporaryFile() as zip_tmp: + with urllib.request.urlopen(url) as image_zip: + zip_tmp.write(image_zip.read()) + + with zipfile.ZipFile(zip_tmp, 'r') as zip_ref: + zip_ref.extractall(destination) + + @staticmethod + def get_xls_file_name(directory): + xls_file_names = [file_name for file_name in listdir(directory) if file_name.endswith('.xls')] + if len(xls_file_names) != 1: + return Response( + data={'error': 'Expected exactly 1 .xls file, but {} files were provided'.format(len(xls_file_names))}, + status=400 + ) + + return xls_file_names[0] + + @staticmethod + def get_worksheet_header_mappings(directory, xls_file): + workbook = xlrd.open_workbook('{}{}{}'.format(directory, sep, xls_file)) + worksheet = workbook.sheet_by_index(0) + header_row = [x.value.replace(' ', '').lower() for x in worksheet.row(0)] + + if not set(header_row) >= required_headers: + raise serializers.ValidationError('Missing required headers {}'.format(required_headers-set(header_row))) + if PATH not in header_row and FILE not in header_row: + raise serializers.ValidationError('Missing required header path/file') + + header_to_index = {COMMENT: set()} + for index, header in enumerate(header_row): + if header == COMMENT: + header_to_index[COMMENT].add(index) + else: + header_to_index[header] = index + return worksheet, header_to_index + + @staticmethod + def parse_dropbox_url(url): + # first add the ?dl=1 to the url if not present or if 0 + parsed_url = list(urlparse(url)) + drop_box_params = {'dl': '1'} + parsed_url[4] = urlencode(drop_box_params) + return urlunparse(parsed_url) + + def create_image(self, base_directory, path, image_container, image_type, collector, scale, + sample, subsample, public_data): + with open(sep.join((base_directory, path)), 'rb') as file_contents: + try: + image_type_obj = ImageType.objects.get(image_type__iexact=image_type) + except ImageType.DoesNotExist: + image_type_obj = None + if not image_type_obj: + image_type_obj = ImageType.objects.get(abbreviation__iexact=image_type) + + owner = None + if self.initial_data.get('owner'): + owner = User.objects.get(pk=self.initial_data['owner']) + + new_image = Image.objects.create(image_container=image_container, + image_type=image_type_obj, + collector=collector, + scale=scale, + sample=sample, + subsample=subsample, + public_data=public_data, + owner=owner) + + file_name = path[path.rfind(sep) + 1:] + new_image.image.save(file_name, File(file_contents)) + new_image.save() + return new_image + + def get_sample_subsample_public_data(self, subsample_name, sample_number, subsample_type): + sample, subsample, public_data = None, None, False + if self.initial_data.get('owner'): + owner = User.objects.get(pk=self.initial_data['owner']); + if not subsample_name or len(subsample_name) == 0: + sample = Sample.objects.get(number=sample_number,owner=owner) + public_data = sample.public_data + else: + subsample_sample = Sample.objects.get(number=sample_number,owner=owner) + try: + subsample = Subsample.objects.get(sample=subsample_sample, name=subsample_name) + except Subsample.DoesNotExist: + owner = None + if self.initial_data.get('owner'): + owner = User.objects.get(pk=self.initial_data['owner']) + subsample = Subsample.objects.create( + name=subsample_name, + sample=subsample_sample, + subsample_type=SubsampleType.objects.get(name=subsample_type), + owner=owner + ) + public_data = subsample.public_data + return sample, subsample, public_data + + @staticmethod + def create_xray_image(image_type, values, header_to_index, created_image, dwell_time, current, voltage): + image_type_value = re.sub('[^a-z]+', '', image_type.lower()) + #print(image_type_value) + if image_type_value == XRAY_IMAGE: + element = values[header_to_index[ELEMENT]] + if not element: + raise serializers.ValidationError('Expected element for xray image, but none provided') + xray_image = XrayImage.objects.create(image=created_image, + dwelltime=dwell_time, + current=current, + voltage=voltage, + element=element) + xray_image.save() + + @staticmethod + def create_image_comments(image, comments): + ImageComments.objects.bulk_create([ImageComments( + image=image, + comment_text=comment + ) for comment in comments]) + + def process_worksheet(self, worksheet, header_to_index, base_directory, image_container): + num_rows = worksheet.nrows + + for i in range(1, num_rows): + values = ImageContainerSerializer.get_worksheet_row_values(worksheet, i) + sample_number = values[header_to_index[SAMPLE_NUMBER]] # required + subsample_name = values[header_to_index[SUBSAMPLE]] if SUBSAMPLE in header_to_index else None + subsample_type = values[header_to_index[SUBSAMPLE_TYPE]] if SUBSAMPLE_TYPE in header_to_index else None + path_to_replace = values[header_to_index[PATH]] if PATH in header_to_index else values[header_to_index[FILE]] + path = re.sub(r'[\\/]', sep, path_to_replace) + image_type = values[header_to_index[IMAGE_TYPE]] # required + scale = values[header_to_index[SCALE]] if SCALE in header_to_index else None + collector = values[header_to_index[COLLECTOR]] if COLLECTOR in header_to_index else None + comments = {values[comment_index] for comment_index in header_to_index[COMMENT] if len(values[comment_index].strip()) > 0} if COMMENT in header_to_index else None + dwell_time = values[header_to_index[DWELL_TIME]] if DWELL_TIME in header_to_index else None + current = values[header_to_index[CURRENT]] if CURRENT in header_to_index else None + voltage = values[header_to_index[VOLTAGE]] if VOLTAGE in header_to_index else None + + if not scale: + scale = None + + sample, subsample, public_data = self.get_sample_subsample_public_data(subsample_name, + sample_number, subsample_type) + + created_image = self.create_image(base_directory, path, image_container, image_type, + collector, scale, sample, subsample, public_data) + + if image_type: + ImageContainerSerializer.create_xray_image(image_type, values, header_to_index, created_image, + dwell_time, current, voltage) + + if comments: + ImageContainerSerializer.create_image_comments(created_image, comments) + + @transaction.atomic + def create(self, validated_data): + new_image_container = ImageContainer.objects.create(**validated_data) + url = self.parse_dropbox_url(validated_data['url']) + + with tempfile.TemporaryDirectory() as tmp_dir: + self.extract_zip_file_from_url(url, tmp_dir) + xls_file = self.get_xls_file_name(tmp_dir) + worksheet, header_to_index = self.get_worksheet_header_mappings(tmp_dir, xls_file) + self.process_worksheet(worksheet, header_to_index, tmp_dir, new_image_container) + + return new_image_container diff --git a/metpetdb_api/api/images/v1/tests.py b/metpetdb_api/api/images/v1/tests.py new file mode 100644 index 0000000..e69de29 diff --git a/metpetdb_api/api/images/v1/views.py b/metpetdb_api/api/images/v1/views.py new file mode 100644 index 0000000..a9af749 --- /dev/null +++ b/metpetdb_api/api/images/v1/views.py @@ -0,0 +1,61 @@ +from rest_framework import viewsets, permissions, status +from rest_framework.response import Response +from rest_framework.views import APIView + +from api.images.v1.serializers import ImageContainerSerializer, ImageSerializer, ImageTypeSerializer +from apps.images.models import ImageContainer, Image, ImageType +from api.lib.permissions import IsOwnerOrReadOnly, IsSuperuserOrReadOnly + + +class ImageContainerViewSet(viewsets.ModelViewSet): + queryset = ImageContainer.objects.all() + serializer_class = ImageContainerSerializer + permission_classes = (permissions.IsAuthenticatedOrReadOnly, + IsOwnerOrReadOnly,) + +class ImageTypeViewSet(viewsets.ModelViewSet): + queryset = ImageType.objects.all() + serializer_class = ImageTypeSerializer + permission_classes = (permissions.IsAuthenticatedOrReadOnly, + IsSuperuserOrReadOnly,) + + +class ImageViewSet(viewsets.ModelViewSet): + queryset = Image.objects.all() + serializer_class = ImageSerializer + permission_classes = (permissions.IsAuthenticatedOrReadOnly, + IsOwnerOrReadOnly) + + def get_serializer(self, *args, **kwargs): + # FIXME not sure what this is doing yet + # if self.request.method == 'PUT': + # kwargs['partial'] = True + return super().get_serializer(*args, **kwargs) + + def perform_create(self, serializer): + return serializer.save() + + def create(self, request, *args, **kwargs): + img_file = request.FILES.get('image') + if not img_file: + err = "no image file" + return Response(data={'error':err},status=404) + + img_data = request.data + img_data['image'] = img_file + + + serializer = self.get_serializer(data=img_data) + try: + serializer.is_valid(raise_exception=True) + except Exception as err: + return Response( + data={'error':str(err)}, + status=404) + + instance = self.perform_create(serializer) + + headers = self.get_success_headers(serializer.data) + return Response(serializer.data, + status=status.HTTP_201_CREATED, + headers=headers) diff --git a/metpetdb_api/api/samples/lib/query.py b/metpetdb_api/api/samples/lib/query.py index 0bf789b..38805a8 100644 --- a/metpetdb_api/api/samples/lib/query.py +++ b/metpetdb_api/api/samples/lib/query.py @@ -2,7 +2,7 @@ from django.contrib.auth.models import AnonymousUser from django.contrib.gis.geos import Polygon, GEOSException -from django.db.models import Q, F +from django.db.models import Q, F, Count def sample_query(user, params, qs): @@ -13,26 +13,33 @@ def sample_query(user, params, qs): qs = qs.filter(Q(owner=user) | Q(public_data=True)) if params.get('provenance'): - if params['provenance']=="Public": + if params['provenance'] == "Public": qs = qs.filter(Q(public_data=True)) elif params['provenance'] == "Private": qs = qs.filter(Q(public_data=False)) - if params.get('ids'): qs = qs.filter(pk__in=params['ids'].split(',')) if params.get('collectors'): - qs = qs.filter(collector_name__in=params['collectors'].split(',')) + if params.get('collectors_exact') == 'True': + qs = qs.filter(collector_name__in=params['collectors'].split(',')) + else: + rx = r'(' + params['collectors'].replace(',','|') + ')' + qs = qs.filter(Q(collector_name__regex=rx)) if params.get('numbers'): - qs = qs.filter(number__in=params['numbers'].split(',')) + if params.get('numbers_exact') == 'True': + qs = qs.filter(number__in=params['numbers'].split(',')) + else: + rx = r'(' + params['numbers'].replace(',','|') + ')' + qs = qs.filter(Q(number__regex=rx)) if params.get('countries'): qs = qs.filter(country__in=params['countries'].split(',')) if params.get('location_bbox'): - bbox = Polygon.from_bbox(params['location_bbox'].split(',')) + bbox = Polygon.from_bbox(params['location_bbox'].split(',')) qs = qs.filter(location_coords__contained=bbox) if params.get('polygon_coords'): @@ -49,7 +56,7 @@ def sample_query(user, params, qs): if params.get('metamorphic_regions'): metamorphic_regions = params['metamorphic_regions'].split(',') - qs =qs.filter(metamorphic_regions__name__in=metamorphic_regions) + qs = qs.filter(metamorphic_regions__name__in=metamorphic_regions) if params.get('minerals'): minerals = params['minerals'].split(',') @@ -69,13 +76,21 @@ def sample_query(user, params, qs): qs = qs.filter(minerals__name__in=minerals) if params.get('owners'): - qs = qs.filter(owner__name__in=params['owners'].split(',')) + if params.get('owners_exact') == 'True': + qs = qs.filter(owner__name__in=params['owners'].split(',')) + else: + rx = r'(' + params['owners'].replace(',','|') + ')' + qs = qs.filter(Q(owner__name__regex=rx)) if params.get('emails'): qs = qs.filter(owner__email__in=params['emails'].split(',')) if params.get('references'): - qs = qs.filter(references__name__in=params['references'].split(',')) + if params.get('references_exact') == 'True': + qs = qs.filter(references__name__in=params['references'].split(',')) + else: + rx = r'(' + params['references'].replace(',','|') + ')' + qs = qs.filter(Q(references__name__regex=rx)) if params.get('regions'): qs = qs.filter(regions__overlap=params['regions'].split(',')) @@ -93,6 +108,22 @@ def sample_query(user, params, qs): qs = qs.filter(sesar_number__in=params['sesar_number'].split(',')) if params.get('ordering'): - qs = qs.order_by(params['ordering']) + if params['ordering'] == 'images': + # FIX ME WHEN FRONTEND ALLOWS TOGGLE + qs = qs.annotate(image_count=Count('images')).order_by('-image_count') + elif params['ordering'] == '-images': + qs = qs.annotate(image_count=Count('images')).order_by('-image_count') + elif params['ordering'] == 'chemical_analyses': + # FIX ME + qs = qs.annotate(chem_count=Count('subsamples__chemical_analyses')).order_by('-chem_count') + elif params['ordering'] == '-chemical_analyses': + qs = qs.annotate(chem_count=Count('subsamples__chemical_analyses')).order_by('-chem_count') + elif params['ordering'] == 'subsamples': + # FIX ME + qs = qs.annotate(subsample_count=Count('subsamples')).order_by('-subsample_count') + elif params['ordering'] == '-subsamples': + qs = qs.annotate(subsample_count=Count('subsamples')).order_by('-subsample_count') + else: + qs = qs.order_by(params['ordering']) return qs diff --git a/metpetdb_api/api/samples/v1/renderers.py b/metpetdb_api/api/samples/v1/renderers.py index ca97fb1..54a63d8 100644 --- a/metpetdb_api/api/samples/v1/renderers.py +++ b/metpetdb_api/api/samples/v1/renderers.py @@ -4,25 +4,21 @@ class SampleCSVRenderer (r.CSVRenderer): def __init__(self): - self.header = ['number', - 'rock_type', - 'description', - 'latitude', - 'longitude', - 'location_error', - 'country', - 'collector_name', - 'collection_date', - 'location_name', - # 'references.0', - # 'metamorphic_grades.0', - 'minerals', - # 'igsn', - # 'subsample_ids', - # 'chemical_analyses_ids', - ] + self.header = [ + 'number', + 'rock_type', + 'description', + 'latitude', + 'longitude', + 'location_error', + 'country', + 'collector_name', + 'collection_date', + 'location_name', + 'minerals', + ] self.labels = { - 'number': 'Sample', + 'number': 'Sample Number', 'rock_type': 'Rock Type', 'description': 'Comment', 'latitude': 'Latitude', @@ -32,17 +28,15 @@ def __init__(self): 'collector_name': 'Collector', 'collection_date': 'Date of Collection', 'location_name': 'Present Sample Location', - # 'references.0': 'Reference', - # 'metamorphic_grades.0': 'Metamorphic Grade', 'minerals': 'Mineral', - # 'Subsamples': 'Number of Subsamples', - # 'Chemical_Analyses': 'Number of Chemical Analyses' } self.num_regions = 0 + self.num_meta_regions = 0 self.num_refs = 0 self.num_grades = 0 self.minerals = set() + self.fields = set() def tablize(self, data, header=None, labels=None): if data: @@ -60,11 +54,17 @@ def tablize(self, data, header=None, labels=None): region_headers.append('regions.' + str(i)) labels[region_headers[-1]] = 'Region' header[6:6] = region_headers + meta_region_headers = [] + for i in range(self.num_meta_regions): + meta_region_headers.append('metamorphic_regions.' + str(i)) + labels[meta_region_headers[-1]] = 'Metamorphic Region' + offset = 6 + self.num_regions + header[offset:offset] = meta_region_headers ref_headers = [] for i in range(self.num_refs): ref_headers.append('references.' + str(i)) labels[ref_headers[-1]] = 'Reference' - offset = 10 + self.num_regions + offset += 4 + self.num_meta_regions header[offset:offset] = ref_headers grade_headers = [] for i in range(self.num_grades): @@ -75,6 +75,7 @@ def tablize(self, data, header=None, labels=None): mins = list(self.minerals) mins.sort() header.extend(mins) + header = [x for x in header if (x in self.fields or x in self.minerals or x in region_headers or x in meta_region_headers or x in ref_headers or x in grade_headers)] if labels: yield [labels.get(x,x) for x in header] @@ -96,13 +97,18 @@ def tablize(self, data, header=None, labels=None): def flatten_data(self,data): for item in data: - self.handle_regions(item) - self.handle_minerals(item) - self.handle_references(item) - self.handle_meta_grades(item) - # print(item) + self.fields |= item.keys() + if (item.get('regions')): + self.handle_regions(item) + if (item.get('metamorphic_regions')): + self.handle_meta_regions(item) + if (item.get('minerals')): + self.handle_minerals(item) + if (item.get('references')): + self.handle_references(item) + if (item.get('metamorphic_grades')): + self.handle_meta_grades(item) flat_item = self.flatten_item(item) - # print(flat_item) yield flat_item def handle_minerals(self,item): @@ -111,9 +117,13 @@ def handle_minerals(self,item): self.minerals.add(m) def handle_regions(self,item): - regions = {r.title() for r in item['regions']} | {r.title() for r in item['metamorphic_regions']} + regions = {r.title() for r in item['regions']} self.num_regions = max(self.num_regions,len(regions)) + def handle_meta_regions(self,item): + meta_regions = {r.title() for r in item['metamorphic_regions']} + self.num_meta_regions = max(self.num_meta_regions,len(meta_regions)) + def handle_references(self,item): refs = {r for r in item['references']} self.num_refs = max(self.num_refs, len(refs)) diff --git a/metpetdb_api/api/samples/v1/serializers.py b/metpetdb_api/api/samples/v1/serializers.py index 07b4bd6..497e0ac 100644 --- a/metpetdb_api/api/samples/v1/serializers.py +++ b/metpetdb_api/api/samples/v1/serializers.py @@ -2,6 +2,7 @@ from api.lib.serializers import DynamicFieldsModelSerializer from api.users.v1.serializers import UserSerializer +from api.images.v1.serializers import ImageSerializer from apps.chemical_analyses.models import ChemicalAnalysis from apps.samples.models import ( @@ -20,21 +21,46 @@ ) from apps.users.models import User -SAMPLE_FIELDS = ('number', 'aliases', 'collection_date', 'description', - 'location_name', 'location_coords', 'location_error', - 'date_precision', 'country', 'regions', 'collector_name', - 'collector_id', 'sesar_number',) +SAMPLE_FIELDS = ('id', # + 'number', + # 'aliases', + 'owner', # + 'regions', + 'country', + 'rock_type', # + 'metamorphic_grades', # + 'metamorphic_regions', # + 'minerals', # + 'references', # + 'longitude', + 'latitude', + #'location_coords', + 'location_error', + 'sesar_number', + 'collector_name', + # 'collector_id', + 'collection_date', + # 'date_precision', + 'location_name', + 'description', + # 'image', + 'images', + 'subsample_ids', # + 'chemical_analyses_ids', # + 'public_data',) SUBSAMPLE_FIELDS = ('name') class RockTypeSerializer(DynamicFieldsModelSerializer): class Meta: model = RockType + fields = '__all__' class MineralSerializer(DynamicFieldsModelSerializer): class Meta: model = Mineral + fields = '__all__' class SampleMineralSerializer(DynamicFieldsModelSerializer): @@ -51,16 +77,16 @@ class Meta: class SampleSerializer(DynamicFieldsModelSerializer): minerals = SampleMineralSerializer(source='samplemineral_set', many=True) - # owner = UserSerializer(read_only=True) owner = serializers.SerializerMethodField(read_only=True) - rock_type = serializers.SerializerMethodField(read_only=True) + rock_type = serializers.ReadOnlyField(source='rock_type.name') metamorphic_grades = serializers.SerializerMethodField(read_only=True) metamorphic_regions = serializers.SerializerMethodField(read_only=True) minerals = serializers.SerializerMethodField(read_only=True) references = serializers.SerializerMethodField(read_only=True) latitude = serializers.SerializerMethodField(read_only=True) longitude = serializers.SerializerMethodField(read_only=True) - collection_date = serializers.SerializerMethodField(read_only=True) + + images = ImageSerializer(many=True, read_only=True) # TODO: figure out if there is a better, more efficient way to do this subsample_ids = serializers.SerializerMethodField() @@ -69,31 +95,16 @@ class SampleSerializer(DynamicFieldsModelSerializer): class Meta: model = Sample depth = 1 - fields = ( - 'number', - 'owner', - 'regions', - 'country', - 'rock_type', - 'metamorphic_grades', - 'metamorphic_regions', - 'minerals', - 'references', - 'longitude', - 'latitude', - 'location_error', - # 'igsn', - 'collector_name', - 'collection_date', - 'location_name', - 'description', - 'subsample_ids', - 'chemical_analyses_ids', - ) + fields = SAMPLE_FIELDS def is_valid(self, raise_exception=False): + if self.initial_data.get('latitude') and self.initial_data.get('longitude'): + self.initial_data['location_coords'] = "SRID=4326;POINT ("+str(self.initial_data["longitude"])+" "+str(self.initial_data["latitude"])+")" + super().is_valid(raise_exception) + self._validated_data.update({'location_coords':self.initial_data['location_coords']}) + if self.initial_data.get('owner'): self._validated_data.update( {'owner': User.objects.get(pk=self.initial_data['owner'])}) @@ -115,20 +126,16 @@ def create(self, validated_data): instance = super().create(validated_data) return instance - def update(self, instance, validated_data): for attr, value in validated_data.items(): - if attr in SAMPLE_FIELDS: + # if attr in SAMPLE_FIELDS: setattr(instance, attr, value) instance.save() return instance def get_owner(self,obj): - return obj.owner.name - - def get_rock_type(self,obj): - return obj.rock_type.name + return {'name':obj.owner.name,'id':obj.owner.id} def get_metamorphic_grades(self,obj): return [g.name for g in obj.metamorphic_grades.all()] @@ -165,10 +172,12 @@ def get_chemical_analyses_ids(self, obj): class SubsampleSerializer(DynamicFieldsModelSerializer): sample = SampleSerializer(read_only=True) owner = UserSerializer(read_only=True) + images = ImageSerializer(many=True, read_only=True) class Meta: model = Subsample depth = 1 + fields = '__all__' def is_valid(self, raise_exception=False): super().is_valid(raise_exception) @@ -201,7 +210,7 @@ def create(self, validated_data): def update(self, instance, validated_data): for attr, value in validated_data.items(): - if attr in SUBSAMPLE_FIELDS: + if attr in self.fields: setattr(instance, attr, value) instance.save() @@ -212,32 +221,40 @@ class SubsampleTypeSerializer(DynamicFieldsModelSerializer): class Meta: model = SubsampleType depth = 1 + fields = '__all__' class MetamorphicGradeSerializer(DynamicFieldsModelSerializer): class Meta: model = MetamorphicGrade + fields = '__all__' class MetamorphicRegionSerializer(DynamicFieldsModelSerializer): class Meta: model = MetamorphicRegion + fields = '__all__' class GeoReferenceSerializer(DynamicFieldsModelSerializer): class Meta: model = GeoReference + fields = '__all__' + class RegionSerializer(DynamicFieldsModelSerializer): class Meta: model = Region + fields = '__all__' class ReferenceSerializer(DynamicFieldsModelSerializer): class Meta: model = Reference + fields = '__all__' class CollectorSerializer(DynamicFieldsModelSerializer): class Meta: - model = Collector \ No newline at end of file + model = Collector + fields = '__all__' diff --git a/metpetdb_api/api/samples/v1/views.py b/metpetdb_api/api/samples/v1/views.py index 2fb7055..29289e9 100644 --- a/metpetdb_api/api/samples/v1/views.py +++ b/metpetdb_api/api/samples/v1/views.py @@ -51,7 +51,6 @@ def get_serializer(self, *args, **kwargs): kwargs['partial'] = True return super().get_serializer(*args, **kwargs) - def list(self, request, *args, **kwargs): params = request.query_params @@ -77,7 +76,12 @@ def list(self, request, *args, **kwargs): if params.get('format') == 'csv': serializer = self.get_serializer(qs, many=True) - return Response(serializer.data) + response = Response(serializer.data) + filename = params.get('filename') + if filename is None: + filename = 'search_results.csv' + response['content-disposition'] = "attachment; filename=%s" % filename + return response else: page = self.paginate_queryset(qs) if page: @@ -87,7 +91,6 @@ def list(self, request, *args, **kwargs): serializer = self.get_serializer(qs, many=True) return Response(serializer.data) - def _handle_metamorphic_regions(self, instance, ids): metamorphic_regions = [] for id in ids: @@ -100,7 +103,6 @@ def _handle_metamorphic_regions(self, instance, ids): metamorphic_regions.append(metamorphic_region) instance.metamorphic_regions = metamorphic_regions - def _handle_metamorphic_grades(self, instance, ids): metamorphic_grades = [] for id in ids: @@ -112,7 +114,6 @@ def _handle_metamorphic_grades(self, instance, ids): metamorphic_grades.append(metamorphic_grade) instance.metamorphic_grades = metamorphic_grades - def _handle_minerals(self, instance, minerals): to_add = [] for record in minerals: @@ -129,7 +130,6 @@ def _handle_minerals(self, instance, minerals): mineral=record['mineral'], amount=record['amount']) - def _handle_references(self, instance, references): to_add = [] @@ -152,11 +152,9 @@ def _handle_references(self, instance, references): instance.references.clear() instance.references.add(*to_add) - def perform_create(self, serializer): return serializer.save() - def create(self, request, *args, **kwargs): serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) @@ -204,7 +202,6 @@ def create(self, request, *args, **kwargs): status=status.HTTP_201_CREATED, headers=headers) - def update(self, request, *args, **kwargs): params = request.data partial = kwargs.pop('partial', False) @@ -277,22 +274,20 @@ def get_serializer(self, *args, **kwargs): kwargs['partial'] = True return super().get_serializer(*args, **kwargs) - def list(self, request, *args, **kwargs): - + params = request.query_params qs = self.get_queryset().distinct() - + page = self.paginate_queryset(qs) if page is not None: serializer = self.get_serializer(page, many=True) return self.get_paginated_response(serializer.data) - + serializer = self.get_serializer(qs, many=True) return Response(serializer.data) - def perform_create(self, serializer): return serializer.save() @@ -322,14 +317,13 @@ def update(self, request, *args, **kwargs): return Response(data={'error': 'Invalid subsample type'}, status=400) else: - instance.subsample_type = subsample_type + instance.subsample_type = subsample_type instance.save() # refresh the data before returning a response serializer = self.get_serializer(instance) return Response(serializer.data) - class SubsampleTypeViewSet(viewsets.ModelViewSet): queryset = SubsampleType.objects.all() @@ -398,10 +392,10 @@ class SampleNumbersView(APIView): def get(self, request, format=None): sample_numbers = ( Sample - .objects - .all() - .values_list('number', flat=True) - .distinct() + .objects + .all() + .values_list('number', flat=True) + .distinct() ) return Response({'sample_numbers': sample_numbers}) @@ -410,10 +404,10 @@ class CountryNamesView(APIView): def get(self, request, format=None): country_names = ( Country - .objects - .all() - .values_list('name', flat=True) - .distinct() + .objects + .all() + .values_list('name', flat=True) + .distinct() ) return Response({'country_names': country_names}) @@ -422,76 +416,9 @@ class SampleOwnerNamesView(APIView): def get(self, request, format=None): sample_owner_names = ( Sample - .objects - .all() - .values_list('owner__name', flat=True) - .distinct() + .objects + .all() + .values_list('owner__name', flat=True) + .distinct() ) - return Response({'sample_owner_names': sample_owner_names}) - -''' -class SampleCSVRenderer (r.CSVRenderer): - header = ['Sample', 'Rock_Type', 'Comment', 'Latitude', 'Longitude', 'Location_Error', 'Region', 'Country', 'Collector', 'Date_of_Collection', 'Present_Sample_Location', 'Reference', 'Metamorphic_Grade', 'Minerals', 'Subsamples', 'Chemical_Analyses'] - labels = { - 'Sample': 'Sample', - 'Rock_Type': 'Rock Type', - 'Comment': 'Comment', - 'Latitude': 'Latitude', - 'Longitude': 'Longitude', - 'Location_Error': 'Location Error', - 'Region': 'Region', - 'Country': 'Country', - 'Collector': 'Collector', - 'Date_of_Collection': 'Date of Collection', - 'Present_Sample_Location': 'Present Sample Location', - 'Reference': 'Reference', - 'Metamorphic_Grade': 'Metamorphic Grade', - 'Minerals': 'Mineral', - 'Subsamples': 'Number of Subsamples', - 'Chemical_Analyses': 'Number of Chemical Analyses' - } - - - -class SampleSearchView(SampleViewSet): - serializer_class = SampleSearchSerializer - renderer_classes = (JSONRenderer, BrowsableAPIRenderer, SampleCSVRenderer) - - def get_serializer(self, *args, **kwargs): - return super().get_serializer(*args,**kwargs) - - def list(self, request, *args, **kwargs): - params = request.query_params - - if params.get('chemical_analyses_filters') == 'True': - chem_qs = ChemicalAnalysis.objects.all() - chem_qs = chemical_analyses_qs_optimizer(params, chem_qs) - chem_ids = (chemical_analysis_query(request.user, params, chem_qs) - .values_list('id')) - qs = (Sample - .objects - .filter(subsamples__chemical_analyses__id__in=chem_ids)) - else: - qs = self.get_queryset().distinct() - try: - qs = sample_query(request.user, params, qs) - except ValueError as err: - return Response( - data={'error': err.args}, - status=400 - ) - - qs = sample_qs_optimizer(params, qs) - - if params.get('format') == 'csv': - serializer = self.get_serializer(qs, many=True) - return Response(serializer.data) - else: - page = self.paginate_queryset(qs) - if page is not None: - serializer = self.get_serializer(page, many=True) - return self.get_paginated_response(serializer.data) - - serializer = self.get_serializer(qs, many=True) - return Response(serializer.data) -''' + return Response({'sample_owner_names': sample_owner_names}) \ No newline at end of file diff --git a/metpetdb_api/apps/chemical_analyses/management/commands/migrate_legacy_chemical_analyses.py b/metpetdb_api/apps/chemical_analyses/management/commands/migrate_legacy_chemical_analyses.py index a5a2be0..ed73de6 100644 --- a/metpetdb_api/apps/chemical_analyses/management/commands/migrate_legacy_chemical_analyses.py +++ b/metpetdb_api/apps/chemical_analyses/management/commands/migrate_legacy_chemical_analyses.py @@ -6,9 +6,8 @@ ChemicalAnalysis, ChemicalAnalysisElement, ChemicalAnalysisOxide, - Element, - Oxide, ) +from apps.chemical_analyses.shared_models import Element, Oxide from apps.common.utils import queryset_iterator from apps.samples.models import ( Mineral, @@ -19,6 +18,8 @@ Reference, ) +from apps.images.models import Image, ImageMapping + from legacy.models import ( ChemicalAnalyses as LegacyChemicalAnalyses, ChemicalAnalysisElements as LegacyChemicalAnalysisElements, @@ -81,6 +82,16 @@ def _migrate_chemical_analyses(self): except AttributeError: reference = None + try: + reference_image = Image.objects.get( + subsample=subsample, + pk=ImageMapping.objects.get( + old_image_id=record.image.pk).new_image_id) + except Image.DoesNotExist: + reference_image = None + except AttributeError: + reference_image = None + chem_analysis = ChemicalAnalysis.objects.create( subsample=subsample, public_data=True if record.public_data == 'Y' else False, @@ -99,6 +110,7 @@ def _migrate_chemical_analyses(self): mineral=mineral, owner=User.objects.get(email=record.user.email), reference=reference, + reference_image=reference_image ) legacy_cae = LegacyChemicalAnalysisElements.objects.filter( diff --git a/metpetdb_api/apps/chemical_analyses/migrations/0004_auto_20180220_0409.py b/metpetdb_api/apps/chemical_analyses/migrations/0004_auto_20180220_0409.py new file mode 100644 index 0000000..8e7cfe5 --- /dev/null +++ b/metpetdb_api/apps/chemical_analyses/migrations/0004_auto_20180220_0409.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-02-20 04:09 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('chemical_analyses', '0003_chemicalanalysis_stage_y'), + ] + + operations = [ + migrations.AlterModelOptions( + name='chemicalanalysis', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='element', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='oxide', + options={'ordering': ['id']}, + ), + ] diff --git a/metpetdb_api/apps/chemical_analyses/migrations/0005_chemicalanalysis_reference_image.py b/metpetdb_api/apps/chemical_analyses/migrations/0005_chemicalanalysis_reference_image.py new file mode 100644 index 0000000..f2ac4e8 --- /dev/null +++ b/metpetdb_api/apps/chemical_analyses/migrations/0005_chemicalanalysis_reference_image.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-08-27 02:29 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0012_remove_image_chemical_analysis'), + ('chemical_analyses', '0004_auto_20180220_0409'), + ] + + operations = [ + migrations.AddField( + model_name='chemicalanalysis', + name='reference_image', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='chemical_analyses', to='images.Image'), + ), + ] diff --git a/metpetdb_api/apps/chemical_analyses/models.py b/metpetdb_api/apps/chemical_analyses/models.py index 5848045..204d527 100644 --- a/metpetdb_api/apps/chemical_analyses/models.py +++ b/metpetdb_api/apps/chemical_analyses/models.py @@ -1,8 +1,9 @@ import uuid -from concurrency.fields import AutoIncVersionField +from concurrency.fields import AutoIncVersionField from django.conf import settings from django.contrib.gis.db import models +from apps.images.models import Image class ChemicalAnalysis(models.Model): @@ -31,39 +32,16 @@ class ChemicalAnalysis(models.Model): through='ChemicalAnalysisElement') oxides = models.ManyToManyField('Oxide', through='ChemicalAnalysisOxide') + reference_image = models.ForeignKey(Image, on_delete=models.CASCADE, blank=True, null=True, + related_name='chemical_analyses') + # Free-text field; stored as an CharField to avoid joining to the # references table every time we retrieve chemical analyses reference = models.CharField(max_length=100, blank=True, null=True) class Meta: db_table = 'chemical_analyses' - - -class Element(models.Model): - id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - name = models.CharField(unique=True, max_length=100) - alternate_name = models.CharField(max_length=100, blank=True, null=True) - symbol = models.CharField(unique=True, max_length=4) - atomic_number = models.IntegerField() - weight = models.FloatField(blank=True, null=True) - order_id = models.IntegerField(blank=True, null=True) - - class Meta: - db_table = 'elements' - - -class Oxide(models.Model): - id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - element = models.ForeignKey('Element') - oxidation_state = models.SmallIntegerField(blank=True, null=True) - species = models.CharField(unique=True, max_length=20, blank=True, null=True) - weight = models.FloatField(blank=True, null=True) - cations_per_oxide = models.SmallIntegerField(blank=True, null=True) - conversion_factor = models.FloatField() - order_id = models.IntegerField(blank=True, null=True) - - class Meta: - db_table = 'oxides' + ordering = ['id'] class ChemicalAnalysisElement(models.Model): diff --git a/metpetdb_api/apps/chemical_analyses/shared_models.py b/metpetdb_api/apps/chemical_analyses/shared_models.py new file mode 100644 index 0000000..ad50414 --- /dev/null +++ b/metpetdb_api/apps/chemical_analyses/shared_models.py @@ -0,0 +1,32 @@ +import uuid + +from django.contrib.gis.db import models + + +class Element(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(unique=True, max_length=100) + alternate_name = models.CharField(max_length=100, blank=True, null=True) + symbol = models.CharField(unique=True, max_length=4) + atomic_number = models.IntegerField() + weight = models.FloatField(blank=True, null=True) + order_id = models.IntegerField(blank=True, null=True) + + class Meta: + db_table = 'elements' + ordering = ['symbol'] + + +class Oxide(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + element = models.ForeignKey('Element') + oxidation_state = models.SmallIntegerField(blank=True, null=True) + species = models.CharField(unique=True, max_length=20, blank=True, null=True) + weight = models.FloatField(blank=True, null=True) + cations_per_oxide = models.SmallIntegerField(blank=True, null=True) + conversion_factor = models.FloatField() + order_id = models.IntegerField(blank=True, null=True) + + class Meta: + db_table = 'oxides' + ordering = ['species'] \ No newline at end of file diff --git a/metpetdb_api/apps/core/pagination.py b/metpetdb_api/apps/core/pagination.py index 583f6fb..974fa23 100644 --- a/metpetdb_api/apps/core/pagination.py +++ b/metpetdb_api/apps/core/pagination.py @@ -3,4 +3,4 @@ class StandardResultsSetPagination(pagination.PageNumberPagination): page_size = 20 page_size_query_param = 'page_size' - max_page_size = 1000 + max_page_size = 3000 diff --git a/metpetdb_api/apps/images/__init__.py b/metpetdb_api/apps/images/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/metpetdb_api/apps/images/migrations/0001_initial.py b/metpetdb_api/apps/images/migrations/0001_initial.py new file mode 100644 index 0000000..ab8eb1b --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0001_initial.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-06-18 06:15 +from __future__ import unicode_literals + +import apps.images.models +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import uuid +import versatileimagefield.fields + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('samples', '0003_auto_20180220_0409'), + ('chemical_analyses', '0004_auto_20180220_0409'), + ] + + operations = [ + migrations.CreateModel( + name='Image', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('image', versatileimagefield.fields.VersatileImageField(blank=True, null=True, upload_to=apps.images.models.Image.generate_filename, verbose_name='Image')), + ('version', models.IntegerField(default=0)), + ('collector', models.CharField(blank=True, max_length=50)), + ('public_data', models.BooleanField(default=False)), + ('chemical_analysis', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='image', to='chemical_analyses.ChemicalAnalysis')), + ], + options={ + 'ordering': ('id',), + 'db_table': 'images', + }, + ), + migrations.CreateModel( + name='ImageContainer', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('description', models.CharField(blank=True, max_length=100, null=True)), + ('url', models.CharField(blank=True, max_length=500, null=True)), + ], + options={ + 'ordering': ('id',), + 'db_table': 'image_container', + }, + ), + migrations.CreateModel( + name='ImageType', + fields=[ + ('id', models.SmallIntegerField(primary_key=True, serialize=False)), + ('image_type', models.CharField(max_length=100)), + ('abbreviation', models.CharField(max_length=10)), + ('comments', models.CharField(max_length=250)), + ], + ), + migrations.AddField( + model_name='image', + name='image_container', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='images', to='images.ImageContainer'), + ), + migrations.AddField( + model_name='image', + name='image_type_id', + field=models.ForeignKey(blank=True, default='', on_delete=django.db.models.deletion.CASCADE, to='images.ImageType'), + ), + migrations.AddField( + model_name='image', + name='owner', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='images', to=settings.AUTH_USER_MODEL), + ), + migrations.AddField( + model_name='image', + name='sample', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='image', to='samples.Sample'), + ), + migrations.AddField( + model_name='image', + name='subsample', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='image', to='samples.Subsample'), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0002_auto_20180618_0640.py b/metpetdb_api/apps/images/migrations/0002_auto_20180618_0640.py new file mode 100644 index 0000000..e1dd390 --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0002_auto_20180618_0640.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-06-18 06:40 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='image', + name='image_type_id', + ), + migrations.AddField( + model_name='image', + name='image_type', + field=models.ForeignKey(blank=True, default=0, on_delete=django.db.models.deletion.CASCADE, to='images.ImageType'), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0003_auto_20180618_0655.py b/metpetdb_api/apps/images/migrations/0003_auto_20180618_0655.py new file mode 100644 index 0000000..1c5fe46 --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0003_auto_20180618_0655.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-06-18 06:55 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0002_auto_20180618_0640'), + ] + + operations = [ + migrations.AlterField( + model_name='image', + name='owner', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='images', to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0004_auto_20180618_0754.py b/metpetdb_api/apps/images/migrations/0004_auto_20180618_0754.py new file mode 100644 index 0000000..838d761 --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0004_auto_20180618_0754.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-06-18 07:54 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0003_auto_20180618_0655'), + ] + + operations = [ + migrations.AlterField( + model_name='image', + name='chemical_analysis', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='images', to='chemical_analyses.ChemicalAnalysis'), + ), + migrations.AlterField( + model_name='image', + name='sample', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='images', to='samples.Sample'), + ), + migrations.AlterField( + model_name='image', + name='subsample', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='images', to='samples.Subsample'), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0005_auto_20180618_0821.py b/metpetdb_api/apps/images/migrations/0005_auto_20180618_0821.py new file mode 100644 index 0000000..c91681b --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0005_auto_20180618_0821.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-06-18 08:21 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0004_auto_20180618_0754'), + ] + + operations = [ + migrations.AlterField( + model_name='imagetype', + name='comments', + field=models.CharField(blank=True, max_length=250, null=True), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0006_auto_20180619_0335.py b/metpetdb_api/apps/images/migrations/0006_auto_20180619_0335.py new file mode 100644 index 0000000..464fb11 --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0006_auto_20180619_0335.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-06-19 03:35 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0005_auto_20180618_0821'), + ] + + operations = [ + migrations.AlterField( + model_name='image', + name='collector', + field=models.CharField(blank=True, max_length=50, null=True), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0007_xrayimage.py b/metpetdb_api/apps/images/migrations/0007_xrayimage.py new file mode 100644 index 0000000..c121c2c --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0007_xrayimage.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-06-19 04:53 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0006_auto_20180619_0335'), + ] + + operations = [ + migrations.CreateModel( + name='XrayImage', + fields=[ + ('image', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='images.Image')), + ('element', models.CharField(blank=True, max_length=256, null=True)), + ('dwelltime', models.SmallIntegerField(blank=True, null=True)), + ('current', models.SmallIntegerField(blank=True, null=True)), + ('voltage', models.SmallIntegerField(blank=True, null=True)), + ], + options={ + 'db_table': 'xray_image', + 'ordering': ('image_id',), + }, + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0008_auto_20180619_0454.py b/metpetdb_api/apps/images/migrations/0008_auto_20180619_0454.py new file mode 100644 index 0000000..6680c43 --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0008_auto_20180619_0454.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-06-19 04:54 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0007_xrayimage'), + ] + + operations = [ + migrations.AlterField( + model_name='xrayimage', + name='image', + field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='images.Image'), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0009_auto_20180619_0454.py b/metpetdb_api/apps/images/migrations/0009_auto_20180619_0454.py new file mode 100644 index 0000000..5211b7d --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0009_auto_20180619_0454.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-06-19 04:54 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0008_auto_20180619_0454'), + ] + + operations = [ + migrations.AlterField( + model_name='xrayimage', + name='image', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='images.Image', unique=True), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0010_auto_20180619_0455.py b/metpetdb_api/apps/images/migrations/0010_auto_20180619_0455.py new file mode 100644 index 0000000..c7478da --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0010_auto_20180619_0455.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-06-19 04:55 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0009_auto_20180619_0454'), + ] + + operations = [ + migrations.AlterField( + model_name='xrayimage', + name='image', + field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='images.Image'), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0011_auto_20180826_1618.py b/metpetdb_api/apps/images/migrations/0011_auto_20180826_1618.py new file mode 100644 index 0000000..41409c1 --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0011_auto_20180826_1618.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-08-26 16:18 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0010_auto_20180619_0455'), + ] + + operations = [ + migrations.AddField( + model_name='image', + name='description', + field=models.CharField(blank=True, max_length=1024, null=True), + ), + migrations.AddField( + model_name='image', + name='scale', + field=models.SmallIntegerField(blank=True, null=True), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0012_remove_image_chemical_analysis.py b/metpetdb_api/apps/images/migrations/0012_remove_image_chemical_analysis.py new file mode 100644 index 0000000..03e3bee --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0012_remove_image_chemical_analysis.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-08-27 02:29 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0011_auto_20180826_1618'), + ] + + operations = [ + migrations.RemoveField( + model_name='image', + name='chemical_analysis', + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0013_imagecomments.py b/metpetdb_api/apps/images/migrations/0013_imagecomments.py new file mode 100644 index 0000000..f423fbe --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0013_imagecomments.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-08-27 03:34 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0012_remove_image_chemical_analysis'), + ] + + operations = [ + migrations.CreateModel( + name='ImageComments', + fields=[ + ('comment_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('comment_text', models.TextField()), + ('version', models.IntegerField(default=0)), + ('image', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='images.Image')), + ], + options={ + 'db_table': 'image_comments', + }, + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0014_auto_20180827_0336.py b/metpetdb_api/apps/images/migrations/0014_auto_20180827_0336.py new file mode 100644 index 0000000..3d86265 --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0014_auto_20180827_0336.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-08-27 03:36 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0013_imagecomments'), + ] + + operations = [ + migrations.AlterField( + model_name='imagecomments', + name='image', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='comments', to='images.Image'), + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0015_auto_20180827_0344.py b/metpetdb_api/apps/images/migrations/0015_auto_20180827_0344.py new file mode 100644 index 0000000..89615b0 --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0015_auto_20180827_0344.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-08-27 03:44 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0014_auto_20180827_0336'), + ] + + operations = [ + migrations.AlterModelOptions( + name='imagecomments', + options={'ordering': ('comment_id',)}, + ), + ] diff --git a/metpetdb_api/apps/images/migrations/0016_imagemapping.py b/metpetdb_api/apps/images/migrations/0016_imagemapping.py new file mode 100644 index 0000000..27151f4 --- /dev/null +++ b/metpetdb_api/apps/images/migrations/0016_imagemapping.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-10-18 06:13 +from __future__ import unicode_literals + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0015_auto_20180827_0344'), + ] + + operations = [ + migrations.CreateModel( + name='ImageMapping', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('old_image_id', models.IntegerField()), + ('new_image_id', models.UUIDField()), + ], + options={ + 'db_table': 'image_mapping', + }, + ), + ] diff --git a/metpetdb_api/apps/images/migrations/__init__.py b/metpetdb_api/apps/images/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/metpetdb_api/apps/images/models.py b/metpetdb_api/apps/images/models.py new file mode 100644 index 0000000..40b7d4d --- /dev/null +++ b/metpetdb_api/apps/images/models.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import os +import uuid + +from apps.samples.models import Sample, Subsample +import apps.chemical_analyses.models +from apps.chemical_analyses.shared_models import Element +from django.conf import settings +from django.contrib.gis.db import models +from django.dispatch import receiver +from versatileimagefield.fields import VersatileImageField +from versatileimagefield.image_warmer import VersatileImageFieldWarmer + + +class ImageContainer(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + description = models.CharField(max_length=100, blank=True, null=True) + url = models.CharField(max_length=500, blank=True, null=True) + + class Meta: + db_table = 'image_container' + ordering = ('id',) + + +class ImageType(models.Model): + id = models.SmallIntegerField(primary_key=True) + image_type = models.CharField(max_length=100, null=False) + abbreviation = models.CharField(max_length=10) + comments = models.CharField(max_length=250, null=True, blank=True) + + +class ImageComments(models.Model): + comment_id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + comment_text = models.TextField(blank=False, null=False) + version = models.IntegerField(null=False, default=0) + image = models.ForeignKey('Image', related_name='comments') + + class Meta: + db_table = 'image_comments' + ordering = ('comment_id',) + + +class Image(models.Model): + def generate_filename(instance, filename): + f_hash = str(uuid.uuid4()).replace('-', '') + assert (len(f_hash) % 2 == 0) + return "{}{}{}".format(os.sep.join(x + y for x, y in zip(f_hash[::2], f_hash[1::2])), + os.sep, filename) + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + image = VersatileImageField('Image', upload_to=generate_filename, blank=True, null=True) + version = models.IntegerField(null=False, default=0) + scale = models.SmallIntegerField(null=True, blank=True) + description = models.CharField(max_length=1024, null=True, blank=True) + image_type = models.ForeignKey(ImageType, null=False, blank=True, default=0) + collector = models.CharField(max_length=50, blank=True, null=True) + owner = models.ForeignKey(settings.AUTH_USER_MODEL, related_name='images', null=True) + public_data = models.BooleanField(null=False, default=False) + image_container = models.ForeignKey(ImageContainer, on_delete=models.CASCADE, blank=True, null=True, + related_name='images') + # move to respective classes + sample = models.ForeignKey(Sample, on_delete=models.CASCADE, blank=True, null=True, related_name='images') + subsample = models.ForeignKey(Subsample, on_delete=models.CASCADE, blank=True, null=True, related_name='images') + chemical_analysis = models.ForeignKey('chemical_analyses.ChemicalAnalysis', + on_delete=models.CASCADE, blank=True, + null=True, related_name='images') + + element = models.ForeignKey(Element,blank=True,null=True) + + element = models.ForeignKey(Element, blank=True, null=True) + + class Meta: + db_table = 'images' + ordering = ('id',) + + +class XrayImage(models.Model): + image = models.OneToOneField(Image, primary_key=True) + element = models.CharField(max_length=256, blank=True, null=True) + dwelltime = models.SmallIntegerField(blank=True, null=True) + current = models.SmallIntegerField(blank=True, null=True) + voltage = models.SmallIntegerField(blank=True, null=True) + + class Meta: + db_table = 'xray_image' + ordering = ('image_id',) + +# A mapping table to help migration of old images to new images. +# needed (I think?) for chemical analysis migration. +class ImageMapping(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + old_image_id = models.IntegerField() + new_image_id = models.UUIDField() + + class Meta: + db_table = 'image_mapping' + + +@receiver(models.signals.post_save, sender=Image) +def warm_images(sender, instance, **kwargs): + """Create all image size on POST""" + if instance.image: + image_warmer = VersatileImageFieldWarmer( + instance_or_queryset=instance, + rendition_key_set='image_sizes', + image_attr='image' + ) + image_warmer.warm() diff --git a/metpetdb_api/apps/samples/management/commands/migrate_legacy_samples.py b/metpetdb_api/apps/samples/management/commands/migrate_legacy_samples.py index f2bfb96..31676db 100644 --- a/metpetdb_api/apps/samples/management/commands/migrate_legacy_samples.py +++ b/metpetdb_api/apps/samples/management/commands/migrate_legacy_samples.py @@ -1,6 +1,7 @@ from django.contrib.auth import get_user_model from django.core.management import BaseCommand from django.db import transaction +import uuid from apps.common.utils import queryset_iterator from apps.samples.models import ( @@ -21,6 +22,13 @@ Subsample, SubsampleType, ) +from apps.images.models import ( + ImageType, + Image, + XrayImage, + ImageComments, + ImageMapping +) from legacy.models import ( Georeference as LegacyGeoreference, Grids as LegacyGrid, @@ -40,7 +48,19 @@ SampleRegions as LegacySampleRegion, SampleReference as LegacySampleReference, Users as LegacyUser, + ImageType as LegacyImageTypes, + Images as LegacyImages, + XrayImage as LegacyXrayImage, + ImageComments as LegacyImageComments +) +from apps.users.models import ( + User ) +from django.core.files import File + +BASE_DIR = '/mnt/volume-nyc1-01/images' +FAILED_IMAGES_FILE = 'failed_images.txt' + class Command(BaseCommand): help = 'Migrates legacy samples to the new data model' @@ -53,75 +73,43 @@ def handle(self, *args, **options): self._migrate_mineral_relationships() self._migrate_subsample_types() self._migrate_references() + self._migrate_image_types() self._migrate_samples() self._migrate_countries() - @transaction.atomic def _migrate_samples(self): print("Migrating samples...") User = get_user_model() - all_collectors = [] - all_regions = [] + all_collectors = set() + all_regions = set() - old_samples = queryset_iterator(LegacySample.objects.all(), - chunksize=1000) + old_samples = queryset_iterator(LegacySample.objects.all(), chunksize=1000) Sample.objects.all().delete() for old_sample in old_samples: - print("Migrating old sample #{0}: {1}" - .format(old_sample.pk, old_sample.number)) - rock_type = (RockType - .objects - .filter(name=old_sample.rock_type.rock_type)[0]) + print("Migrating old sample #{0}: {1}".format(old_sample.pk, old_sample.number)) + rock_type = (RockType.objects.filter(name=old_sample.rock_type.rock_type)[0]) old_user = LegacyUser.objects.get(pk=old_sample.user_id) new_user = User.objects.get(email=old_user.email) if old_sample.collector: - all_collectors.extend([old_sample.collector]) + all_collectors.add(old_sample.collector) - regions = [lsr.region.name - for lsr in LegacySampleRegion - .objects - .filter(sample=old_sample)] + regions = [lsr.region.name for lsr in LegacySampleRegion.objects.filter(sample=old_sample)] if regions: - all_regions.extend(regions) - - references = [lsr.reference.name - for lsr in LegacySampleReference - .objects - .filter(sample=old_sample)] - new_georeferences = GeoReference.objects.filter(name__in=references) - - old_metamorphic_regions = (lsa.metamorphic_region.name - for lsa in - LegacySampleMetamorphicRegion - .objects - .filter(sample=old_sample)) - new_metamorphic_regions = ( - MetamorphicRegion - .objects - .filter(name__in=old_metamorphic_regions) - ) + all_regions.update(set(regions)) - old_metamorphic_grades = (lsa.metamorphic_grade.name - for lsa in - LegacySampleMetamorphicGrades - .objects - .filter(sample=old_sample)) - new_metamorphic_grades = (MetamorphicGrade - .objects - .filter(name__in=old_metamorphic_grades)) + new_georeferences = self._get_georeferences(old_sample) + new_metamorphic_regions = self._get_metamorphic_regions(old_sample) + new_metamorphic_grades = self._get_metamorphic_grades(old_sample) - aliases = [lsa.alias - for lsa in LegacySampleAlias - .objects - .filter(sample=old_sample)] + aliases = [lsa.alias for lsa in LegacySampleAlias.objects.filter(sample=old_sample)] new_sample = Sample.objects.create( - public_data=True if old_sample.public_data == 'Y' else False, + public_data=old_sample.public_data == 'Y', number=old_sample.number, owner=new_user, aliases=aliases, @@ -137,43 +125,56 @@ def _migrate_samples(self): collector_name=old_sample.collector ) - SampleMapping.objects.create(old_sample_id=old_sample.pk, - new_sample_id=new_sample.pk) + SampleMapping.objects.create(old_sample_id=old_sample.pk, new_sample_id=new_sample.pk) + + self._migrate_subsamples(old_sample, new_sample=new_sample) - self._migrate_subsamples(old_sample, new_sample) + old_images = LegacyImages.objects.filter(sample=old_sample) + self._migrate_images(old_images, new_sample) new_sample.metamorphic_regions.add(*new_metamorphic_regions) new_sample.metamorphic_grades.add(*new_metamorphic_grades) new_sample.references.add(*new_georeferences) - - old_minerals = (LegacySampleMineral - .objects - .filter(sample=old_sample)) - old_mineral_names = [om.mineral.name for om in old_minerals] - new_minerals = Mineral.objects.filter(name__in=old_mineral_names) - - for mineral in new_minerals: - old_sample_mineral = (LegacySampleMineral - .objects - .get(mineral__name=mineral.name, - sample=old_sample)) - SampleMineral.objects.create(sample=new_sample, - mineral=mineral, - amount=old_sample_mineral.amount) - + self._migrate_sample_minerals(old_sample, new_sample) Region.objects.all().delete() - Region.objects.bulk_create([Region(name=region) - for region in set(all_regions)]) + Region.objects.bulk_create([Region(name=region) for region in all_regions]) Collector.objects.all().delete() - Collector.objects.bulk_create([Collector(name=collector) - for collector in set(all_collectors)]) - + Collector.objects.bulk_create([Collector(name=collector) for collector in all_collectors]) + + @staticmethod + def _migrate_sample_minerals(old_sample, new_sample): + old_minerals = (LegacySampleMineral.objects.filter(sample=old_sample)) + old_mineral_names = [om.mineral.name for om in old_minerals] + new_minerals = Mineral.objects.filter(name__in=old_mineral_names) + + for mineral in new_minerals: + old_sample_mineral = (LegacySampleMineral.objects.get(mineral__name=mineral.name, sample=old_sample)) + SampleMineral.objects.create(sample=new_sample, mineral=mineral, amount=old_sample_mineral.amount) + + @staticmethod + def _get_metamorphic_regions(old_sample): + old_metamorphic_regions = (lsa.metamorphic_region.name for lsa in + LegacySampleMetamorphicRegion.objects.filter(sample=old_sample)) + return MetamorphicRegion.objects.filter(name__in=old_metamorphic_regions) + + @staticmethod + def _get_metamorphic_grades(old_sample): + old_metamorphic_grades = (lsa.metamorphic_grade.name for lsa in + LegacySampleMetamorphicGrades.objects.filter(sample=old_sample)) + return MetamorphicGrade.objects.filter(name__in=old_metamorphic_grades) + + @staticmethod + def _get_georeferences(old_sample): + references = [lsr.reference.name for lsr in LegacySampleReference.objects.filter(sample=old_sample)] + return GeoReference.objects.filter(name__in=references) + @transaction.atomic def _migrate_references(self): print("Migrating references...") old_georeferences = LegacyGeoreference.objects.all() + GeoReference.objects.all().delete() for record in old_georeferences: Reference.objects.get_or_create(name=record.reference_number) @@ -196,6 +197,53 @@ def _migrate_references(self): Reference.objects.get_or_create(name=record.name) GeoReference.objects.get_or_create(name=record.name) + @staticmethod + def _migrate_images(old_images, new_sample=None, new_subsample=None): + errors = [] + + for old_image in old_images: + old_user = LegacyUser.objects.get(pk=old_image.user_id) + new_user = User.objects.get(email=old_user.email) + new_image = Image.objects.create( + id=uuid.uuid4(), + version=old_image.version, + image=None, + image_type=ImageType.objects.get(id=old_image.image_type_id), + collector=old_image.collector, + owner=new_user, + public_data=old_image.public_data == 'Y', + sample=new_sample, + subsample=new_subsample + ) + + ImageMapping.objects.create(old_image_id=old_image.pk,new_image_id=new_image.pk) + + checksum = old_image.checksum + try: + with open('{}/{}/{}/{}'.format(BASE_DIR, checksum[0:2], checksum[2:4], checksum[4:]), 'rb') as image_file: + new_image.image.convert('RGB').save(old_image.filename, File(image_file)) + new_image.save() + except Exception as ex: + print('ERROR saving image with checksum {}. Errors logged to {}'.format(checksum, FAILED_IMAGES_FILE)) + errors.append(checksum) + + old_xray = LegacyXrayImage.objects.filter(image=old_image).first() + if old_xray: + XrayImage.objects.create( + image=new_image, + element=old_xray.element, + dwelltime=old_xray.dwelltime, + current=old_xray.current, + voltage=old_xray.voltage + ) + old_comments = LegacyImageComments.objects.filter(image=old_image) + + ImageComments.objects.bulk_create([ImageComments(image=new_image, comment_text=old_comment.comment_text, + version=old_comment.version) for old_comment in old_comments]) + + if errors: + with open(FAILED_IMAGES_FILE, 'a+', encoding='utf-8') as failures: + failures.write('{}{}'.format('\n', '\n'.join(errors))) def _migrate_subsamples(self, old_sample, new_sample): old_records = LegacySubsample.objects.filter(sample=old_sample) @@ -208,29 +256,31 @@ def _migrate_subsamples(self, old_sample, new_sample): subsample = Subsample.objects.create( name=record.name, sample=new_sample, - public_data=record.public_data, + public_data=record.public_data == 'Y', owner=new_user, subsample_type=subsample_type ) + old_images = LegacyImages.objects.filter(subsample=record.subsample_id) + self._migrate_images(old_images, new_subsample=subsample) + old_grids = LegacyGrid.objects.filter(subsample=record) if old_grids: for grid in old_grids: Grid.objects.create(subsample=subsample, width=grid.width, height=grid.height, - public_data=grid.public_data) - + public_data=grid.public_data == 'Y') @transaction.atomic def _migrate_subsample_types(self): print("Migrating legacy subsample types...") old_records = LegacySubsampleType.objects.all() + SubsampleType.objects.all().delete() for record in old_records: SubsampleType.objects.create(name=record.subsample_type) - @transaction.atomic def _migrate_rock_types(self): print("Migrating rock types...") @@ -240,7 +290,6 @@ def _migrate_rock_types(self): for record in old_records: RockType.objects.create(name=record.rock_type) - @transaction.atomic def _migrate_metamorphic_grades(self): print("Migrating metamorphic grades...") @@ -250,7 +299,6 @@ def _migrate_metamorphic_grades(self): for record in old_records: MetamorphicGrade.objects.create(name=record.name) - @transaction.atomic def _migrate_metamorphic_regions(self): print("Migrating metamorphic regions...") @@ -265,7 +313,6 @@ def _migrate_metamorphic_regions(self): label_location=record.label_location ) - @transaction.atomic def _migrate_minerals(self): print("Migrating minerals...") @@ -281,12 +328,9 @@ def _migrate_minerals(self): for record in old_records: mineral = Mineral.objects.get(name=record.name) - mineral.real_mineral = (Mineral - .objects - .get(name=record.real_mineral.name)) + mineral.real_mineral = Mineral.objects.get(name=record.real_mineral.name) mineral.save() - @transaction.atomic def _migrate_mineral_relationships(self): print("Migrating mineral relationships...") @@ -295,22 +339,27 @@ def _migrate_mineral_relationships(self): for record in old_records: MineralRelationship.objects.create( - parent_mineral=Mineral.objects.get( - name=record.parent_mineral.name - ), - child_mineral=Mineral.objects.get( - name=record.child_mineral.name - ) + parent_mineral=Mineral.objects.get(name=record.parent_mineral.name), + child_mineral=Mineral.objects.get(name=record.child_mineral.name) + ) + + @transaction.atomic + def _migrate_image_types(self): + print('Migrating image types...') + old_records = LegacyImageTypes.objects.all() + ImageType.objects.all().delete() + + for record in old_records: + ImageType.objects.create( + id=record.image_type_id, + image_type=record.image_type, + abbreviation=record.abbreviation, + comments=record.comments ) @transaction.atomic def _migrate_countries(self): print("Migrating country names...") - country_names = (Sample - .objects - .all() - .values_list('country', flat=True) - .distinct()) - country_names = [name for name in country_names if name is not None] - Country.objects.bulk_create([Country(name=name) - for name in country_names]) + Country.objects.all().delete() + country_names = set(filter(lambda x: x, Sample.objects.all().values_list('country', flat=True).distinct())) + Country.objects.bulk_create([Country(name=name) for name in country_names]) diff --git a/metpetdb_api/apps/samples/migrations/0003_auto_20180220_0409.py b/metpetdb_api/apps/samples/migrations/0003_auto_20180220_0409.py new file mode 100644 index 0000000..761b62e --- /dev/null +++ b/metpetdb_api/apps/samples/migrations/0003_auto_20180220_0409.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-02-20 04:09 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('samples', '0002_auto_20170425_1902'), + ] + + operations = [ + migrations.AlterModelOptions( + name='collector', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='georeference', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='metamorphicgrade', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='metamorphicregion', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='mineral', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='reference', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='region', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='rocktype', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='sample', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='subsample', + options={'ordering': ['id']}, + ), + migrations.AlterModelOptions( + name='subsampletype', + options={'ordering': ['id']}, + ), + ] diff --git a/metpetdb_api/apps/samples/models.py b/metpetdb_api/apps/samples/models.py index ebc8c24..413ec70 100644 --- a/metpetdb_api/apps/samples/models.py +++ b/metpetdb_api/apps/samples/models.py @@ -4,7 +4,8 @@ from django.conf import settings from django.contrib.gis.db import models from django.contrib.postgres.fields import ArrayField -from apps.chemical_analyses.models import Element, Oxide +from apps.chemical_analyses.shared_models import Element, Oxide + class BulkUpload(models.Model): pass @@ -15,6 +16,7 @@ class RockType(models.Model): class Meta: db_table = 'rock_types' + ordering = ['name'] class Sample(models.Model): @@ -26,7 +28,7 @@ class Sample(models.Model): aliases = ArrayField(models.CharField(max_length=35, blank=True), blank=True, null=True) - collection_date = models.DateTimeField(blank=True, null=True) + collection_date = models.DateField(blank=True, null=True) description = models.TextField(blank=True, null=True) location_name = models.CharField(max_length=50, blank=True, null=True) location_coords = models.PointField() @@ -64,6 +66,8 @@ class Sample(models.Model): class Meta: db_table = 'samples' + unique_together = (('number','owner')) + ordering = ['number'] class SubsampleType(models.Model): @@ -72,6 +76,7 @@ class SubsampleType(models.Model): class Meta: db_table = 'subsample_types' + ordering = ['name'] class Subsample(models.Model): @@ -85,6 +90,8 @@ class Subsample(models.Model): class Meta: db_table = 'subsamples' + unique_together = (('name','sample','owner')) + ordering = ['id'] class Grid(models.Model): @@ -105,6 +112,7 @@ class MetamorphicGrade(models.Model): class Meta: db_table = 'metamorphic_grades' + ordering = ['name'] class MetamorphicRegion(models.Model): @@ -116,6 +124,7 @@ class MetamorphicRegion(models.Model): class Meta: db_table = 'metamorphic_regions' + ordering = ['name'] class Mineral(models.Model): @@ -130,6 +139,7 @@ class Mineral(models.Model): class Meta: db_table = 'minerals' + ordering = ['name'] class SampleMineral(models.Model): @@ -179,6 +189,7 @@ class GeoReference(models.Model): class Meta: db_table = 'georeferences' + ordering = ['name'] # Following are models for easy retrieval of sample-related free-text fields @@ -200,6 +211,7 @@ class Country(models.Model): class Meta: db_table = 'countries' + ordering = ['name'] class Region(models.Model): @@ -208,6 +220,7 @@ class Region(models.Model): class Meta: db_table = 'regions' + ordering = ['name'] class Reference(models.Model): @@ -216,6 +229,7 @@ class Reference(models.Model): class Meta: db_table = 'references' + ordering = ['name'] class Collector(models.Model): @@ -224,7 +238,7 @@ class Collector(models.Model): class Meta: db_table = 'collectors' - + ordering = ['name'] # A mapping table to help the migration of old samples to new samples; can # be gotten rid of once thi app goes into production. diff --git a/metpetdb_api/apps/users/models.py b/metpetdb_api/apps/users/models.py index b477b4a..9114759 100644 --- a/metpetdb_api/apps/users/models.py +++ b/metpetdb_api/apps/users/models.py @@ -1,12 +1,12 @@ -from datetime import datetime import uuid +from datetime import datetime -from django.db import models from django.contrib.auth.models import ( AbstractBaseUser, PermissionsMixin, BaseUserManager, ) +from django.db import models from rest_framework.authtoken.models import Token @@ -76,6 +76,7 @@ class User(AbstractBaseUser, PermissionsMixin): class Meta: db_table = 'users' + ordering = ['name'] def get_full_name(self): """ diff --git a/metpetdb_api/legacy/models.py b/metpetdb_api/legacy/models.py index b67678f..3894107 100644 --- a/metpetdb_api/legacy/models.py +++ b/metpetdb_api/legacy/models.py @@ -136,6 +136,7 @@ class Meta: db_table = 'sample_minerals' unique_together = (('mineral', 'sample'),) + class Reference(models.Model): reference_id = models.BigIntegerField(primary_key=True) name = models.CharField(unique=True, max_length=100) @@ -144,6 +145,7 @@ class Meta: managed = False db_table = 'reference' + class SampleReference(models.Model): sample = models.ForeignKey('Samples') reference = models.ForeignKey(Reference) @@ -162,6 +164,42 @@ class Meta: managed = False db_table = 'regions' + +# class ImageTypes(models.Model): +# image_type_id = models.SmallIntegerField(primary_key=True) +# image_type = models.CharField(null=False, max_length=100) +# abbreviation = models.CharField(max_length=10) +# comments = models.CharField(max_length=250) +# +# class Meta: +# managed = False +# db_table = 'image_type' +# +# +# class Images(models.Model): +# image_id = models.BigIntegerField(primary_key=True) +# checksum = models.CharField(max_length=50, null=False) +# version = models.IntegerField(null=False) +# sample_id = models.BigIntegerField +# subsample_id = models.BigIntegerField +# image_format_id = models.SmallIntegerField +# image_type_id = models.SmallIntegerField(null=False) +# width = models.SmallIntegerField(null=False) +# height = models.SmallIntegerField(null=False) +# collector = models.CharField(max_length=50) +# description = models.CharField(max_length=1024) +# scale = models.SmallIntegerField +# user_id = models.IntegerField(null=False) +# public_data = models.CharField(max_length=1, null=False) +# checksum_64x64 = models.CharField(max_length=50, null=False) +# checksum_half = models.CharField(max_length=50, null=False) +# filename = models.CharField(max_length=256, null=False) +# checksum_mobile = models.CharField(max_length=50) +# +# class Meta: +# managed = False +# db_table = 'images' + class SampleRegions(models.Model): sample = models.ForeignKey('Samples') region = models.ForeignKey(Regions) @@ -240,7 +278,7 @@ class ChemicalAnalyses(models.Model): reference_y = models.FloatField(blank=True, null=True) stage_x = models.FloatField(blank=True, null=True) stage_y = models.FloatField(blank=True, null=True) - # image = models.ForeignKey('Images', blank=True, null=True) + image = models.ForeignKey('Images', blank=True, null=True) analysis_method = models.CharField(max_length=50, blank=True, null=True) where_done = models.CharField(max_length=50, blank=True, null=True) analyst = models.CharField(max_length=50, blank=True, null=True) @@ -331,98 +369,94 @@ class Georeference(models.Model): class Meta: managed = False db_table = 'georeference' -# -# -# -# -# class ImageComments(models.Model): -# comment_id = models.BigIntegerField(primary_key=True) -# image = models.ForeignKey('Images') -# comment_text = models.TextField() -# version = models.IntegerField() -# -# class Meta: -# managed = False -# db_table = 'image_comments' -# -# -# class ImageFormat(models.Model): -# image_format_id = models.SmallIntegerField(primary_key=True) -# name = models.CharField(unique=True, max_length=100) -# -# class Meta: -# managed = False -# db_table = 'image_format' -# -# -# class ImageOnGrid(models.Model): -# image_on_grid_id = models.BigIntegerField(primary_key=True) -# grid = models.ForeignKey(Grids) -# image = models.ForeignKey('Images') -# top_left_x = models.FloatField() -# top_left_y = models.FloatField() -# z_order = models.SmallIntegerField() -# opacity = models.SmallIntegerField() -# resize_ratio = models.FloatField() -# width = models.SmallIntegerField() -# height = models.SmallIntegerField() -# checksum = models.CharField(max_length=50) -# checksum_64x64 = models.CharField(max_length=50) -# checksum_half = models.CharField(max_length=50) -# locked = models.CharField(max_length=1) -# angle = models.FloatField(blank=True, null=True) -# -# class Meta: -# managed = False -# db_table = 'image_on_grid' -# -# -# class ImageReference(models.Model): -# image = models.ForeignKey('Images') -# reference = models.ForeignKey('Reference') -# -# class Meta: -# managed = False -# db_table = 'image_reference' -# unique_together = (('image_id', 'reference_id'),) -# -# -# class ImageType(models.Model): -# image_type_id = models.SmallIntegerField(primary_key=True) -# image_type = models.CharField(unique=True, max_length=100) -# abbreviation = models.CharField(unique=True, max_length=10, blank=True, null=True) -# comments = models.CharField(max_length=250, blank=True, null=True) -# -# class Meta: -# managed = False -# db_table = 'image_type' -# -# -# class Images(models.Model): -# image_id = models.BigIntegerField(primary_key=True) -# checksum = models.CharField(max_length=50) -# version = models.IntegerField() -# sample = models.ForeignKey('Samples', blank=True, null=True) -# subsample = models.ForeignKey('Subsamples', blank=True, null=True) -# image_format = models.ForeignKey(ImageFormat, blank=True, null=True) -# image_type = models.ForeignKey(ImageType) -# width = models.SmallIntegerField() -# height = models.SmallIntegerField() -# collector = models.CharField(max_length=50, blank=True, null=True) -# description = models.CharField(max_length=1024, blank=True, null=True) -# scale = models.SmallIntegerField(blank=True, null=True) -# user = models.ForeignKey('Users') -# public_data = models.CharField(max_length=1) -# checksum_64x64 = models.CharField(max_length=50) -# checksum_half = models.CharField(max_length=50) -# filename = models.CharField(max_length=256) -# checksum_mobile = models.CharField(max_length=50, blank=True, null=True) -# -# class Meta: -# managed = False -# db_table = 'images' -# unique_together = (('sample_id', 'None'), ('subsample_id', 'None'),) -# + + +class ImageComments(models.Model): + comment_id = models.BigIntegerField(primary_key=True) + image = models.ForeignKey('Images') + comment_text = models.TextField() + version = models.IntegerField() + + class Meta: + managed = False + db_table = 'image_comments' + + +class ImageFormat(models.Model): + image_format_id = models.SmallIntegerField(primary_key=True) + name = models.CharField(unique=True, max_length=100) + + class Meta: + managed = False + db_table = 'image_format' + + +class ImageOnGrid(models.Model): + image_on_grid_id = models.BigIntegerField(primary_key=True) + grid = models.ForeignKey(Grids) + image = models.ForeignKey('Images') + top_left_x = models.FloatField() + top_left_y = models.FloatField() + z_order = models.SmallIntegerField() + opacity = models.SmallIntegerField() + resize_ratio = models.FloatField() + width = models.SmallIntegerField() + height = models.SmallIntegerField() + checksum = models.CharField(max_length=50) + checksum_64x64 = models.CharField(max_length=50) + checksum_half = models.CharField(max_length=50) + locked = models.CharField(max_length=1) + angle = models.FloatField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'image_on_grid' + + +class ImageReference(models.Model): + image = models.ForeignKey('Images') + reference = models.ForeignKey('Reference') + + class Meta: + managed = False + db_table = 'image_reference' + + +class ImageType(models.Model): + image_type_id = models.SmallIntegerField(primary_key=True) + image_type = models.CharField(unique=True, max_length=100) + abbreviation = models.CharField(unique=True, max_length=10, blank=True, null=True) + comments = models.CharField(max_length=250, blank=True, null=True) + + class Meta: + managed = False + db_table = 'image_type' + + +class Images(models.Model): + image_id = models.BigIntegerField(primary_key=True) + checksum = models.CharField(max_length=50) + version = models.IntegerField() + sample = models.ForeignKey('Samples', blank=True, null=True) + subsample = models.ForeignKey('Subsamples', blank=True, null=True) + image_format = models.ForeignKey(ImageFormat, blank=True, null=True) + image_type = models.ForeignKey(ImageType) + width = models.SmallIntegerField() + height = models.SmallIntegerField() + collector = models.CharField(max_length=50, blank=True, null=True) + description = models.CharField(max_length=1024, blank=True, null=True) + scale = models.SmallIntegerField(blank=True, null=True) + user = models.ForeignKey('Users') + public_data = models.CharField(max_length=1) + checksum_64x64 = models.CharField(max_length=50) + checksum_half = models.CharField(max_length=50) + filename = models.CharField(max_length=256) + checksum_mobile = models.CharField(max_length=50, blank=True, null=True) + + class Meta: + managed = False + db_table = 'images' + # # # @@ -595,13 +629,14 @@ class Meta: # db_table = 'users_roles' # # -# class XrayImage(models.Model): -# image = models.ForeignKey(Images, primary_key=True) -# element = models.CharField(max_length=256, blank=True, null=True) -# dwelltime = models.SmallIntegerField(blank=True, null=True) -# current = models.SmallIntegerField(blank=True, null=True) -# voltage = models.SmallIntegerField(blank=True, null=True) -# -# class Meta: -# managed = False -# db_table = 'xray_image' + +class XrayImage(models.Model): + image = models.OneToOneField(Images, primary_key=True) + element = models.CharField(max_length=256, blank=True, null=True) + dwelltime = models.SmallIntegerField(blank=True, null=True) + current = models.SmallIntegerField(blank=True, null=True) + voltage = models.SmallIntegerField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'xray_image' diff --git a/metpetdb_api/manage.py b/metpetdb_api/manage.py index 939380d..c4b654e 100755 --- a/metpetdb_api/manage.py +++ b/metpetdb_api/manage.py @@ -8,7 +8,7 @@ if __name__ == "__main__": PROJECT_ROOT = os.path.dirname(__file__) sys.path.insert(-1, os.path.join(PROJECT_ROOT, "vendor/djoser")) - + dotenv.read_dotenv('api.env') os.environ.setdefault("DJANGO_SETTINGS_MODULE", env('API_SETTINGS')) diff --git a/metpetdb_api/metpetdb_api/urls.py b/metpetdb_api/metpetdb_api/urls.py index d791ec3..da0b354 100644 --- a/metpetdb_api/metpetdb_api/urls.py +++ b/metpetdb_api/metpetdb_api/urls.py @@ -45,8 +45,13 @@ ) from api.users.v1.views import UserViewSet +from api.images.v1.views import ImageContainerViewSet, ImageViewSet, ImageTypeViewSet + from api.bulk_upload.v1.views import BulkUploadViewSet +from django.conf import settings +from django.conf.urls.static import static + router = routers.DefaultRouter() router.register(r'users', UserViewSet) @@ -65,15 +70,19 @@ router.register(r'references', ReferenceViewSet) router.register(r'collectors', CollectorViewSet) router.register(r'bulk_upload', BulkUploadViewSet) +router.register(r'image_sets', ImageContainerViewSet) +router.register(r'images', ImageViewSet) +router.register(r'image_types', ImageTypeViewSet) urlpatterns = [ url(r'^api/', include(router.urls)), url(r'^api/admin/', include(admin.site.urls)), + url(r'^api/auth/', include('djoser.urls')), url(r'^api/auth/', include('djoser.urls.authtoken')), url(r'^api/sample_numbers/$', SampleNumbersView.as_view()), url(r'^api/country_names/$', CountryNamesView.as_view()), url(r'^api/sample_owner_names/$', SampleOwnerNamesView.as_view()), -] +] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/metpetdb_api/requirements/dev.txt b/metpetdb_api/requirements/dev.txt index 3e087fe..bd0b633 100644 --- a/metpetdb_api/requirements/dev.txt +++ b/metpetdb_api/requirements/dev.txt @@ -1,13 +1,15 @@ -Django>=1.8.4,<=1.9 +Django>=1.11.0,<=1.11.10 django-concurrency>=0.9 django-debug-toolbar>=1.3.2,<=1.4 django-dotenv>=1.3.0 django-extensions>=1.5.5,<=2.0 django-getenv>=1.3.1 -djangorestframework>=3.2,<=3.3 djangorestframework-csv>=2.1 -psycopg2>=2.6.1 +djangorestframework>=3.7,<=3.8 +psycopg2-binary>=2.6.1 six>=1.9.0 sqlparse>=0.1.15 wheel>=0.24.0 pathlib +django-versatileimagefield +xlrd diff --git a/metpetdb_api/requirements/staging.txt b/metpetdb_api/requirements/staging.txt index e751e6d..db55d4d 100644 --- a/metpetdb_api/requirements/staging.txt +++ b/metpetdb_api/requirements/staging.txt @@ -1,13 +1,15 @@ -Django>=1.8.4,<=1.9 +Django>=1.11.0,<=1.11.10 django-concurrency>=0.9 django-dotenv>=1.3.0 django-extensions>=1.5.5,<=2.0 django-getenv>=1.3.1 -djangorestframework>=3.2,<=3.3 djangorestframework-csv>=2.1 +djangorestframework>=3.7,<=3.8 gunicorn>=19.3.0 -psycopg2>=2.6.1 +psycopg2-binary>=2.6.1 six>=1.9.0 sqlparse>=0.1.15 wheel>=0.24.0 pathlib==1.0.1 +django-versatileimagefield +xlrd diff --git a/metpetdb_api/settings/staging.py b/metpetdb_api/settings/staging.py index fd99421..42d30f0 100644 --- a/metpetdb_api/settings/staging.py +++ b/metpetdb_api/settings/staging.py @@ -36,7 +36,7 @@ EMAIL_PORT = env('EMAIL_PORT') -ALLOWED_HOSTS = [] +ALLOWED_HOSTS = ['165.227.94.236','api.metpetdb.com','api.metpetdb.org'] AUTH_USER_MODEL = 'users.User' @@ -62,6 +62,8 @@ 'apps.samples', 'apps.users', 'apps.core', + 'versatileimagefield', + 'apps.images', 'rest_framework_csv', ) @@ -177,8 +179,64 @@ STATIC_ROOT = os.path.join(BASE_DIR, "static/") DJOSER = { - 'DOMAIN': env('FRONT_END_URL'), 'PASSWORD_RESET_CONFIRM_URL': 'reset-password#/{uid}/{token}', 'ACTIVATION_URL': 'login#/activate/{uid}/{token}', 'SEND_ACTIVATION_EMAIL' : True } + +DOMAIN = env('FRONT_END_URL') +SITE_NAME = env('FRONT_END_SITE_NAME') + +# A dictionary that allows you to fine-tune how django-versatileimagefield works: +VERSATILEIMAGEFIELD_SETTINGS = { + # The amount of time, in seconds, that references to created images + # should be stored in the cache. Defaults to `2592000` (30 days) + 'cache_length': 2592000, + # The name of the cache you'd like `django-versatileimagefield` to use. + # Defaults to 'versatileimagefield_cache'. If no cache exists with the name + # provided, the 'default' cache will be used instead. + 'cache_name': 'versatileimagefield_cache', + # The save quality of modified JPEG images. More info here: + # https://pillow.readthedocs.io/en/latest/handbook/image-file-formats.html#jpeg + # Defaults to 70 + 'jpeg_resize_quality': 70, + # The name of the top-level folder within storage classes to save all + # sized images. Defaults to '__sized__' + 'sized_directory_name': '__sized__', + # The name of the directory to save all filtered images within. + # Defaults to '__filtered__': + 'filtered_directory_name': '__filtered__', + # The name of the directory to save placeholder images within. + # Defaults to '__placeholder__': + 'placeholder_directory_name': '__placeholder__', + # Whether or not to create new images on-the-fly. Set this to `False` for + # speedy performance but don't forget to 'pre-warm' to ensure they're + # created and available at the appropriate URL. + 'create_images_on_demand': False, + # A dot-notated python path string to a function that processes sized + # image keys. Typically used to md5-ify the 'image key' portion of the + # filename, giving each a uniform length. + # `django-versatileimagefield` ships with two post processors: + # 1. 'versatileimagefield.processors.md5' Returns a full length (32 char) + # md5 hash of `image_key`. + # 2. 'versatileimagefield.processors.md5_16' Returns the first 16 chars + # of the 32 character md5 hash of `image_key`. + # By default, image_keys are unprocessed. To write your own processor, + # just define a function (that can be imported from your project's + # python path) that takes a single argument, `image_key` and returns + # a string. + 'image_key_post_processor': None, + # Whether to create progressive JPEGs. Read more about progressive JPEGs + # here: https://optimus.io/support/progressive-jpeg/ + 'progressive_jpeg': False +} + +VERSATILEIMAGEFIELD_RENDITION_KEY_SETS = { + 'image_sizes': [ + ('full_size', 'url'), + ('thumbnail', 'thumbnail__100x100') + ] +} + +MEDIA_ROOT = os.path.join(BASE_DIR, 'images/') +MEDIA_URL = '/api/images/' diff --git a/metpetdb_api/vendor/djoser b/metpetdb_api/vendor/djoser index 0e3b5d6..eb26c90 160000 --- a/metpetdb_api/vendor/djoser +++ b/metpetdb_api/vendor/djoser @@ -1 +1 @@ -Subproject commit 0e3b5d62d88cec8c6d949785dce487753d8edf59 +Subproject commit eb26c909125d5b1c32773a62d1830c40684430c3