diff --git a/components/tools/OmeroPy/src/omero/gateway/__init__.py b/components/tools/OmeroPy/src/omero/gateway/__init__.py index a5bf6085c76..beaa744421d 100644 --- a/components/tools/OmeroPy/src/omero/gateway/__init__.py +++ b/components/tools/OmeroPy/src/omero/gateway/__init__.py @@ -3020,7 +3020,7 @@ def getObject(self, obj_type, oid=None, params=None, attributes=None, If more than one object found, raises ome.conditions.ApiUsageException See :meth:`getObjects` for more info. - :param obj_type: Object type. E.g. "Project" see above + :param obj_type: Object type, e.g. "Project" see above :type obj_type: String :param ids: object IDs :type ids: List of Long @@ -3048,7 +3048,7 @@ def getObjects(self, obj_type, ids=None, params=None, attributes=None, be returned. i.e. listObjects() Filter objects by attributes. E.g. attributes={'name':name} - :param obj_type: Object type. E.g. "Project" see above + :param obj_type: Object type, e.g. "Project" see above :type obj_type: String :param ids: object IDs :type ids: List of Long @@ -3087,7 +3087,7 @@ def buildQuery(self, obj_type, ids=None, params=None, attributes=None, can be used with the appropriate query method. Used by :meth:`getObjects` and :meth:`getObject` above. - :param obj_type: Object type. E.g. "Project" see above + :param obj_type: Object type, e.g. "Project" see above :type obj_type: String :param ids: object IDs :type ids: List of Long @@ -3181,6 +3181,36 @@ def buildQuery(self, obj_type, ids=None, params=None, attributes=None, return (query, baseParams, wrapper) + def buildCountQuery(self, obj_type, opts=None): + """ + Prepares a 'projection' query to count objects. + + Based on buildQuery(), we modify the query to only return a count. + Modified query does not 'fetch' any data or add any other + unnecessary objects to query. + We return just the query and omero.sys.ParametersI for the query. + + :param obj_type: Object type, e.g. "Project" see above + :param opts: Dict of options for filtering by + offset, limit and owner for all objects. + Additional opts handled by _getQueryString() + e.g. filter Dataset by 'project' + :return: (query, params) + """ + # We disable pagination since we want to count ALL results + opts_copy = opts.copy() + if 'limit' in opts_copy: + del opts_copy['limit'] + + # Get query with other options + query, params, wrapper = self.buildQuery(obj_type, opts=opts_copy) + + # Modify query to only select count() + query = query.replace("select obj ", "select count(distinct obj) ") + query = query.replace("fetch", "") + query = query.split("order by")[0] + return query, params + def listFileAnnotations(self, eid=None, toInclude=[], toExclude=[]): """ Lists FileAnnotations created by users, filtering by namespaces if @@ -3222,7 +3252,7 @@ def getAnnotationLinks(self, parent_type, parent_ids=None, ann_ids=None, If parent_ids is None, all available objects will be returned. i.e. listObjects() - :param obj_type: Object type. E.g. "Project" see above + :param obj_type: Object type, e.g. "Project" see above :type obj_type: String :param ids: object IDs :type ids: List of Long diff --git a/components/tools/OmeroWeb/omeroweb/api/api_query.py b/components/tools/OmeroWeb/omeroweb/api/api_query.py index 5e22d326496..dd08b8fe47a 100644 --- a/components/tools/OmeroWeb/omeroweb/api/api_query.py +++ b/components/tools/OmeroWeb/omeroweb/api/api_query.py @@ -22,11 +22,16 @@ from omero.rtypes import unwrap, wrap from omero.sys import ParametersI +from . import api_settings from api_marshal import marshal_objects from copy import deepcopy +MAX_LIMIT = max(1, api_settings.API_MAX_LIMIT) +DEFAULT_LIMIT = max(1, api_settings.API_LIMIT) + + def get_child_counts(conn, link_class, parent_ids): """ Count child links for the specified parent_ids. @@ -50,6 +55,18 @@ def get_child_counts(conn, link_class, parent_ids): return counts +def validate_opts(opts): + """Check that opts dict has valid 'limit' and 'offset'.""" + if opts is None: + opts = {} + if opts.get('limit') is None or opts.get('limit') < 0: + opts['limit'] = DEFAULT_LIMIT + opts['limit'] = min(opts['limit'], MAX_LIMIT) + if opts.get('offset') is None or opts.get('offset') < 0: + opts['offset'] = 0 + return opts + + def query_objects(conn, object_type, group=None, opts=None, @@ -66,6 +83,7 @@ def query_objects(conn, object_type, @param opts: Options dict for conn.buildQuery() @param normalize: If true, marshal groups and experimenters separately """ + opts = validate_opts(opts) # buildQuery is used by conn.getObjects() query, params, wrapper = conn.buildQuery(object_type, opts=opts) # Set the desired group context @@ -79,7 +97,10 @@ def query_objects(conn, object_type, objects = [] extras = {} - result = qs.findAllByQuery(query, params, ctx) + if opts['limit'] == 0: + result = [] + else: + result = qs.findAllByQuery(query, params, ctx) for obj in result: objects.append(obj) @@ -91,4 +112,16 @@ def query_objects(conn, object_type, count = counts[obj_id] if obj_id in counts else 0 extras[obj_id] = {'omero:childCount': count} - return marshal_objects(objects, extras=extras, normalize=normalize) + # Query the count() of objects & add to 'meta' dict + count_query, params = conn.buildCountQuery(object_type, opts=opts) + result = qs.projection(count_query, params, ctx) + + meta = {} + meta['offset'] = opts['offset'] + meta['limit'] = opts['limit'] + meta['maxLimit'] = MAX_LIMIT + meta['totalCount'] = result[0][0].val + + marshalled = marshal_objects(objects, extras=extras, normalize=normalize) + marshalled['meta'] = meta + return marshalled diff --git a/components/tools/OmeroWeb/omeroweb/api/api_settings.py b/components/tools/OmeroWeb/omeroweb/api/api_settings.py new file mode 100644 index 00000000000..3c8ec797b1f --- /dev/null +++ b/components/tools/OmeroWeb/omeroweb/api/api_settings.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (C) 2017 University of Dundee & Open Microscopy Environment. +# All rights reserved. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Settings for the OMERO JSON api app.""" + +import sys +from omeroweb.settings import process_custom_settings, report_settings, \ + str_slash + +# load settings +API_SETTINGS_MAPPING = { + + "omero.web.api.limit": + ["API_LIMIT", + 200, + int, + "Default number of items returned from json api."], + "omero.web.api.max_limit": + ["API_MAX_LIMIT", + 500, + int, + "Maximum number of items returned from json api."], + "omero.web.api.absolute_url": + ["API_ABSOLUTE_URL", + None, + str_slash, + ("URL to use for generating urls within API json responses. " + "By default this is None, and we use Django's " + "request.build_absolute_uri() to generate absolute urls " + "based on each request. If set to a string or empty string, " + "this will be used as prefix to relative urls.")], +} + +process_custom_settings(sys.modules[__name__], 'API_SETTINGS_MAPPING') +report_settings(sys.modules[__name__]) + +# For any given release of api, we may support +# one or more versions of the api. +# E.g. /api/v0/ +# TODO - need to decide how this is configured, strategy for extending etc. +API_VERSIONS = ('0',) diff --git a/components/tools/OmeroWeb/omeroweb/api/urls.py b/components/tools/OmeroWeb/omeroweb/api/urls.py index f320700b2bd..fe04b93a5cb 100644 --- a/components/tools/OmeroWeb/omeroweb/api/urls.py +++ b/components/tools/OmeroWeb/omeroweb/api/urls.py @@ -22,11 +22,11 @@ from django.conf.urls import url, patterns from omeroweb.api import views from omeroweb.webgateway.views import LoginView -from django.conf import settings +from . import api_settings import re versions = '|'.join([re.escape(v) - for v in settings.API_VERSIONS]) + for v in api_settings.API_VERSIONS]) api_versions = url(r'^$', views.api_versions, name='api_versions') diff --git a/components/tools/OmeroWeb/omeroweb/api/views.py b/components/tools/OmeroWeb/omeroweb/api/views.py index 31df70e2567..4c7add6389e 100644 --- a/components/tools/OmeroWeb/omeroweb/api/views.py +++ b/components/tools/OmeroWeb/omeroweb/api/views.py @@ -23,7 +23,7 @@ from django.middleware import csrf from django.utils.decorators import method_decorator from django.core.urlresolvers import reverse -from django.conf import settings +from . import api_settings import traceback import json @@ -54,11 +54,11 @@ def build_url(request, name, api_version, **kwargs): """ kwargs['api_version'] = api_version url = reverse(name, kwargs=kwargs) - if settings.API_ABSOLUTE_URL is None: + if api_settings.API_ABSOLUTE_URL is None: return request.build_absolute_uri(url) else: # remove trailing slash - prefix = settings.API_ABSOLUTE_URL.rstrip('/') + prefix = api_settings.API_ABSOLUTE_URL.rstrip('/') return "%s%s" % (prefix, url) @@ -66,7 +66,7 @@ def build_url(request, name, api_version, **kwargs): def api_versions(request, **kwargs): """Base url of the webgateway json api.""" versions = [] - for v in settings.API_VERSIONS: + for v in api_settings.API_VERSIONS: versions.append({ 'version': v, 'url:base': build_url(request, 'api_base', v) @@ -304,8 +304,8 @@ class ObjectsView(ApiView): def get_opts(self, request, **kwargs): """Return an options dict based on request parameters.""" try: - page = getIntOrDefault(request, 'page', 1) - limit = getIntOrDefault(request, 'limit', settings.PAGE) + offset = getIntOrDefault(request, 'offset', 0) + limit = getIntOrDefault(request, 'limit', None) owner = getIntOrDefault(request, 'owner', None) child_count = request.GET.get('childCount', False) == 'true' orphaned = request.GET.get('orphaned', False) == 'true' @@ -313,7 +313,7 @@ def get_opts(self, request, **kwargs): raise BadRequestError(str(ex)) # orphaned and child_count not used by every subclass - opts = {'offset': (page - 1) * limit, + opts = {'offset': offset, 'limit': limit, 'owner': owner, 'orphaned': orphaned, diff --git a/components/tools/OmeroWeb/omeroweb/settings.py b/components/tools/OmeroWeb/omeroweb/settings.py index 98f1baff576..235a47fae4a 100644 --- a/components/tools/OmeroWeb/omeroweb/settings.py +++ b/components/tools/OmeroWeb/omeroweb/settings.py @@ -474,15 +474,6 @@ def leave_none_unset_int(s): ("Workers silent for more than this many seconds are killed " "and restarted. Check Gunicorn Documentation " "http://docs.gunicorn.org/en/stable/settings.html#timeout")], - "omero.web.api.absolute_url": - ["API_ABSOLUTE_URL", - None, - str_slash, - ("URL to use for generating urls within API json responses. " - "By default this is None, and we use Django's " - "request.build_absolute_uri() to generate absolute urls " - "based on each request. If set to a string or empty string, " - "this will be used as prefix to relative urls.")], # Public user "omero.web.public.enabled": @@ -1075,6 +1066,7 @@ def report_settings(module): 'omeroweb.webclient', 'omeroweb.webgateway', 'omeroweb.webredirect', + 'omeroweb.api', 'pipeline', ) @@ -1155,12 +1147,6 @@ def report_settings(module): # FEEDBACK_APP: 6 = OMERO.web FEEDBACK_APP = 6 -# For any given release of api, we may support -# one or more versions of the api. -# E.g. /api/v1.0/ -# TODO - need to decide how this is configured, strategy for extending etc. -API_VERSIONS = ('0',) - # IGNORABLE_404_STARTS: # Default: ('/cgi-bin/', '/_vti_bin', '/_vti_inf') # IGNORABLE_404_ENDS: diff --git a/components/tools/OmeroWeb/omeroweb/webgateway/util.py b/components/tools/OmeroWeb/omeroweb/webgateway/util.py index ec006cd9486..953e42dd84b 100644 --- a/components/tools/OmeroWeb/omeroweb/webgateway/util.py +++ b/components/tools/OmeroWeb/omeroweb/webgateway/util.py @@ -28,12 +28,9 @@ # helper method def getIntOrDefault(request, name, default): - try: - index = request.GET.get(name, request.POST.get(name, default)) - if index is not None: - index = int(index) - except ValueError: - index = 0 + index = request.GET.get(name, request.POST.get(name, default)) + if index is not None: + index = int(index) return index diff --git a/components/tools/OmeroWeb/test/integration/test_api_containers.py b/components/tools/OmeroWeb/test/integration/test_api_containers.py index 90555ebffa5..4b5c6fef5d1 100644 --- a/components/tools/OmeroWeb/test/integration/test_api_containers.py +++ b/components/tools/OmeroWeb/test/integration/test_api_containers.py @@ -22,7 +22,7 @@ from omeroweb.testlib import IWebTest, _get_response_json, \ _csrf_post_json, _csrf_put_json, _csrf_delete_response_json from django.core.urlresolvers import reverse -from django.conf import settings +from omeroweb.api import api_settings import pytest from test_api_projects import cmp_name_insensitive, get_update_service, \ get_connection, marshal_objects @@ -50,7 +50,7 @@ def build_url(client, url_name, url_kwargs): def add_image_urls(expected, client): """Add urls to expected Images within Well dict.""" - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] if 'WellSamples' in expected: for ws in expected['WellSamples']: image_id = ws['Image']['@id'] @@ -188,7 +188,7 @@ def test_create_update_unsupported(self, user1, dtype, method): conn = get_connection(user1) user_name = conn.getUser().getName() django_client = self.new_django_client(user_name, user_name) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] save_url = reverse('api_save', kwargs={'api_version': version}) payload = {'Name': 'test', '@type': OME_SCHEMA_URL + '#%s' % dtype} @@ -203,7 +203,7 @@ def test_delete_unsupported(self, user1, dtype): conn = get_connection(user1) user_name = conn.getUser().getName() django_client = self.new_django_client(user_name, user_name) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] # Delete (fake url - image doesn't need to exist for test) url_name = 'api_%s' % dtype.lower() delete_url = reverse(url_name, kwargs={'api_version': version, @@ -225,7 +225,7 @@ def test_container_crud(self, dtype): """ django_client = self.django_root_client group = self.ctx.groupId - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] # Need to get the Schema url to create @type base_url = reverse('api_base', kwargs={'api_version': version}) rsp = _get_response_json(django_client, base_url, {}) @@ -272,7 +272,7 @@ def test_datasets_plates(self, user1, dtype, child_count, conn = get_connection(user1) user_name = conn.getUser().getName() django_client = self.new_django_client(user_name, user_name) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] # Handle parametrized dtype, setting up other variables if dtype == 'Dataset': @@ -320,12 +320,20 @@ def test_datasets_plates(self, user1, dtype, child_count, # List ALL Datasets or Plates rsp = _get_response_json(django_client, request_url, payload) assert len(rsp['data']) == 6 + assert rsp['meta'] == {'totalCount': 6, + 'limit': api_settings.API_LIMIT, + 'maxLimit': api_settings.API_MAX_LIMIT, + 'offset': 0} # Filter Datasets or Plates by Orphaned payload = {'orphaned': 'true', 'childCount': str(child_count).lower()} rsp = _get_response_json(django_client, request_url, payload) assert_objects(conn, rsp['data'], [orphaned], dtype=dtype, extra=orph_ds_pl_children) + assert rsp['meta'] == {'totalCount': 1, + 'limit': api_settings.API_LIMIT, + 'maxLimit': api_settings.API_MAX_LIMIT, + 'offset': 0} # Filter Datasets by Project or Plates by Screen children.sort(cmp_name_insensitive) @@ -335,6 +343,10 @@ def test_datasets_plates(self, user1, dtype, child_count, assert len(rsp['data']) == 5 assert_objects(conn, rsp['data'], children, dtype=dtype, extra=ds_or_pl_children) + assert rsp['meta'] == {'totalCount': 5, + 'limit': api_settings.API_LIMIT, + 'maxLimit': api_settings.API_MAX_LIMIT, + 'offset': 0} # Single (first) Dataset or Plate payload = {'childCount': str(child_count).lower()} @@ -355,19 +367,27 @@ def test_datasets_plates(self, user1, dtype, child_count, extra = ds_or_pl_children[0:limit] assert_objects(conn, rsp['data'], children[0:limit], dtype=dtype, extra=extra) - payload['page'] = 2 + assert rsp['meta'] == {'totalCount': 5, + 'limit': limit, + 'maxLimit': api_settings.API_MAX_LIMIT, + 'offset': 0} + payload['offset'] = limit # page 2 rsp = _get_response_json(django_client, request_url, payload) if ds_or_pl_children is not None: extra = ds_or_pl_children[limit:limit * 2] assert_objects(conn, rsp['data'], children[limit:limit * 2], dtype=dtype, extra=extra) + assert rsp['meta'] == {'totalCount': 5, + 'limit': limit, + 'maxLimit': api_settings.API_MAX_LIMIT, + 'offset': limit} def test_screens(self, user1, user_screens): """Test listing of Screens.""" conn = get_connection(user1) user_name = conn.getUser().getName() client = self.new_django_client(user_name, user_name) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] request_url = reverse('api_screens', kwargs={'api_version': version}) # List ALL Screens @@ -392,7 +412,7 @@ def test_spw_urls(self, user1, screen_plates): conn = get_connection(user1) user_name = conn.getUser().getName() client = self.new_django_client(user_name, user_name) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] base_url = reverse('api_base', kwargs={'api_version': version}) base_rsp = _get_response_json(client, base_url, {}) @@ -453,7 +473,7 @@ def test_pdi_urls(self, user1, project_datasets): conn = get_connection(user1) user_name = conn.getUser().getName() client = self.new_django_client(user_name, user_name) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] base_url = reverse('api_base', kwargs={'api_version': version}) base_rsp = _get_response_json(client, base_url, {}) diff --git a/components/tools/OmeroWeb/test/integration/test_api_errors.py b/components/tools/OmeroWeb/test/integration/test_api_errors.py index d48d1680c5a..3279bcb2bac 100644 --- a/components/tools/OmeroWeb/test/integration/test_api_errors.py +++ b/components/tools/OmeroWeb/test/integration/test_api_errors.py @@ -19,9 +19,10 @@ """Tests querying & editing Projects with webgateway json api.""" -from omeroweb.testlib import IWebTest, _csrf_post_json, _csrf_put_json +from omeroweb.testlib import IWebTest, _csrf_post_json, _csrf_put_json, \ + _get_response_json from django.core.urlresolvers import reverse -from django.conf import settings +from omeroweb.api import api_settings import pytest from test_api_projects import get_connection from omero.model import ProjectI, TagAnnotationI @@ -56,7 +57,7 @@ def user_A(self, group_A, group_B): def test_save_post_no_id(self): """If POST to /save/ data shouldn't contain @id.""" django_client = self.django_root_client - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] save_url = reverse('api_save', kwargs={'api_version': version}) payload = {'Name': 'test_save_post_no_id', '@type': '%s#Project' % OME_SCHEMA_URL, @@ -69,7 +70,7 @@ def test_save_post_no_id(self): def test_save_put_id(self): """If PUT to /save/ to update, data must contain @id.""" django_client = self.django_root_client - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] save_url = reverse('api_save', kwargs={'api_version': version}) payload = {'Name': 'test_save_put_id', '@type': '%s#Project' % OME_SCHEMA_URL} @@ -81,7 +82,7 @@ def test_save_put_id(self): def test_marshal_type(self): """If no decoder found for @type, get suitable message.""" django_client = self.django_root_client - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] save_url = reverse('api_save', kwargs={'api_version': version}) objType = 'SomeInvalidSchema#Project' payload = {'Name': 'test_marshal_type', @@ -91,10 +92,21 @@ def test_marshal_type(self): assert (rsp['message'] == 'No decoder found for type: %s' % objType) + def test_invalid_parameter(self): + """Test that invalid query parameter gives suitable message.""" + django_client = self.django_root_client + version = api_settings.API_VERSIONS[-1] + projects_url = reverse('api_projects', kwargs={'api_version': version}) + payload = {'limit': 'foo'} + rsp = _get_response_json(django_client, projects_url, payload, + status_code=400) + assert (rsp['message'] == + "invalid literal for int() with base 10: 'foo'") + def test_marshal_validation(self): """Test that we get expected error with invalid @type in json.""" django_client = self.django_root_client - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] save_url = reverse('api_save', kwargs={'api_version': version}) payload = {'Name': 'test_marshal_validation', '@type': OME_SCHEMA_URL + '#Project', @@ -112,7 +124,7 @@ def test_security_violation(self, group_B, user_A): group_A_id = conn.getEventContext().groupId userName = conn.getUser().getName() django_client = self.new_django_client(userName, userName) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] group_B_id = group_B.id.val save_url = reverse('api_save', kwargs={'api_version': version}) # Create project in group_A (default group) @@ -139,7 +151,7 @@ def test_validation_exception(self, user_A): group = conn.getEventContext().groupId userName = conn.getUser().getName() django_client = self.new_django_client(userName, userName) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] save_url = reverse('api_save', kwargs={'api_version': version}) save_url += '?group=' + str(group) diff --git a/components/tools/OmeroWeb/test/integration/test_api_images.py b/components/tools/OmeroWeb/test/integration/test_api_images.py index dc754ea072f..52886269157 100644 --- a/components/tools/OmeroWeb/test/integration/test_api_images.py +++ b/components/tools/OmeroWeb/test/integration/test_api_images.py @@ -21,7 +21,7 @@ from omeroweb.testlib import IWebTest, _get_response_json from django.core.urlresolvers import reverse -from django.conf import settings +from omeroweb.api import api_settings import pytest from test_api_projects import cmp_name_insensitive, get_update_service, \ get_connection, marshal_objects @@ -105,7 +105,7 @@ def test_dataset_images(self, user1, dataset_images): conn = get_connection(user1) user_name = conn.getUser().getName() django_client = self.new_django_client(user_name, user_name) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] dataset = dataset_images[0] images = dataset.linkedImageList() @@ -117,12 +117,20 @@ def test_dataset_images(self, user1, dataset_images): # List ALL Images rsp = _get_response_json(django_client, images_url, {}) assert len(rsp['data']) == 6 + assert rsp['meta'] == {'totalCount': 6, + 'limit': api_settings.API_LIMIT, + 'maxLimit': api_settings.API_MAX_LIMIT, + 'offset': 0} # Filter Images by Orphaned payload = {'orphaned': 'true'} rsp = _get_response_json(django_client, images_url, payload) assert_objects(conn, rsp['data'], [orphaned], dtype='Image', opts={'load_pixels': True}) + assert rsp['meta'] == {'totalCount': 1, + 'limit': api_settings.API_LIMIT, + 'maxLimit': api_settings.API_MAX_LIMIT, + 'offset': 0} # Filter Images by Dataset images.sort(cmp_name_insensitive) @@ -133,6 +141,10 @@ def test_dataset_images(self, user1, dataset_images): assert 'Channels' not in rsp['data'][0]['Pixels'] assert_objects(conn, rsp['data'], images, dtype='Image', opts={'load_pixels': True}) + assert rsp['meta'] == {'totalCount': 5, + 'limit': api_settings.API_LIMIT, + 'maxLimit': api_settings.API_MAX_LIMIT, + 'offset': 0} # Pagination, listing images via /datasets/:id/images/ limit = 3 @@ -141,10 +153,18 @@ def test_dataset_images(self, user1, dataset_images): rsp = _get_response_json(django_client, dataset_images_url, payload) assert_objects(conn, rsp['data'], images[0:limit], dtype='Image', opts={'load_pixels': True}) - payload['page'] = 2 + assert rsp['meta'] == {'totalCount': 5, + 'limit': limit, + 'maxLimit': api_settings.API_MAX_LIMIT, + 'offset': 0} + payload['offset'] = limit # page 2 rsp = _get_response_json(django_client, images_url, payload) assert_objects(conn, rsp['data'], images[limit:limit * 2], dtype='Image', opts={'load_pixels': True}) + assert rsp['meta'] == {'totalCount': 5, + 'limit': limit, + 'maxLimit': api_settings.API_MAX_LIMIT, + 'offset': limit} # Show ONLY the orphaned image (channels are loaded by default) img_url = images_url + '%s/' % orphaned.id.val diff --git a/components/tools/OmeroWeb/test/integration/test_api_login.py b/components/tools/OmeroWeb/test/integration/test_api_login.py index 2c316c9e4cd..b161e8cad58 100644 --- a/components/tools/OmeroWeb/test/integration/test_api_login.py +++ b/components/tools/OmeroWeb/test/integration/test_api_login.py @@ -25,7 +25,7 @@ from omeroweb.testlib import IWebTest, _get_response_json, \ _post_response_json, _csrf_post_response_json from django.core.urlresolvers import reverse, NoReverseMatch -from django.conf import settings +from omeroweb.api import api_settings from django.test import Client from omero_marshal import OME_SCHEMA_URL import json @@ -44,9 +44,9 @@ def test_versions(self): request_url = reverse('api_versions') rsp = _get_response_json(django_client, request_url, {}) versions = rsp['data'] - assert len(versions) == len(settings.API_VERSIONS) + assert len(versions) == len(api_settings.API_VERSIONS) for v in versions: - assert v['version'] in settings.API_VERSIONS + assert v['version'] in api_settings.API_VERSIONS def test_base_url(self): """ @@ -54,7 +54,7 @@ def test_base_url(self): """ django_client = self.django_root_client # test the most recent version - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] request_url = reverse('api_base', kwargs={'api_version': version}) rsp = _get_response_json(django_client, request_url, {}) assert 'url:servers' in rsp @@ -71,7 +71,7 @@ def test_base_url_versions_404(self): """ Tests that the base url gives 404 for invalid versions """ - version = settings.API_VERSIONS[-1] + "1" + version = api_settings.API_VERSIONS[-1] + "1" with pytest.raises(NoReverseMatch): reverse('api_base', kwargs={'api_version': version}) @@ -80,7 +80,7 @@ def test_login_get(self): Tests that we get a suitable message if we try to GET login_url """ django_client = self.django_root_client - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] request_url = reverse('api_login', kwargs={'api_version': version}) rsp = _get_response_json(django_client, request_url, {}, status_code=405) @@ -93,7 +93,7 @@ def test_login_csrf(self): """ django_client = self.django_root_client # test the most recent version - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] request_url = reverse('api_login', kwargs={'api_version': version}) rsp = _post_response_json(django_client, request_url, {}, status_code=403) @@ -123,7 +123,7 @@ def test_login_errors(self, credentials): """ django_client = self.django_root_client # test the most recent version - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] request_url = reverse('api_login', kwargs={'api_version': version}) data = credentials[0] message = credentials[1] diff --git a/components/tools/OmeroWeb/test/integration/test_api_projects.py b/components/tools/OmeroWeb/test/integration/test_api_projects.py index 2ca6d48075c..60bdd40ef41 100644 --- a/components/tools/OmeroWeb/test/integration/test_api_projects.py +++ b/components/tools/OmeroWeb/test/integration/test_api_projects.py @@ -24,7 +24,7 @@ from omeroweb.testlib import IWebTest, _get_response_json, \ _csrf_post_json, _csrf_put_json, _csrf_delete_response_json from django.core.urlresolvers import reverse -from django.conf import settings +from omeroweb.api import api_settings from django.test import Client import pytest from omero.gateway import BlitzGateway @@ -168,6 +168,9 @@ def assert_objects(conn, json_objects, omero_ids_objects, dtype="Project", pids.append(long(p)) except TypeError: pids.append(p.id.val) + if len(pids) == 0: + assert len(json_objects) == 0 + return conn.SERVICE_OPTS.setOmeroGroup(group) projects = conn.getObjects(dtype, pids, respect_order=True) projects = [p._obj for p in projects] @@ -265,7 +268,7 @@ def test_marshal_projects_not_logged_in(self): Test marshalling projects without log-in """ django_client = Client() - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] request_url = reverse('api_projects', kwargs={'api_version': version}) rsp = _get_response_json(django_client, request_url, {}, status_code=403) @@ -278,7 +281,7 @@ def test_marshal_projects_no_results(self, user1): conn = get_connection(user1) user_name = conn.getUser().getName() django_client = self.new_django_client(user_name, user_name) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] request_url = reverse('api_projects', kwargs={'api_version': version}) rsp = _get_response_json(django_client, request_url, {}) assert rsp['data'] == [] @@ -290,12 +293,11 @@ def test_marshal_projects_user(self, user1, projects_user1_group1): conn = get_connection(user1) user_name = conn.getUser().getName() django_client = self.new_django_client(user_name, user_name) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] request_url = reverse('api_projects', kwargs={'api_version': version}) rsp = _get_response_json(django_client, request_url, {}) - # Reload projects with group '-1' to get same 'canLink' perms - # on owner and group permissions assert_objects(conn, rsp['data'], projects_user1_group1) + assert rsp['meta']['totalCount'] == len(rsp['data']) def test_marshal_projects_another_user(self, user1, user2, projects_user2_group1): @@ -306,11 +308,12 @@ def test_marshal_projects_another_user(self, user1, user2, conn = get_connection(user1) user_name = conn.getUser().getName() django_client = self.new_django_client(user_name, user_name) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] request_url = reverse('api_projects', kwargs={'api_version': version}) rsp = _get_response_json(django_client, request_url, {}) # user1 reloads user2's projects assert_objects(conn, rsp['data'], projects_user2_group1) + assert rsp['meta']['totalCount'] == len(rsp['data']) def test_marshal_projects_another_group(self, user1, group2, projects_user1_group2): @@ -320,7 +323,7 @@ def test_marshal_projects_another_group(self, user1, group2, conn = get_connection(user1) user_name = conn.getUser().getName() django_client = self.new_django_client(user_name, user_name) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] request_url = reverse('api_projects', kwargs={'api_version': version}) rsp = _get_response_json(django_client, request_url, {}) @@ -329,6 +332,7 @@ def test_marshal_projects_another_group(self, user1, group2, # are same as owner's default group Group 1 (rwra--) instead of # group that the data is in Group 2 (rwr--) assert_objects(conn, rsp['data'], projects_user1_group2) + assert rsp['meta']['totalCount'] == len(rsp['data']) def test_marshal_projects_all_groups(self, user1, group1, group2, projects_user1): @@ -339,20 +343,23 @@ def test_marshal_projects_all_groups(self, user1, group1, group2, conn = get_connection(user1) user_name = conn.getUser().getName() django_client = self.new_django_client(user_name, user_name) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] request_url = reverse('api_projects', kwargs={'api_version': version}) # All groups rsp = _get_response_json(django_client, request_url, {}) assert_objects(conn, rsp['data'], projects_user1) + assert rsp['meta']['totalCount'] == len(projects_user1) # Filter by group A... gid = group1.id.val rsp = _get_response_json(django_client, request_url, {'group': gid}) assert_objects(conn, rsp['data'], projects_user1, group=gid) + assert rsp['meta']['totalCount'] == len(rsp['data']) # ...and group B gid = group2.id.val rsp = _get_response_json(django_client, request_url, {'group': gid}) assert_objects(conn, rsp['data'], projects_user1, group=gid) + assert rsp['meta']['totalCount'] == len(rsp['data']) def test_marshal_projects_all_users(self, user1, user2, projects_user1_group1, @@ -366,20 +373,23 @@ def test_marshal_projects_all_users(self, user1, user2, conn = get_connection(user1) user_name = conn.getUser().getName() django_client = self.new_django_client(user_name, user_name) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] request_url = reverse('api_projects', kwargs={'api_version': version}) # Both users rsp = _get_response_json(django_client, request_url, {}) assert_objects(conn, rsp['data'], projects) + assert rsp['meta']['totalCount'] == len(projects) eid = user1[1].id.val rsp = _get_response_json(django_client, request_url, {'owner': eid}) assert_objects(conn, rsp['data'], projects_user1_group1) + assert rsp['meta']['totalCount'] == len(projects_user1_group1) eid = user2[1].id.val rsp = _get_response_json(django_client, request_url, {'owner': eid}) assert_objects(conn, rsp['data'], projects_user2_group1) + assert rsp['meta']['totalCount'] == len(projects_user2_group1) def test_marshal_projects_pagination(self, user1, user2, projects_user1_group1, @@ -392,20 +402,27 @@ def test_marshal_projects_pagination(self, user1, user2, conn = get_connection(user1) user_name = conn.getUser().getName() django_client = self.new_django_client(user_name, user_name) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] request_url = reverse('api_projects', kwargs={'api_version': version}) - # First page, just 2 projects. Page = 1 by default - limit = 2 - rsp = _get_response_json(django_client, request_url, {'limit': limit}) - assert len(rsp['data']) == limit - assert_objects(conn, rsp['data'], projects[0:limit]) - - # Check that page 2 gives next 2 projects - page = 2 - payload = {'limit': limit, 'page': page} - rsp = _get_response_json(django_client, request_url, payload) - assert_objects(conn, rsp['data'], projects[limit:limit * page]) + # Test a range of limits. offset = 0 by default + for limit in range(-1, 4): + rsp = _get_response_json(django_client, request_url, + {'limit': limit}) + assert rsp['meta']['totalCount'] == len(projects) + if limit == -1: + # if we passed a negative number, expect the default to be used + limit = api_settings.API_LIMIT + else: + assert len(rsp['data']) == limit + assert rsp['meta']['limit'] == limit + assert_objects(conn, rsp['data'], projects[0:limit]) + + # Check that page 2 gives next n projects + payload = {'limit': limit, 'offset': limit} + rsp = _get_response_json(django_client, request_url, payload) + assert_objects(conn, rsp['data'], projects[limit:limit * 2]) + assert rsp['meta']['totalCount'] == len(projects) def test_marshal_projects_params(self, user1, user2, projects_user1_group1, @@ -418,7 +435,7 @@ def test_marshal_projects_params(self, user1, user2, conn = get_connection(user1) user_name = conn.getUser().getName() django_client = self.new_django_client(user_name, user_name) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] request_url = reverse('api_projects', kwargs={'api_version': version}) # Test 'childCount' parameter @@ -440,6 +457,7 @@ def test_marshal_projects_params(self, user1, user2, # Test 'normalize' parameter. payload = {'normalize': 'true'} rsp = _get_response_json(django_client, request_url, payload) + assert rsp['meta']['totalCount'] == len(rsp['data']) for p in rsp['data']: details = p['omero:details'] owner = details['owner'] @@ -463,7 +481,7 @@ def test_project_create_read(self): """ django_client = self.django_root_client group = self.ctx.groupId - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] # Need to get the Schema url to create @type base_url = reverse('api_base', kwargs={'api_version': version}) rsp = _get_response_json(django_client, base_url, {}) @@ -496,7 +514,7 @@ def test_project_create_other_group(self, user1, projects_user1_group2): conn = get_connection(user1) user_name = conn.getUser().getName() django_client = self.new_django_client(user_name, user_name) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] # We're only using projects_user1_group2 to get group2 id group2_id = projects_user1_group2[0].getDetails().group.id.val # This seems to be the minimum details needed to pass group ID @@ -540,7 +558,7 @@ def test_project_update(self, user1): project = get_update_service(user1).saveAndReturnObject(project) # Update Project in 2 ways... - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] project_url = reverse('api_project', kwargs={'api_version': version, 'object_id': project.id.val}) @@ -591,7 +609,7 @@ def test_project_delete(self, user1): project.name = rstring('test_project_delete') project.description = rstring('Test update') project = get_update_service(user1).saveAndReturnObject(project) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] project_url = reverse('api_project', kwargs={'api_version': version, 'object_id': project.id.val}) diff --git a/components/tools/OmeroWeb/test/integration/test_api_wells.py b/components/tools/OmeroWeb/test/integration/test_api_wells.py index d746b2b0cc5..643a399a0f7 100644 --- a/components/tools/OmeroWeb/test/integration/test_api_wells.py +++ b/components/tools/OmeroWeb/test/integration/test_api_wells.py @@ -21,7 +21,7 @@ from omeroweb.testlib import IWebTest, _get_response_json from django.core.urlresolvers import reverse -from django.conf import settings +from omeroweb.api import api_settings import pytest from test_api_projects import get_update_service, \ get_connection, marshal_objects @@ -173,7 +173,7 @@ def test_plate_wells(self, user1, small_plate, bigger_plate): conn = get_connection(user1) user_name = conn.getUser().getName() django_client = self.new_django_client(user_name, user_name) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] wells_url = reverse('api_wells', kwargs={'api_version': version}) @@ -193,6 +193,7 @@ def test_plate_wells(self, user1, small_plate, bigger_plate): rsp = _get_response_json(django_client, wells_url, payload) # Manual check that Images are loaded but Pixels are not assert len(rsp['data']) == well_count + assert rsp['meta']['totalCount'] == well_count well_sample = rsp['data'][0]['WellSamples'][0] assert 'Image' in well_sample assert ('PlateAcquisition' in well_sample) == with_acq @@ -206,7 +207,7 @@ def test_well(self, user1, small_plate): conn = get_connection(user1) user_name = conn.getUser().getName() django_client = self.new_django_client(user_name, user_name) - version = settings.API_VERSIONS[-1] + version = api_settings.API_VERSIONS[-1] small_plate = conn.getObject('Plate', small_plate.id.val) wells = [w._obj for w in small_plate.listChildren()]