Api pagination

components/tools/OmeroPy/src/omero/gateway/__init__.py

@@ -3020,7 +3020,7 @@ def getObject(self, obj_type, oid=None, params=None, attributes=None,
         If more than one object found, raises ome.conditions.ApiUsageException
         See :meth:`getObjects` for more info.
 
-        :param obj_type:    Object type. E.g. "Project" see above
+        :param obj_type:    Object type, e.g. "Project" see above
         :type obj_type:     String
         :param ids:         object IDs
         :type ids:          List of Long
@@ -3048,7 +3048,7 @@ def getObjects(self, obj_type, ids=None, params=None, attributes=None,
         be returned. i.e. listObjects() Filter objects by attributes. E.g.
         attributes={'name':name}
 
-        :param obj_type:    Object type. E.g. "Project" see above
+        :param obj_type:    Object type, e.g. "Project" see above
         :type obj_type:     String
         :param ids:         object IDs
         :type ids:          List of Long
@@ -3087,7 +3087,7 @@ def buildQuery(self, obj_type, ids=None, params=None, attributes=None,
         can be used with the appropriate query method. Used by
         :meth:`getObjects` and :meth:`getObject` above.
 
-        :param obj_type:    Object type. E.g. "Project" see above
+        :param obj_type:    Object type, e.g. "Project" see above
         :type obj_type:     String
         :param ids:         object IDs
         :type ids:          List of Long
@@ -3181,6 +3181,36 @@ def buildQuery(self, obj_type, ids=None, params=None, attributes=None,
 
         return (query, baseParams, wrapper)
 
+    def buildCountQuery(self, obj_type, opts=None):
+        """
+        Prepares a 'projection' query to count objects.
+
+        Based on buildQuery(), we modify the query to only return a count.
+        Modified query does not 'fetch' any data or add any other
+        unnecessary objects to query.
+        We return just the query and omero.sys.ParametersI for the query.
+
+        :param obj_type:    Object type, e.g. "Project" see above
+        :param opts:        Dict of options for filtering by
+                            offset, limit and owner for all objects.
+                            Additional opts handled by _getQueryString()
+                            e.g. filter Dataset by 'project'
+        :return:            (query, params)
+        """
+        # We disable pagination since we want to count ALL results
+        opts_copy = opts.copy()
+        if 'limit' in opts_copy:
+            del opts_copy['limit']
+
+        # Get query with other options
+        query, params, wrapper = self.buildQuery(obj_type, opts=opts_copy)
+
+        # Modify query to only select count()
+        query = query.replace("select obj ", "select count(distinct obj) ")
+        query = query.replace("fetch", "")
+        query = query.split("order by")[0]
+        return query, params
+
     def listFileAnnotations(self, eid=None, toInclude=[], toExclude=[]):
         """
         Lists FileAnnotations created by users, filtering by namespaces if
@@ -3222,7 +3252,7 @@ def getAnnotationLinks(self, parent_type, parent_ids=None, ann_ids=None,
         If parent_ids is None, all available objects will be returned.
         i.e. listObjects()
 
-        :param obj_type:    Object type. E.g. "Project" see above
+        :param obj_type:    Object type, e.g. "Project" see above
         :type obj_type:     String
         :param ids:         object IDs
         :type ids:          List of Long

components/tools/OmeroWeb/omeroweb/api/api_query.py

@@ -22,11 +22,16 @@
 
 from omero.rtypes import unwrap, wrap
 from omero.sys import ParametersI
+from . import api_settings
 
 from api_marshal import marshal_objects
 from copy import deepcopy
 
 
+MAX_LIMIT = max(1, api_settings.API_MAX_LIMIT)
+DEFAULT_LIMIT = max(1, api_settings.API_LIMIT)
+
+
 def get_child_counts(conn, link_class, parent_ids):
     """
     Count child links for the specified parent_ids.
@@ -50,6 +55,18 @@ def get_child_counts(conn, link_class, parent_ids):
     return counts
 
 
+def validate_opts(opts):
+    """Check that opts dict has valid 'limit' and 'offset'."""
+    if opts is None:
+        opts = {}
+    if opts.get('limit') is None or opts.get('limit') < 0:
+        opts['limit'] = DEFAULT_LIMIT
+    opts['limit'] = min(opts['limit'], MAX_LIMIT)
+    if opts.get('offset') is None or opts.get('offset') < 0:
+        opts['offset'] = 0
+    return opts
+
+
 def query_objects(conn, object_type,
                   group=None,
                   opts=None,
@@ -66,6 +83,7 @@ def query_objects(conn, object_type,
     @param opts:        Options dict for conn.buildQuery()
     @param normalize:   If true, marshal groups and experimenters separately
     """
+    opts = validate_opts(opts)
     # buildQuery is used by conn.getObjects()
     query, params, wrapper = conn.buildQuery(object_type, opts=opts)
     # Set the desired group context
@@ -79,7 +97,10 @@ def query_objects(conn, object_type,
     objects = []
     extras = {}
 
-    result = qs.findAllByQuery(query, params, ctx)
+    if opts['limit'] == 0:
+        result = []
+    else:
+        result = qs.findAllByQuery(query, params, ctx)
     for obj in result:
         objects.append(obj)
 
@@ -91,4 +112,16 @@ def query_objects(conn, object_type,
             count = counts[obj_id] if obj_id in counts else 0
             extras[obj_id] = {'omero:childCount': count}
 
-    return marshal_objects(objects, extras=extras, normalize=normalize)
+    # Query the count() of objects & add to 'meta' dict
+    count_query, params = conn.buildCountQuery(object_type, opts=opts)
+    result = qs.projection(count_query, params, ctx)
+
+    meta = {}
+    meta['offset'] = opts['offset']
+    meta['limit'] = opts['limit']
+    meta['maxLimit'] = MAX_LIMIT
+    meta['totalCount'] = result[0][0].val
+
+    marshalled = marshal_objects(objects, extras=extras, normalize=normalize)
+    marshalled['meta'] = meta
+    return marshalled

components/tools/OmeroWeb/omeroweb/api/api_settings.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2017 University of Dundee & Open Microscopy Environment.
+# All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Settings for the OMERO JSON api app."""
+
+import sys
+from omeroweb.settings import process_custom_settings, report_settings, \
+    str_slash
+
+# load settings
+API_SETTINGS_MAPPING = {
+
+    "omero.web.api.limit":
+        ["API_LIMIT",
+         200,
+         int,
+         "Default number of items returned from json api."],
+    "omero.web.api.max_limit":
+        ["API_MAX_LIMIT",
+         500,
+         int,
+         "Maximum number of items returned from json api."],
+    "omero.web.api.absolute_url":
+        ["API_ABSOLUTE_URL",
+         None,
+         str_slash,
+         ("URL to use for generating urls within API json responses. "
+          "By default this is None, and we use Django's "
+          "request.build_absolute_uri() to generate absolute urls "
+          "based on each request. If set to a string or empty string, "
+          "this will be used as prefix to relative urls.")],
+}
+
+process_custom_settings(sys.modules[__name__], 'API_SETTINGS_MAPPING')
+report_settings(sys.modules[__name__])
+
+# For any given release of api, we may support
+# one or more versions of the api.
+# E.g. /api/v0/
+# TODO - need to decide how this is configured, strategy for extending etc.
+API_VERSIONS = ('0',)

components/tools/OmeroWeb/omeroweb/api/urls.py

@@ -22,11 +22,11 @@
 from django.conf.urls import url, patterns
 from omeroweb.api import views
 from omeroweb.webgateway.views import LoginView
-from django.conf import settings
+from . import api_settings
 import re
 
 versions = '|'.join([re.escape(v)
-                    for v in settings.API_VERSIONS])
+                    for v in api_settings.API_VERSIONS])
 
 api_versions = url(r'^$', views.api_versions, name='api_versions')
 

components/tools/OmeroWeb/omeroweb/api/views.py

@@ -23,7 +23,7 @@
 from django.middleware import csrf
 from django.utils.decorators import method_decorator
 from django.core.urlresolvers import reverse
-from django.conf import settings
+from . import api_settings
 
 import traceback
 import json
@@ -54,19 +54,19 @@ def build_url(request, name, api_version, **kwargs):
     """
     kwargs['api_version'] = api_version
     url = reverse(name, kwargs=kwargs)
-    if settings.API_ABSOLUTE_URL is None:
+    if api_settings.API_ABSOLUTE_URL is None:
         return request.build_absolute_uri(url)
     else:
         # remove trailing slash
-        prefix = settings.API_ABSOLUTE_URL.rstrip('/')
+        prefix = api_settings.API_ABSOLUTE_URL.rstrip('/')
         return "%s%s" % (prefix, url)
 
 
 @json_response()
 def api_versions(request, **kwargs):
     """Base url of the webgateway json api."""
     versions = []
-    for v in settings.API_VERSIONS:
+    for v in api_settings.API_VERSIONS:
         versions.append({
             'version': v,
             'url:base': build_url(request, 'api_base', v)
@@ -304,16 +304,16 @@ class ObjectsView(ApiView):
     def get_opts(self, request, **kwargs):
         """Return an options dict based on request parameters."""
         try:
-            page = getIntOrDefault(request, 'page', 1)
-            limit = getIntOrDefault(request, 'limit', settings.PAGE)
+            offset = getIntOrDefault(request, 'offset', 0)
+            limit = getIntOrDefault(request, 'limit', None)
             owner = getIntOrDefault(request, 'owner', None)
             child_count = request.GET.get('childCount', False) == 'true'
             orphaned = request.GET.get('orphaned', False) == 'true'
         except ValueError as ex:
             raise BadRequestError(str(ex))
 
         # orphaned and child_count not used by every subclass
-        opts = {'offset': (page - 1) * limit,
+        opts = {'offset': offset,
                 'limit': limit,
                 'owner': owner,
                 'orphaned': orphaned,

components/tools/OmeroWeb/omeroweb/settings.py

@@ -474,15 +474,6 @@ def leave_none_unset_int(s):
          ("Workers silent for more than this many seconds are killed "
           "and restarted. Check Gunicorn Documentation "
           "http://docs.gunicorn.org/en/stable/settings.html#timeout")],
-    "omero.web.api.absolute_url":
-        ["API_ABSOLUTE_URL",
-         None,
-         str_slash,
-         ("URL to use for generating urls within API json responses. "
-          "By default this is None, and we use Django's "
-          "request.build_absolute_uri() to generate absolute urls "
-          "based on each request. If set to a string or empty string, "
-          "this will be used as prefix to relative urls.")],
 
     # Public user
     "omero.web.public.enabled":
@@ -1075,6 +1066,7 @@ def report_settings(module):
     'omeroweb.webclient',
     'omeroweb.webgateway',
     'omeroweb.webredirect',
+    'omeroweb.api',
     'pipeline',
 )
 
@@ -1155,12 +1147,6 @@ def report_settings(module):
 # FEEDBACK_APP: 6 = OMERO.web
 FEEDBACK_APP = 6
 
-# For any given release of api, we may support
-# one or more versions of the api.
-# E.g. /api/v1.0/
-# TODO - need to decide how this is configured, strategy for extending etc.
-API_VERSIONS = ('0',)
-
 # IGNORABLE_404_STARTS:
 # Default: ('/cgi-bin/', '/_vti_bin', '/_vti_inf')
 # IGNORABLE_404_ENDS:

components/tools/OmeroWeb/omeroweb/webgateway/util.py

@@ -28,12 +28,9 @@
 
 # helper method
 def getIntOrDefault(request, name, default):
-    try:
-        index = request.GET.get(name, request.POST.get(name, default))
-        if index is not None:
-            index = int(index)
-    except ValueError:
-        index = 0
+    index = request.GET.get(name, request.POST.get(name, default))
+    if index is not None:
+        index = int(index)
     return index