Remove filtered JSON export support from the search API

complaint_search/defaults.py

@@ -64,10 +64,7 @@
 
 EXCLUDE_PREFIX = "not_"
 
-EXPORT_FORMATS = (
-    "csv",
-    "json",
-)
+EXPORT_FORMATS = ("csv",)
 
 CSV_ORDERED_HEADERS = OrderedDict(
     [
@@ -95,7 +92,6 @@
 CHUNK_SIZE = 512
 
 FORMAT_CONTENT_TYPE_MAP = {
-    "json": "application/json",
     "csv": "text/csv",
 }
 

complaint_search/es_interface.py

@@ -265,7 +265,7 @@ def search(agg_exclude=None, **kwargs):
     - Assemble pagination break points if needed.
 
     The response is finalized based on whether the results are to be viewed
-    in a browser or exported as CSV or JSON.
+    in a browser or exported as CSV.
     Exportable results are produced with "scroll" OpenSearch searches,
     and are never paginated.
     """
@@ -356,8 +356,6 @@ def search(agg_exclude=None, **kwargs):
 
         if params.get("format") == "csv":
             res = exporter.export_csv(scan_response, CSV_ORDERED_HEADERS, hit_total)
-        elif params.get("format") == "json":
-            res = exporter.export_json(scan_response, hit_total)
 
     return res
 

complaint_search/export.py

@@ -1,14 +1,9 @@
 import csv
-import json
 from csv import DictWriter
 from io import StringIO
 
 from django.http import StreamingHttpResponse
 
-from rest_framework.exceptions import ValidationError
-
-from complaint_search.defaults import MAX_DOWNLOAD_SIZE
-
 
 class OpenSearchExporter(object):
     def _check_download_size(self, total_count):
@@ -75,30 +70,3 @@ def stream():
         response = StreamingHttpResponse(stream(), content_type="text/csv")
         response["Content-Disposition"] = "attachment; filename=file.csv"
         return response
-
-    # export_json - Stream an OpenSearch response as a JSON file
-    #
-    # Parameters:
-    # - scanResponse (generator)
-    #   The response from an OpenSearch scan query
-    # - total_count (int)
-    #   The total number of records to be output
-    def export_json(self, scanResponse, total_count):
-        self._check_download_size(total_count)
-
-        def stream():
-            count = 0
-            # Write JSON
-            yield "["
-            for row in scanResponse:
-                count += 1
-                if count < total_count:
-                    yield "{},".format(json.dumps(row))
-                else:
-                    yield json.dumps(row)
-
-            yield "]"
-
-        response = StreamingHttpResponse(stream(), content_type="text/json")
-        response["Content-Disposition"] = "attachment; filename=file.json"
-        return response

complaint_search/serializer.py

@@ -8,12 +8,10 @@ class SearchInputSerializer(serializers.Serializer):
 
     # Format Choices
     FORMAT_DEFAULT = "default"
-    FORMAT_JSON = "json"
     FORMAT_CSV = "csv"
 
     FORMAT_CHOICES = (
         (FORMAT_DEFAULT, "DEFAULT"),
-        (FORMAT_JSON, "JSON"),
         (FORMAT_CSV, "CSV"),
     )
 

complaint_search/stream_content.py

@@ -13,70 +13,3 @@ def __next__(self):
             return self.header
         else:
             return next(self.content)
-
-
-class StreamJSONContent(object):
-    def __init__(self, content):
-        self.content = content
-        self.complaint_in_progress = ""
-        self.is_streaming_started = False
-        self.is_streaming_stopped = False
-
-    def get_next_complaint(self):
-        self.complaint_in_progress = self.complaint_in_progress.lstrip()
-        # see if first line is reached
-        try:
-            first_eol_index = self.complaint_in_progress.index("\n")
-
-            # see if we have 2nd completed line, and that's the complaint we
-            # want to return assuming at the EOF there's also a '\n' as seen
-            # from data format plugin so far
-            second_eol_index = self.complaint_in_progress.index(
-                "\n", first_eol_index + 1
-            )
-
-            complaint = self.complaint_in_progress[
-                first_eol_index + 1 : second_eol_index + 1
-            ].strip()
-            # save the rest for next iteration
-            self.complaint_in_progress = self.complaint_in_progress[
-                second_eol_index + 1 :
-            ]
-            return complaint
-        except ValueError:
-            # This means cannot find two \n, complaint is not ready, need more
-            # data
-            return None
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        while True:
-            if not self.is_streaming_started:
-                # This is the beginning
-                self.is_streaming_started = True
-                return "["
-            try:
-                next_chunk = next(self.content)
-                self.complaint_in_progress += next_chunk
-                # peek ahead, it will raise StopIteration if no more chunk
-                next2_chunk = next(self.content)
-                complaint = self.get_next_complaint()
-                self.complaint_in_progress += next2_chunk
-                if complaint and self.complaint_in_progress.strip():
-                    return complaint + ","
-                elif complaint and not self.complaint_in_progress.strip():
-                    return complaint
-            except StopIteration:
-                complaint = self.get_next_complaint()
-                if complaint and not self.complaint_in_progress.strip():
-                    return complaint
-                elif complaint and self.complaint_in_progress.strip():
-                    return complaint + ","
-                elif not self.is_streaming_stopped:
-                    # This is the end
-                    self.is_streaming_stopped = True
-                    return "]"
-                else:
-                    raise StopIteration

complaint_search/tests/test_es_interface.py

@@ -6,7 +6,6 @@
 from django.test import SimpleTestCase, TestCase
 
 from opensearchpy import OpenSearch
-from parameterized import parameterized
 
 from complaint_search.defaults import AGG_EXCLUDE_FIELDS
 from complaint_search.es_builders import AggregationBuilder, SearchBuilder
@@ -259,37 +258,26 @@ def test_search_agg_exclude__valid(self, mock_rget):
         )
         mock_rget.assert_not_called()
 
-    @parameterized.expand([["csv"], ["json"]])
     @mock.patch.object(OpenSearchExporter, "export_csv")
-    @mock.patch.object(OpenSearchExporter, "export_json")
     @mock.patch.object(OpenSearch, "search")
     @mock.patch("opensearchpy.helpers.scan")
     def test_search_with_format__valid(
         self,
-        export_type,
         mock_es_helper,
         mock_search,
-        mock_exporter_json,
         mock_exporter_csv,
     ):
         mock_search_side_effect = copy.deepcopy(self.MOCK_SEARCH_SIDE_EFFECT)
         mock_search_side_effect[0]["hits"]["total"]["value"] = 4
         mock_search.side_effect = mock_search_side_effect
 
         mock_exporter_csv.return_value = StreamingHttpResponse()
-        mock_exporter_json.return_value = StreamingHttpResponse()
 
-        res = search(format=export_type)
+        res = search(format="csv")
 
         self.assertIsInstance(res, StreamingHttpResponse)
         self.assertEqual(1, mock_es_helper.call_count)
-        if export_type == "csv":
-            self.assertEqual(1, mock_exporter_csv.call_count)
-            self.assertEqual(0, mock_exporter_json.call_count)
-        else:
-            self.assertEqual(1, mock_search.call_count)
-            self.assertEqual(1, mock_exporter_json.call_count)
-            self.assertEqual(0, mock_exporter_csv.call_count)
+        self.assertEqual(1, mock_exporter_csv.call_count)
 
     @mock.patch.object(OpenSearch, "search")
     @mock.patch("requests.get", ok=True, content="RGET_OK")
@@ -300,6 +288,15 @@ def test_search_with_format__invalid(self, mock_rget, mock_search):
         mock_search.assert_not_called()
         mock_rget.assert_not_called()
 
+    @mock.patch.object(OpenSearch, "search")
+    @mock.patch("requests.get", ok=True, content="RGET_OK")
+    def test_search_with_format_json__invalid(self, mock_rget, mock_search):
+        mock_search.return_value = "OK"
+        res = search(format="json")
+        self.assertEqual(res, {})
+        mock_search.assert_not_called()
+        mock_rget.assert_not_called()
+
     def test_search_with_size__valid(self):
         self.request_test("search_with_size__valid", size=40)
 

complaint_search/tests/test_export.py

@@ -7,9 +7,7 @@
 from django.test import TestCase
 
 from parameterized import parameterized
-from rest_framework.exceptions import ValidationError
 
-from complaint_search.defaults import MAX_DOWNLOAD_SIZE
 from complaint_search.export import OpenSearchExporter
 
 
@@ -66,41 +64,6 @@ def test_csv_export_limit(self):
             es_exporter.export_csv(gen, TEST_HEADERS, length)
         self.assertEqual(context.exception.get_codes(), {"size": ["invalid"]})
 
-    @parameterized.expand([[10], [5010], [100000]])
-    def test_export_json_request_response(self, length):
-        # arrange
-        es_exporter = OpenSearchExporter()
-        gen = es_generator(length)
-
-        # act
-        res = es_exporter.export_json(gen, length)
-
-        # assert
-        self.assertTrue(isinstance(res, StreamingHttpResponse))
-
-        # mock_search.assert_not_called()
-        self.assertEqual(
-            res.get("Content-Disposition"), "attachment; filename=file.json"
-        )
-        self.assertTrue("map" in str(type(res.streaming_content)))
-        downloaded_file = io.BytesIO(b"".join(res.streaming_content))
-        self.assertFalse(downloaded_file is None)
-
-    def test_json_export_limit(self):
-        length = MAX_DOWNLOAD_SIZE + 1
-        es_exporter = OpenSearchExporter()
-        gen = es_generator(length)
-        with self.assertRaises(ValidationError) as context:
-            es_exporter.export_json(gen, length)
-        self.assertEqual(context.exception.get_codes(), {"size": ["invalid"]})
-
-    def test_json_export_empty(self):
-        es_exporter = OpenSearchExporter()
-        res = es_exporter.export_json(es_generator(0), 0)
-        self.assertIsInstance(res, StreamingHttpResponse)
-        content = io.BytesIO(b"".join(res.streaming_content)).read()
-        self.assertEqual(content, b"[]")
-
 
 class TestCSVExportWithUnicodeCharacters(TestCase):
     def test_export_contains_unicode_chacter(self):

complaint_search/tests/test_stream_content.py

@@ -1,6 +1,6 @@
 from django.test import TestCase
 
-from complaint_search.stream_content import StreamCSVContent, StreamJSONContent
+from complaint_search.stream_content import StreamCSVContent
 
 
 class StreamCSVContentTests(TestCase):
@@ -20,46 +20,3 @@ def test_next_header(self):
         sc = StreamCSVContent("header", iter([1, 2, 3]))
         content = [item for item in sc]
         self.assertListEqual(["header", 1, 2, 3], content)
-
-
-class StreamJSONContentTests(TestCase):
-    def setUp(self):
-        self.content = (
-            '{"index": {"_index": "test", "_id": 12345}}\n'
-            '{"product": "mortgage", "complaint_id": 12345, "tags": null}\n'
-            '{"index": {"_index": "test", "_id": 23456}}\n'
-            '{"product": "test", "complaint_id": 23456, '
-            '"tags": "Older Americans"}\n'
-            '{"index": {"_index": "test", "_id": 45678}}\n'
-            '{"product": "loan", "complaint_id": 45678, "tags": null} \n'
-        )
-
-        # pretend this is broken up randomly every 20 chars
-        self.content_list = [
-            self.content[(i * 20) : (i * 20 + 20)]
-            for i in range(int(len(self.content) / 20 + 1))
-        ]
-
-    def test_iter(self):
-        sc = StreamJSONContent(iter(self.content_list))
-        self.assertTrue(isinstance(iter(sc), StreamJSONContent))
-
-    def test_next_complete(self):
-        for size in range(1, 1024):
-            content_list = [
-                self.content[(i * size) : (i * size + size)]
-                for i in range(int(len(self.content) / size + 1))
-            ]
-            sc = StreamJSONContent(iter(content_list))
-            result = ""
-            for json_in_progress in sc:
-                result += json_in_progress
-
-            exp_result = (
-                '[{"product": "mortgage", "complaint_id": 12345, '
-                '"tags": null},'
-                '{"product": "test", "complaint_id": 23456, '
-                '"tags": "Older Americans"},'
-                '{"product": "loan", "complaint_id": 45678, "tags": null}]'
-            )
-            self.assertEqual(exp_result, result)

complaint_search/views.py

@@ -9,7 +9,6 @@
     renderer_classes,
     throttle_classes,
 )
-from rest_framework.renderers import JSONRenderer
 from rest_framework.response import Response
 
 from complaint_search import es_interface
@@ -127,7 +126,6 @@ def _build_headers():
 @renderer_classes(
     (
         DefaultRenderer,
-        JSONRenderer,
         CSVRenderer,
     )
 )

swagger-config.yaml

@@ -332,9 +332,8 @@ components:
       schema:
         type: string
         enum:
-          - json
           - csv
-        default: json
+        default: csv
     from:
       name: frm
       in: query