diff --git a/complaint_search/defaults.py b/complaint_search/defaults.py index cace984..3256e21 100644 --- a/complaint_search/defaults.py +++ b/complaint_search/defaults.py @@ -64,10 +64,7 @@ EXCLUDE_PREFIX = "not_" -EXPORT_FORMATS = ( - "csv", - "json", -) +EXPORT_FORMATS = ("csv",) CSV_ORDERED_HEADERS = OrderedDict( [ @@ -95,7 +92,6 @@ CHUNK_SIZE = 512 FORMAT_CONTENT_TYPE_MAP = { - "json": "application/json", "csv": "text/csv", } diff --git a/complaint_search/es_interface.py b/complaint_search/es_interface.py index 23bb74c..9083752 100644 --- a/complaint_search/es_interface.py +++ b/complaint_search/es_interface.py @@ -265,7 +265,7 @@ def search(agg_exclude=None, **kwargs): - Assemble pagination break points if needed. The response is finalized based on whether the results are to be viewed - in a browser or exported as CSV or JSON. + in a browser or exported as CSV. Exportable results are produced with "scroll" OpenSearch searches, and are never paginated. """ @@ -356,8 +356,6 @@ def search(agg_exclude=None, **kwargs): if params.get("format") == "csv": res = exporter.export_csv(scan_response, CSV_ORDERED_HEADERS, hit_total) - elif params.get("format") == "json": - res = exporter.export_json(scan_response, hit_total) return res diff --git a/complaint_search/export.py b/complaint_search/export.py index 1a5f94d..8af81f6 100644 --- a/complaint_search/export.py +++ b/complaint_search/export.py @@ -1,14 +1,9 @@ import csv -import json from csv import DictWriter from io import StringIO from django.http import StreamingHttpResponse -from rest_framework.exceptions import ValidationError - -from complaint_search.defaults import MAX_DOWNLOAD_SIZE - class OpenSearchExporter(object): def _check_download_size(self, total_count): @@ -75,30 +70,3 @@ def stream(): response = StreamingHttpResponse(stream(), content_type="text/csv") response["Content-Disposition"] = "attachment; filename=file.csv" return response - - # export_json - Stream an OpenSearch response as a JSON file - # - # Parameters: - # - scanResponse (generator) - # The response from an OpenSearch scan query - # - total_count (int) - # The total number of records to be output - def export_json(self, scanResponse, total_count): - self._check_download_size(total_count) - - def stream(): - count = 0 - # Write JSON - yield "[" - for row in scanResponse: - count += 1 - if count < total_count: - yield "{},".format(json.dumps(row)) - else: - yield json.dumps(row) - - yield "]" - - response = StreamingHttpResponse(stream(), content_type="text/json") - response["Content-Disposition"] = "attachment; filename=file.json" - return response diff --git a/complaint_search/serializer.py b/complaint_search/serializer.py index 8247501..d88f212 100644 --- a/complaint_search/serializer.py +++ b/complaint_search/serializer.py @@ -8,12 +8,10 @@ class SearchInputSerializer(serializers.Serializer): # Format Choices FORMAT_DEFAULT = "default" - FORMAT_JSON = "json" FORMAT_CSV = "csv" FORMAT_CHOICES = ( (FORMAT_DEFAULT, "DEFAULT"), - (FORMAT_JSON, "JSON"), (FORMAT_CSV, "CSV"), ) diff --git a/complaint_search/stream_content.py b/complaint_search/stream_content.py index 4762fe3..a392dd7 100644 --- a/complaint_search/stream_content.py +++ b/complaint_search/stream_content.py @@ -13,70 +13,3 @@ def __next__(self): return self.header else: return next(self.content) - - -class StreamJSONContent(object): - def __init__(self, content): - self.content = content - self.complaint_in_progress = "" - self.is_streaming_started = False - self.is_streaming_stopped = False - - def get_next_complaint(self): - self.complaint_in_progress = self.complaint_in_progress.lstrip() - # see if first line is reached - try: - first_eol_index = self.complaint_in_progress.index("\n") - - # see if we have 2nd completed line, and that's the complaint we - # want to return assuming at the EOF there's also a '\n' as seen - # from data format plugin so far - second_eol_index = self.complaint_in_progress.index( - "\n", first_eol_index + 1 - ) - - complaint = self.complaint_in_progress[ - first_eol_index + 1 : second_eol_index + 1 - ].strip() - # save the rest for next iteration - self.complaint_in_progress = self.complaint_in_progress[ - second_eol_index + 1 : - ] - return complaint - except ValueError: - # This means cannot find two \n, complaint is not ready, need more - # data - return None - - def __iter__(self): - return self - - def __next__(self): - while True: - if not self.is_streaming_started: - # This is the beginning - self.is_streaming_started = True - return "[" - try: - next_chunk = next(self.content) - self.complaint_in_progress += next_chunk - # peek ahead, it will raise StopIteration if no more chunk - next2_chunk = next(self.content) - complaint = self.get_next_complaint() - self.complaint_in_progress += next2_chunk - if complaint and self.complaint_in_progress.strip(): - return complaint + "," - elif complaint and not self.complaint_in_progress.strip(): - return complaint - except StopIteration: - complaint = self.get_next_complaint() - if complaint and not self.complaint_in_progress.strip(): - return complaint - elif complaint and self.complaint_in_progress.strip(): - return complaint + "," - elif not self.is_streaming_stopped: - # This is the end - self.is_streaming_stopped = True - return "]" - else: - raise StopIteration diff --git a/complaint_search/tests/test_es_interface.py b/complaint_search/tests/test_es_interface.py index 1e79459..8784255 100644 --- a/complaint_search/tests/test_es_interface.py +++ b/complaint_search/tests/test_es_interface.py @@ -6,7 +6,6 @@ from django.test import SimpleTestCase, TestCase from opensearchpy import OpenSearch -from parameterized import parameterized from complaint_search.defaults import AGG_EXCLUDE_FIELDS from complaint_search.es_builders import AggregationBuilder, SearchBuilder @@ -259,17 +258,13 @@ def test_search_agg_exclude__valid(self, mock_rget): ) mock_rget.assert_not_called() - @parameterized.expand([["csv"], ["json"]]) @mock.patch.object(OpenSearchExporter, "export_csv") - @mock.patch.object(OpenSearchExporter, "export_json") @mock.patch.object(OpenSearch, "search") @mock.patch("opensearchpy.helpers.scan") def test_search_with_format__valid( self, - export_type, mock_es_helper, mock_search, - mock_exporter_json, mock_exporter_csv, ): mock_search_side_effect = copy.deepcopy(self.MOCK_SEARCH_SIDE_EFFECT) @@ -277,19 +272,12 @@ def test_search_with_format__valid( mock_search.side_effect = mock_search_side_effect mock_exporter_csv.return_value = StreamingHttpResponse() - mock_exporter_json.return_value = StreamingHttpResponse() - res = search(format=export_type) + res = search(format="csv") self.assertIsInstance(res, StreamingHttpResponse) self.assertEqual(1, mock_es_helper.call_count) - if export_type == "csv": - self.assertEqual(1, mock_exporter_csv.call_count) - self.assertEqual(0, mock_exporter_json.call_count) - else: - self.assertEqual(1, mock_search.call_count) - self.assertEqual(1, mock_exporter_json.call_count) - self.assertEqual(0, mock_exporter_csv.call_count) + self.assertEqual(1, mock_exporter_csv.call_count) @mock.patch.object(OpenSearch, "search") @mock.patch("requests.get", ok=True, content="RGET_OK") @@ -300,6 +288,15 @@ def test_search_with_format__invalid(self, mock_rget, mock_search): mock_search.assert_not_called() mock_rget.assert_not_called() + @mock.patch.object(OpenSearch, "search") + @mock.patch("requests.get", ok=True, content="RGET_OK") + def test_search_with_format_json__invalid(self, mock_rget, mock_search): + mock_search.return_value = "OK" + res = search(format="json") + self.assertEqual(res, {}) + mock_search.assert_not_called() + mock_rget.assert_not_called() + def test_search_with_size__valid(self): self.request_test("search_with_size__valid", size=40) diff --git a/complaint_search/tests/test_export.py b/complaint_search/tests/test_export.py index e7e2f25..0180a52 100644 --- a/complaint_search/tests/test_export.py +++ b/complaint_search/tests/test_export.py @@ -7,9 +7,7 @@ from django.test import TestCase from parameterized import parameterized -from rest_framework.exceptions import ValidationError -from complaint_search.defaults import MAX_DOWNLOAD_SIZE from complaint_search.export import OpenSearchExporter @@ -66,41 +64,6 @@ def test_csv_export_limit(self): es_exporter.export_csv(gen, TEST_HEADERS, length) self.assertEqual(context.exception.get_codes(), {"size": ["invalid"]}) - @parameterized.expand([[10], [5010], [100000]]) - def test_export_json_request_response(self, length): - # arrange - es_exporter = OpenSearchExporter() - gen = es_generator(length) - - # act - res = es_exporter.export_json(gen, length) - - # assert - self.assertTrue(isinstance(res, StreamingHttpResponse)) - - # mock_search.assert_not_called() - self.assertEqual( - res.get("Content-Disposition"), "attachment; filename=file.json" - ) - self.assertTrue("map" in str(type(res.streaming_content))) - downloaded_file = io.BytesIO(b"".join(res.streaming_content)) - self.assertFalse(downloaded_file is None) - - def test_json_export_limit(self): - length = MAX_DOWNLOAD_SIZE + 1 - es_exporter = OpenSearchExporter() - gen = es_generator(length) - with self.assertRaises(ValidationError) as context: - es_exporter.export_json(gen, length) - self.assertEqual(context.exception.get_codes(), {"size": ["invalid"]}) - - def test_json_export_empty(self): - es_exporter = OpenSearchExporter() - res = es_exporter.export_json(es_generator(0), 0) - self.assertIsInstance(res, StreamingHttpResponse) - content = io.BytesIO(b"".join(res.streaming_content)).read() - self.assertEqual(content, b"[]") - class TestCSVExportWithUnicodeCharacters(TestCase): def test_export_contains_unicode_chacter(self): diff --git a/complaint_search/tests/test_stream_content.py b/complaint_search/tests/test_stream_content.py index 98e8cd7..6ef5baf 100644 --- a/complaint_search/tests/test_stream_content.py +++ b/complaint_search/tests/test_stream_content.py @@ -1,6 +1,6 @@ from django.test import TestCase -from complaint_search.stream_content import StreamCSVContent, StreamJSONContent +from complaint_search.stream_content import StreamCSVContent class StreamCSVContentTests(TestCase): @@ -20,46 +20,3 @@ def test_next_header(self): sc = StreamCSVContent("header", iter([1, 2, 3])) content = [item for item in sc] self.assertListEqual(["header", 1, 2, 3], content) - - -class StreamJSONContentTests(TestCase): - def setUp(self): - self.content = ( - '{"index": {"_index": "test", "_id": 12345}}\n' - '{"product": "mortgage", "complaint_id": 12345, "tags": null}\n' - '{"index": {"_index": "test", "_id": 23456}}\n' - '{"product": "test", "complaint_id": 23456, ' - '"tags": "Older Americans"}\n' - '{"index": {"_index": "test", "_id": 45678}}\n' - '{"product": "loan", "complaint_id": 45678, "tags": null} \n' - ) - - # pretend this is broken up randomly every 20 chars - self.content_list = [ - self.content[(i * 20) : (i * 20 + 20)] - for i in range(int(len(self.content) / 20 + 1)) - ] - - def test_iter(self): - sc = StreamJSONContent(iter(self.content_list)) - self.assertTrue(isinstance(iter(sc), StreamJSONContent)) - - def test_next_complete(self): - for size in range(1, 1024): - content_list = [ - self.content[(i * size) : (i * size + size)] - for i in range(int(len(self.content) / size + 1)) - ] - sc = StreamJSONContent(iter(content_list)) - result = "" - for json_in_progress in sc: - result += json_in_progress - - exp_result = ( - '[{"product": "mortgage", "complaint_id": 12345, ' - '"tags": null},' - '{"product": "test", "complaint_id": 23456, ' - '"tags": "Older Americans"},' - '{"product": "loan", "complaint_id": 45678, "tags": null}]' - ) - self.assertEqual(exp_result, result) diff --git a/complaint_search/views.py b/complaint_search/views.py index c1af432..b73427c 100644 --- a/complaint_search/views.py +++ b/complaint_search/views.py @@ -9,7 +9,6 @@ renderer_classes, throttle_classes, ) -from rest_framework.renderers import JSONRenderer from rest_framework.response import Response from complaint_search import es_interface @@ -127,7 +126,6 @@ def _build_headers(): @renderer_classes( ( DefaultRenderer, - JSONRenderer, CSVRenderer, ) ) diff --git a/swagger-config.yaml b/swagger-config.yaml index d40563d..544987c 100644 --- a/swagger-config.yaml +++ b/swagger-config.yaml @@ -332,9 +332,8 @@ components: schema: type: string enum: - - json - csv - default: json + default: csv from: name: frm in: query