diff --git a/django/evaluate_m2/filters.py b/django/evaluate_m2/filters.py index 61c779e4..ee74822b 100644 --- a/django/evaluate_m2/filters.py +++ b/django/evaluate_m2/filters.py @@ -1,6 +1,7 @@ import django_filters.rest_framework from evaluate_m2.models import EvaluatorResult +from parse_m2.models import AccountActivity class AnyCharFilter(django_filters.BaseInFilter, django_filters.CharFilter): @@ -186,3 +187,21 @@ class Meta: "smpa", "sort", ] + + +class AccountListFilterSet(django_filters.rest_framework.FilterSet): + cons_acct_num = django_filters.BaseInFilter( + field_name="cons_acct_num", + lookup_expr="in" + ) + + class Meta: + model = AccountActivity + fields = ["cons_acct_num"] + + # Require a specific list of accounts to filter, otherwise this + # filter will return an empty queryset. + def filter_queryset(self, queryset): + if not self.form.cleaned_data.get("cons_acct_num"): + return queryset.none() + return super().filter_queryset(queryset) diff --git a/django/evaluate_m2/managers.py b/django/evaluate_m2/managers.py index 235dad22..1ba2d1da 100644 --- a/django/evaluate_m2/managers.py +++ b/django/evaluate_m2/managers.py @@ -1,5 +1,6 @@ from django.db import models -from django.db.models import Q +from django.db.models import Count, IntegerField, OuterRef, Q, Subquery, Value +from django.db.models.functions import Coalesce, TruncMonth class AccountActivityQuerySet(models.QuerySet): @@ -9,3 +10,49 @@ def no_previous_bankruptcy_indicators(self): Q(previous_values__cons_info_ind_assoc__isnull=True) ) & Q(previous_values__cons_info_ind='') ) + + def with_inconsistency_counts(self, event): + from evaluate_m2.models import EvaluatorResult + + # Subquery for the number of inconsistencies (evaluators hit) for + # each account. This is just a count of hits. + subquery = EvaluatorResult.objects.filter( + result_summary__event=event, + acct_num=OuterRef("cons_acct_num"), + ).order_by().values("acct_num").annotate( + n=Count("id"), + ).values("n")[:1] + + return self.annotate( + total_inconsistencies=Coalesce( + Subquery(subquery, output_field=IntegerField()), + Value(0), + ), + ) + + def with_months_of_data(self, event): + # Subquery for the number of months of data. This counts **distinct** + # months, not the number of months in the total range. + # I.e. if there's data for Feb, March, and May, but not April, that's + # three months of data, not four. + subquery = self.model._default_manager.filter( + data_file__event=event, + cons_acct_num=OuterRef("cons_acct_num"), + ).order_by().annotate( + month=TruncMonth("activity_date"), + ).values("cons_acct_num").annotate( + n=Count("month", distinct=True), + ).values("n")[:1] + + return self.annotate( + months_of_data=Coalesce( + Subquery(subquery, output_field=IntegerField()), + Value(0), + ) + ) + + def distinct_accounts(self): + return self.order_by( + "cons_acct_num", + "-activity_date", + ).distinct("cons_acct_num") diff --git a/django/evaluate_m2/serializers.py b/django/evaluate_m2/serializers.py index 1313c251..0a7c8539 100644 --- a/django/evaluate_m2/serializers.py +++ b/django/evaluate_m2/serializers.py @@ -6,8 +6,8 @@ parse_fields_from_csv, plain_to_code_field_map, ) - -from .models import EvaluatorMetadata, EvaluatorResultSummary +from evaluate_m2.models import EvaluatorMetadata, EvaluatorResultSummary +from parse_m2.serializers import AccountActivitySerializer class EventsViewSerializer(serializers.ModelSerializer): @@ -224,3 +224,17 @@ def validate(self, data): if invalid_fields: raise serializers.ValidationError(f"Invalid field names: {invalid_fields}") return data + + +class AccountListSerializer(AccountActivitySerializer): + total_inconsistencies = serializers.IntegerField(read_only=True) + months_of_data = serializers.IntegerField(read_only=True) + + class Meta(AccountActivitySerializer.Meta): + default_fields = [ + "cons_acct_num", + "port_type", + "acct_type", + "total_inconsistencies", + "months_of_data", + ] diff --git a/django/evaluate_m2/tests/evaluator_test_helper.py b/django/evaluate_m2/tests/evaluator_test_helper.py index fbd1301a..635fdb09 100644 --- a/django/evaluate_m2/tests/evaluator_test_helper.py +++ b/django/evaluate_m2/tests/evaluator_test_helper.py @@ -1,6 +1,11 @@ from datetime import date from evaluate_m2.evaluate import evaluator +from evaluate_m2.models import ( + EvaluatorMetadata, + EvaluatorResult, + EvaluatorResultSummary, +) from parse_m2.models import J1, J2, K2, K4, L1, AccountActivity, M2DataFile, Metro2Event @@ -106,6 +111,7 @@ def acct_record(file: M2DataFile, custom_values: dict) -> AccountActivity: acct_activity.save() return acct_activity + def k2_record(custom_values: dict): """ Returns a K2 record for use in tests, using the values @@ -131,6 +137,7 @@ def k2_record(custom_values: dict): k2.save() return k2 + def k4_record(custom_values: dict): """ Returns a K4 record for use in tests, using the values @@ -160,6 +167,7 @@ def k4_record(custom_values: dict): k4.save() return k4 + def l1_record(custom_values: dict): """ Returns a L1 record for use in tests, using the values @@ -187,6 +195,7 @@ def l1_record(custom_values: dict): l1.save() return l1 + def create_bulk_acct_record(file: M2DataFile, value_list: dict, size: int): """ Returns a list of AccountActivity records for use in tests, using the values @@ -208,6 +217,7 @@ def create_bulk_acct_record(file: M2DataFile, value_list: dict, size: int): account_activities.append(acct_activity) return account_activities + def create_bulk_JSegments(j_type: str, value_list: dict, size: int): """ Returns a list of J1/J2 records for use in tests, using the values @@ -240,6 +250,39 @@ def create_bulk_JSegments(j_type: str, value_list: dict, size: int): return J2.objects.bulk_create(j_segments) +def evaluator_result_record( + event: Metro2Event, + evaluator_id: str, + source_record: AccountActivity +) -> EvaluatorResult: + """ + Returns an EvaluatorResult for use in tests, creating the parent + EvaluatorResultSummary (one per evaluator/event) as needed. + + Inputs: + - event: the Metro2Event the result belongs to + - evaluator_id: id for the EvaluatorMetadata (created if absent) + - source_record: the AccountActivity the result points to + """ + evaluator, _ = EvaluatorMetadata.objects.get_or_create( + id=evaluator_id + ) + result_summary, _ = EvaluatorResultSummary.objects.get_or_create( + event=event, + evaluator=evaluator, + defaults={ + "hits": 0, + "sample_ids": [] + } + ) + return EvaluatorResult.objects.create( + result_summary=result_summary, + acct_num=source_record.cons_acct_num, + source_record=source_record, + date=source_record.activity_date, + ) + + class EvaluatorTestHelper: evaluators = evaluator.evaluators diff --git a/django/evaluate_m2/tests/test_managers.py b/django/evaluate_m2/tests/test_managers.py index f59632ac..38d11571 100644 --- a/django/evaluate_m2/tests/test_managers.py +++ b/django/evaluate_m2/tests/test_managers.py @@ -2,7 +2,7 @@ from django.test import TestCase -from evaluate_m2.tests.evaluator_test_helper import acct_record +from evaluate_m2.tests.evaluator_test_helper import acct_record, evaluator_result_record from parse_m2.models import AccountActivity, M2DataFile, Metro2Event @@ -110,3 +110,75 @@ def test_j1_has_previous_bankruptcy_indicators(self): result = AccountActivity.objects.no_previous_bankruptcy_indicators().count() self.assertEqual(result, 0) + + +class AccountActivityQuerySetAnnotationsTest(TestCase): + def setUp(self) -> None: + self.event = Metro2Event.objects.create(name = "test") + self.file = M2DataFile.objects.create(event=self.event, file_name="test") + + def test_inconsistency_counts(self): + acct = acct_record( + self.file, { + "id":"1", + "activity_date": date(2022, 5, 31), + "cons_acct_num": "0032", + } + ) + evaluator_result_record(self.event, "Sample-1", acct) + evaluator_result_record(self.event, "Sample-2", acct) + + qs = AccountActivity.objects.with_inconsistency_counts(self.event) + query_acct = qs.filter(cons_acct_num=acct.cons_acct_num).first() + self.assertEqual(query_acct.total_inconsistencies, 2) + + def test_inconsistency_counts_same_eval(self): + acct1 = acct_record( + self.file, { + "id":"1", + "activity_date": date(2022, 5, 31), + "cons_acct_num": "0032", + } + ) + acct2 = acct_record( + self.file, { + "id":"2", + "activity_date": date(2022, 6, 30), + "cons_acct_num": "0032", + } + ) + # One hit across two months + evaluator_result_record(self.event, "Sample-1", acct1) + evaluator_result_record(self.event, "Sample-1", acct2) + + qs = AccountActivity.objects.with_inconsistency_counts(self.event) + query_acct = qs.filter(cons_acct_num=acct1.cons_acct_num).first() + self.assertEqual(query_acct.total_inconsistencies, 2) + + def test_months_of_data(self): + acct_record( + self.file, { + "id":"1", + "activity_date": date(2022, 6, 5), + "cons_acct_num": "0032", + } + ) + acct_record( + self.file, { + "id":"2", + "activity_date": date(2022, 6, 20), + "cons_acct_num": "0032", + } + ) + acct_record( + self.file, { + "id":"3", + "activity_date": date(2022, 8, 1), + "cons_acct_num": "0032", + } + ) + + qs = AccountActivity.objects.with_months_of_data(self.event) + query_acct = qs.filter(cons_acct_num="0032").first() + # June + August + self.assertEqual(query_acct.months_of_data, 2) diff --git a/django/evaluate_m2/tests/test_views.py b/django/evaluate_m2/tests/test_views.py index 663db4d3..b238f413 100644 --- a/django/evaluate_m2/tests/test_views.py +++ b/django/evaluate_m2/tests/test_views.py @@ -303,6 +303,43 @@ def test_evaluator_results_view_with_error_no_evaluator_results_summary(self): 'EvaluatorResultSummary record(s) not found for event ID 1.', status_code=404) + ######################################## + # Tests for Account list view API endpoint + def test_account_list_view_no_acct_num(self): + self.create_activity_data() + expected = [] + response = self.client.get("/api/events/1/account/") + + # the response should be a JSON + self.assertEqual(response.status_code, 200) + self.assertEqual(response.headers["Content-Type"], "application/json") + self.assertEqual(response.json(), expected) + + def test_account_list_view(self): + self.create_activity_data() + expected = [ + { + "acct_type": "", + "cons_acct_num": "0032", + "months_of_data": 1, + "port_type": "A", + "total_inconsistencies": 2 + }, + { + "acct_type": "", + "cons_acct_num": "0033", + "months_of_data": 1, + "port_type": "A", + "total_inconsistencies": 1 + } + ] + response = self.client.get("/api/events/1/account/?cons_acct_num=0033,0032") + + # the response should be a JSON + self.assertEqual(response.status_code, 200) + self.assertEqual(response.headers["Content-Type"], "application/json") + self.assertEqual(response.json(), expected) + ######################################## # Tests for Account Summary view API endpoint def test_account_summary_view_single_results(self): diff --git a/django/evaluate_m2/urls.py b/django/evaluate_m2/urls.py index 93a2d290..25b4b911 100644 --- a/django/evaluate_m2/urls.py +++ b/django/evaluate_m2/urls.py @@ -9,9 +9,11 @@ eval_views.download_evaluator_results_csv), path('/evaluator//', eval_views.EvaluatorResultsView().as_view()), + path('/account/', + eval_views.AccountsListView().as_view()), path('/account//', - eval_views.account_summary_view), + eval_views.account_summary_view), path('/account//account_holder/', - eval_views.account_pii_view), + eval_views.account_pii_view), path('/', eval_views.events_view), ] diff --git a/django/evaluate_m2/views.py b/django/evaluate_m2/views.py index f2c5054d..98b45661 100644 --- a/django/evaluate_m2/views.py +++ b/django/evaluate_m2/views.py @@ -6,6 +6,7 @@ from django.conf import settings from django.http import Http404, HttpResponse, StreamingHttpResponse from django.shortcuts import get_list_or_404 +from django.utils.functional import cached_property import botocore import django_filters.rest_framework @@ -16,7 +17,7 @@ from django_application.s3_utils import s3_session from evaluate_m2 import upload_utils from evaluate_m2.exception_utils import get_evaluate_m2_not_found_exception -from evaluate_m2.filters import EvaluatorResultFilterSet +from evaluate_m2.filters import AccountListFilterSet, EvaluatorResultFilterSet from evaluate_m2.models import ( EvaluatorMetadata, EvaluatorResult, @@ -24,6 +25,7 @@ ) from evaluate_m2.pagination import EvaluatorResultsPaginator from evaluate_m2.serializers import ( + AccountListSerializer, EvaluatorMetadataSerializer, EventsViewSerializer, ) @@ -58,6 +60,7 @@ def download_evaluator_metadata_csv(request): return response + @api_view(('GET',)) def download_evaluator_results_csv(request, event_id, evaluator_id): logger = logging.getLogger('views.download_evaluator_results_csv') @@ -131,6 +134,7 @@ def account_summary_view(request, event_id, account_number): logger.error(error['message']) return Response(error, status=status.HTTP_404_NOT_FOUND) + @api_view(('GET',)) def account_pii_view(request, event_id, account_number): logger = logging.getLogger('views.account_pii_view') @@ -153,6 +157,7 @@ def account_pii_view(request, event_id, account_number): logger.error(error['message']) return Response(error, status=status.HTTP_404_NOT_FOUND) + @api_view() def events_view(request, event_id): logger = logging.getLogger('views.evaluator_results_view') @@ -188,6 +193,7 @@ def events_view(request, event_id): logger.error(error['message']) return Response(error, status=status.HTTP_404_NOT_FOUND) + ########################################### ## Helper methods for eval results when S3_ENABLED == True def fetch_csv_results_from_s3(request, event_id, evaluator_id): @@ -209,6 +215,7 @@ def fetch_csv_results_from_s3(request, event_id, evaluator_id): logger.error(error['message']) return Response(error, status=status.HTTP_404_NOT_FOUND) + def fetch_json_results_from_s3(request, event_id, evaluator_id): logger = logging.getLogger('views.fetch_json_results_from_s3') s3 = s3_session() @@ -319,3 +326,35 @@ def list(self, request, *args, **kwargs): many=True, ) return self.get_paginated_response(serializer.data) + + +class AccountsListView(generics.ListAPIView): + filter_backends = [ + django_filters.rest_framework.DjangoFilterBackend, + ] + filterset_class = AccountListFilterSet + + @cached_property + def event(self): + event_id = self.kwargs["event_id"] + return Metro2Event.objects.get(id=event_id) + + def get_queryset(self): + return self.event.get_all_account_activity( + ).with_inconsistency_counts( + self.event + ).with_months_of_data( + self.event + ) + + def list(self, request, *args, **kwargs): + # TODO: replace using DRF permissions/check_permissions() + if not has_permissions_for_request(request, self.event): + return HttpResponse('Unauthorized', status=401) + + queryset = self.filter_queryset( + self.get_queryset() + ).distinct_accounts() + + serializer = AccountListSerializer(queryset, many=True) + return Response(serializer.data)