From 5a225e2794afe6446f5b0214b41d4188d01925f6 Mon Sep 17 00:00:00 2001 From: Will Barton Date: Thu, 23 Jul 2026 10:01:22 -0400 Subject: [PATCH 1/7] Initial account list endpoint This adds an account listing endpoint to an event that serializes the accounts within the event and annotates them with total number of inconsistencies (evaluators hit), and months of data for the account. I've added these annotations to the `AccountActivityQuerySet` object management for `AccountActivity` (which is defined in `evaluate_m2`), so they're available on any `AccountActivity.objects` query. --- django/evaluate_m2/managers.py | 49 ++++++++++++++++++++++++++++++- django/evaluate_m2/serializers.py | 15 ++++++++-- django/evaluate_m2/urls.py | 6 ++-- django/evaluate_m2/views.py | 47 +++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 5 deletions(-) diff --git a/django/evaluate_m2/managers.py b/django/evaluate_m2/managers.py index 235dad22..a3bee787 100644 --- a/django/evaluate_m2/managers.py +++ b/django/evaluate_m2/managers.py @@ -1,5 +1,6 @@ from django.db import models -from django.db.models import Q +from django.db.models import Count, IntegerField, OuterRef, Q, Subquery, Value +from django.db.models.functions import Coalesce, TruncMonth class AccountActivityQuerySet(models.QuerySet): @@ -9,3 +10,49 @@ def no_previous_bankruptcy_indicators(self): Q(previous_values__cons_info_ind_assoc__isnull=True) ) & Q(previous_values__cons_info_ind='') ) + + def with_inconsistency_counts(self, event): + from evaluate_m2.models import EvaluatorResult + + # Subquery for the number of inconsistencies (evaluators hit) for + # each account. This is just a count of hits. + subquery = EvaluatorResult.objects.filter( + result_summary__event=event, + acct_num=OuterRef("cons_acct_num"), + ).order_by().values("acct_num").annotate( + n=Count("result_summary__evaluator_id", distinct=True), + ).values("n")[:1] + + return self.annotate( + total_inconsistencies=Coalesce( + Subquery(subquery, output_field=IntegerField()), + Value(0), + ), + ) + + def with_months_of_data(self, event): + # Subquery for the number of months of data. This counts **distinct** + # months, not the number of months in the total range. + # I.e. if there's data for Feb, March, and May, but not April, that's + # three months of data, not four. + subquery = self.model._default_manager.filter( + data_file__event=event, + cons_acct_num=OuterRef("cons_acct_num"), + ).order_by().annotate( + month=TruncMonth("activity_date"), + ).values("cons_acct_num").annotate( + n=Count("month", distinct=True), + ).values("n")[:1] + + return self.annotate( + months_of_data=Coalesce( + Subquery(subquery, output_field=IntegerField()), + Value(0), + ) + ) + + def distinct_accounts(self): + return self.order_by( + "cons_acct_num", + "-activity_date", + ).distinct("cons_acct_num") diff --git a/django/evaluate_m2/serializers.py b/django/evaluate_m2/serializers.py index 1313c251..f87634be 100644 --- a/django/evaluate_m2/serializers.py +++ b/django/evaluate_m2/serializers.py @@ -6,8 +6,8 @@ parse_fields_from_csv, plain_to_code_field_map, ) - -from .models import EvaluatorMetadata, EvaluatorResultSummary +from evaluate_m2.models import EvaluatorMetadata, EvaluatorResultSummary +from parse_m2.serializers import AccountActivitySerializer class EventsViewSerializer(serializers.ModelSerializer): @@ -224,3 +224,14 @@ def validate(self, data): if invalid_fields: raise serializers.ValidationError(f"Invalid field names: {invalid_fields}") return data + + +class AccountListSerializer(AccountActivitySerializer): + total_inconsistencies = serializers.IntegerField(read_only=True) + months_of_data = serializers.IntegerField(read_only=True) + + class Meta(AccountActivitySerializer.Meta): + default_fields = list(AccountActivitySerializer.Meta.default_fields) + [ + "total_inconsistencies", + "months_of_data", + ] diff --git a/django/evaluate_m2/urls.py b/django/evaluate_m2/urls.py index 93a2d290..25b4b911 100644 --- a/django/evaluate_m2/urls.py +++ b/django/evaluate_m2/urls.py @@ -9,9 +9,11 @@ eval_views.download_evaluator_results_csv), path('/evaluator//', eval_views.EvaluatorResultsView().as_view()), + path('/account/', + eval_views.AccountsListView().as_view()), path('/account//', - eval_views.account_summary_view), + eval_views.account_summary_view), path('/account//account_holder/', - eval_views.account_pii_view), + eval_views.account_pii_view), path('/', eval_views.events_view), ] diff --git a/django/evaluate_m2/views.py b/django/evaluate_m2/views.py index f2c5054d..a6ea3c06 100644 --- a/django/evaluate_m2/views.py +++ b/django/evaluate_m2/views.py @@ -24,6 +24,7 @@ ) from evaluate_m2.pagination import EvaluatorResultsPaginator from evaluate_m2.serializers import ( + AccountListSerializer, EvaluatorMetadataSerializer, EventsViewSerializer, ) @@ -58,6 +59,7 @@ def download_evaluator_metadata_csv(request): return response + @api_view(('GET',)) def download_evaluator_results_csv(request, event_id, evaluator_id): logger = logging.getLogger('views.download_evaluator_results_csv') @@ -131,6 +133,7 @@ def account_summary_view(request, event_id, account_number): logger.error(error['message']) return Response(error, status=status.HTTP_404_NOT_FOUND) + @api_view(('GET',)) def account_pii_view(request, event_id, account_number): logger = logging.getLogger('views.account_pii_view') @@ -153,6 +156,7 @@ def account_pii_view(request, event_id, account_number): logger.error(error['message']) return Response(error, status=status.HTTP_404_NOT_FOUND) + @api_view() def events_view(request, event_id): logger = logging.getLogger('views.evaluator_results_view') @@ -188,6 +192,7 @@ def events_view(request, event_id): logger.error(error['message']) return Response(error, status=status.HTTP_404_NOT_FOUND) + ########################################### ## Helper methods for eval results when S3_ENABLED == True def fetch_csv_results_from_s3(request, event_id, evaluator_id): @@ -209,6 +214,7 @@ def fetch_csv_results_from_s3(request, event_id, evaluator_id): logger.error(error['message']) return Response(error, status=status.HTTP_404_NOT_FOUND) + def fetch_json_results_from_s3(request, event_id, evaluator_id): logger = logging.getLogger('views.fetch_json_results_from_s3') s3 = s3_session() @@ -319,3 +325,44 @@ def list(self, request, *args, **kwargs): many=True, ) return self.get_paginated_response(serializer.data) + + + + +class AccountsListView(generics.ListAPIView): + filter_backends = [ + django_filters.rest_framework.DjangoFilterBackend, + ] + # filterset_class = EvaluatorResultFilterSet + + def get_queryset(self): + event_id = self.kwargs["event_id"] + event = Metro2Event.objects.get(id=event_id) + + queryset = event.get_all_account_activity().select_related( + "k2", + "k4", + "l1" + ).with_inconsistency_counts( + event + ).with_months_of_data( + event + ) + + return queryset + + def list(self, request, *args, **kwargs): + event_id = self.kwargs["event_id"] + event = Metro2Event.objects.get(id=event_id) + + # TODO: replace using DRF permissions/check_permissions() + if not has_permissions_for_request(request, event): + return HttpResponse('Unauthorized', status=401) + + queryset = self.filter_queryset( + self.get_queryset() + ).distinct_accounts() + + # Paginate the results + serializer = AccountListSerializer(queryset, many=True) + return Response(serializer.data) From 4878b63c9a9802370919cfbf2b11a7fb239bec6d Mon Sep 17 00:00:00 2001 From: Will Barton Date: Thu, 23 Jul 2026 11:45:09 -0400 Subject: [PATCH 2/7] Limit the account list serializer fields --- django/evaluate_m2/serializers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/django/evaluate_m2/serializers.py b/django/evaluate_m2/serializers.py index f87634be..eff66bf7 100644 --- a/django/evaluate_m2/serializers.py +++ b/django/evaluate_m2/serializers.py @@ -231,7 +231,11 @@ class AccountListSerializer(AccountActivitySerializer): months_of_data = serializers.IntegerField(read_only=True) class Meta(AccountActivitySerializer.Meta): - default_fields = list(AccountActivitySerializer.Meta.default_fields) + [ + default_fields = [ + # list(AccountActivitySerializer.Meta.default_fields) + [ + "cons_acct_num", + "port_type", + "acct_type", "total_inconsistencies", "months_of_data", ] From 24f9285ac6dcf550551d2a2375b79e5d830055f5 Mon Sep 17 00:00:00 2001 From: Will Barton Date: Thu, 23 Jul 2026 11:45:15 -0400 Subject: [PATCH 3/7] Enable filtering accounts by account number This also requires a `cons_acct_num` parameter to be given, otherwise no results will be returned. --- django/evaluate_m2/filters.py | 19 +++++++++++++++++++ django/evaluate_m2/views.py | 6 ++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/django/evaluate_m2/filters.py b/django/evaluate_m2/filters.py index 61c779e4..ee74822b 100644 --- a/django/evaluate_m2/filters.py +++ b/django/evaluate_m2/filters.py @@ -1,6 +1,7 @@ import django_filters.rest_framework from evaluate_m2.models import EvaluatorResult +from parse_m2.models import AccountActivity class AnyCharFilter(django_filters.BaseInFilter, django_filters.CharFilter): @@ -186,3 +187,21 @@ class Meta: "smpa", "sort", ] + + +class AccountListFilterSet(django_filters.rest_framework.FilterSet): + cons_acct_num = django_filters.BaseInFilter( + field_name="cons_acct_num", + lookup_expr="in" + ) + + class Meta: + model = AccountActivity + fields = ["cons_acct_num"] + + # Require a specific list of accounts to filter, otherwise this + # filter will return an empty queryset. + def filter_queryset(self, queryset): + if not self.form.cleaned_data.get("cons_acct_num"): + return queryset.none() + return super().filter_queryset(queryset) diff --git a/django/evaluate_m2/views.py b/django/evaluate_m2/views.py index a6ea3c06..e7a34e6b 100644 --- a/django/evaluate_m2/views.py +++ b/django/evaluate_m2/views.py @@ -16,7 +16,7 @@ from django_application.s3_utils import s3_session from evaluate_m2 import upload_utils from evaluate_m2.exception_utils import get_evaluate_m2_not_found_exception -from evaluate_m2.filters import EvaluatorResultFilterSet +from evaluate_m2.filters import AccountListFilterSet, EvaluatorResultFilterSet from evaluate_m2.models import ( EvaluatorMetadata, EvaluatorResult, @@ -327,13 +327,11 @@ def list(self, request, *args, **kwargs): return self.get_paginated_response(serializer.data) - - class AccountsListView(generics.ListAPIView): filter_backends = [ django_filters.rest_framework.DjangoFilterBackend, ] - # filterset_class = EvaluatorResultFilterSet + filterset_class = AccountListFilterSet def get_queryset(self): event_id = self.kwargs["event_id"] From 65d0529e3f7d3856f050bc35b76f1e7bbd354cdc Mon Sep 17 00:00:00 2001 From: Will Barton Date: Wed, 5 Aug 2026 11:49:21 -0400 Subject: [PATCH 4/7] Add tests for account view and annotations --- .../tests/evaluator_test_helper.py | 43 +++++++++++ django/evaluate_m2/tests/test_managers.py | 74 ++++++++++++++++++- django/evaluate_m2/tests/test_views.py | 37 ++++++++++ 3 files changed, 153 insertions(+), 1 deletion(-) diff --git a/django/evaluate_m2/tests/evaluator_test_helper.py b/django/evaluate_m2/tests/evaluator_test_helper.py index fbd1301a..635fdb09 100644 --- a/django/evaluate_m2/tests/evaluator_test_helper.py +++ b/django/evaluate_m2/tests/evaluator_test_helper.py @@ -1,6 +1,11 @@ from datetime import date from evaluate_m2.evaluate import evaluator +from evaluate_m2.models import ( + EvaluatorMetadata, + EvaluatorResult, + EvaluatorResultSummary, +) from parse_m2.models import J1, J2, K2, K4, L1, AccountActivity, M2DataFile, Metro2Event @@ -106,6 +111,7 @@ def acct_record(file: M2DataFile, custom_values: dict) -> AccountActivity: acct_activity.save() return acct_activity + def k2_record(custom_values: dict): """ Returns a K2 record for use in tests, using the values @@ -131,6 +137,7 @@ def k2_record(custom_values: dict): k2.save() return k2 + def k4_record(custom_values: dict): """ Returns a K4 record for use in tests, using the values @@ -160,6 +167,7 @@ def k4_record(custom_values: dict): k4.save() return k4 + def l1_record(custom_values: dict): """ Returns a L1 record for use in tests, using the values @@ -187,6 +195,7 @@ def l1_record(custom_values: dict): l1.save() return l1 + def create_bulk_acct_record(file: M2DataFile, value_list: dict, size: int): """ Returns a list of AccountActivity records for use in tests, using the values @@ -208,6 +217,7 @@ def create_bulk_acct_record(file: M2DataFile, value_list: dict, size: int): account_activities.append(acct_activity) return account_activities + def create_bulk_JSegments(j_type: str, value_list: dict, size: int): """ Returns a list of J1/J2 records for use in tests, using the values @@ -240,6 +250,39 @@ def create_bulk_JSegments(j_type: str, value_list: dict, size: int): return J2.objects.bulk_create(j_segments) +def evaluator_result_record( + event: Metro2Event, + evaluator_id: str, + source_record: AccountActivity +) -> EvaluatorResult: + """ + Returns an EvaluatorResult for use in tests, creating the parent + EvaluatorResultSummary (one per evaluator/event) as needed. + + Inputs: + - event: the Metro2Event the result belongs to + - evaluator_id: id for the EvaluatorMetadata (created if absent) + - source_record: the AccountActivity the result points to + """ + evaluator, _ = EvaluatorMetadata.objects.get_or_create( + id=evaluator_id + ) + result_summary, _ = EvaluatorResultSummary.objects.get_or_create( + event=event, + evaluator=evaluator, + defaults={ + "hits": 0, + "sample_ids": [] + } + ) + return EvaluatorResult.objects.create( + result_summary=result_summary, + acct_num=source_record.cons_acct_num, + source_record=source_record, + date=source_record.activity_date, + ) + + class EvaluatorTestHelper: evaluators = evaluator.evaluators diff --git a/django/evaluate_m2/tests/test_managers.py b/django/evaluate_m2/tests/test_managers.py index f59632ac..c6aa05ce 100644 --- a/django/evaluate_m2/tests/test_managers.py +++ b/django/evaluate_m2/tests/test_managers.py @@ -2,7 +2,7 @@ from django.test import TestCase -from evaluate_m2.tests.evaluator_test_helper import acct_record +from evaluate_m2.tests.evaluator_test_helper import acct_record, evaluator_result_record from parse_m2.models import AccountActivity, M2DataFile, Metro2Event @@ -110,3 +110,75 @@ def test_j1_has_previous_bankruptcy_indicators(self): result = AccountActivity.objects.no_previous_bankruptcy_indicators().count() self.assertEqual(result, 0) + + +class AccountActivityQuerySetAnnotationsTest(TestCase): + def setUp(self) -> None: + self.event = Metro2Event.objects.create(name = "test") + self.file = M2DataFile.objects.create(event=self.event, file_name="test") + + def test_inconsistency_counts(self): + acct = acct_record( + self.file, { + "id":"1", + "activity_date": date(2022, 5, 31), + "cons_acct_num": "0032", + } + ) + evaluator_result_record(self.event, "Sample-1", acct) + evaluator_result_record(self.event, "Sample-2", acct) + + qs = AccountActivity.objects.with_inconsistency_counts(self.event) + query_acct = qs.filter(cons_acct_num=acct.cons_acct_num).first() + self.assertEqual(query_acct.total_inconsistencies, 2) + + def test_inconsistency_counts_same_eval(self): + acct1 = acct_record( + self.file, { + "id":"1", + "activity_date": date(2022, 5, 31), + "cons_acct_num": "0032", + } + ) + acct2 = acct_record( + self.file, { + "id":"2", + "activity_date": date(2022, 6, 30), + "cons_acct_num": "0032", + } + ) + # One hit across two months + evaluator_result_record(self.event, "Sample-1", acct1) + evaluator_result_record(self.event, "Sample-1", acct2) + + qs = AccountActivity.objects.with_inconsistency_counts(self.event) + query_acct = qs.filter(cons_acct_num=acct1.cons_acct_num).first() + self.assertEqual(query_acct.total_inconsistencies, 1) + + def test_months_of_data(self): + acct_record( + self.file, { + "id":"1", + "activity_date": date(2022, 6, 5), + "cons_acct_num": "0032", + } + ) + acct_record( + self.file, { + "id":"2", + "activity_date": date(2022, 6, 20), + "cons_acct_num": "0032", + } + ) + acct_record( + self.file, { + "id":"3", + "activity_date": date(2022, 8, 1), + "cons_acct_num": "0032", + } + ) + + qs = AccountActivity.objects.with_months_of_data(self.event) + query_acct = qs.filter(cons_acct_num="0032").first() + # June + August + self.assertEqual(query_acct.months_of_data, 2) diff --git a/django/evaluate_m2/tests/test_views.py b/django/evaluate_m2/tests/test_views.py index 663db4d3..b238f413 100644 --- a/django/evaluate_m2/tests/test_views.py +++ b/django/evaluate_m2/tests/test_views.py @@ -303,6 +303,43 @@ def test_evaluator_results_view_with_error_no_evaluator_results_summary(self): 'EvaluatorResultSummary record(s) not found for event ID 1.', status_code=404) + ######################################## + # Tests for Account list view API endpoint + def test_account_list_view_no_acct_num(self): + self.create_activity_data() + expected = [] + response = self.client.get("/api/events/1/account/") + + # the response should be a JSON + self.assertEqual(response.status_code, 200) + self.assertEqual(response.headers["Content-Type"], "application/json") + self.assertEqual(response.json(), expected) + + def test_account_list_view(self): + self.create_activity_data() + expected = [ + { + "acct_type": "", + "cons_acct_num": "0032", + "months_of_data": 1, + "port_type": "A", + "total_inconsistencies": 2 + }, + { + "acct_type": "", + "cons_acct_num": "0033", + "months_of_data": 1, + "port_type": "A", + "total_inconsistencies": 1 + } + ] + response = self.client.get("/api/events/1/account/?cons_acct_num=0033,0032") + + # the response should be a JSON + self.assertEqual(response.status_code, 200) + self.assertEqual(response.headers["Content-Type"], "application/json") + self.assertEqual(response.json(), expected) + ######################################## # Tests for Account Summary view API endpoint def test_account_summary_view_single_results(self): From 8c90463ba378623d98569693b5b8266ae3466db7 Mon Sep 17 00:00:00 2001 From: Will Barton Date: Wed, 5 Aug 2026 11:49:42 -0400 Subject: [PATCH 5/7] Cache event in account list view --- django/evaluate_m2/views.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/django/evaluate_m2/views.py b/django/evaluate_m2/views.py index e7a34e6b..046e43b7 100644 --- a/django/evaluate_m2/views.py +++ b/django/evaluate_m2/views.py @@ -6,6 +6,7 @@ from django.conf import settings from django.http import Http404, HttpResponse, StreamingHttpResponse from django.shortcuts import get_list_or_404 +from django.utils.functional import cached_property import botocore import django_filters.rest_framework @@ -333,34 +334,30 @@ class AccountsListView(generics.ListAPIView): ] filterset_class = AccountListFilterSet - def get_queryset(self): + @cached_property + def event(self): event_id = self.kwargs["event_id"] - event = Metro2Event.objects.get(id=event_id) + return Metro2Event.objects.get(id=event_id) - queryset = event.get_all_account_activity().select_related( + def get_queryset(self): + return self.event.get_all_account_activity().select_related( "k2", "k4", "l1" ).with_inconsistency_counts( - event + self.event ).with_months_of_data( - event + self.event ) - return queryset - def list(self, request, *args, **kwargs): - event_id = self.kwargs["event_id"] - event = Metro2Event.objects.get(id=event_id) - # TODO: replace using DRF permissions/check_permissions() - if not has_permissions_for_request(request, event): + if not has_permissions_for_request(request, self.event): return HttpResponse('Unauthorized', status=401) queryset = self.filter_queryset( self.get_queryset() ).distinct_accounts() - # Paginate the results serializer = AccountListSerializer(queryset, many=True) return Response(serializer.data) From 65153069525faa2343597c1f35faf64d202e80cb Mon Sep 17 00:00:00 2001 From: Will Barton Date: Thu, 6 Aug 2026 08:53:44 -0400 Subject: [PATCH 6/7] Remove unnecessary select_related and comment --- django/evaluate_m2/serializers.py | 1 - django/evaluate_m2/views.py | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/django/evaluate_m2/serializers.py b/django/evaluate_m2/serializers.py index eff66bf7..0a7c8539 100644 --- a/django/evaluate_m2/serializers.py +++ b/django/evaluate_m2/serializers.py @@ -232,7 +232,6 @@ class AccountListSerializer(AccountActivitySerializer): class Meta(AccountActivitySerializer.Meta): default_fields = [ - # list(AccountActivitySerializer.Meta.default_fields) + [ "cons_acct_num", "port_type", "acct_type", diff --git a/django/evaluate_m2/views.py b/django/evaluate_m2/views.py index 046e43b7..98b45661 100644 --- a/django/evaluate_m2/views.py +++ b/django/evaluate_m2/views.py @@ -340,10 +340,7 @@ def event(self): return Metro2Event.objects.get(id=event_id) def get_queryset(self): - return self.event.get_all_account_activity().select_related( - "k2", - "k4", - "l1" + return self.event.get_all_account_activity( ).with_inconsistency_counts( self.event ).with_months_of_data( From 1317617b5ef01550a310b1c04e7423610a953850 Mon Sep 17 00:00:00 2001 From: Will Barton Date: Thu, 6 Aug 2026 09:34:32 -0400 Subject: [PATCH 7/7] Inconsistencies are cumulative, including duplicates --- django/evaluate_m2/managers.py | 2 +- django/evaluate_m2/tests/test_managers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/django/evaluate_m2/managers.py b/django/evaluate_m2/managers.py index a3bee787..1ba2d1da 100644 --- a/django/evaluate_m2/managers.py +++ b/django/evaluate_m2/managers.py @@ -20,7 +20,7 @@ def with_inconsistency_counts(self, event): result_summary__event=event, acct_num=OuterRef("cons_acct_num"), ).order_by().values("acct_num").annotate( - n=Count("result_summary__evaluator_id", distinct=True), + n=Count("id"), ).values("n")[:1] return self.annotate( diff --git a/django/evaluate_m2/tests/test_managers.py b/django/evaluate_m2/tests/test_managers.py index c6aa05ce..38d11571 100644 --- a/django/evaluate_m2/tests/test_managers.py +++ b/django/evaluate_m2/tests/test_managers.py @@ -153,7 +153,7 @@ def test_inconsistency_counts_same_eval(self): qs = AccountActivity.objects.with_inconsistency_counts(self.event) query_acct = qs.filter(cons_acct_num=acct1.cons_acct_num).first() - self.assertEqual(query_acct.total_inconsistencies, 1) + self.assertEqual(query_acct.total_inconsistencies, 2) def test_months_of_data(self): acct_record(