Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
"rotate-keys": "node --env-file-if-exists=.env --import tsx scripts/rotate-keys.ts",
"backfill-clean-names": "node --env-file-if-exists=.env --import tsx scripts/backfill-clean-names.ts",
"backfill-transfers": "node --env-file-if-exists=.env --import tsx scripts/backfill-transfers.ts",
"backfill-investment-activity": "node --env-file-if-exists=.env --import tsx scripts/backfill-investment-activity.ts",
"backfill-balances": "node --env-file-if-exists=.env --import tsx scripts/backfill-balances.ts",
"test": "vitest run",
"test:watch": "vitest",
Expand Down
19 changes: 19 additions & 0 deletions scripts/backfill-investment-activity.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
/**
* Operator entry point for tagging historical investment-account activity
* (brokerage fills, clearing fees) that predates the investment-account
* exclusion.
* Usage: pnpm backfill-investment-activity
* Requires DATABASE_URL (loaded from .env when present).
*/
import { backfillInvestmentActivity } from "@/lib/jobs/backfill-investment-activity";

async function main() {
const { households, tagged } = await backfillInvestmentActivity();
console.log(`[backfill-investment-activity] households=${households} tagged=${tagged}`);
process.exit(0);
}

main().catch((err) => {
console.error(err);
process.exit(1);
});
14 changes: 14 additions & 0 deletions src/components/molecules/category-pill-label.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,18 @@ describe("categoryPillLabel", () => {
variant: "category",
});
});

it("labels investment-account activity as Investment, not Transfer", () => {
expect(categoryPillLabel(null, true, "investment_account")).toEqual({
text: "Investment",
variant: "investment",
});
});

it("prefers an assigned category name even for investment-account activity", () => {
expect(categoryPillLabel("Dividends", true, "investment_account")).toEqual({
text: "Dividends",
variant: "category",
});
});
});
12 changes: 10 additions & 2 deletions src/components/molecules/category-pill-label.ts
Original file line number Diff line number Diff line change
@@ -1,18 +1,26 @@
import { UNCATEGORIZED } from "@/lib/labels";

export type CategoryPillVariant = "category" | "transfer" | "uncategorized";
export type CategoryPillVariant = "category" | "transfer" | "investment" | "uncategorized";

/**
* Decides what a transaction's category pill should read. Transfers (CC
* autopay, inter-account moves, P2P) legitimately have no spending category,
* so an uncategorized transfer reads "Transfer" rather than "Uncategorized" —
* it isn't a categorization gap. An assigned category always wins.
* it isn't a categorization gap. Investment-account activity (brokerage
* fills, clearing fees) is tagged the same way under the hood but reads
* "Investment" — it's excluded from spend/income for the same reason, but
* it isn't a transfer in the user-facing sense. An assigned category always
* wins over either label.
*/
export function categoryPillLabel(
categoryName: string | null,
isTransfer: boolean,
transferSource?: string | null,
): { text: string; variant: CategoryPillVariant } {
if (categoryName) return { text: categoryName, variant: "category" };
if (isTransfer && transferSource === "investment_account") {
return { text: "Investment", variant: "investment" };
}
if (isTransfer) return { text: "Transfer", variant: "transfer" };
return { text: UNCATEGORIZED, variant: "uncategorized" };
}
4 changes: 3 additions & 1 deletion src/components/molecules/category-pill.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ interface CategoryPillProps {
categories: CategoryGroup[];
disabled?: boolean;
isTransfer?: boolean;
transferSource?: string | null;
merchantId?: string | null;
merchantName?: string | null;
onCategoryChange?: (categoryId: string | null, categoryName: string | null) => void;
Expand All @@ -50,6 +51,7 @@ export function CategoryPill({
categories,
disabled = false,
isTransfer = false,
transferSource = null,
merchantId,
merchantName,
onCategoryChange,
Expand Down Expand Up @@ -138,7 +140,7 @@ export function CategoryPill({
>
<span className="truncate">
{(() => {
const { text, variant } = categoryPillLabel(categoryName, isTransfer);
const { text, variant } = categoryPillLabel(categoryName, isTransfer, transferSource);
if (variant === "category") return text;
return (
<span className={cn("text-muted-foreground", variant === "uncategorized" && "italic")}>
Expand Down
1 change: 1 addition & 0 deletions src/components/molecules/review-card.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ export function ReviewCard({
currentCategoryName={transaction.categoryName}
categories={categories}
isTransfer={transaction.isTransfer}
transferSource={transaction.transferSource}
onCategoryChange={onCategoryChange}
open={categoryOpen}
onOpenChange={onCategoryOpenChange}
Expand Down
1 change: 1 addition & 0 deletions src/components/molecules/transaction-row.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ export const TransactionRow = memo(function TransactionRow({
categories={categories}
disabled={txn.hasSplits}
isTransfer={txn.isTransfer}
transferSource={txn.transferSource}
merchantId={txn.merchantId}
merchantName={txn.merchantName}
/>
Expand Down
1 change: 1 addition & 0 deletions src/components/organisms/transaction-detail-panel.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ export function TransactionDetailPanel({
currentCategoryName={txn.categoryName}
categories={categories}
isTransfer={txn.isTransfer}
transferSource={txn.transferSource}
merchantId={txn.merchantId}
merchantName={txn.merchantName}
/>
Expand Down
9 changes: 7 additions & 2 deletions src/db/schema/transactions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,14 @@ export const transactions = pgTable(
// tagged separately since it never has a transferPairId. `suggested` is a
// single-leg, low-confidence match (e.g. a bare P2P processor name) —
// isTransfer stays false until a human confirms via the review queue, so
// it keeps counting toward spend/income until then.
// it keeps counting toward spend/income until then. `investment_account`
// is a deterministic, non-ambiguous tag applied to every transaction on
// an account.type="investment" account (brokerage fills, clearing fees)
// — those are never spending/income, but unlike `pattern`/`auto` they
// aren't a "transfer" in the UI sense, so the pill label branches on this
// source to show "Investment" instead of "Transfer".
transferSource: text("transfer_source", {
enum: ["pfc", "auto", "pattern", "suggested", "manual", "manual_rejected"],
enum: ["pfc", "auto", "pattern", "suggested", "manual", "manual_rejected", "investment_account"],
}),
deletedAt: timestamp("deleted_at", { withTimezone: true }),
createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(),
Expand Down
60 changes: 60 additions & 0 deletions src/lib/investment-account-tagging.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
import { eq, and, ne, or, isNull, inArray } from "drizzle-orm";
import { db as defaultDb, type LedgrDb } from "@/db";
import { transactions, accounts } from "@/db/schema";
import { scopedQuery } from "@/lib/scoped-query";
import { notDeleted } from "@/lib/query-helpers";
import { withHousehold } from "@/lib/household-context";

/**
* Tags every not-yet-decided transaction on one of the household's
* investment-type accounts (brokerage fills, clearing fees) as non-spending,
* the same way sync.ts tags them going forward. Deterministic and
* idempotent: only touches rows that are still `isTransfer=false` and whose
* transferSource isn't a user decision (manual/manual_rejected), so a repeat
* call naturally skips already-tagged or user-corrected rows.
*/
export async function applyInvestmentAccountTagging(
householdId: string,
db: LedgrDb = defaultDb,
): Promise<{ tagged: number }> {
return withHousehold(
householdId,
async (tx) => {
const scoped = scopedQuery(householdId, tx);

const investmentAccounts = await tx
.select({ id: accounts.id })
.from(accounts)
.where(and(eq(accounts.householdId, householdId), eq(accounts.type, "investment")));

if (investmentAccounts.length === 0) return { tagged: 0 };
const investmentAccountIds = investmentAccounts.map((a) => a.id);

const untagged = () =>
scoped.where(
transactions,
notDeleted(transactions),
inArray(transactions.accountId, investmentAccountIds),
eq(transactions.isTransfer, false),
or(
isNull(transactions.transferSource),
ne(transactions.transferSource, "manual_rejected"),
),
);

const candidates = await tx
.select({ id: transactions.id })
.from(transactions)
.where(untagged());

if (candidates.length === 0) return { tagged: 0 };

await tx.update(transactions)
.set({ isTransfer: true, transferSource: "investment_account", updatedAt: new Date() })
.where(untagged());

return { tagged: candidates.length };
},
db,
);
}
65 changes: 65 additions & 0 deletions src/lib/jobs/backfill-investment-activity.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
import { describe, it, expect } from "vitest";
import { eq } from "drizzle-orm";
import { createTestDb } from "../../../tests/integration/setup";
import { insertHousehold, insertAccount, insertTransaction } from "../../../tests/integration/helpers";
import { backfillInvestmentActivity } from "./backfill-investment-activity";
import { transactions } from "@/db/schema";

describe("backfillInvestmentActivity", () => {
it("tags historical investment-account transactions and skips manual_rejected rows", async () => {
const { db, close } = await createTestDb();
try {
const { householdId } = await insertHousehold(db);
const { accountId: investmentAccountId } = await insertAccount(db, householdId, { type: "investment" });
const { accountId: checkingId } = await insertAccount(db, householdId, { type: "checking" });

const { transactionId: fillId } = await insertTransaction(db, householdId, investmentAccountId, {
name: "Buy XPO Fill",
});
const { transactionId: correctedId } = await insertTransaction(db, householdId, investmentAccountId, {
name: "Buy Sphr Fill",
isTransfer: false,
transferSource: "manual_rejected",
});
const { transactionId: checkingTxnId } = await insertTransaction(db, householdId, checkingId, {
name: "Grocery Store",
});

const result = await backfillInvestmentActivity(db);

expect(result.households).toBe(1);
expect(result.tagged).toBe(1);

const [fill] = await db.select().from(transactions).where(eq(transactions.id, fillId));
expect(fill.isTransfer).toBe(true);
expect(fill.transferSource).toBe("investment_account");

const [corrected] = await db.select().from(transactions).where(eq(transactions.id, correctedId));
expect(corrected.isTransfer).toBe(false);
expect(corrected.transferSource).toBe("manual_rejected");

const [checkingTxn] = await db.select().from(transactions).where(eq(transactions.id, checkingTxnId));
expect(checkingTxn.isTransfer).toBe(false);
expect(checkingTxn.transferSource).toBeNull();
} finally {
await close();
}
});

it("is safe to re-run — already-tagged rows are skipped", async () => {
const { db, close } = await createTestDb();
try {
const { householdId } = await insertHousehold(db);
const { accountId: investmentAccountId } = await insertAccount(db, householdId, { type: "investment" });
await insertTransaction(db, householdId, investmentAccountId, { name: "Buy XPO Fill" });

const first = await backfillInvestmentActivity(db);
expect(first.tagged).toBe(1);

const second = await backfillInvestmentActivity(db);
expect(second.tagged).toBe(0);
} finally {
await close();
}
});
});
29 changes: 29 additions & 0 deletions src/lib/jobs/backfill-investment-activity.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
import { db as defaultDb, type LedgrDb } from "@/db";
import { households } from "@/db/schema";
import { applyInvestmentAccountTagging } from "@/lib/investment-account-tagging";
import { assertCanEnumerateHouseholds } from "@/lib/jobs/cross-household";

/**
* One-time operator job: runs applyInvestmentAccountTagging for every
* household, so historical transactions synced before the investment-account
* exclusion existed get tagged too. Non-destructive and idempotent — safe to
* re-run.
*
* Cross-household by design (an operator maintenance job, run via `pnpm
* backfill-investment-activity` — not reachable from the app), same shape as
* backfill-transfers.ts.
*/
export async function backfillInvestmentActivity(
db: LedgrDb = defaultDb,
): Promise<{ households: number; tagged: number }> {
await assertCanEnumerateHouseholds(db);
const allHouseholds = await db.select({ id: households.id }).from(households);

let tagged = 0;
for (const { id: householdId } of allHouseholds) {
const result = await applyInvestmentAccountTagging(householdId, db);
tagged += result.tagged;
}

return { households: allHouseholds.length, tagged };
}
25 changes: 18 additions & 7 deletions src/lib/plaid/sync.ts
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,12 @@ async function applyToDb(
? merchantNameToId.get(row.merchantName) ?? null
: null;

// Every transaction Plaid returns for an investment account (brokerage
// fills, clearing fees) is deterministically non-spending — unlike the
// PFC/pattern transfer heuristics, there's no ambiguity to preserve a
// user override for on a brand-new row.
const isInvestmentAccount = typeByInternalId.get(internalAccountId) === "investment";

insertRows.push({
id: uuid(),
accountId: internalAccountId,
Expand All @@ -361,8 +367,8 @@ async function applyToDb(
pending: row.pending,
pfcPrimary: row.pfcPrimary,
pfcDetailed: row.pfcDetailed,
isTransfer: row.isTransfer,
transferSource: row.isTransfer ? "pfc" : null,
isTransfer: isInvestmentAccount ? true : row.isTransfer,
transferSource: isInvestmentAccount ? "investment_account" : row.isTransfer ? "pfc" : null,
createdAt: now,
updatedAt: now,
});
Expand Down Expand Up @@ -397,6 +403,10 @@ async function applyToDb(
? merchantNameToId.get(row.merchantName) ?? null
: null;

const isInvestmentAccount = typeByInternalId.get(internalAccountId) === "investment";
const computedIsTransfer = isInvestmentAccount ? true : row.isTransfer;
const computedTransferSource = isInvestmentAccount ? "investment_account" : row.isTransfer ? "pfc" : null;

const existingId = existingIdByExternalId.get(row.externalId);
if (existingId) {
await tx.update(transactions)
Expand All @@ -416,11 +426,12 @@ async function applyToDb(
// Plaid's PFC re-derives isTransfer on every modified row, which
// would otherwise silently revert a user's own transfer decision
// (and orphan any transferPairId). Keep manual decisions; let PFC
// refresh everything else.
// (or the investment-account override above) refresh everything
// else.
isTransfer: sql`CASE WHEN ${transactions.transferSource} IN ('manual','manual_rejected')
THEN ${transactions.isTransfer} ELSE ${row.isTransfer} END`,
THEN ${transactions.isTransfer} ELSE ${computedIsTransfer} END`,
transferSource: sql`CASE WHEN ${transactions.transferSource} IN ('manual','manual_rejected')
THEN ${transactions.transferSource} ELSE ${row.isTransfer ? "pfc" : null} END`,
THEN ${transactions.transferSource} ELSE ${computedTransferSource} END`,
updatedAt: now,
// Preserve user's manual categorization and reviewed status
})
Expand All @@ -443,8 +454,8 @@ async function applyToDb(
pending: row.pending,
pfcPrimary: row.pfcPrimary,
pfcDetailed: row.pfcDetailed,
isTransfer: row.isTransfer,
transferSource: row.isTransfer ? "pfc" : null,
isTransfer: computedIsTransfer,
transferSource: computedTransferSource,
createdAt: now,
updatedAt: now,
});
Expand Down
4 changes: 4 additions & 0 deletions src/queries/transactions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ export interface TransactionRow {
hasSplits: boolean;
isTransfer: boolean;
transferPairId: string | null;
transferSource: string | null;
categorySource: CategorySource | null;
externalId: string | null;
}
Expand Down Expand Up @@ -72,6 +73,7 @@ const transactionSelectFields = {
notes: transactions.notes,
isTransfer: transactions.isTransfer,
transferPairId: transactions.transferPairId,
transferSource: transactions.transferSource,
categorySource: transactions.categorySource,
externalId: transactions.externalId,
};
Expand Down Expand Up @@ -200,6 +202,7 @@ export async function fetchTransactionPage(
hasSplits: splitSet.has(row.id),
isTransfer: Boolean(row.isTransfer),
transferPairId: row.transferPairId ?? null,
transferSource: row.transferSource ?? null,
categorySource: row.categorySource ?? null,
externalId: row.externalId ?? null,
}));
Expand Down Expand Up @@ -336,6 +339,7 @@ export async function getTransactionDetail(
reviewed: Boolean(row.reviewed),
isTransfer: Boolean(row.isTransfer),
transferPairId: row.transferPairId ?? null,
transferSource: row.transferSource ?? null,
categorySource: row.categorySource ?? null,
externalId: row.externalId ?? null,
hasSplits: splits.length > 0,
Expand Down
Loading
Loading