feat: integrate mailing list service (CM-1318)

.github/workflows/backend-lint.yaml

@@ -98,6 +98,48 @@ jobs:
         if: steps.changes.outputs.python_changed == 'true'
         run: uv sync --group dev --frozen
 
+      - name: Check Python linting and formatting
+        if: steps.changes.outputs.python_changed == 'true'
+        run: |
+          uv run ruff check src/ --output-format=github
+
+  lint-python-mailing-list-integration:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+        working-directory: ./services/apps/mailing_list_integration
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Check for Python file changes
+        id: changes
+        run: |
+          if git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -q "^services/apps/mailing_list_integration/.*\.py$"; then
+            echo "python_changed=true" >> $GITHUB_OUTPUT
+          else
+            echo "python_changed=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Install uv
+        if: steps.changes.outputs.python_changed == 'true'
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-dependency-glob: "services/apps/mailing_list_integration/uv.lock"
+
+      - name: Set up Python
+        if: steps.changes.outputs.python_changed == 'true'
+        run: uv python install 3.13
+
+      - name: Install dependencies
+        if: steps.changes.outputs.python_changed == 'true'
+        run: uv sync --group dev --frozen
+
       - name: Check Python linting and formatting
         if: steps.changes.outputs.python_changed == 'true'
         run: |

backend/package.json

@@ -33,7 +33,11 @@
     "script:fix-duplicate-members": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/fix-duplicate-members.ts",
     "script:fix-members-activities-after-unaffilation": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/fix-members-activities-after-unaffilation.ts",
     "script:process-bot-members": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/process-bot-members.ts",
-    "script:backfill-email-domain-member-organization-dates": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/backfill-email-domain-member-organization-dates.ts"
+    "script:backfill-email-domain-member-organization-dates": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/backfill-email-domain-member-organization-dates.ts",
+    "script:onboard-default-tenant": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/onboard-default-tenant.ts",
+    "script:onboard-default-tenant:local": "set -a && . ./.env.dist.local && . ./.env.override.local && set +a && pnpm run script:onboard-default-tenant",
+    "script:create-mailing-list-integration": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/create-mailing-list-integration.ts",
+    "script:create-mailing-list-integration:local": "set -a && . ./.env.dist.local && . ./.env.override.local && set +a && SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/create-mailing-list-integration.ts"
   },
   "lint-staged": {
     "**/*.ts": [

backend/src/api/integration/helpers/mailingListAuthenticate.ts

@@ -0,0 +1,25 @@
+import { z } from 'zod'
+
+import Permissions from '../../../security/permissions'
+import IntegrationService from '../../../services/integrationService'
+import PermissionChecker from '../../../services/user/permissionChecker'
+import { validateOrThrow } from '../../../utils/validation'
+
+const bodySchema = z.object({
+  lists: z
+    .array(
+      z.object({
+        name: z.string().trim().min(1),
+        sourceUrl: z.string().trim().min(1),
+      }),
+    )
+    .default([]),
+})
+
+export default async (req, res) => {
+  new PermissionChecker(req).validateHas(Permissions.values.tenantEdit)
+  const integrationData = validateOrThrow(bodySchema, req.body)
+
+  const payload = await new IntegrationService(req).mailingListConnectOrUpdate(integrationData)
+  await req.responseHandler.success(req, res, payload)
+}

backend/src/api/integration/index.ts

@@ -77,6 +77,8 @@ export default (app) => {
 
   // Git
   app.put(`/git-connect`, safeWrap(require('./helpers/gitAuthenticate').default))
+
+  app.put(`/mailing-list-connect`, safeWrap(require('./helpers/mailingListAuthenticate').default))
   app.put(`/confluence-connect`, safeWrap(require('./helpers/confluenceAuthenticate').default))
   app.put(`/gerrit-connect`, safeWrap(require('./helpers/gerritAuthenticate').default))
   app.get('/devto-validate', safeWrap(require('./helpers/devtoValidators').default))

backend/src/bin/scripts/create-mailing-list-integration.ts

@@ -0,0 +1,110 @@
+/* eslint-disable no-console */
+
+/* eslint-disable import/no-extraneous-dependencies */
+import commandLineArgs from 'command-line-args'
+import commandLineUsage from 'command-line-usage'
+import * as fs from 'fs'
+
+import { DEFAULT_TENANT_ID, generateUUIDv1 } from '@crowd/common'
+
+import SegmentRepository from '@/database/repositories/segmentRepository'
+import SequelizeRepository from '@/database/repositories/sequelizeRepository'
+import IntegrationService from '@/services/integrationService'
+
+const options = [
+  {
+    name: 'help',
+    alias: 'h',
+    type: Boolean,
+    description: 'Print this usage guide.',
+  },
+  {
+    name: 'file',
+    alias: 'f',
+    type: String,
+    description:
+      'Path to a JSON file with the lists array, each entry shaped as name + sourceUrl ' +
+      '(e.g. name "linux-serial", sourceUrl https://lore.kernel.org/linux-serial).',
+  },
+  {
+    name: 'segment',
+    alias: 's',
+    type: String,
+    description: "Segment id. Optional — defaults to the tenant's first subproject.",
+  },
+]
+
+const sections = [
+  {
+    header: 'Create mailing list integration',
+    content:
+      'Calls IntegrationService.mailingListConnectOrUpdate() directly to create/update the ' +
+      'mailinglist integration and onboard its lists for processing, until the frontend connect ' +
+      'UI ships (CM-1318).',
+  },
+  {
+    header: 'Options',
+    optionList: options,
+  },
+]
+
+const usage = commandLineUsage(sections)
+// pnpm forwards a literal `--` separator when args are passed via `pnpm run ... -- <args>`;
+// strip it so command-line-args doesn't choke on it.
+const argv = process.argv.slice(2).filter((arg) => arg !== '--')
+const parameters = commandLineArgs(options, { argv })
+
+if (parameters.help || !parameters.file) {
+  console.log(usage)
+  process.exit(parameters.help ? 0 : 1)
+} else {
+  setImmediate(async () => {
+    let fileContents: string
+    try {
+      fileContents = fs.readFileSync(parameters.file, 'utf-8')
+    } catch (err) {
+      console.error(`Could not read file at ${parameters.file}: ${(err as Error).message}`)
+      process.exit(1)
+    }
+
+    let lists
+    try {
+      lists = JSON.parse(fileContents)
+    } catch (err) {
+      console.error(`File at ${parameters.file} is not valid JSON: ${(err as Error).message}`)
+      process.exit(1)
+    }
+
+    const repoOptions = await SequelizeRepository.getDefaultIRepositoryOptions()
+    repoOptions.currentTenant = { id: DEFAULT_TENANT_ID }
+    ;(repoOptions as unknown as { requestId: string }).requestId = generateUUIDv1()
+
+    const adminUser = await repoOptions.database.user.findOne({ where: {} })
+    if (!adminUser) {
+      console.error('No user found — run script:onboard-default-tenant first.')
+      process.exit(1)
+    }
+    repoOptions.currentUser = adminUser
+
+    const segmentRepository = new SegmentRepository(repoOptions)
+    repoOptions.currentSegments = parameters.segment
+      ? await segmentRepository.findInIds([parameters.segment])
+      : (await segmentRepository.querySubprojects({ limit: 1, offset: 0 })).rows
+
+    if (repoOptions.currentSegments.length === 0) {
+      console.error('No segment found/resolved — pass --segment explicitly.')
+      process.exit(1)
+    }
+
+    console.log(`Segment: ${repoOptions.currentSegments[0].id} (${repoOptions.currentSegments[0].name})`)
+    console.log('Lists:', JSON.stringify(lists, null, 2))
+
+    const integration = await new IntegrationService(repoOptions).mailingListConnectOrUpdate(
+      { lists },
+      repoOptions,
+    )
+
+    console.log('Integration:', JSON.stringify(integration, null, 2))
+    process.exit(0)
+  })
+}

backend/src/bin/scripts/onboard-default-tenant.ts

@@ -0,0 +1,74 @@
+/* eslint-disable no-console */
+
+/* eslint-disable import/no-extraneous-dependencies */
+import commandLineArgs from 'command-line-args'
+import commandLineUsage from 'command-line-usage'
+
+import SequelizeRepository from '@/database/repositories/sequelizeRepository'
+import AuthService from '@/services/auth/authService'
+
+const options = [
+  {
+    name: 'help',
+    alias: 'h',
+    type: Boolean,
+    description: 'Print this usage guide.',
+  },
+  {
+    name: 'email',
+    alias: 'e',
+    type: String,
+    defaultValue: 'local-dev@example.com',
+    description: 'Email for the dev user driving the onboarding (default: local-dev@example.com)',
+  },
+]
+
+const sections = [
+  {
+    header: 'Onboard default tenant',
+    content:
+      'Runs the same onboarding logic as a real Auth0 signup (AuthService.signinFromSSO -> ' +
+      'handleOnboard -> TenantService.createOrJoinDefault), without needing a real Auth0 token. ' +
+      'Creates the default tenant, default segment, settings, and an admin user/tenantUser row. ' +
+      'Safe to re-run — createOrJoinDefault joins the existing tenant instead of duplicating it.',
+  },
+  {
+    header: 'Options',
+    optionList: options,
+  },
+]
+
+const usage = commandLineUsage(sections)
+const parameters = commandLineArgs(options)
+
+if (parameters.help) {
+  console.log(usage)
+} else {
+  setImmediate(async () => {
+    const repoOptions = await SequelizeRepository.getDefaultIRepositoryOptions()
+
+    await AuthService.signinFromSSO(
+      'auth0',
+      `dev|${parameters.email}`,
+      parameters.email,
+      true,
+      'Local',
+      'Dev',
+      'Local Dev',
+      null,
+      null,
+      null,
+      repoOptions,
+    )
+
+    const tenant = await repoOptions.database.tenant.findOne({ where: {} })
+    const segment = await repoOptions.database.segment.findOne({
+      where: { tenantId: tenant.id },
+    })
+
+    console.log(`Tenant: ${tenant.id} (${tenant.name})`)
+    console.log(`Segment: ${segment.id} (${segment.name})`)
+
+    process.exit(0)
+  })
+}

backend/src/database/migrations/V1784048135__mailinglist-schema.sql

@@ -0,0 +1,50 @@
+-- Create the mailinglist schema
+CREATE SCHEMA IF NOT EXISTS mailinglist;
+
+-- Mailing lists (lore/public-inbox lists onboarded into CDP)
+CREATE TABLE mailinglist.lists (
+    id UUID PRIMARY KEY NOT NULL DEFAULT uuid_generate_v4(),
+    "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+    "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+    "deletedAt" TIMESTAMP WITH TIME ZONE,
+
+    name TEXT NOT NULL,
+    "sourceUrl" TEXT NOT NULL,
+
+    "segmentId" UUID NOT NULL REFERENCES segments (id),
+    "integrationId" UUID NOT NULL REFERENCES public."integrations" (id),
+
+    UNIQUE ("sourceUrl")
+);
+
+-- Per-list processing state (public-inbox shard head tracking)
+CREATE TABLE mailinglist."listProcessing" (
+    "listId" UUID PRIMARY KEY NOT NULL REFERENCES mailinglist.lists (id) ON DELETE CASCADE,
+
+    "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+    "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+
+    state VARCHAR(50) NOT NULL,
+    priority INTEGER NOT NULL DEFAULT 2, -- 0=urgent, 1=high, 2=normal
+
+    "lockedAt" TIMESTAMP WITH TIME ZONE,
+    "lastProcessedAt" TIMESTAMP WITH TIME ZONE,
+    "lastProcessedHeads" JSONB NOT NULL DEFAULT '{}'
+);
+
+-- Indexes for optimal query performance
+
+CREATE INDEX "ix_mailinglist_lists_segmentId" ON mailinglist.lists ("segmentId");
+CREATE INDEX "ix_mailinglist_lists_integrationId" ON mailinglist.lists ("integrationId");
+
+CREATE INDEX "ix_mailinglist_listProcessing_state" ON mailinglist."listProcessing" (state);
+CREATE INDEX "ix_mailinglist_listProcessing_state_priority" ON mailinglist."listProcessing" (state, priority);
+
+-- Comments for documentation
+COMMENT ON SCHEMA mailinglist IS 'Schema for mailing list integration system that manages mailing list processing state';
+COMMENT ON TABLE mailinglist.lists IS 'Stores onboarded mailing lists (public-inbox/lore) and their segment/integration associations';
+COMMENT ON TABLE mailinglist."listProcessing" IS 'Per-list processing state including public-inbox shard head tracking';
+
+COMMENT ON COLUMN mailinglist."listProcessing".priority IS 'Processing priority: 0=urgent, 1=high, 2=normal';
+COMMENT ON COLUMN mailinglist."listProcessing".state IS 'Current processing state of the list';
+COMMENT ON COLUMN mailinglist."listProcessing"."lastProcessedHeads" IS 'JSONB map of shard index to last processed commit SHA, e.g. {"0": sha, "1": sha}';

backend/src/database/migrations/V1784291394__add_mailinglist_activity_type.sql

@@ -0,0 +1,3 @@
+-- Mailing list (Collaboration)
+INSERT INTO "activityTypes" ("activityType", platform, "isCodeContribution", "isCollaboration", description) VALUES
+('message', 'mailinglist', false, true, 'Sent a message to a mailing list');