From eb844bda96ca15bbd8574a35327fed6b7ac1f063 Mon Sep 17 00:00:00 2001 From: Chris Bongers Date: Fri, 19 Jun 2026 13:55:38 +0200 Subject: [PATCH 1/2] fix: normalize trailing slashes in standardizeURL Treat `.../foo` and `.../foo/` as the same article so they no longer create separate posts. Trims trailing slashes from non-root paths, matching yggdrasil's trailing-slash normalization (root slash preserved). Co-Authored-By: Claude Opus 4.8 --- __tests__/common/standardizeURL.ts | 30 ++++++++++++++++++++++++++++++ src/common/links.ts | 4 +++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/__tests__/common/standardizeURL.ts b/__tests__/common/standardizeURL.ts index e0b87ace00..6141bbe271 100644 --- a/__tests__/common/standardizeURL.ts +++ b/__tests__/common/standardizeURL.ts @@ -13,6 +13,36 @@ describe('standardizeURL', () => { expect(canonicalUrl).toBe('https://test.com/posts/1'); }); + it('should strip a trailing slash', () => { + const { url, canonicalUrl } = standardizeURL( + 'https://github.com/versity/versitygw/', + ); + expect(url).toBe('https://github.com/versity/versitygw'); + expect(canonicalUrl).toBe('https://github.com/versity/versitygw'); + }); + + it('should strip multiple trailing slashes', () => { + const { url, canonicalUrl } = standardizeURL( + 'https://github.com/versity/versitygw//', + ); + expect(url).toBe('https://github.com/versity/versitygw'); + expect(canonicalUrl).toBe('https://github.com/versity/versitygw'); + }); + + it('should preserve a root trailing slash', () => { + const { url, canonicalUrl } = standardizeURL('https://github.com/'); + expect(url).toBe('https://github.com/'); + expect(canonicalUrl).toBe('https://github.com/'); + }); + + it('should strip trailing slash before query params', () => { + const { url, canonicalUrl } = standardizeURL( + 'https://test.com/posts/1/?sk=google', + ); + expect(url).toBe('https://test.com/posts/1?sk=google'); + expect(canonicalUrl).toBe('https://test.com/posts/1'); + }); + it('should clean query params', () => { const { url, canonicalUrl } = standardizeURL( 'https://test.com/posts/1?utm_source=google', diff --git a/src/common/links.ts b/src/common/links.ts index 512697c2b8..19e5f4d65a 100644 --- a/src/common/links.ts +++ b/src/common/links.ts @@ -102,7 +102,9 @@ export const standardizeURL = ( ): { url: string; canonicalUrl: string } => { const domain = subtractDomain(inputUrl); - const [canonicalUrl, params] = inputUrl.split('?'); + const [path, params] = inputUrl.split('?'); + // Trim trailing slashes from non-root paths, matching yggdrasil's normalization + const canonicalUrl = path.replace(/^(https?:\/\/[^/]+\/.+?)\/+$/i, '$1'); const searchParams = new URLSearchParams(params); const isAllowedDomain = domain && domain in domainAllowedSearchParams; From 8c89f3e11a1cb3ecd55f82223df3643be298ec5d Mon Sep 17 00:00:00 2001 From: Chris Bongers Date: Fri, 19 Jun 2026 15:46:02 +0200 Subject: [PATCH 2/2] fix: dedup trailing-slash urls without rewriting the submitted url Revert the standardizeURL trailing-slash trimming so the submitted url is persisted verbatim. Instead, dedup at lookup time: getUrlTrailingSlashVariants crosses with the existing www variants (getUrlDedupVariants), so a single lookup matches both slash and no-slash forms without changing what we store. Co-Authored-By: Claude Opus 4.8 (1M context) --- __tests__/common/standardizeURL.ts | 92 +++++++++++++++++++++--------- src/common/links.ts | 35 +++++++++++- src/common/post.ts | 10 ++-- 3 files changed, 103 insertions(+), 34 deletions(-) diff --git a/__tests__/common/standardizeURL.ts b/__tests__/common/standardizeURL.ts index 6141bbe271..5104e2a8dd 100644 --- a/__tests__/common/standardizeURL.ts +++ b/__tests__/common/standardizeURL.ts @@ -1,4 +1,9 @@ -import { domainAllowedSearchParams, standardizeURL } from '../../src/common'; +import { + domainAllowedSearchParams, + getUrlDedupVariants, + getUrlTrailingSlashVariants, + standardizeURL, +} from '../../src/common'; describe('standardizeURL', () => { it('should keep url without query', () => { @@ -13,34 +18,12 @@ describe('standardizeURL', () => { expect(canonicalUrl).toBe('https://test.com/posts/1'); }); - it('should strip a trailing slash', () => { + it('should preserve a trailing slash on the submitted url', () => { const { url, canonicalUrl } = standardizeURL( 'https://github.com/versity/versitygw/', ); - expect(url).toBe('https://github.com/versity/versitygw'); - expect(canonicalUrl).toBe('https://github.com/versity/versitygw'); - }); - - it('should strip multiple trailing slashes', () => { - const { url, canonicalUrl } = standardizeURL( - 'https://github.com/versity/versitygw//', - ); - expect(url).toBe('https://github.com/versity/versitygw'); - expect(canonicalUrl).toBe('https://github.com/versity/versitygw'); - }); - - it('should preserve a root trailing slash', () => { - const { url, canonicalUrl } = standardizeURL('https://github.com/'); - expect(url).toBe('https://github.com/'); - expect(canonicalUrl).toBe('https://github.com/'); - }); - - it('should strip trailing slash before query params', () => { - const { url, canonicalUrl } = standardizeURL( - 'https://test.com/posts/1/?sk=google', - ); - expect(url).toBe('https://test.com/posts/1?sk=google'); - expect(canonicalUrl).toBe('https://test.com/posts/1'); + expect(url).toBe('https://github.com/versity/versitygw/'); + expect(canonicalUrl).toBe('https://github.com/versity/versitygw/'); }); it('should clean query params', () => { @@ -81,3 +64,60 @@ describe('standardizeURL', () => { }); }); }); + +describe('getUrlTrailingSlashVariants', () => { + it('should return both slash forms for a url without a trailing slash', () => { + expect( + getUrlTrailingSlashVariants('https://github.com/versity/versitygw'), + ).toEqual([ + 'https://github.com/versity/versitygw', + 'https://github.com/versity/versitygw/', + ]); + }); + + it('should return both slash forms for a url with a trailing slash', () => { + expect( + getUrlTrailingSlashVariants('https://github.com/versity/versitygw/'), + ).toEqual([ + 'https://github.com/versity/versitygw', + 'https://github.com/versity/versitygw/', + ]); + }); + + it('should collapse multiple trailing slashes', () => { + expect( + getUrlTrailingSlashVariants('https://github.com/versity/versitygw//'), + ).toEqual([ + 'https://github.com/versity/versitygw', + 'https://github.com/versity/versitygw/', + ]); + }); + + it('should preserve query params on both forms', () => { + expect( + getUrlTrailingSlashVariants('https://test.com/posts/1/?sk=google'), + ).toEqual([ + 'https://test.com/posts/1?sk=google', + 'https://test.com/posts/1/?sk=google', + ]); + }); + + it('should not toggle a root url', () => { + expect(getUrlTrailingSlashVariants('https://github.com/')).toEqual([ + 'https://github.com/', + ]); + }); +}); + +describe('getUrlDedupVariants', () => { + it('should cross www and trailing-slash variants', () => { + expect( + getUrlDedupVariants('https://github.com/versity/versitygw/'), + ).toEqual([ + 'https://github.com/versity/versitygw', + 'https://github.com/versity/versitygw/', + 'https://www.github.com/versity/versitygw', + 'https://www.github.com/versity/versitygw/', + ]); + }); +}); diff --git a/src/common/links.ts b/src/common/links.ts index 19e5f4d65a..f40b14ee89 100644 --- a/src/common/links.ts +++ b/src/common/links.ts @@ -97,14 +97,43 @@ export const getUrlWwwVariants = (url: string): string[] => { return [url, www ? `${scheme}${rest}` : `${scheme}www.${rest}`]; }; +// The same article is often reachable with or without a trailing slash. +// Return the url alongside its trailing-slash-toggled form so dedup lookups +// can match either without rewriting the url we persist. Root-only urls +// (`https://host/`) are left untouched. +export const getUrlTrailingSlashVariants = (url: string): string[] => { + const [path, params] = url.split('?'); + const trimmed = path.replace(/^(https?:\/\/[^/]+\/.+?)\/+$/i, '$1'); + + // No real (non-root) path to toggle. + if (!/^https?:\/\/[^/]+\/.+/i.test(trimmed)) { + return [url]; + } + + const suffix = params !== undefined ? `?${params}` : ''; + return [`${trimmed}${suffix}`, `${trimmed}/${suffix}`]; +}; + +// All url variants we treat as the same article for dedup lookups: every +// www.-toggled form crossed with its trailing-slash-toggled form. +export const getUrlDedupVariants = (url: string): string[] => { + const variants = new Set(); + + for (const wwwVariant of getUrlWwwVariants(url)) { + for (const slashVariant of getUrlTrailingSlashVariants(wwwVariant)) { + variants.add(slashVariant); + } + } + + return [...variants]; +}; + export const standardizeURL = ( inputUrl: string, ): { url: string; canonicalUrl: string } => { const domain = subtractDomain(inputUrl); - const [path, params] = inputUrl.split('?'); - // Trim trailing slashes from non-root paths, matching yggdrasil's normalization - const canonicalUrl = path.replace(/^(https?:\/\/[^/]+\/.+?)\/+$/i, '$1'); + const [canonicalUrl, params] = inputUrl.split('?'); const searchParams = new URLSearchParams(params); const isAllowedDomain = domain && domain in domainAllowedSearchParams; diff --git a/src/common/post.ts b/src/common/post.ts index 3b1c8d643a..0788f18735 100644 --- a/src/common/post.ts +++ b/src/common/post.ts @@ -25,7 +25,7 @@ import { WelcomePost, } from '../entity'; import { ForbiddenError, ValidationError } from 'apollo-server-errors'; -import { getUrlWwwVariants, isValidHttpUrl, standardizeURL } from './links'; +import { getUrlDedupVariants, isValidHttpUrl, standardizeURL } from './links'; import { findMarkdownTag, markdown, @@ -916,8 +916,8 @@ export const getExistingPost = async ( .createQueryBuilder(Post, 'post') .select(['post.id', 'post.deleted', 'post.visible']) .where([ - { canonicalUrl: In(getUrlWwwVariants(canonicalUrl)) }, - { url: In(getUrlWwwVariants(url)) }, + { canonicalUrl: In(getUrlDedupVariants(canonicalUrl)) }, + { url: In(getUrlDedupVariants(url)) }, ]) .getOne(); @@ -966,8 +966,8 @@ export const findPostByUrl = async ( ]); } else { queryBuilder = queryBuilder.andWhere([ - { canonicalUrl: In(getUrlWwwVariants(canonicalUrl)) }, - { url: In(getUrlWwwVariants(cleanUrl)) }, + { canonicalUrl: In(getUrlDedupVariants(canonicalUrl)) }, + { url: In(getUrlDedupVariants(cleanUrl)) }, ]); }